From 8c71fd284065f4405ae4f8c341b07c365029992f Mon Sep 17 00:00:00 2001 From: pbrw Date: Tue, 25 Apr 2023 23:14:21 +0200 Subject: [PATCH 01/20] Update deprecated mc admin commands --- minio/minioadmin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 5127acfe8..d30dd6de5 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -127,7 +127,7 @@ def group_list(self): def policy_add(self, policy_name, policy_file): """Add new policy.""" return self._run( - ["policy", "add", self._target, policy_name, policy_file], + ["policy", "create", self._target, policy_name, policy_file], ) def policy_remove(self, policy_name): @@ -147,8 +147,8 @@ def policy_set(self, policy_name, user=None, group=None): if (user is not None) ^ (group is not None): return self._run( [ - "policy", "set", self._target, policy_name, - ("user=" if user else "group=") + (user or group), + "policy", "attach", self._target, policy_name, + "--user" if user else "--group", user or group, ], ) raise ValueError("either user or group must be set") @@ -158,8 +158,8 @@ def policy_unset(self, policy_name, user=None, group=None): if (user is not None) ^ (group is not None): return self._run( [ - "policy", "unset", self._target, policy_name, - ("user=" if user else "group=") + (user or group), + "policy", "detach", self._target, policy_name, + "--user" if user else "--group", user or group, ], ) raise ValueError("either user or group must be set") From aca09558f8727fdd90bf2c42ffcd01ae2497fef7 Mon Sep 17 00:00:00 2001 From: pbrw Date: Tue, 25 Jul 2023 01:42:20 +0200 Subject: [PATCH 02/20] Cryptography to read/write encrypted MinIO Admin payload --- .github/workflows/ci.yml | 2 +- Makefile | 2 +- minio/crypto.py | 146 ++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- tests/unit/crypto_test.py | 35 +++++++++ 5 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 minio/crypto.py create mode 100644 tests/unit/crypto_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33b636bb3..328209484 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install certifi urllib3 mock pytest + pip install certifi urllib3 mock pytest argon2-cffi pycryptodome - name: Run check if Ubuntu if: matrix.os == 'ubuntu-latest' run: | diff --git a/Makefile b/Makefile index ee466b8f5..fc1375653 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ default: tests getdeps: @echo "Installing required dependencies" - @pip install --user --upgrade autopep8 certifi pytest pylint urllib3 + @pip install --user --upgrade autopep8 certifi pytest pylint urllib3 argon2-cffi pycryptodome check: getdeps @echo "Running checks" diff --git a/minio/crypto.py b/minio/crypto.py new file mode 100644 index 000000000..b8611076f --- /dev/null +++ b/minio/crypto.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2015, 2016, 2017 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=too-many-lines,disable=too-many-branches,too-many-statements +# pylint: disable=too-many-arguments + +"""Cryptography to read and write encrypted MinIO Admin payload""" + +import os +from Crypto.Cipher import AES +from Crypto.Cipher import ChaCha20_Poly1305 +from argon2.low_level import hash_secret_raw, Type + +_NONCE_LEN = 8 +_SALT_LEN = 32 + + +class AesGcmCipherProvider: + """AES-GCM cipher provider""" + @staticmethod + def get_cipher(key: bytes, nonce: bytes): + """Get cipher""" + return AES.new(key, AES.MODE_GCM, nonce) + + +class ChaCha20Poly1305CipherProvider: + """ChaCha20Poly1305 cipher provider""" + @staticmethod + def get_cipher(key: bytes, nonce: bytes): + """Get cipher""" + return ChaCha20_Poly1305.new(key=key, nonce=nonce) + + +def encrypt(payload: bytes, password: str) -> bytes: + """ + Encrypts data using AES-GCM using a 256-bit Argon2ID key. + To see the original implementation in Go, check out the madmin-go library + (https://github.com/minio/madmin-go/blob/main/encrypt.go#L38) + """ + cipher_provider = AesGcmCipherProvider() + nonce = os.urandom(_NONCE_LEN) + salt = os.urandom(_SALT_LEN) + + padded_nonce = [0] * (_NONCE_LEN + 4) + padded_nonce[:_NONCE_LEN] = nonce + + key = _generate_key(password.encode(), salt) + additional_data = _generate_additional_data( + cipher_provider, key, bytes(padded_nonce)) + + padded_nonce[8] = 0x01 + padded_nonce = bytes(padded_nonce) + + cipher = cipher_provider.get_cipher(key, padded_nonce) + cipher.update(additional_data) + encrypted_data, mac = cipher.encrypt_and_digest(payload) + + payload = salt + payload += bytes([0x00]) + payload += nonce + payload += encrypted_data + payload += mac + + return bytes(payload) + + +def decrypt(payload: bytes, password: str) -> bytes: + """ + Decrypts data using AES-GCM or ChaCha20Poly1305 using a + 256-bit Argon2ID key. To see the original implementation in Go, + check out the madmin-go library + (https://github.com/minio/madmin-go/blob/main/encrypt.go#L38) + """ + pos = 0 + salt = payload[pos:pos+_SALT_LEN] + pos += _SALT_LEN + + cipher_id = payload[pos] + if cipher_id == 0: + cipher_provider = AesGcmCipherProvider() + elif cipher_id == 1: + cipher_provider = ChaCha20Poly1305CipherProvider() + else: + return None + + pos += 1 + + nonce = payload[pos:pos+_NONCE_LEN] + pos += _NONCE_LEN + + encrypted_data = payload[pos:-16] + hmac_tag = payload[-16:] + + key = _generate_key(password.encode(), salt) + + padded_nonce = [0] * 12 + padded_nonce[:_NONCE_LEN] = nonce + + additional_data = _generate_additional_data( + cipher_provider, key, bytes(padded_nonce)) + padded_nonce[8] = 1 + + cipher = cipher_provider.get_cipher(key, bytes(padded_nonce)) + + cipher.update(additional_data) + decrypted_data = cipher.decrypt_and_verify(encrypted_data, hmac_tag) + + return decrypted_data + + +def _generate_additional_data(cipher_provider, key: bytes, + padded_nonce: bytes) -> bytes: + """Generate additional data""" + cipher = cipher_provider.get_cipher(key, padded_nonce) + tag = cipher.digest() + new_tag = [0] * 17 + new_tag[1:] = tag + new_tag[0] = 0x80 + return bytes(new_tag) + + +def _generate_key(password: bytes, salt: bytes) -> bytes: + """Generate 256-bit Argon2ID key""" + return hash_secret_raw( + secret=password, + salt=salt, + time_cost=1, + memory_cost=65536, + parallelism=4, + hash_len=32, + type=Type.ID, + version=19 + ) diff --git a/setup.py b/setup.py index 544d68d00..28b01e8ea 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ long_description_content_type="text/markdown", package_dir={"minio": "minio"}, packages=["minio", "minio.credentials"], - install_requires=["certifi", "urllib3"], + install_requires=["certifi", "urllib3", "argon2-cffi", "pycryptodome"], tests_require=[], license="Apache-2.0", classifiers=[ diff --git a/tests/unit/crypto_test.py b/tests/unit/crypto_test.py new file mode 100644 index 000000000..c0d3325d0 --- /dev/null +++ b/tests/unit/crypto_test.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, +# (C) 2015 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import TestCase + +from minio.crypto import encrypt, decrypt + + +class CryptoTest(TestCase): + def test_correct(self): + secret = "topsecret" + plaintext = "Hello MinIO!" + encrypted = encrypt(plaintext.encode(), secret) + decrypted = decrypt(encrypted, secret).decode() + self.assertEquals(plaintext, decrypted) + + def test_wrong(self): + secret = "topsecret" + secret2 = "othersecret" + plaintext = "Hello MinIO!" + encrypted = encrypt(plaintext.encode(), secret) + self.assertRaises(ValueError, decrypt, encrypted, secret2) From 26b12582b2d417a7133b29703cb14fba7ce19f58 Mon Sep 17 00:00:00 2001 From: pbrw Date: Mon, 21 Aug 2023 13:55:09 +0200 Subject: [PATCH 03/20] Basic MinIO user management API --- minio/error.py | 16 ++ minio/helpers.py | 65 +++++ minio/minioadmin.py | 619 ++++++++++++++++++++++++++------------------ 3 files changed, 452 insertions(+), 248 deletions(-) diff --git a/minio/error.py b/minio/error.py index f0fa86467..7b9bed9d8 100644 --- a/minio/error.py +++ b/minio/error.py @@ -53,6 +53,22 @@ def __reduce__(self): return type(self), (self._code, self._content_type, self._body) +class AdminResponseError(Exception): + """Raised to indicate error response from server.""" + + def __init__(self, code, content_type, body): + self._code = code + self._content_type = content_type + self._body = body + super().__init__( + f"Error admin response from server; " + f"Response code: {code}, Body: {body}" + ) + + def __reduce__(self): + return type(self), (self._code, self._content_type, self._body) + + class ServerError(MinioException): """Raised to indicate that S3 service returning HTTP server error.""" diff --git a/minio/helpers.py b/minio/helpers.py index 712050a2c..2448de42c 100644 --- a/minio/helpers.py +++ b/minio/helpers.py @@ -650,6 +650,71 @@ def build( return url_replace(url, netloc=netloc, path=path) +class AdminURL: + """URL of Minio Admin endpoint""" + + def __init__(self, endpoint): + url = urllib.parse.urlsplit(endpoint) + host = url.hostname + + if url.scheme.lower() not in ["http", "https"]: + raise ValueError("scheme in endpoint must be http or https") + + url = url_replace(url, scheme=url.scheme.lower()) + + if url.path and url.path != "/": + raise ValueError("path in endpoint is not allowed") + + url = url_replace(url, path="") + + if url.query: + raise ValueError("query in endpoint is not allowed") + + if url.fragment: + raise ValueError("fragment in endpoint is not allowed") + + try: + url.port + except ValueError as exc: + raise ValueError("invalid port") from exc + + if url.username: + raise ValueError("username in endpoint is not allowed") + + if url.password: + raise ValueError("password in endpoint is not allowed") + + if ( + (url.scheme == "http" and url.port == 80) or + (url.scheme == "https" and url.port == 443) + ): + url = url_replace(url, netloc=host) + + self._url = url + + @property + def is_https(self): + """Check if scheme is HTTPS.""" + return self._url.scheme == "https" + + def build( + self, path, query_params=None + ): + """Build URL for given information.""" + url = url_replace(self._url, path=path) + + query = [] + for key, values in sorted((query_params or {}).items()): + values = values if isinstance(values, (list, tuple)) else [values] + query += [ + f"{queryencode(key)}={queryencode(value)}" + for value in sorted(values) + ] + url = url_replace(url, query="&".join(query)) + + return url + + class ObjectWriteResult: """Result class of any APIs doing object creation.""" diff --git a/minio/minioadmin.py b/minio/minioadmin.py index d30dd6de5..dd3ee70ae 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -20,278 +20,401 @@ from __future__ import absolute_import -import json -import subprocess +from datetime import timedelta +from urllib.parse import urlunsplit +import os + +import certifi +import urllib3 + +from . import time + +from .credentials.providers import StaticProvider +from .error import AdminResponseError +from .helpers import AdminURL, sha256_hash +from .signer import sign_v4_s3 + + +_ADMIN_PATH_PREFIX = "/minio/admin/v3" class MinioAdmin: """MinIO Admin wrapper using MinIO Client (mc) tool.""" - def __init__( - self, target, - binary_path=None, config_dir=None, ignore_cert_check=False, - timeout=None, env=None, - ): - self._target = target - self._timeout = timeout - self._env = env - self._base_args = [binary_path or "mc", "--json"] - if config_dir: - self._base_args += ["--config-dir", config_dir] - if ignore_cert_check: - self._base_args.append("--insecure") - self._base_args.append("admin") - - def _run(self, args, multiline=False): - """Execute mc command and return JSON output.""" - proc = subprocess.run( - self._base_args + args, - capture_output=True, - timeout=self._timeout, - env=self._env, - check=True, - text=True, + def __init__(self, endpoint, access_key, + secret_key, + secure=True, + cert_check=True): + self._base_url = AdminURL( + ("https://" if secure else "http://") + endpoint ) - if not proc.stdout: - return [] if multiline else {} - if multiline: - return [json.loads(line) for line in proc.stdout.splitlines()] - return json.loads(proc.stdout) - - def service_restart(self): - """Restart MinIO service.""" - return self._run(["service", "restart", self._target]) - - def service_stop(self): - """Stop MinIO service.""" - return self._run(["service", "stop", self._target]) - - def update(self): - """Update MinIO.""" - return self._run(["update", self._target]) - - def info(self): - """Get MinIO server information.""" - return self._run(["info", self._target]) - - def user_add(self, access_key, secret_key): - """Add a new user.""" - return self._run(["user", "add", self._target, access_key, secret_key]) - - def user_disable(self, access_key): - """Disable user.""" - return self._run(["user", "disable", self._target, access_key]) - - def user_enable(self, access_key): - """Enable user.""" - return self._run(["user", "enable", self._target, access_key]) - - def user_remove(self, access_key): - """Remove user.""" - return self._run(["user", "remove", self._target, access_key]) - - def user_info(self, access_key): - """Get user information.""" - return self._run(["user", "info", self._target, access_key]) - - def user_list(self): - """List users.""" - return self._run(["user", "list", self._target], multiline=True) - - def group_add(self, group_name, members): - """Add users a new or existing group.""" - return self._run(["group", "add", self._target, group_name] + members) - - def group_disable(self, group_name): - """Disable group.""" - return self._run(["group", "disable", self._target, group_name]) - - def group_enable(self, group_name): - """Enable group.""" - return self._run(["group", "enable", self._target, group_name]) - - def group_remove(self, group_name, members=None): - """Remove group or members from a group.""" - return self._run( - ["group", "remove", self._target, group_name] + (members or []), + self._credentials = StaticProvider(access_key, secret_key).retrieve() + + timeout = timedelta(minutes=5).seconds + self._http = urllib3.PoolManager( + timeout=urllib3.util.Timeout(connect=timeout, read=timeout), + maxsize=10, + cert_reqs='CERT_REQUIRED' if cert_check else 'CERT_NONE', + ca_certs=os.environ.get('SSL_CERT_FILE') or certifi.where(), + retries=urllib3.Retry( + total=5, + backoff_factor=0.2, + status_forcelist=[500, 502, 503, 504] + ) ) - def group_info(self, group_name): - """Get group information.""" - return self._run(["group", "info", self._target, group_name]) + def __del__(self): + self._http.clear() + + def _build_headers(self, host, body): + """Build headers with given parameters.""" + headers = {} + headers["Host"] = host + sha256 = None + + if self._base_url.is_https: + sha256 = "UNSIGNED-PAYLOAD" + else: + sha256 = sha256_hash(body) + if sha256: + headers["x-amz-content-sha256"] = sha256 + date = time.utcnow() + headers["x-amz-date"] = time.to_amz_date(date) + return headers, date + + def _build_signed_headers(self, url, body, method): + """Build signed headers""" + headers, date = self._build_headers(url.netloc, body) + headers = sign_v4_s3( + method, + url, + '', + headers, + self._credentials, + headers.get("x-amz-content-sha256"), + date, + ) - def group_list(self): - """List groups.""" - return self._run(["group", "list", self._target], multiline=True) + return headers - def policy_add(self, policy_name, policy_file): - """Add new policy.""" - return self._run( - ["policy", "create", self._target, policy_name, policy_file], + def _send_request(self, method, url, body): + """Send HTTP request with given parameters""" + + headers = self._build_signed_headers( + url, + body, + method ) - def policy_remove(self, policy_name): - """Remove policy.""" - return self._run(["policy", "remove", self._target, policy_name]) - - def policy_info(self, policy_name): - """Get policy information.""" - return self._run(["policy", "info", self._target, policy_name]) - - def policy_list(self): - """List policies.""" - return self._run(["policy", "list", self._target], multiline=True) - - def policy_set(self, policy_name, user=None, group=None): - """Set IAM policy on a user or group.""" - if (user is not None) ^ (group is not None): - return self._run( - [ - "policy", "attach", self._target, policy_name, - "--user" if user else "--group", user or group, - ], - ) - raise ValueError("either user or group must be set") - - def policy_unset(self, policy_name, user=None, group=None): - """Unset an IAM policy for a user or group.""" - if (user is not None) ^ (group is not None): - return self._run( - [ - "policy", "detach", self._target, policy_name, - "--user" if user else "--group", user or group, - ], - ) - raise ValueError("either user or group must be set") + return self._http.urlopen( + method=method, + url=urlunsplit(url), + body=body, + headers=_convert_to_urllib3_headers(headers) + ) - def config_get(self, key=None): - """Get configuration parameters.""" - return self._run( - ["config", "get", self._target] + [key] if key else [], - key, + def _url_open( + self, + method, + path, + body=None, + query_params=None, + ): + """Execute HTTP request.""" + url = self._base_url.build( + path=_ADMIN_PATH_PREFIX + path, + query_params=query_params, ) - def config_set(self, key, config): - """Set configuration parameters.""" - args = [name + "=" + value for name, value in config.items()] - return self._run(["config", "set", self._target, key] + args) - - def config_reset(self, key, name=None): - """Reset configuration parameters.""" - if name: - key += ":" + name - return self._run(["config", "reset", self._target, key]) - - def config_remove(self, access_key): - """Remove config.""" - return self._run(["config", "remove", self._target, access_key]) - - def config_history(self): - """Get historic configuration changes.""" - return self._run(["config", "history", self._target], multiline=True) - - def config_restore(self, restore_id): - """Restore to a specific configuration history.""" - return self._run(["config", "restore", self._target, restore_id]) - - def profile_start(self, profilers=()): - """Start recording profile data.""" - args = ["profile", "start"] - if profilers: - args += ["--type", ",".join(profilers)] - args.append(self._target) - return self._run(args) - - def profile_stop(self): - """Stop and download profile data.""" - return self._run(["profile", "stop", self._target]) - - def top_locks(self): - """Get a list of the 10 oldest locks on a MinIO cluster.""" - return self._run(["top", "locks", self._target], multiline=True) - - def prometheus_generate(self): - """Generate prometheus configuration.""" - return self._run(["prometheus", "generate", self._target]) - - def kms_key_create(self, key=None): - """Create a new KMS master key.""" - return self._run( - [ - "kms", "key", "create", self._target, key - ] + ([key] if key else []), + response = self._send_request( + method, + url, + body, ) - def kms_key_status(self, key=None): - """Get status information of a KMS master key.""" - return self._run( - [ - "kms", "key", "status", self._target, key - ] + ([key] if key else []), + if response.status in [200, 204, 206]: + return response + + raise AdminResponseError( + response.status, + response.headers.get("content-type"), + response.data.decode() if response.data else None ) - def bucket_remote_add( - self, src_bucket, dest_url, - path=None, region=None, bandwidth=None, service=None, + def _execute( + self, + method, + path, + body=None, + query_params=None, ): - """Add a new remote target.""" - args = [ - "bucket", "remote", "add", self._target + "/" + src_bucket, - dest_url, "--service", service or "replication", - ] - if path: - args += ["--path", path] - if region: - args += ["--region", region] - if bandwidth: - args += ["--bandwidth", bandwidth] - return self._run(args) - - def bucket_remote_edit(self, src_bucket, dest_url, arn): - """Edit credentials of remote target.""" - return self._run( - [ - "bucket", "remote", "edit", self._target + "/" + src_bucket, - dest_url, "--arn", arn, - ], + """Execute HTTP request.""" + url = self._base_url.build( + path=_ADMIN_PATH_PREFIX + path, + query_params=query_params, ) - def bucket_remote_list(self, src_bucket=None, service=None): - """List remote targets.""" - return self._run( - [ - "bucket", "remote", "ls", - self._target + ("/" + src_bucket if src_bucket else ""), - "--service", service or "replication", - ], + response = self._send_request( + method, + url, + body, ) - def bucket_remote_remove(self, src_bucket, arn): - """Remove configured remote target.""" - return self._run( - [ - "bucket", "remote", "rm", self._target + "/" + src_bucket, - "--arn", arn, - ], - ) + if response.status in [200, 204, 206]: + return response - def bucket_quota_set(self, bucket, fifo=None, hard=None): - """Set bucket quota configuration.""" - if fifo is None and hard is None: - raise ValueError("fifo or hard must be set") - args = ["bucket", "quota", self._target + "/" + bucket] - if fifo: - args += ["--fifo", fifo] - if hard: - args += ["--hard", hard] - return self._run(args) - - def bucket_quota_clear(self, bucket): - """Clear bucket quota configuration.""" - return self._run( - ["bucket", "quota", self._target + "/" + bucket, "--clear"], + raise AdminResponseError( + response.status, + response.headers.get("content-type"), + response.data.decode() if response.data else None ) - def bucket_quota_get(self, bucket): - """Get bucket quota configuration.""" - return self._run(["bucket", "quota", self._target + "/" + bucket]) + # def service_restart(self): + # """Restart MinIO service.""" + # return self._run(["service", "restart", self._target]) + + # def service_stop(self): + # """Stop MinIO service.""" + # return self._run(["service", "stop", self._target]) + + # def update(self): + # """Update MinIO.""" + # return self._run(["update", self._target]) + + # def info(self): + # """Get MinIO server information.""" + # return self._run(["info", self._target]) + + # def user_add(self, access_key, secret_key): + # """Add a new user.""" + # return self._run(["user", "add", self._target, + # access_key, secret_key]) + + # def user_disable(self, access_key): + # """Disable user.""" + # return self._run(["user", "disable", self._target, access_key]) + + # def user_enable(self, access_key): + # """Enable user.""" + # return self._run(["user", "enable", self._target, access_key]) + + # def user_remove(self, access_key): + # """Remove user.""" + # return self._run(["user", "remove", self._target, access_key]) + + # def user_info(self, access_key): + # """Get user information.""" + # return self._run(["user", "info", self._target, access_key]) + + # def user_list(self): + # """List users.""" + # return self._run(["user", "list", self._target], multiline=True) + + # def group_add(self, group_name, members): + # """Add users a new or existing group.""" + # return self._run(["group", "add", self._target, group_name] + members) + + # def group_disable(self, group_name): + # """Disable group.""" + # return self._run(["group", "disable", self._target, group_name]) + + # def group_enable(self, group_name): + # """Enable group.""" + # return self._run(["group", "enable", self._target, group_name]) + + # def group_remove(self, group_name, members=None): + # """Remove group or members from a group.""" + # return self._run( + # ["group", "remove", self._target, group_name] + (members or []), + # ) + + # def group_info(self, group_name): + # """Get group information.""" + # return self._run(["group", "info", self._target, group_name]) + + # def group_list(self): + # """List groups.""" + # return self._run(["group", "list", self._target], multiline=True) + + # def policy_add(self, policy_name, policy_file): + # """Add new policy.""" + # return self._run( + # ["policy", "create", self._target, policy_name, policy_file], + # ) + + # def policy_remove(self, policy_name): + # """Remove policy.""" + # return self._run(["policy", "remove", self._target, policy_name]) + + # def policy_info(self, policy_name): + # """Get policy information.""" + # return self._run(["policy", "info", self._target, policy_name]) + + # def policy_list(self): + # """List policies.""" + # return self._run(["policy", "list", self._target], multiline=True) + + # def policy_set(self, policy_name, user=None, group=None): + # """Set IAM policy on a user or group.""" + # if (user is not None) ^ (group is not None): + # return self._run( + # [ + # "policy", "attach", self._target, policy_name, + # "--user" if user else "--group", user or group, + # ], + # ) + # raise ValueError("either user or group must be set") + + # def policy_unset(self, policy_name, user=None, group=None): + # """Unset an IAM policy for a user or group.""" + # if (user is not None) ^ (group is not None): + # return self._run( + # [ + # "policy", "detach", self._target, policy_name, + # "--user" if user else "--group", user or group, + # ], + # ) + # raise ValueError("either user or group must be set") + + # def config_get(self, key=None): + # """Get configuration parameters.""" + # return self._run( + # ["config", "get", self._target] + [key] if key else [], + # key, + # ) + + # def config_set(self, key, config): + # """Set configuration parameters.""" + # args = [name + "=" + value for name, value in config.items()] + # return self._run(["config", "set", self._target, key] + args) + + # def config_reset(self, key, name=None): + # """Reset configuration parameters.""" + # if name: + # key += ":" + name + # return self._run(["config", "reset", self._target, key]) + + # def config_remove(self, access_key): + # """Remove config.""" + # return self._run(["config", "remove", self._target, access_key]) + + # def config_history(self): + # """Get historic configuration changes.""" + # return self._run(["config", "history", self._target], multiline=True) + + # def config_restore(self, restore_id): + # """Restore to a specific configuration history.""" + # return self._run(["config", "restore", self._target, restore_id]) + + # def profile_start(self, profilers=()): + # """Start recording profile data.""" + # args = ["profile", "start"] + # if profilers: + # args += ["--type", ",".join(profilers)] + # args.append(self._target) + # return self._run(args) + + # def profile_stop(self): + # """Stop and download profile data.""" + # return self._run(["profile", "stop", self._target]) + + # def top_locks(self): + # """Get a list of the 10 oldest locks on a MinIO cluster.""" + # return self._run(["top", "locks", self._target], multiline=True) + + # def prometheus_generate(self): + # """Generate prometheus configuration.""" + # return self._run(["prometheus", "generate", self._target]) + + # def kms_key_create(self, key=None): + # """Create a new KMS master key.""" + # return self._run( + # [ + # "kms", "key", "create", self._target, key + # ] + ([key] if key else []), + # ) + + # def kms_key_status(self, key=None): + # """Get status information of a KMS master key.""" + # return self._run( + # [ + # "kms", "key", "status", self._target, key + # ] + ([key] if key else []), + # ) + + # def bucket_remote_add( + # self, src_bucket, dest_url, + # path=None, region=None, bandwidth=None, service=None, + # ): + # """Add a new remote target.""" + # args = [ + # "bucket", "remote", "add", self._target + "/" + src_bucket, + # dest_url, "--service", service or "replication", + # ] + # if path: + # args += ["--path", path] + # if region: + # args += ["--region", region] + # if bandwidth: + # args += ["--bandwidth", bandwidth] + # return self._run(args) + + # def bucket_remote_edit(self, src_bucket, dest_url, arn): + # """Edit credentials of remote target.""" + # return self._run( + # [ + # "bucket", "remote", "edit", self._target + "/" + src_bucket, + # dest_url, "--arn", arn, + # ], + # ) + + # def bucket_remote_list(self, src_bucket=None, service=None): + # """List remote targets.""" + # return self._run( + # [ + # "bucket", "remote", "ls", + # self._target + ("/" + src_bucket if src_bucket else ""), + # "--service", service or "replication", + # ], + # ) + + # def bucket_remote_remove(self, src_bucket, arn): + # """Remove configured remote target.""" + # return self._run( + # [ + # "bucket", "remote", "rm", self._target + "/" + src_bucket, + # "--arn", arn, + # ], + # ) + + # def bucket_quota_set(self, bucket, fifo=None, hard=None): + # """Set bucket quota configuration.""" + # if fifo is None and hard is None: + # raise ValueError("fifo or hard must be set") + # args = ["bucket", "quota", self._target + "/" + bucket] + # if fifo: + # args += ["--fifo", fifo] + # if hard: + # args += ["--hard", hard] + # return self._run(args) + + # def bucket_quota_clear(self, bucket): + # """Clear bucket quota configuration.""" + # return self._run( + # ["bucket", "quota", self._target + "/" + bucket, "--clear"], + # ) + + # def bucket_quota_get(self, bucket): + # """Get bucket quota configuration.""" + # return self._run(["bucket", "quota", self._target + "/" + bucket]) + + +def _convert_to_urllib3_headers(headers): + """Convert headers to urllib3 format""" + http_headers = urllib3.HTTPHeaderDict() + for key, value in (headers or {}).items(): + if isinstance(value, (list, tuple)): + _ = [http_headers.add(key, val) for val in value] + else: + http_headers.add(key, value) + return http_headers From e2b3c9076ae67c89766a5017d5a0b1e9070beaa1 Mon Sep 17 00:00:00 2001 From: pbrw Date: Mon, 21 Aug 2023 18:29:20 +0200 Subject: [PATCH 04/20] User: Add, Info, Remove, List --- minio/minioadmin.py | 44 +++++++++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index dd3ee70ae..b6bfcc344 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -21,12 +21,15 @@ from __future__ import absolute_import from datetime import timedelta +import json from urllib.parse import urlunsplit import os import certifi import urllib3 +from minio.crypto import decrypt, encrypt + from . import time from .credentials.providers import StaticProvider @@ -185,10 +188,15 @@ def _execute( # """Get MinIO server information.""" # return self._run(["info", self._target]) - # def user_add(self, access_key, secret_key): - # """Add a new user.""" - # return self._run(["user", "add", self._target, - # access_key, secret_key]) + def user_add(self, access_key, secret_key): + """Create user with access and secret keys""" + params = {"accessKey": access_key} + body = {"status": "enabled", "secretKey": secret_key} + plain_body = json.dumps(body).encode() + cipher_body = encrypt(plain_body, self._credentials.secret_key) + response = self._url_open("PUT", "/add-user", + query_params=params, body=cipher_body) + return response.data.decode() # def user_disable(self, access_key): # """Disable user.""" @@ -198,17 +206,23 @@ def _execute( # """Enable user.""" # return self._run(["user", "enable", self._target, access_key]) - # def user_remove(self, access_key): - # """Remove user.""" - # return self._run(["user", "remove", self._target, access_key]) - - # def user_info(self, access_key): - # """Get user information.""" - # return self._run(["user", "info", self._target, access_key]) - - # def user_list(self): - # """List users.""" - # return self._run(["user", "list", self._target], multiline=True) + def user_remove(self, access_key): + """Delete user""" + params = {"accessKey": access_key} + response = self._url_open("DELETE", "/remove-user", query_params=params) + return response.data.decode() + + def user_info(self, access_key): + """Get information about user""" + params = {"accessKey": access_key} + response = self._url_open("GET", "/user-info", query_params=params) + return response.data.decode() + + def user_list(self): + """List all users""" + response = self._url_open("GET", "/list-users") + plain_data = decrypt(response.data, self._credentials.secret_key) + return plain_data.decode() # def group_add(self, group_name, members): # """Add users a new or existing group.""" From b2e92e962fee36b5a93426f5aad5c162005a29e3 Mon Sep 17 00:00:00 2001 From: "Bala.FA" Date: Wed, 23 Aug 2023 07:37:00 +0530 Subject: [PATCH 05/20] simplify api execution Signed-off-by: Bala.FA --- minio/api.py | 15 +- minio/crypto.py | 6 +- minio/error.py | 30 ++-- minio/helpers.py | 140 ++++++----------- minio/minioadmin.py | 311 ++++++++++++++++++++++---------------- tests/unit/crypto_test.py | 2 +- 6 files changed, 247 insertions(+), 257 deletions(-) diff --git a/minio/api.py b/minio/api.py index 72364774a..cfe2343f8 100644 --- a/minio/api.py +++ b/minio/api.py @@ -25,7 +25,6 @@ import itertools import os -import platform import tarfile from datetime import timedelta from io import BytesIO @@ -47,10 +46,11 @@ parse_copy_object, parse_list_objects) from .deleteobjects import DeleteError, DeleteRequest, DeleteResult from .error import InvalidResponseError, S3Error, ServerError -from .helpers import (MAX_MULTIPART_COUNT, MAX_MULTIPART_OBJECT_SIZE, - MAX_PART_SIZE, MIN_PART_SIZE, BaseURL, ObjectWriteResult, - ThreadPool, check_bucket_name, check_non_empty_string, - check_sse, check_ssec, genheaders, get_part_info, +from .helpers import (_DEFAULT_USER_AGENT, MAX_MULTIPART_COUNT, + MAX_MULTIPART_OBJECT_SIZE, MAX_PART_SIZE, MIN_PART_SIZE, + BaseURL, ObjectWriteResult, ThreadPool, + check_bucket_name, check_non_empty_string, check_sse, + check_ssec, genheaders, get_part_info, headers_to_strings, is_valid_policy_type, makedirs, md5sum_hash, read_part_data, sha256_hash) from .legalhold import LegalHold @@ -67,11 +67,6 @@ from .versioningconfig import VersioningConfig from .xml import Element, SubElement, findtext, getbytes, marshal, unmarshal -_DEFAULT_USER_AGENT = ( - f"MinIO ({platform.system()}; {platform.machine()}) " - f"{__title__}/{__version__}" -) - class Minio: # pylint: disable=too-many-public-methods """ diff --git a/minio/crypto.py b/minio/crypto.py index b8611076f..cafdad96d 100644 --- a/minio/crypto.py +++ b/minio/crypto.py @@ -20,9 +20,9 @@ """Cryptography to read and write encrypted MinIO Admin payload""" import os -from Crypto.Cipher import AES -from Crypto.Cipher import ChaCha20_Poly1305 -from argon2.low_level import hash_secret_raw, Type + +from argon2.low_level import Type, hash_secret_raw +from Crypto.Cipher import AES, ChaCha20_Poly1305 _NONCE_LEN = 8 _SALT_LEN = 32 diff --git a/minio/error.py b/minio/error.py index 7b9bed9d8..f73691324 100644 --- a/minio/error.py +++ b/minio/error.py @@ -53,22 +53,6 @@ def __reduce__(self): return type(self), (self._code, self._content_type, self._body) -class AdminResponseError(Exception): - """Raised to indicate error response from server.""" - - def __init__(self, code, content_type, body): - self._code = code - self._content_type = content_type - self._body = body - super().__init__( - f"Error admin response from server; " - f"Response code: {code}, Body: {body}" - ) - - def __reduce__(self): - return type(self), (self._code, self._content_type, self._body) - - class ServerError(MinioException): """Raised to indicate that S3 service returning HTTP server error.""" @@ -160,3 +144,17 @@ def copy(self, code, message): self._bucket_name, self._object_name, ) + + +class MinioAdminException(Exception): + """Raised to indicate admin API execution error.""" + + def __init__(self, code, body): + self._code = code + self._body = body + super().__init__( + f"admin request failed; Status: {code}, Body: {body}", + ) + + def __reduce__(self): + return type(self), (self._code, self._body) diff --git a/minio/helpers.py b/minio/helpers.py index 2448de42c..8b11f3b14 100644 --- a/minio/helpers.py +++ b/minio/helpers.py @@ -23,15 +23,22 @@ import hashlib import math import os +import platform import re import urllib.parse from queue import Queue from threading import BoundedSemaphore, Thread +from . import __title__, __version__ from .sse import Sse, SseCustomerKey from .time import to_iso8601utc # Constants +_DEFAULT_USER_AGENT = ( + f"MinIO ({platform.system()}; {platform.machine()}) " + f"{__title__}/{__version__}" +) + MAX_MULTIPART_COUNT = 10000 # 10000 parts MAX_MULTIPART_OBJECT_SIZE = 5 * 1024 * 1024 * 1024 * 1024 # 5TiB MAX_PART_SIZE = 5 * 1024 * 1024 * 1024 # 5GiB @@ -427,53 +434,61 @@ def _get_aws_info(host, https, region): "dualstack": dualstack}, None) -class BaseURL: - """Base URL of S3 endpoint.""" +def _parse_url(endpoint): + """Parse url string.""" - def __init__(self, endpoint, region): - url = urllib.parse.urlsplit(endpoint) - host = url.hostname + url = urllib.parse.urlsplit(endpoint) + host = url.hostname - if url.scheme.lower() not in ["http", "https"]: - raise ValueError("scheme in endpoint must be http or https") + if url.scheme.lower() not in ["http", "https"]: + raise ValueError("scheme in endpoint must be http or https") - url = url_replace(url, scheme=url.scheme.lower()) + url = url_replace(url, scheme=url.scheme.lower()) - if url.path and url.path != "/": - raise ValueError("path in endpoint is not allowed") + if url.path and url.path != "/": + raise ValueError("path in endpoint is not allowed") - url = url_replace(url, path="") + url = url_replace(url, path="") - if url.query: - raise ValueError("query in endpoint is not allowed") + if url.query: + raise ValueError("query in endpoint is not allowed") - if url.fragment: - raise ValueError("fragment in endpoint is not allowed") + if url.fragment: + raise ValueError("fragment in endpoint is not allowed") - try: - url.port - except ValueError as exc: - raise ValueError("invalid port") from exc + try: + url.port + except ValueError as exc: + raise ValueError("invalid port") from exc - if url.username: - raise ValueError("username in endpoint is not allowed") + if url.username: + raise ValueError("username in endpoint is not allowed") - if url.password: - raise ValueError("password in endpoint is not allowed") + if url.password: + raise ValueError("password in endpoint is not allowed") - if ( - (url.scheme == "http" and url.port == 80) or - (url.scheme == "https" and url.port == 443) - ): - url = url_replace(url, netloc=host) + if ( + (url.scheme == "http" and url.port == 80) or + (url.scheme == "https" and url.port == 443) + ): + url = url_replace(url, netloc=host) + + return url + + +class BaseURL: + """Base URL of S3 endpoint.""" + + def __init__(self, endpoint, region): + url = _parse_url(endpoint) if region and not _REGION_REGEX.match(region): raise ValueError(f"invalid region {region}") self._aws_info, region_in_host = _get_aws_info( - host, url.scheme == "https", region) + url.hostname, url.scheme == "https", region) self._virtual_style_flag = ( - self._aws_info or host.endswith("aliyuncs.com") + self._aws_info or url.hostname.endswith("aliyuncs.com") ) self._url = url self._region = region or region_in_host @@ -650,71 +665,6 @@ def build( return url_replace(url, netloc=netloc, path=path) -class AdminURL: - """URL of Minio Admin endpoint""" - - def __init__(self, endpoint): - url = urllib.parse.urlsplit(endpoint) - host = url.hostname - - if url.scheme.lower() not in ["http", "https"]: - raise ValueError("scheme in endpoint must be http or https") - - url = url_replace(url, scheme=url.scheme.lower()) - - if url.path and url.path != "/": - raise ValueError("path in endpoint is not allowed") - - url = url_replace(url, path="") - - if url.query: - raise ValueError("query in endpoint is not allowed") - - if url.fragment: - raise ValueError("fragment in endpoint is not allowed") - - try: - url.port - except ValueError as exc: - raise ValueError("invalid port") from exc - - if url.username: - raise ValueError("username in endpoint is not allowed") - - if url.password: - raise ValueError("password in endpoint is not allowed") - - if ( - (url.scheme == "http" and url.port == 80) or - (url.scheme == "https" and url.port == 443) - ): - url = url_replace(url, netloc=host) - - self._url = url - - @property - def is_https(self): - """Check if scheme is HTTPS.""" - return self._url.scheme == "https" - - def build( - self, path, query_params=None - ): - """Build URL for given information.""" - url = url_replace(self._url, path=path) - - query = [] - for key, values in sorted((query_params or {}).items()): - values = values if isinstance(values, (list, tuple)) else [values] - query += [ - f"{queryencode(key)}={queryencode(value)}" - for value in sorted(values) - ] - url = url_replace(url, query="&".join(query)) - - return url - - class ObjectWriteResult: """Result class of any APIs doing object creation.""" diff --git a/minio/minioadmin.py b/minio/minioadmin.py index b6bfcc344..46d193cf0 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -16,161 +16,211 @@ # pylint: disable=too-many-public-methods -"""MinIO Admin wrapper using MinIO Client (mc) tool.""" +"""MinIO Admin Client to perform MinIO administration operations.""" from __future__ import absolute_import -from datetime import timedelta import json -from urllib.parse import urlunsplit import os +from datetime import timedelta +from enum import Enum +from urllib.parse import urlunsplit import certifi import urllib3 +from urllib3._collections import HTTPHeaderDict from minio.crypto import decrypt, encrypt from . import time - -from .credentials.providers import StaticProvider -from .error import AdminResponseError -from .helpers import AdminURL, sha256_hash +from .credentials.providers import Provider +from .error import MinioAdminException +from .helpers import (_DEFAULT_USER_AGENT, _REGION_REGEX, _parse_url, + headers_to_strings, queryencode, sha256_hash, + url_replace) from .signer import sign_v4_s3 - -_ADMIN_PATH_PREFIX = "/minio/admin/v3" +_COMMAND = Enum( + "Command", + { + "ADD_USER": "add-user", + "USER_INFO": "user-info", + "LIST_USERS": "list-users", + "REMOVE_USER": "remove-user", + "ADD_CANNED_POLICY": "add-canned-policy", + "SET_USER_OR_GROUP_POLICY": "set-user-or-group-policy", + "LIST_CANNED_POLICIES": "list-canned-policies", + "REMOVE_CANNED_POLICY": "remove-canned-policy", + "SET_BUCKET_QUOTA": "set-bucket-quota", + "GET_BUCKET_QUOTA": "get-bucket-quota", + "DATA_USAGE_INFO": "datausageinfo", + "ADD_UPDATE_REMOVE_GROUP": "update-group-members", + "GROUP_INFO": "group", + "LIST_GROUPS": "groups", + "INFO": "info", + }, +) class MinioAdmin: - """MinIO Admin wrapper using MinIO Client (mc) tool.""" + """Client to perform MinIO administration operations.""" - def __init__(self, endpoint, access_key, - secret_key, + def __init__(self, endpoint, + credentials, + region="", secure=True, - cert_check=True): - self._base_url = AdminURL( - ("https://" if secure else "http://") + endpoint - ) - self._credentials = StaticProvider(access_key, secret_key).retrieve() - - timeout = timedelta(minutes=5).seconds - self._http = urllib3.PoolManager( - timeout=urllib3.util.Timeout(connect=timeout, read=timeout), - maxsize=10, - cert_reqs='CERT_REQUIRED' if cert_check else 'CERT_NONE', - ca_certs=os.environ.get('SSL_CERT_FILE') or certifi.where(), - retries=urllib3.Retry( - total=5, - backoff_factor=0.2, - status_forcelist=[500, 502, 503, 504] + cert_check=True, + http_client=None): + url = _parse_url(("https://" if secure else "http://") + endpoint) + if not isinstance(credentials, Provider): + raise ValueError("valid credentials must be provided") + if region and not _REGION_REGEX.match(region): + raise ValueError(f"invalid region {region}") + if http_client: + if not isinstance(http_client, urllib3.poolmanager.PoolManager): + raise ValueError( + "HTTP client should be instance of " + "`urllib3.poolmanager.PoolManager`" + ) + else: + timeout = timedelta(minutes=5).seconds + http_client = urllib3.PoolManager( + timeout=urllib3.util.Timeout(connect=timeout, read=timeout), + maxsize=10, + cert_reqs='CERT_REQUIRED' if cert_check else 'CERT_NONE', + ca_certs=os.environ.get('SSL_CERT_FILE') or certifi.where(), + retries=urllib3.Retry( + total=5, + backoff_factor=0.2, + status_forcelist=[500, 502, 503, 504] + ) ) - ) + + self._url = url + self._provider = credentials + self._region = region + self._secure = secure + self._cert_check = cert_check + self._http = http_client + self._user_agent = _DEFAULT_USER_AGENT + self._trace_stream = None def __del__(self): self._http.clear() - def _build_headers(self, host, body): - """Build headers with given parameters.""" - headers = {} - headers["Host"] = host - sha256 = None + def _url_open(self, method, command, query_params=None, body=None): + """Execute HTTP request.""" + creds = self._provider.retrieve() + + url = url_replace(self._url, path="/minio/admin/v3/"+command.value) + query = [] + for key, values in sorted((query_params or {}).items()): + values = values if isinstance(values, (list, tuple)) else [values] + query += [ + f"{queryencode(key)}={queryencode(value)}" + for value in sorted(values) + ] + url = url_replace(url, query="&".join(query)) - if self._base_url.is_https: - sha256 = "UNSIGNED-PAYLOAD" - else: - sha256 = sha256_hash(body) - if sha256: - headers["x-amz-content-sha256"] = sha256 date = time.utcnow() - headers["x-amz-date"] = time.to_amz_date(date) - return headers, date + headers = { + "Host": url.netloc, + "User-Agent": self._user_agent, + "x-amz-date": time.to_amz_date(date), + "x-amz-content-sha256": sha256_hash(body), + } + if creds.session_token: + headers["X-Amz-Security-Token"] = creds.session_token + if body: + headers["Content-Length"] = str(len(body)) - def _build_signed_headers(self, url, body, method): - """Build signed headers""" - headers, date = self._build_headers(url.netloc, body) headers = sign_v4_s3( method, url, - '', + self._region, headers, - self._credentials, + creds, headers.get("x-amz-content-sha256"), date, ) - return headers - - def _send_request(self, method, url, body): - """Send HTTP request with given parameters""" - - headers = self._build_signed_headers( - url, - body, - method - ) - - return self._http.urlopen( - method=method, - url=urlunsplit(url), - body=body, - headers=_convert_to_urllib3_headers(headers) - ) - - def _url_open( - self, + if self._trace_stream: + self._trace_stream.write("---------START-HTTP---------\n") + query = ("?" + url.query) if url.query else "" + self._trace_stream.write(f"{method} {url.path}{query} HTTP/1.1\n") + self._trace_stream.write( + headers_to_strings(headers, titled_key=True), + ) + self._trace_stream.write("\n") + if body is not None: + self._trace_stream.write("\n") + self._trace_stream.write( + body.decode() if isinstance(body, bytes) else str(body), + ) + self._trace_stream.write("\n") + self._trace_stream.write("\n") + + http_headers = HTTPHeaderDict() + for key, value in headers.items(): + if isinstance(value, (list, tuple)): + _ = [http_headers.add(key, val) for val in value] + else: + http_headers.add(key, value) + + response = self._http.urlopen( method, - path, - body=None, - query_params=None, - ): - """Execute HTTP request.""" - url = self._base_url.build( - path=_ADMIN_PATH_PREFIX + path, - query_params=query_params, + urlunsplit(url), + body=body, + headers=http_headers, + preload_content=True, ) - response = self._send_request( - method, - url, - body, - ) + if self._trace_stream: + self._trace_stream.write(f"HTTP/1.1 {response.status}\n") + self._trace_stream.write( + headers_to_strings(response.headers), + ) + self._trace_stream.write("\n") + self._trace_stream.write("\n") + self._trace_stream.write(response.data.decode()) + self._trace_stream.write("\n") + self._trace_stream.write("----------END-HTTP----------\n") if response.status in [200, 204, 206]: return response - raise AdminResponseError( - response.status, - response.headers.get("content-type"), - response.data.decode() if response.data else None - ) + raise MinioAdminException(response.status, response.data.decode()) - def _execute( - self, - method, - path, - body=None, - query_params=None, - ): - """Execute HTTP request.""" - url = self._base_url.build( - path=_ADMIN_PATH_PREFIX + path, - query_params=query_params, - ) + def set_app_info(self, app_name, app_version): + """ + Set your application name and version to user agent header. - response = self._send_request( - method, - url, - body, - ) + :param app_name: Application name. + :param app_version: Application version. - if response.status in [200, 204, 206]: - return response + Example:: + client.set_app_info('my_app', '1.0.2') + """ + if not (app_name and app_version): + raise ValueError("Application name/version cannot be empty.") + self._user_agent = f"{_DEFAULT_USER_AGENT} {app_name}/{app_version}" - raise AdminResponseError( - response.status, - response.headers.get("content-type"), - response.data.decode() if response.data else None - ) + def trace_on(self, stream): + """ + Enable http trace. + + :param stream: Stream for writing HTTP call tracing. + """ + if not stream: + raise ValueError('Input stream for trace output is invalid.') + # Save new output stream. + self._trace_stream = stream + + def trace_off(self): + """ + Disable HTTP trace. + """ + self._trace_stream = None # def service_restart(self): # """Restart MinIO service.""" @@ -190,12 +240,14 @@ def _execute( def user_add(self, access_key, secret_key): """Create user with access and secret keys""" - params = {"accessKey": access_key} - body = {"status": "enabled", "secretKey": secret_key} - plain_body = json.dumps(body).encode() - cipher_body = encrypt(plain_body, self._credentials.secret_key) - response = self._url_open("PUT", "/add-user", - query_params=params, body=cipher_body) + body = json.dumps( + {"status": "enabled", "secretKey": secret_key}).encode() + response = self._url_open( + "PUT", + _COMMAND.ADD_USER, + query_params={"accessKey": access_key}, + body=encrypt(body, self._provider.secret_key), + ) return response.data.decode() # def user_disable(self, access_key): @@ -208,20 +260,26 @@ def user_add(self, access_key, secret_key): def user_remove(self, access_key): """Delete user""" - params = {"accessKey": access_key} - response = self._url_open("DELETE", "/remove-user", query_params=params) + response = self._url_open( + "DELETE", + _COMMAND.REMOVE_USER, + query_params={"accessKey": access_key}, + ) return response.data.decode() def user_info(self, access_key): """Get information about user""" - params = {"accessKey": access_key} - response = self._url_open("GET", "/user-info", query_params=params) + response = self._url_open( + "GET", + _COMMAND.USER_INFO, + query_params={"accessKey": access_key}, + ) return response.data.decode() def user_list(self): """List all users""" - response = self._url_open("GET", "/list-users") - plain_data = decrypt(response.data, self._credentials.secret_key) + response = self._url_open("GET", _COMMAND.LIST_USERS) + plain_data = decrypt(response.data, self._provider.secret_key) return plain_data.decode() # def group_add(self, group_name, members): @@ -421,14 +479,3 @@ def user_list(self): # def bucket_quota_get(self, bucket): # """Get bucket quota configuration.""" # return self._run(["bucket", "quota", self._target + "/" + bucket]) - - -def _convert_to_urllib3_headers(headers): - """Convert headers to urllib3 format""" - http_headers = urllib3.HTTPHeaderDict() - for key, value in (headers or {}).items(): - if isinstance(value, (list, tuple)): - _ = [http_headers.add(key, val) for val in value] - else: - http_headers.add(key, value) - return http_headers diff --git a/tests/unit/crypto_test.py b/tests/unit/crypto_test.py index c0d3325d0..367399a46 100644 --- a/tests/unit/crypto_test.py +++ b/tests/unit/crypto_test.py @@ -16,7 +16,7 @@ from unittest import TestCase -from minio.crypto import encrypt, decrypt +from minio.crypto import decrypt, encrypt class CryptoTest(TestCase): From e502e1868a8ad14a5e4b3a380408039a5f0b6da8 Mon Sep 17 00:00:00 2001 From: Piotr Date: Thu, 24 Aug 2023 15:47:14 +0200 Subject: [PATCH 06/20] fix retrieve credentials --- minio/minioadmin.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 46d193cf0..187a05811 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -246,7 +246,7 @@ def user_add(self, access_key, secret_key): "PUT", _COMMAND.ADD_USER, query_params={"accessKey": access_key}, - body=encrypt(body, self._provider.secret_key), + body=encrypt(body, self._provider.retrieve().secret_key), ) return response.data.decode() @@ -279,7 +279,9 @@ def user_info(self, access_key): def user_list(self): """List all users""" response = self._url_open("GET", _COMMAND.LIST_USERS) - plain_data = decrypt(response.data, self._provider.secret_key) + plain_data = decrypt( + response.data, self._provider.retrieve().secret_key + ) return plain_data.decode() # def group_add(self, group_name, members): From a0a55630dd047d1ba7f485120ce8f30a5a59b6f6 Mon Sep 17 00:00:00 2001 From: Piotr Date: Thu, 24 Aug 2023 15:49:26 +0200 Subject: [PATCH 07/20] Policy management commands --- minio/minioadmin.py | 105 ++++++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 38 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 187a05811..28ac8bb58 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -51,6 +51,8 @@ "SET_USER_OR_GROUP_POLICY": "set-user-or-group-policy", "LIST_CANNED_POLICIES": "list-canned-policies", "REMOVE_CANNED_POLICY": "remove-canned-policy", + "UNSET_USER_OR_GROUP_POLICY": "idp/builtin/policy/detach", + "CANNED_POLICY_INFO": "info-canned-policy", "SET_BUCKET_QUOTA": "set-bucket-quota", "GET_BUCKET_QUOTA": "get-bucket-quota", "DATA_USAGE_INFO": "datausageinfo", @@ -128,6 +130,7 @@ def _url_open(self, method, command, query_params=None, body=None): "User-Agent": self._user_agent, "x-amz-date": time.to_amz_date(date), "x-amz-content-sha256": sha256_hash(body), + "Content-Type": "application/octet-stream" } if creds.session_token: headers["X-Amz-Security-Token"] = creds.session_token @@ -310,45 +313,71 @@ def user_list(self): # """List groups.""" # return self._run(["group", "list", self._target], multiline=True) - # def policy_add(self, policy_name, policy_file): - # """Add new policy.""" - # return self._run( - # ["policy", "create", self._target, policy_name, policy_file], - # ) + def policy_add(self, policy_name, policy_file): + """Add new policy.""" + with open(policy_file, encoding='utf-8') as file: + response = self._url_open( + "PUT", + _COMMAND.ADD_CANNED_POLICY, + query_params={"name": policy_name}, + body=file.read().encode(), + ) + return response.data.decode() + + def policy_remove(self, policy_name): + """Remove policy.""" + response = self._url_open( + "DELETE", + _COMMAND.REMOVE_CANNED_POLICY, + query_params={"name": policy_name}, + ) + return response.data.decode() + + def policy_info(self, policy_name): + """Get policy information.""" + response = self._url_open( + "GET", + _COMMAND.CANNED_POLICY_INFO, + query_params={"name": policy_name}, + ) + return response.data.decode() - # def policy_remove(self, policy_name): - # """Remove policy.""" - # return self._run(["policy", "remove", self._target, policy_name]) - - # def policy_info(self, policy_name): - # """Get policy information.""" - # return self._run(["policy", "info", self._target, policy_name]) - - # def policy_list(self): - # """List policies.""" - # return self._run(["policy", "list", self._target], multiline=True) - - # def policy_set(self, policy_name, user=None, group=None): - # """Set IAM policy on a user or group.""" - # if (user is not None) ^ (group is not None): - # return self._run( - # [ - # "policy", "attach", self._target, policy_name, - # "--user" if user else "--group", user or group, - # ], - # ) - # raise ValueError("either user or group must be set") - - # def policy_unset(self, policy_name, user=None, group=None): - # """Unset an IAM policy for a user or group.""" - # if (user is not None) ^ (group is not None): - # return self._run( - # [ - # "policy", "detach", self._target, policy_name, - # "--user" if user else "--group", user or group, - # ], - # ) - # raise ValueError("either user or group must be set") + def policy_list(self): + """List policies.""" + response = self._url_open("GET", _COMMAND.LIST_CANNED_POLICIES) + return response.data.decode() + + def policy_set(self, policy_name, user=None, group=None): + """Set IAM policy on a user or group.""" + if (user is not None) ^ (group is not None): + response = self._url_open( + "PUT", + _COMMAND.SET_USER_OR_GROUP_POLICY, + query_params={"userOrGroup": user or group, + "isGroup": "true" if group else "false", + "policyName": policy_name}, + ) + return response.data.decode() + raise ValueError("either user or group must be set") + + def policy_unset(self, policy_name, user=None, group=None): + """Unset an IAM policy for a user or group.""" + body = json.dumps({ + "policies": [policy_name], + "group": group, + "user": user + }).encode() + if (user is not None) ^ (group is not None): + response = self._url_open( + "POST", + _COMMAND.UNSET_USER_OR_GROUP_POLICY, + body=encrypt(body, self._provider.retrieve().secret_key), + ) + plain_data = decrypt( + response.data, self._provider.retrieve().secret_key + ) + return plain_data.decode() + raise ValueError("either user or group must be set") # def config_get(self, key=None): # """Get configuration parameters.""" From 0d3e28e50421e027c52a750eec01f30eff38bd25 Mon Sep 17 00:00:00 2001 From: Piotr Date: Fri, 25 Aug 2023 12:05:06 +0200 Subject: [PATCH 08/20] Group management commands --- minio/minioadmin.py | 75 +++++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 20 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 28ac8bb58..15295759a 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -57,6 +57,7 @@ "GET_BUCKET_QUOTA": "get-bucket-quota", "DATA_USAGE_INFO": "datausageinfo", "ADD_UPDATE_REMOVE_GROUP": "update-group-members", + "SET_GROUP_STATUS": "set-group-status", "GROUP_INFO": "group", "LIST_GROUPS": "groups", "INFO": "info", @@ -287,31 +288,65 @@ def user_list(self): ) return plain_data.decode() - # def group_add(self, group_name, members): - # """Add users a new or existing group.""" - # return self._run(["group", "add", self._target, group_name] + members) + def group_add(self, group_name, members): + """Add users a new or existing group.""" + body = json.dumps({ + "group": group_name, + "members": members, + "isRemove": False + }).encode() + response = self._url_open( + "PUT", + _COMMAND.ADD_UPDATE_REMOVE_GROUP, + body=body, + ) + return response.data.decode() - # def group_disable(self, group_name): - # """Disable group.""" - # return self._run(["group", "disable", self._target, group_name]) + def group_disable(self, group_name): + """Disable group.""" + response = self._url_open( + "PUT", + _COMMAND.SET_GROUP_STATUS, + query_params={"group": group_name, "status": "disabled"} + ) + return response.data.decode() - # def group_enable(self, group_name): - # """Enable group.""" - # return self._run(["group", "enable", self._target, group_name]) + def group_enable(self, group_name): + """Enable group.""" + response = self._url_open( + "PUT", + _COMMAND.SET_GROUP_STATUS, + query_params={"group": group_name, "status": "enabled"} + ) + return response.data.decode() - # def group_remove(self, group_name, members=None): - # """Remove group or members from a group.""" - # return self._run( - # ["group", "remove", self._target, group_name] + (members or []), - # ) + def group_remove(self, group_name, members=None): + """Remove group or members from a group.""" + body = json.dumps({ + "group": group_name, + "members": members, + "isRemove": True + }).encode() + response = self._url_open( + "PUT", + _COMMAND.ADD_UPDATE_REMOVE_GROUP, + body=body, + ) + return response.data.decode() - # def group_info(self, group_name): - # """Get group information.""" - # return self._run(["group", "info", self._target, group_name]) + def group_info(self, group_name): + """Get group information.""" + response = self._url_open( + "GET", + _COMMAND.GROUP_INFO, + query_params={"group": group_name}, + ) + return response.data.decode() - # def group_list(self): - # """List groups.""" - # return self._run(["group", "list", self._target], multiline=True) + def group_list(self): + """List groups.""" + response = self._url_open("GET", _COMMAND.LIST_GROUPS) + return response.data.decode() def policy_add(self, policy_name, policy_file): """Add new policy.""" From be865a922cff1e6f99acaf4856dde38c321ca97b Mon Sep 17 00:00:00 2001 From: Piotr Date: Fri, 25 Aug 2023 12:10:52 +0200 Subject: [PATCH 09/20] User enable/disable commands --- minio/minioadmin.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 15295759a..255f037db 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -47,6 +47,7 @@ "USER_INFO": "user-info", "LIST_USERS": "list-users", "REMOVE_USER": "remove-user", + "SET_USER_STATUS": "set-user-status", "ADD_CANNED_POLICY": "add-canned-policy", "SET_USER_OR_GROUP_POLICY": "set-user-or-group-policy", "LIST_CANNED_POLICIES": "list-canned-policies", @@ -254,13 +255,23 @@ def user_add(self, access_key, secret_key): ) return response.data.decode() - # def user_disable(self, access_key): - # """Disable user.""" - # return self._run(["user", "disable", self._target, access_key]) + def user_disable(self, access_key): + """Disable user.""" + response = self._url_open( + "PUT", + _COMMAND.SET_USER_STATUS, + query_params={"accessKey": access_key, "status": "disabled"} + ) + return response.data.decode() - # def user_enable(self, access_key): - # """Enable user.""" - # return self._run(["user", "enable", self._target, access_key]) + def user_enable(self, access_key): + """Enable user.""" + response = self._url_open( + "PUT", + _COMMAND.SET_USER_STATUS, + query_params={"accessKey": access_key, "status": "enabled"} + ) + return response.data.decode() def user_remove(self, access_key): """Delete user""" From fa1345054b3964479af40237ce8b110a78505c7e Mon Sep 17 00:00:00 2001 From: Piotr Date: Fri, 25 Aug 2023 12:18:25 +0200 Subject: [PATCH 10/20] Service commands --- minio/minioadmin.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 255f037db..d42c055e4 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -62,6 +62,7 @@ "GROUP_INFO": "group", "LIST_GROUPS": "groups", "INFO": "info", + "SERVICE": "service" }, ) @@ -227,13 +228,23 @@ def trace_off(self): """ self._trace_stream = None - # def service_restart(self): - # """Restart MinIO service.""" - # return self._run(["service", "restart", self._target]) + def service_restart(self): + """Restart MinIO service.""" + response = self._url_open( + "POST", + _COMMAND.SERVICE, + query_params={"action": "restart"} + ) + return response.data.decode() - # def service_stop(self): - # """Stop MinIO service.""" - # return self._run(["service", "stop", self._target]) + def service_stop(self): + """Stop MinIO service.""" + response = self._url_open( + "POST", + _COMMAND.SERVICE, + query_params={"action": "stop"} + ) + return response.data.decode() # def update(self): # """Update MinIO.""" From e7a64bc737cabcec9ff97f532811a4f128c3655c Mon Sep 17 00:00:00 2001 From: Piotr Date: Fri, 25 Aug 2023 13:21:55 +0200 Subject: [PATCH 11/20] Service, update, info, top locks commands --- minio/minioadmin.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index d42c055e4..df82e016d 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -62,7 +62,9 @@ "GROUP_INFO": "group", "LIST_GROUPS": "groups", "INFO": "info", - "SERVICE": "service" + "SERVICE": "service", + "UPDATE": "update", + "TOP_LOCKS": "top/locks" }, ) @@ -246,13 +248,22 @@ def service_stop(self): ) return response.data.decode() - # def update(self): - # """Update MinIO.""" - # return self._run(["update", self._target]) + def update(self): + """Update MinIO.""" + response = self._url_open( + "POST", + _COMMAND.UPDATE, + query_params={"updateURL": ""} + ) + return response.data.decode() - # def info(self): - # """Get MinIO server information.""" - # return self._run(["info", self._target]) + def info(self): + """Get MinIO server information.""" + response = self._url_open( + "GET", + _COMMAND.INFO, + ) + return response.data.decode() def user_add(self, access_key, secret_key): """Create user with access and secret keys""" @@ -478,9 +489,13 @@ def policy_unset(self, policy_name, user=None, group=None): # """Stop and download profile data.""" # return self._run(["profile", "stop", self._target]) - # def top_locks(self): - # """Get a list of the 10 oldest locks on a MinIO cluster.""" - # return self._run(["top", "locks", self._target], multiline=True) + def top_locks(self): + """Get a list of the 10 oldest locks on a MinIO cluster.""" + response = self._url_open( + "GET", + _COMMAND.TOP_LOCKS, + ) + return response.data.decode() # def prometheus_generate(self): # """Generate prometheus configuration.""" From e59936d279ee42987ceb0f1107344b407cde0e25 Mon Sep 17 00:00:00 2001 From: Piotr Date: Mon, 28 Aug 2023 15:05:10 +0200 Subject: [PATCH 12/20] Config commands --- minio/minioadmin.py | 93 ++++++++++++++++++++++++++++++++------------- 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index df82e016d..1000f04e2 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -64,7 +64,13 @@ "INFO": "info", "SERVICE": "service", "UPDATE": "update", - "TOP_LOCKS": "top/locks" + "TOP_LOCKS": "top/locks", + "HELP_CONFIG": "help-config-kv", + "GET_CONFIG": "get-config-kv", + "SET_CONFIG": "set-config-kv", + "DELETE_CONFIG": "del-config-kv", + "LIST_CONFIG_HISTORY": "list-config-history-kv", + "RESOTRE_CONFIG_HISTORY": "restore-config-history-kv" }, ) @@ -447,35 +453,70 @@ def policy_unset(self, policy_name, user=None, group=None): return plain_data.decode() raise ValueError("either user or group must be set") - # def config_get(self, key=None): - # """Get configuration parameters.""" - # return self._run( - # ["config", "get", self._target] + [key] if key else [], - # key, - # ) - - # def config_set(self, key, config): - # """Set configuration parameters.""" - # args = [name + "=" + value for name, value in config.items()] - # return self._run(["config", "set", self._target, key] + args) + def config_get(self, key=None): + """Get configuration parameters.""" + if not key: + response = self._url_open( + "GET", + _COMMAND.HELP_CONFIG, + query_params={"key": "", "subSys": ""}, + ) + return response.data.decode() + else: + response = self._url_open( + "GET", + _COMMAND.GET_CONFIG, + query_params={"key": key, "subSys": ""}, + ) + plain_text = decrypt( + response.data, self._provider.retrieve().secret_key + ) + return plain_text.decode() - # def config_reset(self, key, name=None): - # """Reset configuration parameters.""" - # if name: - # key += ":" + name - # return self._run(["config", "reset", self._target, key]) + def config_set(self, key=None, config=None): + """Set configuration parameters.""" + body = " ".join( + [key] + [f"{name}={value}" for name, value in config.items()] + ).encode() + response = self._url_open( + "PUT", + _COMMAND.SET_CONFIG, + body=encrypt(body, self._provider.retrieve().secret_key), + ) + return response.data.decode() - # def config_remove(self, access_key): - # """Remove config.""" - # return self._run(["config", "remove", self._target, access_key]) + def config_reset(self, key, name=None): + """Reset configuration parameters.""" + if name: + key += ":" + name + body = key.encode() + response = self._url_open( + "DELETE", + _COMMAND.DELETE_CONFIG, + body=encrypt(body, self._provider.retrieve().secret_key), + ) + return response.data.decode() - # def config_history(self): - # """Get historic configuration changes.""" - # return self._run(["config", "history", self._target], multiline=True) + def config_history(self): + """Get historic configuration changes.""" + response = self._url_open( + "GET", + _COMMAND.LIST_CONFIG_HISTORY, + query_params={"count": "10"} + ) + plain_text = decrypt( + response.data, self._provider.retrieve().secret_key + ) + return plain_text.decode() - # def config_restore(self, restore_id): - # """Restore to a specific configuration history.""" - # return self._run(["config", "restore", self._target, restore_id]) + def config_restore(self, restore_id): + """Restore to a specific configuration history.""" + response = self._url_open( + "PUT", + _COMMAND.RESOTRE_CONFIG_HISTORY, + query_params={"restoreId": restore_id} + ) + return response.data.decode() # def profile_start(self, profilers=()): # """Start recording profile data.""" From 68f610ae2e04f346e26c13e01c63e18332a03304 Mon Sep 17 00:00:00 2001 From: Piotr Date: Tue, 29 Aug 2023 12:15:09 +0200 Subject: [PATCH 13/20] Remove unnecessary 'else' --- minio/minioadmin.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 1000f04e2..9a92a78fd 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -462,16 +462,16 @@ def config_get(self, key=None): query_params={"key": "", "subSys": ""}, ) return response.data.decode() - else: - response = self._url_open( - "GET", - _COMMAND.GET_CONFIG, - query_params={"key": key, "subSys": ""}, - ) - plain_text = decrypt( - response.data, self._provider.retrieve().secret_key - ) - return plain_text.decode() + + response = self._url_open( + "GET", + _COMMAND.GET_CONFIG, + query_params={"key": key, "subSys": ""}, + ) + plain_text = decrypt( + response.data, self._provider.retrieve().secret_key + ) + return plain_text.decode() def config_set(self, key=None, config=None): """Set configuration parameters.""" From 255da0e8a18931b4b3bd731d984bc3c05543d257 Mon Sep 17 00:00:00 2001 From: Piotr Date: Tue, 29 Aug 2023 14:07:45 +0200 Subject: [PATCH 14/20] Quota, Profile commands --- minio/minioadmin.py | 68 +++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 9a92a78fd..22e31ec72 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -70,7 +70,8 @@ "SET_CONFIG": "set-config-kv", "DELETE_CONFIG": "del-config-kv", "LIST_CONFIG_HISTORY": "list-config-history-kv", - "RESOTRE_CONFIG_HISTORY": "restore-config-history-kv" + "RESOTRE_CONFIG_HISTORY": "restore-config-history-kv", + "START_PROFILE": "profile" }, ) @@ -518,17 +519,14 @@ def config_restore(self, restore_id): ) return response.data.decode() - # def profile_start(self, profilers=()): - # """Start recording profile data.""" - # args = ["profile", "start"] - # if profilers: - # args += ["--type", ",".join(profilers)] - # args.append(self._target) - # return self._run(args) - - # def profile_stop(self): - # """Stop and download profile data.""" - # return self._run(["profile", "stop", self._target]) + def profile_start(self, profilers=()): + """Runs a system profile""" + response = self._url_open( + "POST", + _COMMAND.START_PROFILE, + query_params={"profilerType;": ",".join(profilers)}, + ) + return response.data def top_locks(self): """Get a list of the 10 oldest locks on a MinIO cluster.""" @@ -603,23 +601,33 @@ def top_locks(self): # ], # ) - # def bucket_quota_set(self, bucket, fifo=None, hard=None): - # """Set bucket quota configuration.""" - # if fifo is None and hard is None: - # raise ValueError("fifo or hard must be set") - # args = ["bucket", "quota", self._target + "/" + bucket] - # if fifo: - # args += ["--fifo", fifo] - # if hard: - # args += ["--hard", hard] - # return self._run(args) + def bucket_quota_set(self, bucket, size): + """Set bucket quota configuration.""" + body = json.dumps({"quota": size, "quotatype": "hard"}).encode() + response = self._url_open( + "PUT", + _COMMAND.SET_BUCKET_QUOTA, + query_params={"bucket": bucket}, + body=body + ) + return response.data.decode() - # def bucket_quota_clear(self, bucket): - # """Clear bucket quota configuration.""" - # return self._run( - # ["bucket", "quota", self._target + "/" + bucket, "--clear"], - # ) + def bucket_quota_clear(self, bucket): + """Clear bucket quota configuration.""" + body = json.dumps({"quota": 0, "quotatype": "hard"}).encode() + response = self._url_open( + "PUT", + _COMMAND.SET_BUCKET_QUOTA, + query_params={"bucket": bucket}, + body=body + ) + return response.data.decode() - # def bucket_quota_get(self, bucket): - # """Get bucket quota configuration.""" - # return self._run(["bucket", "quota", self._target + "/" + bucket]) + def bucket_quota_get(self, bucket): + """Get bucket quota configuration.""" + response = self._url_open( + "GET", + _COMMAND.GET_BUCKET_QUOTA, + query_params={"bucket": bucket} + ) + return response.data.decode() From 3660126dfe1ff05227c5c557eadad0cdb5204763 Mon Sep 17 00:00:00 2001 From: Piotr Date: Tue, 29 Aug 2023 14:31:13 +0200 Subject: [PATCH 15/20] KMS key status commands --- minio/minioadmin.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 22e31ec72..0b6e2f80f 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -71,7 +71,9 @@ "DELETE_CONFIG": "del-config-kv", "LIST_CONFIG_HISTORY": "list-config-history-kv", "RESOTRE_CONFIG_HISTORY": "restore-config-history-kv", - "START_PROFILE": "profile" + "START_PROFILE": "profile", + "CREATE_KMS_KEY": "kms/key/create", + "GET_KMS_KEY_STATUS": "kms/key/status" }, ) @@ -540,21 +542,23 @@ def top_locks(self): # """Generate prometheus configuration.""" # return self._run(["prometheus", "generate", self._target]) - # def kms_key_create(self, key=None): - # """Create a new KMS master key.""" - # return self._run( - # [ - # "kms", "key", "create", self._target, key - # ] + ([key] if key else []), - # ) + def kms_key_create(self, key=None): + """Create a new KMS master key.""" + response = self._url_open( + "POST", + _COMMAND.CREATE_KMS_KEY, + query_params={"key-id": key}, + ) + return response.data.decode() - # def kms_key_status(self, key=None): - # """Get status information of a KMS master key.""" - # return self._run( - # [ - # "kms", "key", "status", self._target, key - # ] + ([key] if key else []), - # ) + def kms_key_status(self, key=None): + """Get status information of a KMS master key.""" + response = self._url_open( + "GET", + _COMMAND.GET_KMS_STATUS, + query_params={"key-id": key or ""} + ) + return response.data.decode() # def bucket_remote_add( # self, src_bucket, dest_url, From cd02b4232d47ff4f58d1a78e9f9b7fe386009f91 Mon Sep 17 00:00:00 2001 From: Piotr Date: Tue, 19 Sep 2023 14:50:09 +0200 Subject: [PATCH 16/20] Typo fix --- minio/minioadmin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 0b6e2f80f..4e0d840c7 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -555,7 +555,7 @@ def kms_key_status(self, key=None): """Get status information of a KMS master key.""" response = self._url_open( "GET", - _COMMAND.GET_KMS_STATUS, + _COMMAND.GET_KMS_KEY_STATUS, query_params={"key-id": key or ""} ) return response.data.decode() From a67ffbf610f0705ad12faac2e0a0c5cf88f16fca Mon Sep 17 00:00:00 2001 From: pbrw Date: Tue, 26 Sep 2023 14:16:48 +0200 Subject: [PATCH 17/20] Prometheus generate command --- .github/workflows/ci.yml | 2 +- Makefile | 2 +- minio/minioadmin.py | 33 +++++++++++++++++++++++++++++---- setup.py | 2 +- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 328209484..4e6deecea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install certifi urllib3 mock pytest argon2-cffi pycryptodome + pip install certifi urllib3 mock pytest argon2-cffi pycryptodome pyjwt - name: Run check if Ubuntu if: matrix.os == 'ubuntu-latest' run: | diff --git a/Makefile b/Makefile index fc1375653..b81e1e75a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ default: tests getdeps: @echo "Installing required dependencies" - @pip install --user --upgrade autopep8 certifi pytest pylint urllib3 argon2-cffi pycryptodome + @pip install --user --upgrade autopep8 certifi pytest pylint urllib3 argon2-cffi pycryptodome pyjwt check: getdeps @echo "Running checks" diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 4e0d840c7..fcf556923 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -22,11 +22,12 @@ import json import os -from datetime import timedelta +from datetime import datetime, timedelta from enum import Enum from urllib.parse import urlunsplit import certifi +import jwt import urllib3 from urllib3._collections import HTTPHeaderDict @@ -538,9 +539,33 @@ def top_locks(self): ) return response.data.decode() - # def prometheus_generate(self): - # """Generate prometheus configuration.""" - # return self._run(["prometheus", "generate", self._target]) + def prometheus_generate(self): + """Generate prometheus configuration.""" + default_expire_time = 100 * 365 + default_job_name = "minio-job" + default_metrics_path = "/minio/v2/metrics/cluster" + expire_time = datetime.utcnow() + timedelta(days=default_expire_time) + token = jwt.encode({ + "iss": "prometheus", + "sub": self._provider.retrieve().access_key, + "exp": expire_time + }, self._provider.retrieve().secret_key, algorithm="HS512") + + return { + "scrape_configs": [ + { + "job_name": default_job_name, + "bearer_token": token, + "metrics_path": default_metrics_path, + "scheme": "https" if self._secure else "http", + "static_configs": [ + { + "targets": [self._url.netloc] + } + ] + } + ] + } def kms_key_create(self, key=None): """Create a new KMS master key.""" diff --git a/setup.py b/setup.py index 28b01e8ea..27caec274 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ long_description_content_type="text/markdown", package_dir={"minio": "minio"}, packages=["minio", "minio.credentials"], - install_requires=["certifi", "urllib3", "argon2-cffi", "pycryptodome"], + install_requires=["certifi", "urllib3", "argon2-cffi", "pycryptodome", "pyjwt"], tests_require=[], license="Apache-2.0", classifiers=[ From c60cd4a104ff75cd7a39185f79cf2934eb7c8aa2 Mon Sep 17 00:00:00 2001 From: "Bala.FA" Date: Tue, 26 Sep 2023 19:11:42 +0530 Subject: [PATCH 18/20] Revert "Prometheus generate command" This reverts commit a67ffbf610f0705ad12faac2e0a0c5cf88f16fca. --- .github/workflows/ci.yml | 2 +- Makefile | 2 +- minio/minioadmin.py | 33 ++++----------------------------- setup.py | 2 +- 4 files changed, 7 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e6deecea..328209484 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install certifi urllib3 mock pytest argon2-cffi pycryptodome pyjwt + pip install certifi urllib3 mock pytest argon2-cffi pycryptodome - name: Run check if Ubuntu if: matrix.os == 'ubuntu-latest' run: | diff --git a/Makefile b/Makefile index b81e1e75a..fc1375653 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ default: tests getdeps: @echo "Installing required dependencies" - @pip install --user --upgrade autopep8 certifi pytest pylint urllib3 argon2-cffi pycryptodome pyjwt + @pip install --user --upgrade autopep8 certifi pytest pylint urllib3 argon2-cffi pycryptodome check: getdeps @echo "Running checks" diff --git a/minio/minioadmin.py b/minio/minioadmin.py index fcf556923..4e0d840c7 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -22,12 +22,11 @@ import json import os -from datetime import datetime, timedelta +from datetime import timedelta from enum import Enum from urllib.parse import urlunsplit import certifi -import jwt import urllib3 from urllib3._collections import HTTPHeaderDict @@ -539,33 +538,9 @@ def top_locks(self): ) return response.data.decode() - def prometheus_generate(self): - """Generate prometheus configuration.""" - default_expire_time = 100 * 365 - default_job_name = "minio-job" - default_metrics_path = "/minio/v2/metrics/cluster" - expire_time = datetime.utcnow() + timedelta(days=default_expire_time) - token = jwt.encode({ - "iss": "prometheus", - "sub": self._provider.retrieve().access_key, - "exp": expire_time - }, self._provider.retrieve().secret_key, algorithm="HS512") - - return { - "scrape_configs": [ - { - "job_name": default_job_name, - "bearer_token": token, - "metrics_path": default_metrics_path, - "scheme": "https" if self._secure else "http", - "static_configs": [ - { - "targets": [self._url.netloc] - } - ] - } - ] - } + # def prometheus_generate(self): + # """Generate prometheus configuration.""" + # return self._run(["prometheus", "generate", self._target]) def kms_key_create(self, key=None): """Create a new KMS master key.""" diff --git a/setup.py b/setup.py index 27caec274..28b01e8ea 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ long_description_content_type="text/markdown", package_dir={"minio": "minio"}, packages=["minio", "minio.credentials"], - install_requires=["certifi", "urllib3", "argon2-cffi", "pycryptodome", "pyjwt"], + install_requires=["certifi", "urllib3", "argon2-cffi", "pycryptodome"], tests_require=[], license="Apache-2.0", classifiers=[ From 518f659fd4384ea40eb8f911fef728f7ae8ea318 Mon Sep 17 00:00:00 2001 From: "Bala.FA" Date: Tue, 26 Sep 2023 19:05:17 +0530 Subject: [PATCH 19/20] add site replication apis Signed-off-by: Bala.FA --- minio/__init__.py | 2 +- minio/api.py | 2 +- minio/datatypes.py | 244 ++++++++++++++++++++++++++++++++++++++++++++ minio/minioadmin.py | 109 +++++++++++--------- 4 files changed, 306 insertions(+), 51 deletions(-) diff --git a/minio/__init__.py b/minio/__init__.py index 876da6c26..d7cc1dbe3 100644 --- a/minio/__init__.py +++ b/minio/__init__.py @@ -33,7 +33,7 @@ __title__ = "minio-py" __author__ = "MinIO, Inc." -__version__ = "7.1.17" +__version__ = "7.2.0" __license__ = "Apache 2.0" __copyright__ = "Copyright 2015, 2016, 2017, 2018, 2019, 2020 MinIO, Inc." diff --git a/minio/api.py b/minio/api.py index abc585055..2dcffa854 100644 --- a/minio/api.py +++ b/minio/api.py @@ -55,7 +55,7 @@ check_bucket_name, check_non_empty_string, check_sse, check_ssec, genheaders, get_part_info, headers_to_strings, is_valid_policy_type, makedirs, - md5sum_hash, read_part_data, sha256_hash, queryencode) + md5sum_hash, queryencode, read_part_data, sha256_hash) from .legalhold import LegalHold from .lifecycleconfig import LifecycleConfig from .notificationconfig import NotificationConfig diff --git a/minio/datatypes.py b/minio/datatypes.py index 8f86bbb39..49b1d6a2c 100644 --- a/minio/datatypes.py +++ b/minio/datatypes.py @@ -14,6 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# pylint: disable=too-many-lines + """ Response of ListBuckets, ListObjects, ListObjectsV2 and ListObjectVersions API. """ @@ -24,6 +26,7 @@ import datetime import json from collections import OrderedDict +from enum import Enum from urllib.parse import unquote_plus from xml.etree import ElementTree as ET @@ -850,3 +853,244 @@ def __enter__(self): def __exit__(self, exc_type, value, traceback): self._close_response() + + +class PeerSite: + """Represents a cluster/site to be added to the set of replicated sites.""" + + def __init__(self, name, endpoint, access_key, secret_key): + self._name = name + self._endpoint = endpoint + self._access_key = access_key + self._secret_key = secret_key + + def to_dict(self): + """Convert to dictionary.""" + return { + "name": self._name, + "endpoints": self._endpoint, + "accessKey": self._access_key, + "secretKey": self._secret_key, + } + + +class SiteReplicationStatusOptions: + """Represents site replication status options.""" + ENTITY_TYPE = Enum( + "ENTITY_TYPE", + { + "BUCKET": "bucket", + "POLICY": "policy", + "USER": "user", + "GROUP": "group", + }, + ) + + def __init__(self): + self._buckets = False + self._policies = False + self._users = False + self._groups = False + self._metrics = False + self._entity = None + self._entity_value = None + self._show_deleted = False + + @property + def buckets(self): + """Get buckets.""" + return self._buckets + + @buckets.setter + def buckets(self, value): + """Set buckets.""" + self._buckets = value + + @property + def policies(self): + """Get policies.""" + return self._policies + + @policies.setter + def policies(self, value): + """Set policies.""" + self._policies = value + + @property + def users(self): + """Get users.""" + return self._users + + @users.setter + def users(self, value): + """Set users.""" + self._users = value + + @property + def groups(self): + """Get groups.""" + return self._groups + + @groups.setter + def groups(self, value): + """Set groups.""" + self._groups = value + + @property + def metrics(self): + """Get metrics.""" + return self._metrics + + @metrics.setter + def metrics(self, value): + """Set metrics.""" + self._metrics = value + + @property + def entity(self): + """Get entity.""" + return self._entity + + @entity.setter + def entity(self, value): + """Set entity.""" + self._entity = value + + @property + def entity_value(self): + """Get entity vaue.""" + return self._entity_value + + @entity_value.setter + def entity_value(self, value): + """Set entity vaue.""" + self._entity_value = value + + @property + def show_deleted(self): + """Get show deleted.""" + return self._show_deleted + + @show_deleted.setter + def show_deleted(self, value): + """Set show deleted.""" + self._show_deleted = value + + def to_query_params(self): + """Convert this options to query parameters.""" + params = { + "buckets": str(self._buckets).lower(), + "policies": str(self._policies).lower(), + "users": str(self._users).lower(), + "groups": str(self._groups).lower(), + "metrics": str(self._metrics).lower(), + "showDeleted": str(self._show_deleted).lower(), + } + if self._entity and self._entity_value: + params["entityvalue"] = self._entity_value + params["entity"] = self._entity.value + return params + + +class PeerInfo: + """Site replication peer information.""" + + def __init__(self, deployment_id, endpoint, bucket_bandwidth_limit, + bucket_bandwidth_set): + self._deployment_id = deployment_id + self._endpoint = endpoint + self._name = None + self._sync_status = None + self._bucket_bandwidth_limit = bucket_bandwidth_limit + self._bucket_bandwidth_set = bucket_bandwidth_set + self._bucket_bandwidth_updated_at = None + + @property + def deployment_id(self): + """Get deployment ID.""" + return self._deployment_id + + @deployment_id.setter + def deployment_id(self, value): + """Set deployment ID.""" + self._deployment_id = value + + @property + def endpoint(self): + """Get endpoint.""" + return self._endpoint + + @endpoint.setter + def endpoint(self, value): + """Set endpoint.""" + self._endpoint = value + + @property + def name(self): + """Get name.""" + return self._name + + @name.setter + def name(self, value): + """Set name.""" + self._name = value + + @property + def sync_status(self): + """Get sync status.""" + return self._sync_status + + @sync_status.setter + def sync_status(self, value): + """Set sync status.""" + self._sync_status = value + + @property + def bucket_bandwidth_limit(self): + """Get bucket bandwidth limit.""" + return self._bucket_bandwidth_limit + + @bucket_bandwidth_limit.setter + def bucket_bandwidth_limit(self, value): + """Set bucket bandwidth limit.""" + self._bucket_bandwidth_limit = value + + @property + def bucket_bandwidth_set(self): + """Get bucket bandwidth set.""" + return self._bucket_bandwidth_set + + @bucket_bandwidth_set.setter + def bucket_bandwidth_set(self, value): + """Set bucket bandwidth set.""" + self._bucket_bandwidth_set = value + + @property + def bucket_bandwidth_updated_at(self): + """Get bucket bandwidth updated at.""" + return self._bucket_bandwidth_updated_at + + @bucket_bandwidth_updated_at.setter + def bucket_bandwidth_updated_at(self, value): + """Set bucket bandwidth updated at.""" + self._bucket_bandwidth_updated_at = value + + def to_dict(self): + """Converts peer information to dictionary.""" + data = { + "endpoint": self._endpoint, + "deploymentID": self._deployment_id, + "defaultbandwidth": { + "bandwidthLimitPerBucket": self._bucket_bandwidth_limit, + "set": self._bucket_bandwidth_set, + }, + } + if self._name: + data["name"] = self._name + if self._sync_status is not None: + data["sync"] = "enable" if self._sync_status else "disable" + if self._bucket_bandwidth_updated_at: + data["defaultbandwidth"]["updatedAt"] = to_iso8601utc( + self._bucket_bandwidth_updated_at, + ) + return data diff --git a/minio/minioadmin.py b/minio/minioadmin.py index 4e0d840c7..19f5ba4f1 100644 --- a/minio/minioadmin.py +++ b/minio/minioadmin.py @@ -73,7 +73,12 @@ "RESOTRE_CONFIG_HISTORY": "restore-config-history-kv", "START_PROFILE": "profile", "CREATE_KMS_KEY": "kms/key/create", - "GET_KMS_KEY_STATUS": "kms/key/status" + "GET_KMS_KEY_STATUS": "kms/key/status", + "SITE_REPLICATION_ADD": "site-replication/add", + "SITE_REPLICATION_INFO": "site-replication/info", + "SITE_REPLICATION_STATUS": "site-replication/status", + "SITE_REPLICATION_EDIT": "site-replication/edit", + "SITE_REPLICATION_REMOVE": "site-replication/remove", }, ) @@ -538,10 +543,6 @@ def top_locks(self): ) return response.data.decode() - # def prometheus_generate(self): - # """Generate prometheus configuration.""" - # return self._run(["prometheus", "generate", self._target]) - def kms_key_create(self, key=None): """Create a new KMS master key.""" response = self._url_open( @@ -560,50 +561,60 @@ def kms_key_status(self, key=None): ) return response.data.decode() - # def bucket_remote_add( - # self, src_bucket, dest_url, - # path=None, region=None, bandwidth=None, service=None, - # ): - # """Add a new remote target.""" - # args = [ - # "bucket", "remote", "add", self._target + "/" + src_bucket, - # dest_url, "--service", service or "replication", - # ] - # if path: - # args += ["--path", path] - # if region: - # args += ["--region", region] - # if bandwidth: - # args += ["--bandwidth", bandwidth] - # return self._run(args) - - # def bucket_remote_edit(self, src_bucket, dest_url, arn): - # """Edit credentials of remote target.""" - # return self._run( - # [ - # "bucket", "remote", "edit", self._target + "/" + src_bucket, - # dest_url, "--arn", arn, - # ], - # ) - - # def bucket_remote_list(self, src_bucket=None, service=None): - # """List remote targets.""" - # return self._run( - # [ - # "bucket", "remote", "ls", - # self._target + ("/" + src_bucket if src_bucket else ""), - # "--service", service or "replication", - # ], - # ) - - # def bucket_remote_remove(self, src_bucket, arn): - # """Remove configured remote target.""" - # return self._run( - # [ - # "bucket", "remote", "rm", self._target + "/" + src_bucket, - # "--arn", arn, - # ], - # ) + def add_site_replication(self, peer_sites): + """Add peer sites to site replication.""" + body = json.dumps( + [peer_site.to_dict() for peer_site in peer_sites]).encode() + response = self._url_open( + "PUT", + _COMMAND.SITE_REPLICATION_ADD, + query_params={"api-version": "1"}, + body=encrypt(body, self._provider.retrieve().secret_key), + ) + return response.data.decode() + + def get_site_replication_info(self): + """Get site replication information.""" + response = self._url_open("GET", _COMMAND.SITE_REPLICATION_INFO) + return response.data.decode() + + def get_site_replication_status(self, options): + """Get site replication information.""" + response = self._url_open( + "GET", + _COMMAND.SITE_REPLICATION_STATUS, + query_params=options.to_query_params(), + ) + return response.data.decode() + + def edit_site_replication(self, peer_info): + """Edit site replication with given peer information.""" + body = json.dumps(peer_info.to_dict()).encode() + response = self._url_open( + "PUT", + _COMMAND.SITE_REPLICATION_EDIT, + query_params={"api-version": "1"}, + body=encrypt(body, self._provider.retrieve().secret_key), + ) + return response.data.decode() + + def remove_site_replication(self, sites=None, all_sites=False): + """Remove given sites or all sites from site replication.""" + data = {} + if all_sites: + data.update({"all": True}) + elif sites: + data.update({"sites": sites}) + else: + raise ValueError("either sites or all flag must be given") + body = json.dumps(data).encode() + response = self._url_open( + "PUT", + _COMMAND.SITE_REPLICATION_REMOVE, + query_params={"api-version": "1"}, + body=encrypt(body, self._provider.retrieve().secret_key), + ) + return response.data.decode() def bucket_quota_set(self, bucket, size): """Set bucket quota configuration.""" From 5bd31ac55a994aa8e7a4b36ab5b6aa44223bef8f Mon Sep 17 00:00:00 2001 From: pbrw Date: Wed, 27 Sep 2023 09:37:04 +0200 Subject: [PATCH 20/20] Typo fix --- minio/datatypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/minio/datatypes.py b/minio/datatypes.py index 49b1d6a2c..1d2842674 100644 --- a/minio/datatypes.py +++ b/minio/datatypes.py @@ -958,12 +958,12 @@ def entity(self, value): @property def entity_value(self): - """Get entity vaue.""" + """Get entity value.""" return self._entity_value @entity_value.setter def entity_value(self, value): - """Set entity vaue.""" + """Set entity value.""" self._entity_value = value @property