Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/upload #82

Merged
merged 18 commits into from
Jan 7, 2025
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
- default reviewer added in `dependabot.yml`
- `ReserverBaseError` added in `reserver/__init__.py`
### Changed
- `upload` method in `reserver_obj.py`
- `README.md` updated
- `AUTHORS.md` updated
- GitHub actions are limited to the `dev` and `main` branches
- `generate_template_setup_py` method in `reserver_func.py`
- `generate_template_setup_py` function in `reserver_func.py`
- `Python 3.13` added to `test.yml`
### Removed
- `does_package_exist` function in `reserver_func.py`
## [0.3] - 2024-08-28
### Added
- CLI tests added
- `param` arg in CLI Handler
- more testcases in conflict cases
- `batch_upload` tests
- `read_json` method in `util.py`
- `read_json` function in `util.py`
### Changed
- `README.md` updated
- `upload` method in `reserver_obj.py`
Expand All @@ -29,14 +32,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
### Added
- `CLI` handler
- `Python 3.6` support
- `has_named_parameter` method in `util.py`
- `has_named_parameter` function in `util.py`
- `feature_request.yml` template
- `config.yml` for issue template
- `batch_upload` method in `PyPIUploader`
- `SECURITY.md`
### Changed
- `upload` method in `reserver_obj.py`
- `does_package_exist` method in `reserver_func.py`
- `does_package_exist` function in `reserver_func.py`
- `test.yml` for `Python 3.6` support
- Logo updated
- Bug report template modified
Expand Down
2 changes: 1 addition & 1 deletion dev-requirements.txt
sepandhaghighi marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ bandit>=1.5.1
pydocstyle>=3.0.0
pytest>=4.3.1
pytest-cov>=2.6.1
chardet==5.2.0
chardet==5.2.0
3 changes: 1 addition & 2 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
art>=5.3
requests>=1.0.0
setuptools>=40.8.0
wheel>=0.35.0
twine>=3.5.0
chardet>=4.0.0
chardet>=4.0.0
41 changes: 4 additions & 37 deletions reserver/reserver_func.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
# -*- coding: utf-8 -*-
"""Reserver functions."""
import re
import requests
import requests.adapters
from .reserver_param import PYPI_TEST_URL, PYPI_MAIN_URL, PACKAGE_PARAMETERS, VALIDATIONS, OVERVIEW
from .reserver_param import INVALID_PACKAGE_PARAMETER_NAME_ERROR, INVALID_PACKAGE_PARAMETER_VALUE_ERROR
from .reserver_errors import ReserverBaseError
from hashlib import sha256
from time import time
from hashlib import sha256
from os import mkdir, rmdir
from .reserver_param import PACKAGE_PARAMETERS, VALIDATIONS, OVERVIEW
from .reserver_param import INVALID_PACKAGE_PARAMETER_NAME_ERROR, INVALID_PACKAGE_PARAMETER_VALUE_ERROR
from .reserver_errors import ReserverBaseError


def get_random_name():
Expand All @@ -20,37 +18,6 @@ def get_random_name():
return sha256(str(time()).encode("utf-8")).hexdigest()


def does_package_exist(suggested_name, test_pypi):
"""
Check whether a package with the given name exists or not.

:param suggested_name: given name to search in pypi(or test.pypi)
:type suggested_name: str
:param test_pypi: indicates to search in test.pypi or not
:type test_pypi: bool
:return: whether given name does exist in the pypi or not(as a boolean value)
"""
if not isinstance(suggested_name, str):
suggested_name = str(suggested_name)
if test_pypi:
url = PYPI_TEST_URL + "/" + suggested_name + "/"
else:
url = PYPI_MAIN_URL + "/" + suggested_name + "/"

s = requests.Session()
retries = requests.adapters.Retry(
total=5,
backoff_factor=0.1,
status_forcelist=[500, 502, 503, 504]
)

s.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
s.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))

response = s.get(url, timeout=5)
return not response.status_code == 404


def get_package_parameter(parameter, user_parameters, regex=None):
"""
Get the value for the associated package parameter.
Expand Down
35 changes: 13 additions & 22 deletions reserver/reserver_obj.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
# -*- coding: utf-8 -*-
"""Reserver modules."""
import chardet
import platform
from re import sub
from sys import executable
from os import environ, path, getcwd, remove
from .reserver_errors import ReserverBaseError
from .reserver_func import generate_template_setup_py
from subprocess import check_output, CalledProcessError
from .reserver_param import UNEQUAL_PARAM_NAME_LENGTH_ERROR
from .util import has_named_parameter, remove_dir, read_json
from .reserver_func import does_package_exist, generate_template_setup_py


class PyPIUploader:
Expand Down Expand Up @@ -71,15 +72,11 @@ def upload(self, package_name, user_parameters=None):

:param package_name: package name
:type package_name: str
:param user_parameters: path to the .json file containing user-defined package parameters
:type user_parameters: str
:param user_parameters: json file or path to the .json file containing user-defined package parameters
:type user_parameters: str | dict
:return: True if the package is successfully reserved, False otherwise
"""
if does_package_exist(package_name, self.test_pypi):
print("This package already exists in PyPI.")
return False

if user_parameters is not None:
if not isinstance(user_parameters, dict) and user_parameters is not None:
user_parameters = read_json(user_parameters)

generate_template_setup_py(package_name, user_parameters)
Expand All @@ -98,16 +95,19 @@ def upload(self, package_name, user_parameters=None):
# prevent from uploading any other previously build library in this path.
if path.exists(generated_dist_folder):
remove_dir(generated_dist_folder)
commands = [f'"{executable}" "{generated_setup_file_path}" sdist bdist_wheel']
if platform.system() == "Windows":
commands = [f'"{executable}" "{generated_setup_file_path}" sdist bdist_wheel > nul 2>&1']
else:
commands = [f'"{executable}" "{generated_setup_file_path}" sdist bdist_wheel > /dev/null 2>&1']
if self.test_pypi:
commands += [
f'"{executable}" -m twine upload --repository testpypi "{generated_tar_gz_file}"',
f'"{executable}" -m twine upload --repository testpypi "{generated_wheel_file}"',
]
else:
commands += [
f'"{executable}" -m twine upload --verbose "{generated_tar_gz_file}"',
f'"{executable}" -m twine upload --verbose "{generated_wheel_file}"',
f'"{executable}" -m twine upload "{generated_tar_gz_file}"',
f'"{executable}" -m twine upload "{generated_wheel_file}"',
]
# Run the commands
publish_failed = False
Expand All @@ -125,22 +125,13 @@ def upload(self, package_name, user_parameters=None):
error = error.decode(chardet.detect(error)['encoding'])
except BaseException:
error = error.decode('utf-8')
if command == commands[-2]:
if "403" in error and "Invalid or non-existent authentication information" in error:
error = "Invalid or non-existent authentication information(PyPI API Key)."
if "400" in error and "too similar to an existing project" in error:
error = "Given package name is too similar to an existing project in PyPI."
if "400" in error and "isn't allowed." in error:
error = "Given package name has conflict with the module name of a previously taken package."
if "400" in error and "isn't allowed (conflict with Python Standard Library" in error:
error = "Given package name has conflict with Python Standard Library module name."
break

# remove credential from env variables
if "TWINE_USERNAME" in environ:
environ.pop("TWINE_USERNAME")
if "TWINE_PASSWORD" in environ:
environ.pop("TWINE_PASSWORD")

# remove previously generated files
remove(generated_setup_file_path)
remove_dir(generated_package_folder)
remove_dir(generated_egginfo_file_path)
Expand Down
2 changes: 0 additions & 2 deletions reserver/reserver_param.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
"""
RESERVER_VERSION = "0.3"
RESERVER_NAME = "reserver"
PYPI_TEST_URL = "https://test.pypi.org/project"
PYPI_MAIN_URL = "https://pypi.org/project"
PACKAGE_PARAMETERS = {
"description": "This name has been reserved using Reserver",
"author": "Development Team",
Expand Down
24 changes: 12 additions & 12 deletions tests/test_reserver.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import os
from reserver import PyPIUploader
from reserver.reserver_func import get_random_name
import os

test_pypi_token = os.environ.get("TWINE_TEST_PASSWORD")
pypi_token = os.environ.get("TWINE_PASSWORD")
test_pypi_token = os.environ.get("TWINE_TEST_PASSWORD")

def test_package_exists():
# test reserved name
Expand All @@ -15,31 +15,31 @@ def test_standard_module_conflict():
uploader = PyPIUploader(test_pypi_token, test_pypi=True)
assert uploader.upload("os") == False

def test_batch_packages_names():
# test batch of package names
uploader = PyPIUploader(test_pypi_token, test_pypi=True)
assert uploader.batch_upload(["numpy", "scikit-learn"]) == 0

def test_valid_package_invalid_credentials():
# test not reserved name -> wrong credentials
wrong_pypi_token = "pypi-wrong-api-token"
uploader = PyPIUploader(wrong_pypi_token, test_pypi=True)
uploader = PyPIUploader("pypi-wrong-api-token", test_pypi=True)
assert uploader.upload(get_random_name()) == False

def test_valid_package_valid_credentials():
# test not reserved name -> correct credentials
# uploader = PyPIUploader(test_pypi_token, test_pypi=True)
# uploader.upload(get_random_name())
# assert uploader.upload(get_random_name()) == True
assert True == True

def test_module_conflict():
# try to reserve a name which conflicts with the module name of a previously taken package (the taken package itself has a different name, but it's module name has conflict)."
uploader = PyPIUploader(pypi_token, test_pypi=False)
uploader = PyPIUploader(test_pypi_token, test_pypi=True)
assert uploader.upload("freeze") == False

def test_batch_packages_names():
# test batch of package names
uploader = PyPIUploader(test_pypi_token, test_pypi=True)
assert uploader.batch_upload(["numpy", "scikit-learn"]) == 0

def test_batch_upload():
# try to reserve two non taken package names with per package custom setup.py parameters
# uploader = PyPIUploader(test_pypi_token, True)
# make sure you are in "tests" directory
# uploader = PyPIUploader(test_pypi_token, test_pypi=True)
# assert uploader.batch_upload(
# [get_random_name(), get_random_name() + get_random_name()],
# ["config.json", "config2.json"]
Expand Down