diff --git a/configure b/configure index 66d939a..bbe87b0 100755 --- a/configure +++ b/configure @@ -53,9 +53,6 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty --find-links https://thirdparty.aboutcode.org/pypi" -if [[ -f "$CFG_ROOT_DIR/requirements.txt" ]] && [[ -f "$CFG_ROOT_DIR/requirements-dev.txt" ]]; then - PIP_EXTRA_ARGS+=" --no-index" -fi ################################ # Set the quiet flag to empty if not defined @@ -161,13 +158,17 @@ install_packages() { # Main command line entry point CFG_DEV_MODE=0 CFG_REQUIREMENTS=$REQUIREMENTS +NO_INDEX="--no-index" case "$CLI_ARGS" in --help) cli_help;; --clean) clean;; --dev) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; + --init) NO_INDEX="";; esac +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" + create_virtualenv "$VIRTUALENV_DIR" install_packages "$CFG_REQUIREMENTS" . "$CFG_BIN_DIR/activate" diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py new file mode 100644 index 0000000..887e407 --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# SPDX-License-Identifier: BSD-2-Clause-Views AND MIT +# Copyright (c) 2010 David Wolever . All rights reserved. +# originally from https://github.com/wolever/pip2pi + +import os +import re +import shutil + +from html import escape +from pathlib import Path + +""" +name: pip compatibility tags +version: 20.3.1 +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE +).match + +sdist_exts = ".tar.gz", ".tar.bz2", ".zip", ".tar.xz", +wheel_ext = ".whl" +app_ext = ".pyz" +dist_exts = sdist_exts + (wheel_ext, app_ext) + + +class InvalidDistributionFilename(Exception): + pass + + +def get_package_name_from_filename(filename, normalize=True): + """ + Return the package name extracted from a package ``filename``. + Optionally ``normalize`` the name according to distribution name rules. + Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: + + >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") + 'foo' + >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl") + 'foo-bar' + >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") + 'cython' + >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") + 'python-ldap' + >>> get_package_name_from_filename("foo.whl") + Traceback (most recent call last): + ... + InvalidDistributionFilename: ... + >>> get_package_name_from_filename("foo.png") + Traceback (most recent call last): + ... + InvalidFilePackageName: ... + """ + if not filename or not filename.endswith(dist_exts): + raise InvalidDistributionFilename(filename) + + filename = os.path.basename(filename) + + if filename.endswith(sdist_exts): + name_ver = None + extension = None + + for ext in sdist_exts: + if filename.endswith(ext): + name_ver, extension, _ = filename.rpartition(ext) + break + + if not extension or not name_ver: + raise InvalidDistributionFilename(filename) + + name, _, version = name_ver.rpartition('-') + + if not (name and version): + raise InvalidDistributionFilename(filename) + + elif filename.endswith(wheel_ext): + + wheel_info = get_wheel_from_filename(filename) + + if not wheel_info: + raise InvalidDistributionFilename(filename) + + name = wheel_info.group('name') + version = wheel_info.group('version') + + if not (name and version): + raise InvalidDistributionFilename(filename) + + elif filename.endswith(app_ext): + name_ver, extension, _ = filename.rpartition(".pyz") + + if "-" in filename: + name, _, version = name_ver.rpartition('-') + else: + name = name_ver + + if not name: + raise InvalidDistributionFilename(filename) + + if normalize: + name = name.lower().replace('_', '-') + return name + + +def build_pypi_index(directory, write_index=False): + """ + Using a ``directory`` directory of wheels and sdists, create the a PyPI simple + directory index at ``directory``/simple/ populated with the proper PyPI simple + index directory structure crafted using symlinks. + + WARNING: The ``directory``/simple/ directory is removed if it exists. + """ + + directory = Path(directory) + + index_dir = directory / "simple" + if index_dir.exists(): + shutil.rmtree(str(index_dir), ignore_errors=True) + + index_dir.mkdir(parents=True) + + if write_index: + simple_html_index = [ + "PyPI Simple Index", + "", + ] + + package_names = set() + for pkg_file in directory.iterdir(): + + pkg_filename = pkg_file.name + + if ( + not pkg_file.is_file() + or not pkg_filename.endswith(dist_exts) + or pkg_filename.startswith(".") + ): + continue + + pkg_name = get_package_name_from_filename(pkg_filename) + pkg_index_dir = index_dir / pkg_name + pkg_index_dir.mkdir(parents=True, exist_ok=True) + pkg_indexed_file = pkg_index_dir / pkg_filename + link_target = Path("../..") / pkg_filename + pkg_indexed_file.symlink_to(link_target) + + if write_index and pkg_name not in package_names: + esc_name = escape(pkg_name) + simple_html_index.append(f'{esc_name}
') + package_names.add(pkg_name) + + if write_index: + simple_html_index.append("") + index_html = index_dir / "index.html" + index_html.write_text("\n".join(simple_html_index)) + + +if __name__ == "__main__": + import sys + pkg_dir = sys.argv[1] + build_pypi_index(pkg_dir) diff --git a/etc/scripts/gen_pypi_simple.py.ABOUT b/etc/scripts/gen_pypi_simple.py.ABOUT new file mode 100644 index 0000000..4de5ded --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py.ABOUT @@ -0,0 +1,8 @@ +about_resource: gen_pypi_simple.py +name: gen_pypi_simple.py +license_expression: bsd-2-clause-views and mit +copyright: Copyright (c) nexB Inc. + Copyright (c) 2010 David Wolever + Copyright (c) The pip developers +notes: Originally from https://github.com/wolever/pip2pi and modified extensivley + Also partially derived from pip code diff --git a/etc/scripts/gen_pypi_simple.py.NOTICE b/etc/scripts/gen_pypi_simple.py.NOTICE new file mode 100644 index 0000000..6e0fbbc --- /dev/null +++ b/etc/scripts/gen_pypi_simple.py.NOTICE @@ -0,0 +1,56 @@ +SPDX-License-Identifier: BSD-2-Clause-Views AND mit + +Copyright (c) nexB Inc. +Copyright (c) 2010 David Wolever +Copyright (c) The pip developers + + +Original code: copyright 2010 David Wolever . All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of David Wolever. + + +Original code: Copyright (c) 2008-2020 The pip developers + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt new file mode 100644 index 0000000..6591e49 --- /dev/null +++ b/etc/scripts/requirements.txt @@ -0,0 +1,12 @@ +aboutcode_toolkit +github-release-retry2 +attrs +commoncode +click +requests +saneyaml +romp +pip +setuptools +twine +wheel \ No newline at end of file diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index d5a6d99..c0613c3 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -899,7 +899,8 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): holder = raw_data['Author'] holder_contact=raw_data['Author-email'] - copyright = f'Copyright {holder} <{holder_contact}>' + copyright = f'Copyright (c) {holder} <{holder_contact}>' + pkginfo_data = dict( name=raw_data['Name'], declared_license=declared_license, @@ -908,8 +909,8 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): homepage_url=raw_data['Home-page'], copyright=copyright, license_expression=license_expression, - holder=raw_data['Author'], - holder_contact=raw_data['Author-email'], + holder=holder, + holder_contact=holder_contact, keywords=raw_data['Keywords'], classifiers=other_classifiers, ) @@ -2949,20 +2950,9 @@ def find_problems( def compute_normalized_license_expression(declared_licenses): if not declared_licenses: return - - from packagedcode import licensing - from packagedcode.utils import combine_expressions - - detected_licenses = [] - for declared in declared_licenses: - try: - license_expression = licensing.get_normalized_expression( - query_string=declared - ) - except Exception: - return 'unknown' - if not license_expression: - continue - detected_licenses.append(license_expression) - if detected_licenses: - return combine_expressions(detected_licenses) + try: + from packagedcode import pypi + return pypi.compute_normalized_license(declared_licenses) + except ImportError: + # Scancode is not installed, we join all license strings and return it + return ' '.join(declared_licenses)