Add --init option to configure seomoz#33

    * This is used for the case where we are starting off a project and have not yet generated requirements files

Signed-off-by: Jono Yang <[email protected]>

configure

@@ -53,9 +53,6 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org
 PIP_EXTRA_ARGS="--find-links $CFG_ROOT_DIR/thirdparty --find-links https://thirdparty.aboutcode.org/pypi"
 
-if [[ -f "$CFG_ROOT_DIR/requirements.txt" ]] && [[ -f "$CFG_ROOT_DIR/requirements-dev.txt" ]]; then
-    PIP_EXTRA_ARGS+=" --no-index"
-fi
 
 ################################
 # Set the quiet flag to empty if not defined
@@ -161,13 +158,17 @@ install_packages() {
 # Main command line entry point
 CFG_DEV_MODE=0
 CFG_REQUIREMENTS=$REQUIREMENTS
+NO_INDEX="--no-index"
 
 case "$CLI_ARGS" in
     --help)    cli_help;;
     --clean)   clean;;
     --dev)     CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;;
+    --init)    NO_INDEX="";;
 esac
 
+PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX"
+
 create_virtualenv "$VIRTUALENV_DIR"
 install_packages "$CFG_REQUIREMENTS"
 . "$CFG_BIN_DIR/activate"

etc/scripts/gen_pypi_simple.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# SPDX-License-Identifier: BSD-2-Clause-Views AND MIT
+# Copyright (c) 2010 David Wolever <[email protected]>. All rights reserved.
+# originally from https://github.com/wolever/pip2pi
+
+import os
+import re
+import shutil
+
+from html import escape
+from pathlib import Path
+
+"""
+name: pip compatibility tags
+version: 20.3.1
+download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py
+copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file)
+license_expression: mit
+notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py
+
+Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+"""
+get_wheel_from_filename = re.compile(
+    r"""^(?P<namever>(?P<name>.+?)-(?P<version>.*?))
+    ((-(?P<build>\d[^-]*?))?-(?P<pyvers>.+?)-(?P<abis>.+?)-(?P<plats>.+?)
+    \.whl)$""",
+    re.VERBOSE
+).match
+
+sdist_exts = ".tar.gz", ".tar.bz2", ".zip", ".tar.xz",
+wheel_ext = ".whl"
+app_ext = ".pyz"
+dist_exts = sdist_exts + (wheel_ext, app_ext)
+
+
+class InvalidDistributionFilename(Exception):
+    pass
+
+
+def get_package_name_from_filename(filename, normalize=True):
+    """
+    Return the package name extracted from a package ``filename``.
+    Optionally ``normalize`` the name according to distribution name rules.
+    Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid::
+
+    >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz")
+    'foo'
+    >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl")
+    'foo-bar'
+    >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl")
+    'cython'
+    >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl")
+    'python-ldap'
+    >>> get_package_name_from_filename("foo.whl")
+    Traceback (most recent call last):
+        ...
+    InvalidDistributionFilename: ...
+    >>> get_package_name_from_filename("foo.png")
+    Traceback (most recent call last):
+        ...
+    InvalidFilePackageName: ...
+    """
+    if not filename or not filename.endswith(dist_exts):
+        raise InvalidDistributionFilename(filename)
+
+    filename = os.path.basename(filename)
+
+    if filename.endswith(sdist_exts):
+        name_ver = None
+        extension = None
+
+        for ext in sdist_exts:
+            if filename.endswith(ext):
+                name_ver, extension, _ = filename.rpartition(ext)
+                break
+
+        if not extension or not name_ver:
+            raise InvalidDistributionFilename(filename)
+
+        name, _, version = name_ver.rpartition('-')
+
+        if not (name and version):
+            raise InvalidDistributionFilename(filename)
+
+    elif filename.endswith(wheel_ext):
+
+        wheel_info = get_wheel_from_filename(filename)
+
+        if not wheel_info:
+            raise InvalidDistributionFilename(filename)
+
+        name = wheel_info.group('name')
+        version = wheel_info.group('version')
+
+        if not (name and version):
+            raise InvalidDistributionFilename(filename)
+
+    elif filename.endswith(app_ext):
+        name_ver, extension, _ = filename.rpartition(".pyz")
+
+        if "-" in filename:
+            name, _, version = name_ver.rpartition('-')
+        else:
+            name = name_ver
+
+        if not name:
+            raise InvalidDistributionFilename(filename)
+
+    if normalize:
+        name = name.lower().replace('_', '-')
+    return name
+
+
+def build_pypi_index(directory, write_index=False):
+    """
+    Using a ``directory`` directory of wheels and sdists, create the a PyPI simple
+    directory index at ``directory``/simple/ populated with the proper PyPI simple
+    index directory structure crafted using symlinks.
+
+    WARNING: The ``directory``/simple/ directory is removed if it exists.
+    """
+
+    directory = Path(directory)
+
+    index_dir = directory / "simple"
+    if index_dir.exists():
+        shutil.rmtree(str(index_dir), ignore_errors=True)
+
+    index_dir.mkdir(parents=True)
+
+    if write_index:
+        simple_html_index = [
+            "<html><head><title>PyPI Simple Index</title>",
+            "<meta name='api-version' value='2' /></head><body>",
+        ]
+
+    package_names = set()
+    for pkg_file in directory.iterdir():
+
+        pkg_filename = pkg_file.name
+
+        if (
+            not pkg_file.is_file()
+            or not pkg_filename.endswith(dist_exts)
+            or pkg_filename.startswith(".")
+        ):
+            continue
+
+        pkg_name = get_package_name_from_filename(pkg_filename)
+        pkg_index_dir = index_dir / pkg_name
+        pkg_index_dir.mkdir(parents=True, exist_ok=True)
+        pkg_indexed_file = pkg_index_dir / pkg_filename
+        link_target = Path("../..") / pkg_filename
+        pkg_indexed_file.symlink_to(link_target)
+
+        if write_index and pkg_name not in package_names:
+            esc_name = escape(pkg_name)
+            simple_html_index.append(f'<a href="{esc_name}/">{esc_name}</a><br/>')
+            package_names.add(pkg_name)
+
+    if write_index:
+        simple_html_index.append("</body></html>")
+        index_html = index_dir / "index.html"
+        index_html.write_text("\n".join(simple_html_index))
+
+
+if __name__ == "__main__":
+    import sys
+    pkg_dir = sys.argv[1]
+    build_pypi_index(pkg_dir)

etc/scripts/gen_pypi_simple.py.ABOUT

@@ -0,0 +1,8 @@
+about_resource: gen_pypi_simple.py
+name: gen_pypi_simple.py
+license_expression: bsd-2-clause-views and mit
+copyright: Copyright (c) nexB Inc.
+   Copyright (c) 2010 David Wolever <[email protected]>
+   Copyright (c) The pip developers
+notes: Originally from https://github.com/wolever/pip2pi and modified extensivley
+ Also partially derived from pip code

etc/scripts/gen_pypi_simple.py.NOTICE

@@ -0,0 +1,56 @@
+SPDX-License-Identifier: BSD-2-Clause-Views AND mit
+
+Copyright (c) nexB Inc.
+Copyright (c) 2010 David Wolever <[email protected]>
+Copyright (c) The pip developers
+
+
+Original code: copyright 2010 David Wolever <[email protected]>. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY <COPYRIGHT HOLDER> ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL <COPYRIGHT HOLDER> OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are those
+of the authors and should not be interpreted as representing official policies,
+either expressed or implied, of David Wolever.
+
+
+Original code: Copyright (c) 2008-2020 The pip developers
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

etc/scripts/requirements.txt

@@ -0,0 +1,12 @@
+aboutcode_toolkit
+github-release-retry2
+attrs
+commoncode
+click
+requests
+saneyaml
+romp
+pip
+setuptools
+twine
+wheel

etc/scripts/utils_thirdparty.py

@@ -899,7 +899,8 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR):
 
         holder = raw_data['Author']
         holder_contact=raw_data['Author-email']
-        copyright = f'Copyright {holder} <{holder_contact}>'
+        copyright = f'Copyright (c) {holder} <{holder_contact}>'
+
         pkginfo_data = dict(
             name=raw_data['Name'],
             declared_license=declared_license,
@@ -908,8 +909,8 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR):
             homepage_url=raw_data['Home-page'],
             copyright=copyright,
             license_expression=license_expression,
-            holder=raw_data['Author'],
-            holder_contact=raw_data['Author-email'],
+            holder=holder,
+            holder_contact=holder_contact,
             keywords=raw_data['Keywords'],
             classifiers=other_classifiers,
         )
@@ -2949,20 +2950,9 @@ def find_problems(
 def compute_normalized_license_expression(declared_licenses):
     if not declared_licenses:
         return
-
-    from packagedcode import licensing
-    from packagedcode.utils import combine_expressions
-
-    detected_licenses = []
-    for declared in declared_licenses:
-        try:
-            license_expression = licensing.get_normalized_expression(
-                query_string=declared
-            )
-        except Exception:
-            return 'unknown'
-        if not license_expression:
-            continue
-        detected_licenses.append(license_expression)
-    if detected_licenses:
-        return combine_expressions(detected_licenses)
+    try:
+        from packagedcode import pypi
+        return pypi.compute_normalized_license(declared_licenses)
+    except ImportError:
+        # Scancode is not installed, we join all license strings and return it
+        return ' '.join(declared_licenses)