diff --git a/.gitignore b/.gitignore index 6e1aa8ad..176131f2 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ docs/_build/ # PyBuilder target/ +dev-release.sh # pyenv python configuration file .python-version @@ -74,3 +75,6 @@ sandbox.py # Virtual Env venv/ .venv/ + +# Mac OS +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index 73ed695b..13f54c06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,73 @@ All notable changes to this project will be documented in this file. The format is partly based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [PEP 440](https://peps.python.org/pep-0440/) +## [3.0.0] - 2024-01-17 + +### Safety 3.0.0 major version release! +- Safety 3.0.0 is a significant update to Safety CLI from 2.x versions, including enhancements to core features, new capabilities, and breaking changes from 2.x. +- See
for more details on Safety 3 and these changes +- See for notes and steps to migrating from Safety 2 to Safety 3 + +### Main updates +- Added scan command, which scans a project’s directory for all Python dependencies and includes many improvements over the `check` command, including automatic Python project scanning, native support for Poetry and Pipenv files, Python virtual environment folders, and more granular configuration options. +- Added auth commands, enabling new browser-based authentication of Safety CLI. +- An updated safety policy file schema to support new scan and system-scan commands. This policy file schema is a breaking change from the policy schema used for `safety check`. To migrate a Safety 2.x policy, see . +- Updated screen output to modern interactive interface, with new help interfaces. +- Updated to new JSON output structure to support new scan command, other ecosystems, and other security findings. +- Added a supporting [safety-schemas project dependency](https://pypi.org/project/safety-schemas/), also published and maintained by Safety, which defines Safety CLI policy file and Safety CLI JSON output schemas as pydantic models, formalizing these into testable and versioned schemas. + +### New scan command: +- New scan command: scans a Python project directory for Python dependencies and security vulnerabilities. Safety scan replaces `safety check` with a more powerful and easier to use command. The scan command: +- Finds and scans Python dependency files and virtual environments inside the target directory without needing to specify file or environment locations. +- Adds native scanning and reporting for Poetry and Pipenv manifest files, and Python virtual environment folders. +- Adds configuration of scanning rules to; + - exclude files and folders from the scan using Unix shell-style wildcards only and glob.. TODO. + - Include files to be scanned + - Max folder depth setting +- Reporting configuration rules + - Reporting rules defining which types and specific vulnerabilities to include or ignore stay the same as safety 2.x, although now in a slightly different structure. +- Failing rules + - Adds ability to configure rules for when safety should return a non-zero (failing) exit code, which can be different from reporting rules under the `report` field. +- Auto-updating rules + - Adds ability to easily update insecure package versions in pip requirements files. + +### Other new commands: +- Added auth command: manages Safety CLI’s authentication in development environments, allowing easy authentication via the browser. + - auth login - adds ability to authenticate safety cli via the browser + - auth register - adds ability to register for a Safety account via the CLI, and get scanning within minutes + - auth status - + - auth logout - + - `safety check` command can still be used with the API key --key argument, and scan and system-scan commands should also be +- Added configure command: configures safety cli using a config.ini file, either saved to the user settings or system settings. This can be used to configure safety’s authentication methods and global proxy details. +- Added system-scan command (beta): Adds the system-scan command, which scans a machine for Python files and environments, reporting these to screen output. system-scan is an experimental beta feature that can scan an entire drive or machine for Python dependency files and Python virtual environments, reporting on packages found and their associated security vulnerabilities. +- Added check-updates command: Check for version updates to Safety CLI, and supports screen and JSON format outputs. Can be used in organizations to test and rollout new version updates as recommended by Safety Cybersecurity. + +### New policy file schema for scan and system-scan commands +- New policy file schema to support safety scan and safety system-scan. +Adds scanning-settings root property, which contains settings to configure rules and settings for how safety traverses the directory and subdirectories being scanned, including “exclude” rules, “include” rules, the max directory depth to scan and which root directories safety system-scan should start from. +- Adds report root property, which defines which vulnerability findings safety should auto-ignore (exclude) in its reporting. Supports excluding vulnerability IDs manually, as well as vulnerability groups to ignore based on CVSS severity score. +- Adds new fail-scan-with-exit-code root property, which defines when safety should exit with a failing exit code. This separates safety’s reporting rules from its failing exit code rules, which is a departure from Safety 2.x which had combined rulesets for these. Failing exit codes can be configured based on CVSS severity score. +- Note that the old `safety check` command still supports and relies on the policy schema from safety 2.3.5 and below, meaning no changes are required when migrating to safety 2.x to Safety 3.0.0 when only using the `safety check` command. + +### New global options and configurations +- Added global --stage option, to set the development lifecycle stage for the `scan` and `system-scan` commands. +- Added global --key option, to set a Safety API key for any command, including scan, system-scan and check. + +### Other +- Safety now requires Python>=3.7. Python 3.7 doesn't have active security support from the Python foundation, and we recommend upgrading to at least Python >= 3.8 whenever possible. Safety’s 3.0.0 Docker image can still be used to scan and secure all Python projects, regardless of Python version. Refer to for details. +- Dropped support for the license command. This legacy command is being replaced by the scan command. Users relying on the license command should continue to use Safety 2.3.5 or 2.4.0b2 until Safety 3 adds license support in an upcoming 3.0.x release. +- Add deprecation notice to `safety check` command, since this is now replaced by `safety scan`, a more comprehensive scanning command. The check command will continue receiving maintenance support until June 2024. +- Add deprecation notice to `safety alert` command, which works in tandem with the `safety check` command. Safety alert functionality is replaced by - Safety Platform . The alert command will continue receiving maintenance support until June 2024. +- `safety validate` will assume 3.0 policy file version by default. + + +### Small updates/ bug fixes +- Fixes [a bug](https://github.com/pyupio/safety/issues/488) related to ignoring vulnerability IDs in Safety’s policy file. +- https://github.com/pyupio/safety/issues/480 +- https://github.com/pyupio/safety/issues/478 +- https://github.com/pyupio/safety/issues/455 +- https://github.com/pyupio/safety/issues/447 + ## [2.4.0b2] - 2023-11-15 - Removed the upper clause restriction for the packaging dependency diff --git a/LICENSE b/LICENSE index 55a1eb03..796a276c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ MIT License -Copyright (c) 2016, pyup.io +Copyright (c) 2016, safetycli.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/README.md b/README.md index a26b2b59..af0defea 100644 --- a/README.md +++ b/README.md @@ -1,619 +1,120 @@ -[![safety](https://raw.githubusercontent.com/pyupio/safety/master/safety.jpg)](https://pyup.io/safety/) +[![safety](https://cdn.safetycli.com/images/cli_readme_header.png)](https://docs.safetycli.com/) +# Introduction +Safety CLI is a Python dependency vulnerability scanner designed to enhance software supply chain security by detecting packages with known vulnerabilities and malicious packages in local development environments, CI/CD, and production systems. +Safety CLI can be deployed in minutes and provides clear, actionable recommendations for remediation of detected vulnerabilities. -[![PyPi](https://img.shields.io/pypi/v/safety.svg)](https://pypi.python.org/pypi/safety) -[![Travis](https://img.shields.io/travis/pyupio/safety.svg)](https://travis-ci.org/pyupio/safety) -[![Updates](https://pyup.io/repos/github/pyupio/safety/shield.svg)](https://pyup.io/repos/github/pyupio/safety/) +Leveraging the industry's most comprehensive database of vulnerabilities and malicious packages, Safety CLI Scanner allows teams to detect vulnerabilities at every stage of the software development lifecycle. -Safety checks Python dependencies for known security vulnerabilities and suggests the proper remediations for vulnerabilities detected. Safety can be run on developer machines, in CI/CD pipelines and on production systems. +# Key Features +- Versatile, comprehensive dependency security scanning for Python packages. +- Leverages Safety DB, the most comprehensive vulnerability data available for Python. +- Clear output with detailed recommendations for vulnerability remediation. +- Automatically updates requirements files to secure versions of dependencies where available, guided by your project's policy settings. +- Scanning of individual requirements files and project directories or system-wide scans on developer machines, CI/CD pipelines, and Production systems to detect vulnerable or malicious dependencies. +- JSON, SBOM, HTML and text output. +- Easy integration with CI/CD pipelines, including GitHub Actions. +- Enterprise Ready: Safety CLI can be deployed to large teams with complex project setups with ease, on-premise or as a SaaS product. -By default it uses the open Python vulnerability database [Safety DB](https://github.com/pyupio/safety-db), which is **licensed for non-commercial use only**. +# Getting Started +## GitHub Action -For all commercial projects, Safely must be upgraded to use a [PyUp API](https://pyup.io) using the `--key` option. +- Test Safety CLI in CI/CD using our [GitHub Action](https://github.com/pyupio/safety-actions). +- Full documentation on the [GitHub Action](https://github.com/pyupio/safety-actions) is available on our [Documentation Hub](https://docs.safetycli.com). -# Supported and Tested Python Versions +## Command Line Interface -Python: `3.6`, `3.7`, `3.8`, `3.9`, `3.10`, `3.11` +### 1. Installation -Safety supports the above versions but only is tested in the latest patch version available at [Python for Actions -](https://github.com/actions/python-versions/blob/main/versions-manifest.json). For instance, in Python `3.6` we only will test with `3.6.15`, which is the latest Python 3.6 version available on GitHub actions. +- Install Safety on your development machine. +- Run `pip install safety`. -Make sure to use the latest patch available for your Python version. +### 2. Log In or Register -# Using Safety as a GitHub Action +- Run your first scan using `safety scan`. +- If not authenticated, Safety will prompt for account creation or login. +- Use `safety auth` to check authentication status. -Safety can be integrated into your existing GitHub CI pipeline as an action. Just add the following as a step in your workflow YAML file after setting your `SAFETY_API_KEY` secret on GitHub under Settings -> Secrets -> Actions: +### 3. Running Your First Scan -```yaml - - uses: pyupio/safety@2.3.5 - with: - api-key: ${{ secrets.SAFETY_API_KEY }} -``` +- Navigate to a project directory and run `safety scan`. +- Safety will perform a scan and present results in the Terminal. -(Don't have an API Key? You can sign up for one with [PyUp](https://pyup.io).) +## Basic Commands -This will run Safety in auto-detect mode which figures out your project's structure and the best configuration to run in automatically. It'll fail your CI pipeline if any vulnerable packages are found. +- `safety --help`: Access help and display all available commands. +- `safety auth`: Start authentication flow or display status. +- `safety scan`: Perform a vulnerability scan in the current directory. +- `safety system-scan`: Perform a scan across the entire development machine. +- `safety scan --apply-fixes`: Update vulnerable dependencies. -If you have something more complicated such as a monorepo; or once you're finished testing, read the [Action Documentation](https://docs.pyup.io/docs/github-actions-safety) for more details on configuring Safety as an action. -# Installation +# Detailed Documentation +Full documentation is available at https://docs.safetycli.com. -Install `safety` with pip. Keep in mind that we support only Python 3.6 and up. +Included in the documentation are the following key topics: -```bash -pip install safety -``` +**Safety CLI 3** +- [Introduction to Safety CLI 3](https://docs.safetycli.com/safety-docs/safety-cli-3/introduction-to-safety-cli-scanner) +- [Quick Start Guide](https://docs.safetycli.com/safety-docs/safety-cli-3/quick-start-guide) +- [Installation and Authentication](https://docs.safetycli.com/safety-docs/safety-cli-3/installation-and-authentication) +- [Scanning for Vulnerable and Malicious Packages](https://docs.safetycli.com/safety-docs/safety-cli-3/scanning-for-vulnerable-and-malicious-packages) +- [System-Wide Developer Machine Scanning](https://docs.safetycli.com/safety-docs/safety-cli-3/system-wide-developer-machine-scanning) +- [Viewing Scan Results](https://docs.safetycli.com/safety-docs/safety-cli-3/viewing-scan-results) +- [Available Commands and Inputs](https://docs.safetycli.com/safety-docs/safety-cli-3/available-commands-and-inputs) +- [Scanning in CI/CD](https://docs.safetycli.com/safety-docs/safety-cli-3/scanning-in-ci-cd) +- [License Scanning](https://docs.safetycli.com/safety-docs/safety-cli-3/license-scanning) +- [Exit Codes](https://docs.safetycli.com/safety-docs/safety-cli-3/exit-codes) -# Documentation +**Vulnerability Remediation** +- [Applying Fixes](https://docs.safetycli.com/safety-docs/vulnerability-remediation/applying-fixes) -For detailed documentation, please see [Safety's documentation portal](https://docs.pyup.io/docs/getting-started-with-safety-cli). +**Integration** +- [Securing Git Repositories](https://docs.safetycli.com/safety-docs/installation/securing-git-repositories) +- [GitHub](https://docs.safetycli.com/safety-docs/installation/github) +- [GitHub Actions](https://docs.safetycli.com/safety-docs/installation/github-actions) +- [GitLab](https://docs.safetycli.com/safety-docs/installation/gitlab) +- [Git Post-Commit Hooks](https://docs.safetycli.com/safety-docs/installation/git-post-commit-hooks) +- [BitBucket](https://docs.safetycli.com/safety-docs/installation/bitbucket) +- [Pipenv](https://docs.safetycli.com/safety-docs/installation/pipenv) +- [Docker Containers](https://docs.safetycli.com/safety-docs/installation/docker-containers) -# Basic Usage +**Administration** +- [Policy Management](https://docs.safetycli.com/safety-docs/administration/policy-management) -To check your currently selected virtual environment for dependencies with known security - vulnerabilities, run: +**Output** +- [Output Options and Recommendations](https://docs.safetycli.com/safety-docs/output/output-options-and-recommendations) +- [JSON Output](https://docs.safetycli.com/safety-docs/output/json-output) +- [SBOM Output](https://docs.safetycli.com/safety-docs/output/sbom-output) +- [HTML Output](https://docs.safetycli.com/safety-docs/output/html-output) -```bash -safety check -``` +**Miscellaneous** +- [Release Notes](https://docs.safetycli.com/safety-docs/miscellaneous/release-notes) +- [Breaking Changes in Safety 3](https://docs.safetycli.com/safety-docs/miscellaneous/release-notes/breaking-changes-in-safety-3) +- [Safety 2.x Documentation](https://docs.safetycli.com/safety-2) +- [Support](https://docs.safetycli.com/safety-docs/miscellaneous/support) -You should get a report similar to this: -```bash -+=================================================================================+ +System status is available at https://status.safetycli.com - /$$$$$$ /$$ - /$$__ $$ | $$ - /$$$$$$$ /$$$$$$ | $$ \__//$$$$$$ /$$$$$$ /$$ /$$ - /$$_____/ |____ $$| $$$$ /$$__ $$|_ $$_/ | $$ | $$ - | $$$$$$ /$$$$$$$| $$_/ | $$$$$$$$ | $$ | $$ | $$ - \____ $$ /$$__ $$| $$ | $$_____/ | $$ /$$| $$ | $$ - /$$$$$$$/| $$$$$$$| $$ | $$$$$$$ | $$$$/| $$$$$$$ - |_______/ \_______/|__/ \_______/ \___/ \____ $$ - /$$ | $$ - | $$$$$$/ - by pyup.io \______/ - -+=================================================================================+ - - REPORT - - Safety v2.0.0 is scanning for Vulnerabilities... - Scanning dependencies in your environment: - - -> /scanned-path/ - - Using non-commercial database - Found and scanned 295 packages - Timestamp 2022-06-28 15:42:04 - 0 vulnerabilities found - 0 vulnerabilities ignored -+=================================================================================+ - - No known security vulnerabilities found. - -+=================================================================================+ -``` - - -Now, let's install something insecure: - -```bash -pip install insecure-package -``` -*Yeah, you can really install that.* - -Run `safety check` again: - -```bash - +=================================================================================+ - - Safety v2.0.0.dev6 is scanning for Vulnerabilities... - Scanning dependencies in your environment: - - -> /scanned-path/ - - Using non-commercial database - Found and scanned 295 packages - Timestamp 2022-06-28 15:42:04 - 1 vulnerabilities found - 0 vulnerabilities ignored - -+=================================================================================+ - VULNERABILITIES FOUND -+=================================================================================+ - --> Vulnerability found in insecure-package version 0.1.0 - Vulnerability ID: 25853 - Affected spec: <0.2.0 - ADVISORY: This is an insecure package with lots of exploitable - security vulnerabilities. - Fixed versions: - PVE-2021-25853 - - For more information, please visit - https://pyup.io/vulnerabilities/PVE-2021-25853/25853/ - - - Scan was completed. - -+=================================================================================+ -``` - - -## Starter documentation - -### Configuring the target of the scan -Safety can scan requirements.txt files, the local environment as well as direct input piped into Safety. - -To scan a requirements file: - -```bash -safety check -r requirements.txt -``` - -To scan the local environment: - -```bash -safety check -``` - -Safety is also able to read from stdin with the `--stdin` flag set. -``` -cat requirements.txt | safety check --stdin -``` - -or the output of `pip freeze`: -``` -pip freeze | safety check --stdin -``` - -or to check a single package: -``` -echo "insecure-package==0.1" | safety check --stdin -``` - -*For more examples, take a look at the [options](#options) section.* - - -### Specifying the output format of the scan - -Safety can output the scan results in a variety of formats and outputs. This includes: screen, text, JSON, and bare outputs. Using the ```--output``` flag to configure this output. The default output is to the screen. - -```--output json``` will output JSON for further processing and analysis. -```--output text``` can be used to save the scan to file to later auditing. -```--output bare``` simply prints out the packages that have known vulnerabilities - -### Exit codes - -Safety by default emits exit codes based on the result of the code, allowing you to run safety inside of CI/CD processes. If no vulnerabilities were found the exit code will be 0. In cases of a vulnerability being found, non-zero exit codes will be returned. - -### Scan a Python-based Docker image - -To scan a docker image `IMAGE_TAG`, you can run - -```console -docker run -it --rm ${IMAGE_TAG} /bin/bash -c "pip install safety && safety check" -``` - -## Using Safety in Docker - -Safety can be easily executed as Docker container. It can be used just as -described in the [examples](#examples) section. - -```console -echo "insecure-package==0.1" | docker run -i --rm pyupio/safety safety check --stdin -cat requirements.txt | docker run -i --rm pyupio/safety safety check --stdin -``` - -## Using the Safety binaries - -The Safety [binaries](https://github.com/pyupio/safety/releases) provide some -[extra security](https://pyup.io/posts/patched-vulnerability/). - -After installation, they can be used just like the regular command line version -of Safety. - -## Using Safety with a CI service - -Safety works great in your CI pipeline. It returns by default meaningful non-zero exit codes: - - -| CODE NAME | MEANING | VALUE | -| ------------- |:-------------:| -----:| -| EXIT_CODE_OK | Successful scan | 0 | -| EXIT_CODE_FAILURE | An unexpected issue happened, please run the debug mode and write to us | 1 | -| EXIT_CODE_VULNERABILITIES_FOUND | Safety found vulnerabilities | 64 | -| EXIT_CODE_INVALID_API_KEY | The API KEY used is invalid | 65 | -| EXIT_CODE_TOO_MANY_REQUESTS | You are making too many request, please wait around 40 seconds | 66 | -| EXIT_CODE_UNABLE_TO_LOAD_LOCAL_VULNERABILITY_DB | The local vulnerability database is malformed | 67 | -| EXIT_CODE_UNABLE_TO_FETCH_VULNERABILITY_DB | Client network or server issues trying to fetch the database | 68 | -| EXIT_CODE_MALFORMED_DB | The fetched vulnerability database is malformed or in the review command case, the report to review is malformed | 69 | - -if you want Safety continues on error (always return zero exit code), you can use `--continue-on-error` flag - -Run it before or after your tests. If Safety finds something, your tests will fail. - -**Travis CI** -```yaml -install: - - pip install safety - -script: - - safety check -``` - -**Gitlab CI** -```yaml -safety: - script: - - pip install safety - - safety check -``` - -**Tox** -```ini -[tox] -envlist = py37 - -[testenv] -deps = - safety - pytest -commands = - safety check - pytest -``` - -**Deep GitHub Integration** - -If you are looking for a deep integration with your GitHub repositories: Safety is available as a -part of [pyup.io](https://pyup.io/), called [Safety CI](https://pyup.io/safety/ci/). Safety CI -checks your commits and pull requests for dependencies with known security vulnerabilities -and displays a status on GitHub. - -![Safety CI](https://github.com/pyupio/safety/raw/master/safety_ci.png) - -# Using Safety in production - -Safety is free and open source (MIT Licensed). The data it relies on from the free Safety-db database is license for non-commercial use only, is limited and only updated once per month. - -**All commercial projects and teams must sign up for a paid plan at [PyUp.io](https://pyup.io)** - -## Options - -### `--key` - -*API Key for pyup.io's vulnerability database. This can also be set as `SAFETY_API_KEY` environment variable.* - -**Example** -```bash -safety check --key=12345-ABCDEFGH -``` - -___ - -### `--db` - -*Path to a directory with a local vulnerability database including `insecure.json` and `insecure_full.json`* - -**Example** -```bash -safety check --db=/home/safety-db/data -``` - -### `--proxy-host` - -*Proxy host IP or DNS* - -### `--proxy-port` - -*Proxy port number* - -### `--proxy-protocol` - -*Proxy protocol (https or http)* - -___ - - -### `--output json` - -*Output a complete report with the vulnerabilities in JSON format.* -The report may be used too with the review command. - -if you are using the PyUp commercial database, Safety will use the same JSON structure but with all the full data for commercial users. - -**Example** -```bash -safety check --output json -``` -```json -{ - "report_meta": { - "scan_target": "environment", - "scanned": [ - "/usr/local/lib/python3.9/site-packages" - ], - "api_key": false, - "packages_found": 1, - "timestamp": "2022-03-23 01:41:25", - "safety_version": "2.0.0.dev6" - }, - "scanned_packages": { - "insecure-package": { - "name": "insecure-package", - "version": "0.1.0" - } - }, - "affected_packages": { - "insecure-package": { - "name": "insecure-package", - "version": "0.1.0", - "found": "/usr/local/lib/python3.9/site-packages", - "insecure_versions": [], - "secure_versions": [], - "latest_version_without_known_vulnerabilities": null, - "latest_version": null, - "more_info_url": "None" - } - }, - "announcements": [], - "vulnerabilities": [ - { - "name": "insecure-package", - "ignored": false, - "reason": "", - "expires": "", - "vulnerable_spec": "<0.2.0", - "all_vulnerable_specs": [ - "<0.2.0" - ], - "analyzed_version": "0.1.0", - "advisory": "This is an insecure package with lots of exploitable security vulnerabilities.", - "vulnerability_id": "25853", - "is_transitive": false, - "published_date": null, - "fixed_versions": [], - "closest_versions_without_known_vulnerabilities": [], - "resources": [], - "CVE": { - "name": "PVE-2021-25853", - "cvssv2": null, - "cvssv3": null - }, - "affected_versions": [], - "more_info_url": "None" - } - ], - "ignored_vulnerabilities": [], - "remediations": { - "insecure-package": { - "vulns_found": 1, - "version": "0.1.0", - "recommended": null, - "other_recommended_versions": [], - "more_info_url": "None" - } - } -} -``` -___ - -### `--full-report` - -*Full reports includes a security advisory. It also shows CVSS values for CVEs (requires a premium PyUp subscription).* - -**Example** -```bash -safety check --full-report -``` - -### `--output bare` - -*Output vulnerable packages only. Useful in combination with other tools.* - -**Example** -```bash -safety check --output bare -``` - -``` -cryptography django -``` -___ - - -### `--stdin` - -*Read input from stdin.* - -**Example** -```bash -cat requirements.txt | safety check --stdin -``` -```bash -pip freeze | safety check --stdin -``` -```bash -echo "insecure-package==0.1" | safety check --stdin -``` -___ - -### `--file`, `-r` - -*Read input from one (or multiple) requirement files.* - -**Example** -```bash -safety check -r requirements.txt -``` -```bash -safety check --file=requirements.txt -``` -```bash -safety check -r req_dev.txt -r req_prod.txt -``` -___ - -### `--ignore`, `-i` - -*Ignore one (or multiple) vulnerabilities by ID* - -**Example** -```bash -safety check -i 1234 -``` -```bash -safety check --ignore=1234 -``` -```bash -safety check -i 1234,4567,89101 -``` -The following is also supported (backward compatibility) -```bash -safety check -i 1234 -i 4567 -i 89101 -``` - -### `--output`, `-o` - -*Save the report to a file* - -**Example** -```bash -safety check --output text > insecure_report.txt -``` -```bash -safety check --output json > insecure_report.json -``` -___ - -# Review - -If you save the report in JSON format you can review in the report format again. - -## Options - -### `--file`, `-f` (REQUIRED) - -*Read an insecure report.* - -**Example** -```bash -safety review -f insecure.json -``` -```bash -safety review --file=insecure.json -``` -___ - -### `--full-report` - -*Full reports include a security advisory (if available).* - -**Example** -```bash -safety review -r insecure.json --full-report -``` - -___ - -### `--bare` - -*Output vulnerable packages only.* - -**Example** -```bash -safety review --file report.json --output bare -``` - -``` -django -``` - - -___ +Further support is available by emailing support@safetycli.com. # License +Safety is released under the MIT License. -Display packages licenses information (requires a premium PyUp subscription). - -## Options - -### `--key` (REQUIRED) - -*API Key for pyup.io's licenses database. Can be set as `SAFETY_API_KEY` environment variable.* - -**Example** -```bash -safety license --key=12345-ABCDEFGH -``` -*Shows the license of each package in the current environment* - - -### `--output json` (Optional) - -This license command can also be used in conjunction with optional arguments `--output bare` and `--output json` for structured, parsable outputs that can be fed into other tools and pipelines. - -___ - -### `--db` - -*Path to a directory with a local licenses database `licenses.json`* - -**Example** -```bash -safety license --key=12345-ABCDEFGH --db /home/safety-db/data -``` -___ - -### `--file`, `-r` - -*Read input from one (or multiple) requirement files.* - -**Example** -```bash -safety license --key=12345-ABCDEFGH -r requirements.txt -``` -```bash -safety license --key=12345-ABCDEFGH --file=requirements.txt -``` -```bash -safety license --key=12345-ABCDEFGH -r req_dev.txt -r req_prod.txt -``` - -___ - - -### `--proxy-host`, `-ph` - -*Proxy host IP or DNS* - -### `--proxy-port`, `-pp` - -*Proxy port number* - -### `--proxy-protocol`, `-pr` - -*Proxy protocol (https or http)* - -**Example** -```bash -safety license --key=12345-ABCDEFGH -ph 127.0.0.1 -pp 8080 -pr https -``` +Upon creating an account, a 7-day free trial of our Team plan is offered to new users, after which they will be downgraded to our Free plan. This plan is limited to a single user and is not recommended for commercial purposes. -___ +Our paid [plans for commercial use](https://safetycli.com/resources/plans) begin at just $25 per seat per month and allow scans to be performed using our full vulnerability database, complete with 3x more tracked vulnerabilities and malicious packages than our free plan and other providers. To learn more about our Team and Enterprise plans, please visit https://safetycli.com/resources/plans or email sales@safetycli.com. -# Python 2.7 +# Supported Python Versions +Safety CLI 3 supports Python versions >=3.7. Further details on supported versions, as well as options to run Safety CLI on versions <3.7 using a Docker image are available in our [Documentation Hub](https://docs.safetycli.com). -This tool requires latest Python patch versions starting with version 3.6. We -did support Python 2.7 in the past but, as for other Python 3.x minor versions, -it reached its End-Of-Life and as such we are not able to support it anymore. +We maintain a policy of supporting all maintained and secure versions of Python, plus one minor version below the oldest maintained and secure version. Details on Python versions that meet these criteria can be found here: https://endoflife.date/python. -We understand you might still have Python < 3.6 projects running. At the same -time, Safety itself has a commitment to encourage developers to keep their -software up-to-date, and it would not make sense for us to work with officially -unsupported Python versions, or even those that reached their end of life. +# Resources -If you still need to run Safety from a Python 2.7 environment, please use -version 1.8.7 available at PyPi. Alternatively, you can run Safety from a -Python 3 environment to check the requirements file for your Python 2.7 -project. +- [Safety Cybersecurity website](https://safetycli.com) +- [Safety Login Page](https://safetycli.com/login) +- [Documentation](https://docs.safetycli.com) +- [Security Research and Blog](https://safetycli.com/blog) +- [GitHub Action](https://github.com/safetycli/action) +- [Support](mailto:support@safetycli.com) +- [Status Page](https://status.safetycli.com) \ No newline at end of file diff --git a/README.old b/README.old deleted file mode 100644 index a1043325..00000000 --- a/README.old +++ /dev/null @@ -1,569 +0,0 @@ -[![safety](https://raw.githubusercontent.com/pyupio/safety/master/safety.jpg)](https://pyup.io/safety/) - -[![PyPi](https://img.shields.io/pypi/v/safety.svg)](https://pypi.python.org/pypi/safety) -[![Travis](https://img.shields.io/travis/pyupio/safety.svg)](https://travis-ci.org/pyupio/safety) -[![Updates](https://pyup.io/repos/github/pyupio/safety/shield.svg)](https://pyup.io/repos/github/pyupio/safety/) - -Safety checks your installed dependencies for known security vulnerabilities. - -By default it uses the open Python vulnerability database [Safety DB](https://github.com/pyupio/safety-db), -but can be upgraded to use pyup.io's [Safety API](https://github.com/pyupio/safety/blob/master/docs/api_key.md) using the `--key` option. - -# Installation - -Install `safety` with pip. Keep in mind that we support only Python 3.5 and up. -Look at *Python 2.7* section at the end of this document. - -```bash -pip install safety -``` - -# Usage - -To check your currently selected virtual environment for dependencies with known security - vulnerabilites, run: - -```bash -safety check -``` - -You should get a report similar to this: -```bash -+==============================================================================+ -| | -| /$$$$$$ /$$ | -| /$$__ $$ | $$ | -| /$$$$$$$ /$$$$$$ | $$ \__//$$$$$$ /$$$$$$ /$$ /$$ | -| /$$_____/ |____ $$| $$$$ /$$__ $$|_ $$_/ | $$ | $$ | -| | $$$$$$ /$$$$$$$| $$_/ | $$$$$$$$ | $$ | $$ | $$ | -| \____ $$ /$$__ $$| $$ | $$_____/ | $$ /$$| $$ | $$ | -| /$$$$$$$/| $$$$$$$| $$ | $$$$$$$ | $$$$/| $$$$$$$ | -| |_______/ \_______/|__/ \_______/ \___/ \____ $$ | -| /$$ | $$ | -| | $$$$$$/ | -| by pyup.io \______/ | -| | -+==============================================================================+ -| REPORT | -+==============================================================================+ -| No known security vulnerabilities found. | -+==============================================================================+ -``` - -Now, let's install something insecure: - -```bash -pip install insecure-package -``` -*Yeah, you can really install that.* - -Run `safety check` again: -```bash -+==============================================================================+ -| | -| /$$$$$$ /$$ | -| /$$__ $$ | $$ | -| /$$$$$$$ /$$$$$$ | $$ \__//$$$$$$ /$$$$$$ /$$ /$$ | -| /$$_____/ |____ $$| $$$$ /$$__ $$|_ $$_/ | $$ | $$ | -| | $$$$$$ /$$$$$$$| $$_/ | $$$$$$$$ | $$ | $$ | $$ | -| \____ $$ /$$__ $$| $$ | $$_____/ | $$ /$$| $$ | $$ | -| /$$$$$$$/| $$$$$$$| $$ | $$$$$$$ | $$$$/| $$$$$$$ | -| |_______/ \_______/|__/ \_______/ \___/ \____ $$ | -| /$$ | $$ | -| | $$$$$$/ | -| by pyup.io \______/ | -| | -+==============================================================================+ -| REPORT | -+==========================+===============+===================+===============+ -| package | installed | affected | source | -+==========================+===============+===================+===============+ -| insecure-package | 0.1.0 | <0.2.0 | changelog | -+==========================+===============+===================+===============+ -``` - -## Examples - -### Read requirement files -Just like pip, Safety is able to read local requirement files: - -```bash -safety check -r requirements.txt -``` - -### Read from stdin -Safety is also able to read from stdin with the `--stdin` flag set. - -To check a local requirements file, run: -``` -cat requirements.txt | safety check --stdin -``` - -or the output of `pip freeze`: -``` -pip freeze | safety check --stdin -``` - -or to check a single package: -``` -echo "insecure-package==0.1" | safety check --stdin -``` - -*For more examples, take a look at the [options](#options) section.* - - -### Scan a Python-based Docker image - -To scan a docker image `IMAGE_TAG`, you can run - -```console -docker run -it --rm ${IMAGE_TAG} "/bin/bash -c \"pip install safety && safety check\" -``` - -## Using Safety in Docker - -Safety can be easily executed as Docker container. It can be used just as -described in the [examples](#examples) section. - -```console -echo "insecure-package==0.1" | docker run -i --rm pyupio/safety safety check --stdin -cat requirements.txt | docker run -i --rm pyupio/safety safety check --stdin -``` - -## Using the Safety binaries - -The Safety [binaries](https://github.com/pyupio/safety/releases) provide some -[extra security](https://pyup.io/posts/patched-vulnerability/). - -After installation, they can be used just like the regular command line version -of Safety. - -## Using Safety with a CI service - -Safety works great in your CI pipeline. It returns a non-zero exit status if it finds a vulnerability. - -Run it before or after your tests. If Safety finds something, your tests will fail. - -**Travis** -```yaml -install: - - pip install safety - -script: - - safety check -``` - -**Gitlab CI** -```yaml -safety: - script: - - pip install safety - - safety check -``` - -**Tox** -```ini -[tox] -envlist = py37 - -[testenv] -deps = - safety - pytest -commands = - safety check - pytest -``` - -**Deep GitHub Integration** - -If you are looking for a deep integration with your GitHub repositories: Safety is available as a -part of [pyup.io](https://pyup.io/), called [Safety CI](https://pyup.io/safety/ci/). Safety CI -checks your commits and pull requests for dependencies with known security vulnerabilities -and displays a status on GitHub. - -![Safety CI](https://github.com/pyupio/safety/raw/master/safety_ci.png) - - -# Using Safety in production - -Safety is free and open source (MIT Licensed). The underlying open vulnerability database is updated once per month. - -To get access to all vulnerabilites as soon as they are added, you need a [Safety API key](https://github.com/pyupio/safety/blob/master/docs/api_key.md) that comes with a paid [pyup.io](https://pyup.io) account, starting at $99. - -## Options - -### `--key` - -*API Key for pyup.io's vulnerability database. Can be set as `SAFETY_API_KEY` environment variable.* - -**Example** -```bash -safety check --key=12345-ABCDEFGH -``` - -___ - -### `--db` - -*Path to a directory with a local vulnerability database including `insecure.json` and `insecure_full.json`* - -**Example** -```bash -safety check --db=/home/safety-db/data -``` - -### `--proxy-host` - -*Proxy host IP or DNS* - -### `--proxy-port` - -*Proxy port number* - -### `--proxy-protocol` - -*Proxy protocol (https or http)* - -___ - -### `--json` - -*Output vulnerabilities in JSON format.* - -**Example** -```bash -safety check --json -``` -```javascript -[ - [ - "django", - "<1.2.2", - "1.2", - "Cross-site scripting (XSS) vulnerability in Django 1.2.x before 1.2.2 allows remote attackers to inject arbitrary web script or HTML via a csrfmiddlewaretoken (aka csrf_token) cookie.", - "25701" - ] -] -``` -___ - -### `--full-report` - -*Full reports includes a security advisory. It also shows CVSS values for CVEs (requires a premium PyUp subscription).* - -**Example** -```bash -safety check --full-report -``` - -``` -+==============================================================================+ -| | -| /$$$$$$ /$$ | -| /$$__ $$ | $$ | -| /$$$$$$$ /$$$$$$ | $$ \__//$$$$$$ /$$$$$$ /$$ /$$ | -| /$$_____/ |____ $$| $$$$ /$$__ $$|_ $$_/ | $$ | $$ | -| | $$$$$$ /$$$$$$$| $$_/ | $$$$$$$$ | $$ | $$ | $$ | -| \____ $$ /$$__ $$| $$ | $$_____/ | $$ /$$| $$ | $$ | -| /$$$$$$$/| $$$$$$$| $$ | $$$$$$$ | $$$$/| $$$$$$$ | -| |_______/ \_______/|__/ \_______/ \___/ \____ $$ | -| /$$ | $$ | -| | $$$$$$/ | -| by pyup.io \______/ | -| | -+==============================================================================+ -| REPORT | -+============================+===========+==========================+==========+ -| package | installed | affected | ID | -+============================+===========+==========================+==========+ -| CVSS v2 | BASE SCORE: 6.5 | IMPACT SCORE: 6.4 | -+============================+===========+==========================+==========+ -| django | 1.2 | <1.2.2 | 25701 | -+==============================================================================+ -| Cross-site scripting (XSS) vulnerability in Django 1.2.x before 1.2.2 allows | -| remote attackers to inject arbitrary web script or HTML via a csrfmiddlewar | -| etoken (aka csrf_token) cookie. | -+==============================================================================+ -``` -___ - -### `--bare` - -*Output vulnerable packages only. Useful in combination with other tools.* - -**Example** -```bash -safety check --bare -``` - -``` -cryptography django -``` -___ - -### `--cache` - -*Cache requests to the vulnerability database locally for 2 hours.* - -**Example** -```bash -safety check --cache -``` -___ - -### `--stdin` - -*Read input from stdin.* - -**Example** -```bash -cat requirements.txt | safety check --stdin -``` -```bash -pip freeze | safety check --stdin -``` -```bash -echo "insecure-package==0.1" | safety check --stdin -``` -___ - -### `--file`, `-r` - -*Read input from one (or multiple) requirement files.* - -**Example** -```bash -safety check -r requirements.txt -``` -```bash -safety check --file=requirements.txt -``` -```bash -safety check -r req_dev.txt -r req_prod.txt -``` -___ - -### `--ignore`, `-i` - -*Ignore one (or multiple) vulnerabilities by ID* - -**Example** -```bash -safety check -i 1234 -``` -```bash -safety check --ignore=1234 -``` -```bash -safety check -i 1234 -i 4567 -i 89101 -``` - -### `--output`, `-o` - -*Save the report to a file* - -**Example** -```bash -safety check -o insecure_report.txt -``` -```bash -safety check --output --json insecure_report.json -``` -___ - -# Review - -If you save the report in JSON format you can review in the report format again. - -## Options - -### `--file`, `-f` (REQUIRED) - -*Read an insecure report.* - -**Example** -```bash -safety review -f insecure.json -``` -```bash -safety review --file=insecure.json -``` -___ - -### `--full-report` - -*Full reports include a security advisory (if available).* - -**Example** -```bash -safety review -r insecure.json --full-report -``` - -``` -+==============================================================================+ -| | -| /$$$$$$ /$$ | -| /$$__ $$ | $$ | -| /$$$$$$$ /$$$$$$ | $$ \__//$$$$$$ /$$$$$$ /$$ /$$ | -| /$$_____/ |____ $$| $$$$ /$$__ $$|_ $$_/ | $$ | $$ | -| | $$$$$$ /$$$$$$$| $$_/ | $$$$$$$$ | $$ | $$ | $$ | -| \____ $$ /$$__ $$| $$ | $$_____/ | $$ /$$| $$ | $$ | -| /$$$$$$$/| $$$$$$$| $$ | $$$$$$$ | $$$$/| $$$$$$$ | -| |_______/ \_______/|__/ \_______/ \___/ \____ $$ | -| /$$ | $$ | -| | $$$$$$/ | -| by pyup.io \______/ | -| | -+==============================================================================+ -| REPORT | -+============================+===========+==========================+==========+ -| package | installed | affected | ID | -+============================+===========+==========================+==========+ -| django | 1.2 | <1.2.2 | 25701 | -+==============================================================================+ -| Cross-site scripting (XSS) vulnerability in Django 1.2.x before 1.2.2 allows | -| remote attackers to inject arbitrary web script or HTML via a csrfmiddlewar | -| etoken (aka csrf_token) cookie. | -+==============================================================================+ -``` -___ - -### `--bare` - -*Output vulnerable packages only.* - -**Example** -```bash -safety review --file report.json --bare -``` - -``` -django -``` - -___ - -# License - -Display packages licenses information (requires a premium PyUp subscription). - -## Options - -### `--key` (REQUIRED) - -*API Key for pyup.io's licenses database. Can be set as `SAFETY_API_KEY` environment variable.* - -**Example** -```bash -safety license --key=12345-ABCDEFGH -``` -*Shows the license of each package in the current environment* - - -``` -+==============================================================================+ -| | -| /$$$$$$ /$$ | -| /$$__ $$ | $$ | -| /$$$$$$$ /$$$$$$ | $$ \__//$$$$$$ /$$$$$$ /$$ /$$ | -| /$$_____/ |____ $$| $$$$ /$$__ $$|_ $$_/ | $$ | $$ | -| | $$$$$$ /$$$$$$$| $$_/ | $$$$$$$$ | $$ | $$ | $$ | -| \____ $$ /$$__ $$| $$ | $$_____/ | $$ /$$| $$ | $$ | -| /$$$$$$$/| $$$$$$$| $$ | $$$$$$$ | $$$$/| $$$$$$$ | -| |_______/ \_______/|__/ \_______/ \___/ \____ $$ | -| /$$ | $$ | -| | $$$$$$/ | -| by pyup.io \______/ | -| | -+==============================================================================+ -| Packages licenses | -+=============================================+===========+====================+ -| package | version | license | -+=============================================+===========+====================+ -| requests | 2.25.0 | Apache-2.0 | -|------------------------------------------------------------------------------| -| click | 7.1.2 | BSD-3-Clause | -|------------------------------------------------------------------------------| -| safety | 1.10.0 | MIT | -+==============================================================================+ -``` - -### `--json` (Optional) - -This license command can also be used in conjuction with optional arguments `--bare` and `--json` for structured, parsable outputs that can be fed into other tools and pipelines. - -___ - -### `--db` - -*Path to a directory with a local licenses database `licenses.json`* - -**Example** -```bash -safety license --key=12345-ABCDEFGH --db=/home/safety-db/data -``` -___ - -### `--no-cache` - -*Since PyUp.io licenses DB is updated once a week, the licenses database is cached locally for 7 days. You can use `--no-cache` to download it once again.* - -**Example** -```bash -safety license --key=12345-ABCDEFGH --no-cache -``` -___ - -### `--file`, `-r` - -*Read input from one (or multiple) requirement files.* - -**Example** -```bash -safety license --key=12345-ABCDEFGH -r requirements.txt -``` -```bash -safety license --key=12345-ABCDEFGH --file=requirements.txt -``` -```bash -safety license --key=12345-ABCDEFGH -r req_dev.txt -r req_prod.txt -``` - -___ - - -### `--proxy-host`, `-ph` - -*Proxy host IP or DNS* - -### `--proxy-port`, `-pp` - -*Proxy port number* - -### `--proxy-protocol`, `-pr` - -*Proxy protocol (https or http)* - -**Example** -```bash -safety license --key=12345-ABCDEFGH -ph 127.0.0.1 -pp 8080 -pr https -``` - -___ - -# Python 2.7 - -This tool requires latest Python patch versions starting with version 3.5. We -did support Python 2.7 in the past but, as for other Python 3.x minor versions, -it reached its End-Of-Life and as such we are not able to support it anymore. - -We understand you might still have Python 2.7 projects running. At the same -time, Safety itself has a commitment to encourage developers to keep their -software up-to-date, and it would not make sense for us to work with officially -unsupported Python versions, or even those that reached their end of life. - -If you still need to run Safety from a Python 2.7 environment, please use -version 1.8.7 available at PyPi. Alternatively, you can run Safety from a -Python 3 environment to check the requirements file for your Python 2.7 -project. diff --git a/docs/api_key.md b/docs/api_key.md index 7a5ad0b0..e39bdb02 100644 --- a/docs/api_key.md +++ b/docs/api_key.md @@ -4,11 +4,11 @@ This is a step by step guide on how to get an API key that can be used for safet with safety gives you access to the latest vulnerabilities. The freely available database is synced only once per month. -In order to get an API Key you need a subscription on [pyup.io](https://pyup.io). +In order to get an API Key you need a subscription on [safetycli.com](https://safetycli.com). ## Step 1 - Sign Up -Go to [pyup.io](https://pyup.io) and click on `sign up`. +Go to [safetycli.com](https://safetycli.com) and click on `sign up`. ## Step 2 - Start your free trial diff --git a/safety/.DS_Store b/safety/.DS_Store new file mode 100644 index 00000000..8eb97cf4 Binary files /dev/null and b/safety/.DS_Store differ diff --git a/safety/VERSION b/safety/VERSION index 66e9d53f..4a36342f 100644 --- a/safety/VERSION +++ b/safety/VERSION @@ -1 +1 @@ -2.4.0b2 +3.0.0 diff --git a/safety/__init__.py b/safety/__init__.py index da2f5d50..6d3d19d3 100644 --- a/safety/__init__.py +++ b/safety/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -__author__ = """pyup.io""" -__email__ = 'support@pyup.io' +__author__ = """safetycli.com""" +__email__ = 'support@safetycli.com' import os diff --git a/safety/alerts/__init__.py b/safety/alerts/__init__.py index 440b8d2a..cf5d4f04 100644 --- a/safety/alerts/__init__.py +++ b/safety/alerts/__init__.py @@ -6,9 +6,11 @@ from dataclasses import dataclass +from safety.cli_util import SafetyCLILegacyGroup + from . import github from safety.util import SafetyPolicyFile - +from safety.scan.constants import CLI_ALERT_COMMAND_HELP LOG = logging.getLogger(__name__) @@ -19,13 +21,13 @@ class Alert: policy: Any = None requirements_files: Any = None -@click.group(help="Send alerts based on the results of a Safety scan.") -@click.option('--check-report', help='JSON output of Safety Check to work with.', type=click.File('r'), default=sys.stdin) -@click.option("--policy-file", type=SafetyPolicyFile(), default='.safety-policy.yml', - help="Define the policy file to be used") +@click.group(cls=SafetyCLILegacyGroup, help=CLI_ALERT_COMMAND_HELP, deprecated=True, utility_command=True) +@click.option('--check-report', help='JSON output of Safety Check to work with.', type=click.File('r'), default=sys.stdin, required=True) @click.option("--key", envvar="SAFETY_API_KEY", - help="API Key for pyup.io's vulnerability database. Can be set as SAFETY_API_KEY " + help="API Key for safetycli.com's vulnerability database. Can be set as SAFETY_API_KEY " "environment variable.", required=True) +@click.option("--policy-file", type=SafetyPolicyFile(), default='.safety-policy.yml', + help="Define the policy file to be used") @click.pass_context def alert(ctx, check_report, policy_file, key): LOG.info('alert started') diff --git a/safety/alerts/github.py b/safety/alerts/github.py index 392b44f1..ddf6b66d 100644 --- a/safety/alerts/github.py +++ b/safety/alerts/github.py @@ -35,9 +35,9 @@ def delete_branch(repo, branch): @utils.require_files_report def github_pr(obj, repo, token, base_url): """ - Create a GitHub PR to fix any vulnerabilities using PyUp's remediation data. + Create a GitHub PR to fix any vulnerabilities using Safety's remediation data. - Normally, this is run by a GitHub action. If you're running this manually, ensure that your local repo is up to date and on HEAD - otherwise you'll see strange results. + This is usally run by a GitHub action. If you're running this manually, ensure that your local repo is up to date and on HEAD - otherwise you'll see strange results. """ if pygithub is None: click.secho("pygithub is not installed. Did you install Safety with GitHub support? Try pip install safety[github]", fg='red') diff --git a/safety/auth/__init__.py b/safety/auth/__init__.py new file mode 100644 index 00000000..4d62ecf7 --- /dev/null +++ b/safety/auth/__init__.py @@ -0,0 +1,11 @@ +from .cli_utils import auth_options, build_client_session, proxy_options, \ + inject_session +from .cli import auth + +__all__ = [ + "build_client_session", + "proxy_options", + "auth_options", + "inject_session", + "auth" +] \ No newline at end of file diff --git a/safety/auth/cli.py b/safety/auth/cli.py new file mode 100644 index 00000000..6d07e2bd --- /dev/null +++ b/safety/auth/cli.py @@ -0,0 +1,254 @@ +from datetime import datetime +import logging +import sys +from safety.auth.models import Auth + +from safety.console import main_console as console +from safety.constants import MSG_FINISH_REGISTRATION_TPL, MSG_VERIFICATION_HINT + +try: + from typing import Annotated +except ImportError: + from typing_extensions import Annotated + +from typing import Optional + +import click +from typer import Typer +import typer + +from safety.auth.main import get_auth_info, get_authorization_data, get_token, clean_session +from safety.auth.server import process_browser_callback +from ..cli_util import get_command_for, pass_safety_cli_obj, SafetyCLISubGroup + +from .constants import MSG_FAIL_LOGIN_AUTHED, MSG_FAIL_REGISTER_AUTHED, MSG_LOGOUT_DONE, MSG_LOGOUT_FAILED, MSG_NON_AUTHENTICATED +from safety.scan.constants import CLI_AUTH_COMMAND_HELP, DEFAULT_EPILOG, CLI_AUTH_LOGIN_HELP, CLI_AUTH_LOGOUT_HELP, CLI_AUTH_STATUS_HELP + + +from rich.padding import Padding + +LOG = logging.getLogger(__name__) + +auth_app = Typer(rich_markup_mode="rich") + + + +CMD_LOGIN_NAME = "login" +CMD_REGISTER_NAME = "register" +CMD_STATUS_NAME = "status" +CMD_LOGOUT_NAME = "logout" +DEFAULT_CMD = CMD_LOGIN_NAME + +@auth_app.callback(invoke_without_command=True, + cls=SafetyCLISubGroup, + help=CLI_AUTH_COMMAND_HELP, + epilog=DEFAULT_EPILOG, + context_settings={"allow_extra_args": True, + "ignore_unknown_options": True}) +@pass_safety_cli_obj +def auth(ctx: typer.Context): + """ + Authenticate Safety CLI with your account + """ + LOG.info('auth started') + + if not ctx.invoked_subcommand: + default_command = get_command_for(name=DEFAULT_CMD, + typer_instance=auth_app) + return ctx.forward(default_command) + + +def fail_if_authenticated(ctx, with_msg: str): + info = get_auth_info(ctx) + + if info: + console.print() + email = f"[green]{ctx.obj.auth.email}[/green]" + if not ctx.obj.auth.email_verified: + email = f"{email} {render_email_note(ctx.obj.auth)}" + + console.print(with_msg.format(email=email)) + sys.exit(0) + +def render_email_note(auth: Auth) -> str: + return "" if auth.email_verified else "[red](email verification required)[/red]" + +def render_successful_login(auth: Auth, + organization: Optional[str] = None): + DEFAULT = "--" + name = auth.name if auth.name else DEFAULT + email = auth.email if auth.email else DEFAULT + email_note = render_email_note(auth) + + console.print("[bold][green]You're authenticated[/green][/bold]") + if name and name != email: + details = [f"[green][bold]Account:[/bold] {name}, {email}[/green] {email_note}"] + else: + details = [f"[green][bold]Account:[/bold] {email}[/green] {email_note}"] + + if organization: + details.insert(0, + "[green][bold]Organization:[/bold] " \ + f"{organization}[green]") + + for msg in details: + console.print(Padding(msg, (0, 0, 0, 1)), emoji=True) + + +@auth_app.command(name=CMD_LOGIN_NAME, help=CLI_AUTH_LOGIN_HELP) +def login(ctx: typer.Context): + """ + Authenticate Safety CLI with your safetycli.com account using your default browser. + """ + LOG.info('login started') + + fail_if_authenticated(ctx, with_msg=MSG_FAIL_LOGIN_AUTHED) + + console.print() + brief_msg: str = "Redirecting your browser to log in; once authenticated, " \ + "return here to start using Safety" + + uri, initial_state = get_authorization_data(client=ctx.obj.auth.client, + code_verifier=ctx.obj.auth.code_verifier, + organization=ctx.obj.auth.org) + + if ctx.obj.auth.org: + console.print(f"Logging into [bold]{ctx.obj.auth.org.name}[/bold] " \ + "organization.") + + click.secho(brief_msg) + click.echo() + + info = process_browser_callback(uri, + initial_state=initial_state, ctx=ctx) + + if info: + if info.get("email", None): + organization = None + if ctx.obj.auth.org and ctx.obj.auth.org.name: + organization = ctx.obj.auth.org.name + ctx.obj.auth.refresh_from(info) + render_successful_login(ctx.obj.auth, organization=organization) + + console.print() + if ctx.obj.auth.org or ctx.obj.auth.email_verified: + console.print( + "[tip]Tip[/tip]: now try [bold]`safety scan`[/bold] in your project’s root " \ + "folder to run a project scan or [bold]`safety -–help`[/bold] to learn more.") + else: + console.print(MSG_FINISH_REGISTRATION_TPL.format(email=ctx.obj.auth.email)) + console.print() + console.print(MSG_VERIFICATION_HINT) + else: + click.secho("Safety is now authenticated but your email is missing.") + else: + msg = ":stop_sign: [red]" + if ctx.obj.auth.org: + msg += f"Error logging into {ctx.obj.auth.org.name} organization " \ + f"with auth ID: {ctx.obj.auth.org.id}." + else: + msg += "Error logging into Safety." + + msg += " Please try again, or use [bold]`safety auth –help`[/bold] " \ + "for more information[/red]" + + console.print(msg, emoji=True) + +@auth_app.command(name=CMD_LOGOUT_NAME, help=CLI_AUTH_LOGOUT_HELP) +def logout(ctx: typer.Context): + """ + Log out of your current session. + """ + LOG.info('logout started') + + id_token = get_token('id_token') + + msg = MSG_NON_AUTHENTICATED + + if id_token: + if clean_session(ctx.obj.auth.client): + msg = MSG_LOGOUT_DONE + else: + msg = MSG_LOGOUT_FAILED + + console.print(msg) + + +@auth_app.command(name=CMD_STATUS_NAME, help=CLI_AUTH_STATUS_HELP) +@click.option("--ensure-auth/--no-ensure-auth", default=False, + help="This will keep running the command until an" \ + "authentication is made.") +@click.option("--login-timeout", "-w", type=int, default=600, + help="Max time allowed to wait for an authentication.") +def status(ctx: typer.Context, ensure_auth: bool = False, + login_timeout: int = 600): + """ + Display Safety CLI's current authentication status. + """ + LOG.info('status started') + current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + from safety.util import get_safety_version + safety_version = get_safety_version() + console.print(f"[{current_time}]: Safety {safety_version}") + + info = get_auth_info(ctx) + + if ensure_auth: + console.print("running: safety auth status --ensure-auth") + console.print() + + if info: + verified = info.get("email_verified", False) + email_status = " [red](email not verified)[/red]" if not verified else "" + + console.print(f'[green]Authenticated as {info["email"]}[/green]{email_status}') + elif ensure_auth: + console.print('Safety is not authenticated. Launching default browser to log in') + console.print() + uri, initial_state = get_authorization_data(client=ctx.obj.auth.client, + code_verifier=ctx.obj.auth.code_verifier, + organization=ctx.obj.auth.org, ensure_auth=ensure_auth) + + info = process_browser_callback(uri, initial_state=initial_state, + timeout=login_timeout, ctx=ctx) + + if not info: + console.print(f'[red]Timeout error ({login_timeout} seconds): not successfully authenticated without the timeout period.[/red]') + sys.exit(1) + + organization = None + if ctx.obj.auth.org and ctx.obj.auth.org.name: + organization = ctx.obj.auth.org.name + + render_successful_login(ctx.obj.auth, organization=organization) + console.print() + + else: + console.print(MSG_NON_AUTHENTICATED) + + +@auth_app.command(name=CMD_REGISTER_NAME) +def register(ctx: typer.Context): + """ + Create a new user account for the safetycli.com service. + """ + LOG.info('register started') + + fail_if_authenticated(ctx, with_msg=MSG_FAIL_REGISTER_AUTHED) + + uri, initial_state = get_authorization_data(client=ctx.obj.auth.client, + code_verifier=ctx.obj.auth.code_verifier, + sign_up=True) + + console.print("Redirecting your browser to register for a free account. Once registered, return here to start using Safety.") + console.print() + + info = process_browser_callback(uri, + initial_state=initial_state, ctx=ctx) + + if info: + console.print(f'[green]Successfully registered {info.get("email")}[/green]') + console.print() + else: + console.print('[red]Unable to register in this time, try again.[/red]') + diff --git a/safety/auth/cli_utils.py b/safety/auth/cli_utils.py new file mode 100644 index 00000000..4cffe1bd --- /dev/null +++ b/safety/auth/cli_utils.py @@ -0,0 +1,191 @@ +import logging +from typing import Dict, Optional + +import click + +from .main import get_auth_info, get_host_config, get_organization, get_proxy_config, \ + get_redirect_url, get_token_data, save_auth_config, get_token, clean_session +from authlib.common.security import generate_token +from safety.auth.constants import CLIENT_ID, OPENID_CONFIG_URL + +from safety.auth.models import Organization, Auth +from safety.auth.utils import S3PresignedAdapter, SafetyAuthSession, get_keys +from safety.constants import REQUEST_TIMEOUT +from safety.scan.constants import CLI_KEY_HELP, CLI_PROXY_HOST_HELP, CLI_PROXY_PORT_HELP, CLI_PROXY_PROTOCOL_HELP, CLI_STAGE_HELP +from safety.scan.util import Stage +from safety.util import DependentOption, SafetyContext, get_proxy_dict + +from functools import wraps + + +LOG = logging.getLogger(__name__) + + +def build_client_session(api_key=None, proxies=None, headers=None): + kwargs = {} + + target_proxies = proxies + + # Global proxy defined in the config.ini + proxy_config, proxy_timeout, proxy_required = get_proxy_config() + + if not proxies: + target_proxies = proxy_config + + def update_token(tokens, **kwargs): + save_auth_config(access_token=tokens['access_token'], id_token=tokens['id_token'], + refresh_token=tokens['refresh_token']) + load_auth_session(click_ctx=click.get_current_context(silent=True)) + + client_session = SafetyAuthSession(client_id=CLIENT_ID, + code_challenge_method='S256', + redirect_uri=get_redirect_url(), + update_token=update_token, + scope='openid email profile offline_access', + **kwargs) + + client_session.mount("https://pyup.io/static-s3/", S3PresignedAdapter()) + + client_session.proxy_required = proxy_required + client_session.proxy_timeout = proxy_timeout + client_session.proxies = target_proxies + client_session.headers = {"Accept": "application/json", "Content-Type": "application/json"} + + try: + openid_config = client_session.get(url=OPENID_CONFIG_URL, timeout=REQUEST_TIMEOUT).json() + except Exception as e: + LOG.exception('Unable to load the openID config: %s', e) + openid_config = {} + + client_session.metadata["token_endpoint"] = openid_config.get("token_endpoint", + None) + + if api_key: + client_session.api_key = api_key + client_session.headers['X-Api-Key'] = api_key + + if headers: + client_session.headers.update(headers) + + return client_session, openid_config + + +def load_auth_session(click_ctx): + if not click_ctx: + LOG.warn("Click context is needed to be able to load the Auth data.") + return + + client = click_ctx.obj.auth.client + keys = click_ctx.obj.auth.keys + + access_token: str = get_token(name='access_token') + refresh_token: str = get_token(name='refresh_token') + id_token: str = get_token(name='id_token') + + if access_token and keys: + try: + token = get_token_data(access_token, keys, silent_if_expired=True) + client.token = {'access_token': access_token, + 'refresh_token': refresh_token, + 'id_token': id_token, + 'token_type': 'bearer', + 'expires_at': token.get('exp', None)} + except Exception as e: + print(e) + clean_session(client) + +def proxy_options(func): + """ + Options defined per command, this will override the proxy settings defined in the + config.ini file. + """ + func = click.option("--proxy-protocol", + type=click.Choice(['http', 'https']), default='https', + cls=DependentOption, required_options=['proxy_host'], + help=CLI_PROXY_PROTOCOL_HELP)(func) + func = click.option("--proxy-port", multiple=False, type=int, default=80, + cls=DependentOption, required_options=['proxy_host'], + help=CLI_PROXY_PORT_HELP)(func) + func = click.option("--proxy-host", multiple=False, type=str, default=None, + help=CLI_PROXY_HOST_HELP)(func) + + return func + +def auth_options(stage=True): + + def decorator(func): + + func = click.option("--key", default=None, envvar="SAFETY_API_KEY", + help=CLI_KEY_HELP)(func) + + if stage: + func = click.option("--stage", default=None, envvar="SAFETY_STAGE", + help=CLI_STAGE_HELP)(func) + + return func + + return decorator + + +def inject_session(func): + """ + Builds the session object to be used in each command. + """ + @wraps(func) + def inner(ctx, proxy_protocol: Optional[str] = None, + proxy_host: Optional[str] = None, + proxy_port: Optional[str] = None, + key: Optional[str] = None, + stage: Optional[Stage] = None, *args, **kwargs): + + if ctx.invoked_subcommand == "configure": + return + + org: Optional[Organization] = get_organization() + + if not stage: + host_stage = get_host_config(key_name="stage") + stage = host_stage if host_stage else Stage.development + + proxy_config: Optional[Dict[str, str]] = get_proxy_dict(proxy_protocol, + proxy_host, proxy_port) + + client_session, openid_config = build_client_session(api_key=key, + proxies=proxy_config) + keys = get_keys(client_session, openid_config) + + auth = Auth( + stage=stage, + keys=keys, + org=org, + client_id=CLIENT_ID, + client=client_session, + code_verifier=generate_token(48) + ) + + if not ctx.obj: + from safety.models import SafetyCLI + ctx.obj = SafetyCLI() + + ctx.obj.auth=auth + + load_auth_session(ctx) + + info = get_auth_info(ctx) + + if info: + ctx.obj.auth.name = info.get("name") + ctx.obj.auth.email = info.get("email") + ctx.obj.auth.email_verified = info.get("email_verified", False) + SafetyContext().account = info["email"] + else: + SafetyContext().account = "" + + @ctx.call_on_close + def clean_up_on_close(): + LOG.debug('Closing requests session.') + ctx.obj.auth.client.close() + + return func(ctx, *args, **kwargs) + + return inner diff --git a/safety/auth/constants.py b/safety/auth/constants.py new file mode 100644 index 00000000..1d99d901 --- /dev/null +++ b/safety/auth/constants.py @@ -0,0 +1,30 @@ +from pathlib import Path + +from safety.constants import USER_CONFIG_DIR, get_config_setting + +AUTH_CONFIG_FILE_NAME = "auth.ini" +AUTH_CONFIG_USER = USER_CONFIG_DIR / Path(AUTH_CONFIG_FILE_NAME) + + +HOST: str = 'localhost' + +CLIENT_ID = get_config_setting("CLIENT_ID") +AUTH_SERVER_URL = get_config_setting("AUTH_SERVER_URL") +SAFETY_PLATFORM_URL = get_config_setting("SAFETY_PLATFORM_URL") + +OPENID_CONFIG_URL = f"{AUTH_SERVER_URL}/.well-known/openid-configuration" + +CLI_AUTH = f'{SAFETY_PLATFORM_URL}/cli/auth' +CLI_AUTH_SUCCESS = f'{SAFETY_PLATFORM_URL}/cli/auth/success' +CLI_AUTH_LOGOUT = f'{SAFETY_PLATFORM_URL}/cli/logout' +CLI_CALLBACK = f'{SAFETY_PLATFORM_URL}/cli/callback' +CLI_LOGOUT_SUCCESS = f'{SAFETY_PLATFORM_URL}/cli/logout/success' + +MSG_NON_AUTHENTICATED = "Safety is not authenticated. Please run 'safety auth login' to log in." +MSG_FAIL_LOGIN_AUTHED = """[green]You are authenticated as[/green] {email}. + +To log into a different account, first logout via: safety auth logout, and then login again.""" +MSG_FAIL_REGISTER_AUTHED = 'You are currently logged in to {email}, please logout using `safety auth logout` before registering a new account.' + +MSG_LOGOUT_DONE = "[green]Logout done.[/green]" +MSG_LOGOUT_FAILED = "[red]Logout failed. Try again.[/red]" \ No newline at end of file diff --git a/safety/auth/main.py b/safety/auth/main.py new file mode 100644 index 00000000..eadbb648 --- /dev/null +++ b/safety/auth/main.py @@ -0,0 +1,178 @@ +import configparser +import json + +from typing import Any, Dict, Optional, Tuple, Union + +from authlib.oidc.core import CodeIDToken +from authlib.jose import jwt +from authlib.jose.errors import ExpiredTokenError + +from safety.auth.models import Organization +from safety.auth.constants import AUTH_SERVER_URL, CLI_AUTH_LOGOUT, CLI_CALLBACK, AUTH_CONFIG_USER, CLI_AUTH +from safety.constants import CONFIG +from safety.errors import NotVerifiedEmailError +from safety.scan.util import Stage +from safety.util import get_proxy_dict + + +def get_authorization_data(client, code_verifier: str, + organization: Optional[Organization] = None, + sign_up: bool = False, ensure_auth: bool = False) -> Tuple[str, str]: + + kwargs = {'sign_up': sign_up, 'locale': 'en', 'ensure_auth': ensure_auth} + if organization: + kwargs['organization'] = organization.id + + return client.create_authorization_url(CLI_AUTH, + code_verifier=code_verifier, + **kwargs) + +def get_logout_url(id_token: str) -> str: + return f'{CLI_AUTH_LOGOUT}?id_token={id_token}' + +def get_redirect_url() -> str: + return CLI_CALLBACK + +def get_organization() -> Optional[Organization]: + config = configparser.ConfigParser() + config.read(CONFIG) + + org_conf: Union[Dict[str, str], configparser.SectionProxy] = config[ + 'organization'] if 'organization' in config.sections() else {} + org_id: Optional[str] = org_conf['id'].replace("\"", "") if org_conf.get('id', None) else None + org_name: Optional[str] = org_conf['name'].replace("\"", "") if org_conf.get('name', None) else None + + if not org_id: + return None + + org = Organization( + id=org_id, + name=org_name + ) + + return org + +def get_auth_info(ctx): + info = None + if ctx.obj.auth.client.token: + try: + info = get_token_data(get_token(name='id_token'), keys=ctx.obj.auth.keys) + + verified = info.get("email_verified", False) + if not verified: + user_info = ctx.obj.auth.client.fetch_user_info() + verified = user_info.get("email_verified", False) + + if verified: + # refresh only if needed + raise ExpiredTokenError + + except ExpiredTokenError as e: + # id_token expired. So fire a manually a refresh + try: + ctx.obj.auth.client.refresh_token(ctx.obj.auth.client.metadata.get('token_endpoint'), + refresh_token=ctx.obj.auth.client.token.get('refresh_token')) + info = get_token_data(get_token(name='id_token'), keys=ctx.obj.auth.keys) + except Exception as _e: + clean_session(ctx.obj.auth.client) + except Exception as _g: + clean_session(ctx.obj.auth.client) + + return info + +def get_token_data(token, keys, silent_if_expired=False) -> Optional[Dict]: + claims = jwt.decode(token, keys, claims_cls=CodeIDToken) + try: + claims.validate() + except ExpiredTokenError as e: + if not silent_if_expired: + raise e + + return claims + +def get_token(name='access_token') -> Optional[str]: + """" + This returns tokens saved in the local auth configuration. + There are two types of tokens: access_token and id_token + """ + config = configparser.ConfigParser() + config.read(AUTH_CONFIG_USER) + + if 'auth' in config.sections() and name in config['auth']: + value = config['auth'][name] + if value: + return value + + return None + +def get_host_config(key_name) -> Optional[Any]: + config = configparser.ConfigParser() + config.read(CONFIG) + + if not config.has_section("host"): + return None + + host_section = dict(config.items("host")) + + if key_name in host_section: + if key_name == "stage": + # Support old alias in the config.ini + if host_section[key_name] == "dev": + host_section[key_name] = "development" + if host_section[key_name] not in {env.value for env in Stage}: + return None + return Stage(host_section[key_name]) + + return None + +def str_to_bool(s): + """Convert a string to a boolean value.""" + if s.lower() == 'true' or s == '1': + return True + elif s.lower() == 'false' or s == '0': + return False + else: + raise ValueError(f"Cannot convert '{s}' to a boolean value.") + +def get_proxy_config() -> Tuple[Dict[str, str], Optional[int], bool]: + config = configparser.ConfigParser() + config.read(CONFIG) + + proxy_dictionary = None + required = False + timeout = None + proxy = None + + if config.has_section("proxy"): + proxy = dict(config.items("proxy")) + + if proxy: + try: + proxy_dictionary = get_proxy_dict(proxy['protocol'], proxy['host'], + proxy['port']) + required = str_to_bool(proxy["required"]) + timeout = proxy["timeout"] + except Exception as e: + pass + + return proxy_dictionary, timeout, required + +def clean_session(client): + config = configparser.ConfigParser() + config['auth'] = {'access_token': '', 'id_token': '', 'refresh_token':''} + + with open(AUTH_CONFIG_USER, 'w') as configfile: + config.write(configfile) + + client.token = None + + return True + +def save_auth_config(access_token=None, id_token=None, refresh_token=None): + config = configparser.ConfigParser() + config.read(AUTH_CONFIG_USER) + config['auth'] = {'access_token': access_token, 'id_token': id_token, + 'refresh_token': refresh_token} + + with open(AUTH_CONFIG_USER, 'w') as configfile: + config.write(configfile) diff --git a/safety/auth/models.py b/safety/auth/models.py new file mode 100644 index 00000000..5a14f11b --- /dev/null +++ b/safety/auth/models.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass +from typing import Any, Optional + +from authlib.integrations.base_client import BaseOAuth + +from safety_schemas.models import Stage + +@dataclass +class Organization: + id: str + name: str + + def to_dict(self): + return {'id': self.id, 'name': self.name} + +@dataclass +class Auth: + org: Optional[Organization] + keys: Any + client: Any + code_verifier: str + client_id: str + stage: Optional[Stage] = Stage.development + email: Optional[str] = None + name: Optional[str] = None + email_verified: bool = False + + def is_valid(self) -> bool: + if not self.client: + return False + + if self.client.api_key: + return True + + return bool(self.client.token and self.email_verified) + + def refresh_from(self, info): + self.name = info.get("name") + self.email = info.get("email") + self.email_verified = info.get("email_verified", False) + +class XAPIKeyAuth(BaseOAuth): + def __init__(self, api_key): + self.api_key = api_key + + def __call__(self, r): + r.headers['X-API-Key'] = self.api_key + return r diff --git a/safety/auth/server.py b/safety/auth/server.py new file mode 100644 index 00000000..45e82241 --- /dev/null +++ b/safety/auth/server.py @@ -0,0 +1,169 @@ +import http.server +import logging +import socket +import sys +import time +from typing import Any, Optional +import urllib.parse +import threading +import click +from safety.auth.cli_utils import load_auth_session + +from safety.console import main_console as console + +from safety.auth.constants import AUTH_SERVER_URL, CLI_AUTH_SUCCESS, CLI_LOGOUT_SUCCESS, HOST +from safety.auth.main import save_auth_config + +LOG = logging.getLogger(__name__) + + +def find_available_port(): + """Find an available port on localhost""" + # Dynamic ports IANA + port_range = range(49152, 65536) + + for port in port_range: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + result = s.connect(('localhost', port)) + # If the connect succeeds, the port is already in use + except socket.error as e: + # If the connect fails, the port is available + return port + + return None + + +class CallbackHandler(http.server.BaseHTTPRequestHandler): + def auth(self, code: str, state: str, err, error_description): + initial_state = self.server.initial_state + ctx = self.server.ctx + + if initial_state is None or initial_state != state: + err = "The state parameter value provided does not match the expected" \ + "value. The state parameter is used to protect against Cross-Site " \ + "Request Forgery (CSRF) attacks. For security reasons, the " \ + "authorization process cannot proceed with an invalid state " \ + "parameter value. Please try again, ensuring that the state " \ + "parameter value provided in the authorization request matches " \ + "the value returned in the callback." + + if err: + click.secho(f'Error: {err}', fg='red') + sys.exit(1) + + try: + tokens = ctx.obj.auth.client.fetch_token(url=f'{AUTH_SERVER_URL}/oauth/token', + code_verifier=ctx.obj.auth.code_verifier, + client_id=ctx.obj.auth.client.client_id, + grant_type='authorization_code', code=code) + + save_auth_config(access_token=tokens['access_token'], + id_token=tokens['id_token'], + refresh_token=tokens['refresh_token']) + self.server.callback = ctx.obj.auth.client.fetch_user_info() + + except Exception as e: + LOG.exception(e) + sys.exit(1) + + self.do_redirect(location=CLI_AUTH_SUCCESS, params={}) + + def logout(self): + ctx = self.server.ctx + uri = CLI_LOGOUT_SUCCESS + + if ctx.obj.auth.org: + uri = f"{uri}&org_id={ctx.obj.auth.org.id}" + + self.do_redirect(location=CLI_LOGOUT_SUCCESS, params={}) + + def do_GET(self): + query = urllib.parse.urlparse(self.path).query + params = urllib.parse.parse_qs(query) + callback_type: Optional[str] = None + + try: + c_type = params.get('type', []) + if isinstance(c_type, list) and len(c_type) == 1 and isinstance(c_type[0], str): + callback_type = c_type[0] + except Exception: + click.secho("Unable to process the callback, try again.") + return + + if callback_type == 'logout': + self.logout() + return + + code = params.get('code', [''])[0] + state = params.get('state', [''])[0] + err = params.get('error', [''])[0] + error_description = params.get('error_description', [''])[0] + + self.auth(code=code, state=state, err=err, error_description=error_description) + + def do_redirect(self, location, params): + self.send_response(301) + self.send_header('Location', location) + self.end_headers() + + def log_message(self, format, *args): + LOG.info(format % args) + + +def process_browser_callback(uri, **kwargs) -> Any: + + class ThreadedHTTPServer(http.server.HTTPServer): + def __init__(self, server_address, RequestHandlerClass): + super().__init__(server_address, RequestHandlerClass) + self.initial_state = None + self.ctx = None + self.callback = None + self.timeout_reached = False + + def handle_timeout(self) -> None: + self.timeout_reached = True + return super().handle_timeout() + + PORT = find_available_port() + + if not PORT: + click.secho("No available ports.") + sys.exit(1) + + try: + server = ThreadedHTTPServer((HOST, PORT), CallbackHandler) + server.initial_state = kwargs.get("initial_state", None) + server.timeout = kwargs.get("timeout", 600) + # timeout = kwargs.get("timeout", None) + # timeout = float(timeout) if timeout else None + server.ctx = kwargs.get("ctx", None) + server_thread = threading.Thread(target=server.handle_request) + server_thread.start() + + target = f"{uri}&port={PORT}" + console.print(f"If the browser does not automatically open in 5 seconds, " \ + "copy and paste this url into your browser: " \ + f"[link={target}]{target}[/link]") + click.echo() + + wait_msg = "waiting for browser authentication" + + with console.status(wait_msg, spinner="bouncingBar"): + time.sleep(2) + click.launch(target) + server_thread.join() + + except OSError as e: + if e.errno == socket.errno.EADDRINUSE: + reason = f"The port {HOST}:{PORT} is currently being used by another" \ + "application or process. Please choose a different port or " \ + "terminate the conflicting application/process to free up " \ + "the port." + else: + reason = "An error occurred while performing this operation." + + click.secho(reason) + sys.exit(1) + + return server.callback diff --git a/safety/auth/utils.py b/safety/auth/utils.py new file mode 100644 index 00000000..0de1c730 --- /dev/null +++ b/safety/auth/utils.py @@ -0,0 +1,263 @@ +import json +import logging +from typing import Any, Optional +from authlib.integrations.requests_client import OAuth2Session +from authlib.integrations.base_client.errors import OAuthError +import requests +from requests.adapters import HTTPAdapter +from safety.auth.constants import AUTH_SERVER_URL +from safety.auth.main import get_auth_info, get_token_data +from safety.constants import PLATFORM_API_CHECK_UPDATES_ENDPOINT, PLATFORM_API_INITIALIZE_SCAN_ENDPOINT, PLATFORM_API_POLICY_ENDPOINT, \ + PLATFORM_API_PROJECT_CHECK_ENDPOINT, PLATFORM_API_PROJECT_ENDPOINT, PLATFORM_API_PROJECT_SCAN_REQUEST_ENDPOINT, \ + PLATFORM_API_PROJECT_UPLOAD_SCAN_ENDPOINT, REQUEST_TIMEOUT +from safety.scan.util import AuthenticationType + +from safety.util import SafetyContext, output_exception +from safety_schemas.models import STAGE_ID_MAPPING, Stage +from safety.errors import InvalidCredentialError, NetworkConnectionError, \ + RequestTimeoutError, ServerError, TooManyRequestsError, SafetyError + +LOG = logging.getLogger(__name__) + +def get_keys(client_session, openid_config): + if "jwks_uri" in openid_config: + return client_session.get(url=openid_config["jwks_uri"], bearer=False).json() + return None + +def parse_response(func): + def wrapper(*args, **kwargs): + try: + r = func(*args, **kwargs) + except OAuthError as e: + LOG.exception('OAuth failed: %s', e) + raise InvalidCredentialError(message="Your token authentication expired, try login again.") + except requests.exceptions.ConnectionError: + raise NetworkConnectionError() + except requests.exceptions.Timeout: + raise RequestTimeoutError() + except requests.exceptions.RequestException as e: + raise e + + if r.status_code == 403: + raise InvalidCredentialError(credential="Failed authentication.", + reason=r.text) + + if r.status_code == 429: + raise TooManyRequestsError(reason=r.text) + + if r.status_code >= 400 and r.status_code < 500: + error_code = None + try: + data = r.json() + reason = data.get('detail', "Unable to find reason.") + error_code = data.get("error_code", None) + except Exception as e: + reason = r.reason + + raise SafetyError(message=reason, error_code=error_code) + + if r.status_code >= 500: + raise ServerError(reason=f"{r.reason} - {r.text}") + + data = None + + try: + data = r.json() + except json.JSONDecodeError as e: + raise SafetyError(message=f"Bad JSON response: {e}") + + return data + + return wrapper + +class SafetyAuthSession(OAuth2Session): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.proxy_required: bool = False + self.proxy_timeout: Optional[int] = None + self.api_key = None + + def get_credential(self) -> Optional[str]: + if self.api_key: + return self.api_key + + if self.token: + return SafetyContext().account + + return None + + def is_using_auth_credentials(self) -> bool: + """This does NOT check if the client is authenticated""" + return self.get_authentication_type() != AuthenticationType.none + + def get_authentication_type(self) -> AuthenticationType: + if self.api_key: + return AuthenticationType.api_key + + if self.token: + return AuthenticationType.token + + return AuthenticationType.none + + def request(self, method, url, withhold_token=False, auth=None, bearer=True, **kwargs): + """Use the right auth parameter for Safety supported auth types""" + # By default use the token_auth + TIMEOUT_KEYWARD = "timeout" + func_timeout = kwargs[TIMEOUT_KEYWARD] if TIMEOUT_KEYWARD in kwargs else REQUEST_TIMEOUT + + if self.api_key: + key_header = {"X-Api-Key": self.api_key} + if not "headers" in kwargs: + kwargs["headers"] = key_header + else: + kwargs["headers"]["X-Api-Key"] = self.api_key + + if not self.token or not bearer: + # Fallback to no token auth + auth = () + + + # Override proxies + if self.proxies: + kwargs['proxies'] = self.proxies + + if self.proxy_timeout: + kwargs['timeout'] = int(self.proxy_timeout) / 1000 + + if ("proxies" not in kwargs or not self.proxies) and self.proxy_required: + output_exception("Proxy connection is required but there is not a proxy setup.", exit_code_output=True) + + request_func = super(SafetyAuthSession, self).request + params = { + 'method': method, + 'url': url, + 'withhold_token': withhold_token, + 'auth': auth, + } + params.update(kwargs) + + try: + return request_func(**params) + except Exception as e: + LOG.debug('Request failed: %s', e) + + if self.proxy_required: + output_exception(f"Proxy is required but the connection failed because: {e}", exit_code_output=True) + + if ("proxies" in kwargs or self.proxies): + params["proxies"] = {} + params['timeout'] = func_timeout + self.proxies = {} + message = "The proxy configuration failed to function and was disregarded." + LOG.debug(message) + if message not in [a['message'] for a in SafetyContext.local_announcements]: + SafetyContext.local_announcements.append({'message': message, 'type': 'warning', 'local': True}) + + return request_func(**params) + + raise e + + @parse_response + def fetch_user_info(self) -> Any: + USER_INFO_ENDPOINT = f"{AUTH_SERVER_URL}/userinfo" + + r = self.get( + url=USER_INFO_ENDPOINT + ) + + return r + + @parse_response + def check_project(self, scan_stage: str, safety_source: str, + project_slug: Optional[str] = None, git_origin: Optional[str] = None, + project_slug_source: Optional[str] = None) -> Any: + + data = {"scan_stage": scan_stage, "safety_source": safety_source, + "project_slug": project_slug, + "project_slug_source": project_slug_source, + "git_origin": git_origin} + + r = self.post( + url=PLATFORM_API_PROJECT_CHECK_ENDPOINT, + json=data + ) + + return r + + @parse_response + def project(self, project_id: str) -> Any: + data = {"project": project_id} + + r = self.get( + url=PLATFORM_API_PROJECT_ENDPOINT, + params=data + ) + + return r + + @parse_response + def download_policy(self, project_id: Optional[str], stage: Stage, branch: Optional[str]) -> Any: + data = {"project": project_id, "stage": STAGE_ID_MAPPING[stage], "branch": branch} + + r = self.get( + url=PLATFORM_API_POLICY_ENDPOINT, + params=data + ) + + return r + + @parse_response + def project_scan_request(self, project_id: str) -> Any: + data = {"project_id": project_id} + + r = self.post( + url=PLATFORM_API_PROJECT_SCAN_REQUEST_ENDPOINT, + json=data + ) + + return r + + @parse_response + def upload_report(self, json_report: str) -> Any: + + headers = { + "Content-Type": "application/json" + } + + r = self.post( + url=PLATFORM_API_PROJECT_UPLOAD_SCAN_ENDPOINT, + data=json_report, + headers=headers + ) + + return r + + @parse_response + def check_updates(self, version: int, safety_version=None, + python_version=None, + os_type=None, + os_release=None, + os_description=None) -> Any: + data = {"version": version, + "safety_version": safety_version, + "python_version": python_version, + "os_type": os_type, + "os_release": os_release, + "os_description": os_description} + + r = self.get( + url=PLATFORM_API_CHECK_UPDATES_ENDPOINT, + params=data + ) + + return r + + @parse_response + def initialize_scan(self) -> Any: + return self.get(url=PLATFORM_API_INITIALIZE_SCAN_ENDPOINT, timeout=2) + +class S3PresignedAdapter(HTTPAdapter): + def send(self, request, **kwargs): + request.headers.pop("Authorization", None) + return super().send(request, **kwargs) diff --git a/safety/cli.py b/safety/cli.py index a99f0041..a2c3768e 100644 --- a/safety/cli.py +++ b/safety/cli.py @@ -1,56 +1,79 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import configparser +from dataclasses import asdict +from enum import Enum import json import logging import os +from pathlib import Path +import platform import sys from functools import wraps +from typing import Dict, Optional import click +import typer from safety import safety +from safety.console import main_console as console from safety.alerts import alert -from safety.constants import EXIT_CODE_VULNERABILITIES_FOUND, EXIT_CODE_OK, EXIT_CODE_FAILURE -from safety.errors import SafetyException, SafetyError, InvalidKeyError +from safety.auth import auth, inject_session, proxy_options, auth_options +from safety.auth.models import Organization +from safety.scan.constants import CLI_MAIN_INTRODUCTION, CLI_DEBUG_HELP, CLI_DISABLE_OPTIONAL_TELEMETRY_DATA_HELP, \ + DEFAULT_EPILOG, DEFAULT_SPINNER, CLI_CHECK_COMMAND_HELP, CLI_CHECK_UPDATES_HELP, CLI_CONFIGURE_HELP, CLI_GENERATE_HELP, \ + CLI_CONFIGURE_PROXY_TIMEOUT, CLI_CONFIGURE_PROXY_REQUIRED, CLI_CONFIGURE_ORGANIZATION_ID, CLI_CONFIGURE_ORGANIZATION_NAME, \ + CLI_CONFIGURE_SAVE_TO_SYSTEM, CLI_CONFIGURE_PROXY_HOST_HELP, CLI_CONFIGURE_PROXY_PORT_HELP, CLI_CONFIGURE_PROXY_PROTOCOL_HELP, \ + CLI_GENERATE_PATH +from .cli_util import SafetyCLICommand, SafetyCLILegacyGroup, SafetyCLILegacyCommand, SafetyCLISubGroup, SafetyCLIUtilityCommand, handle_cmd_exception +from safety.constants import CONFIG_FILE_USER, CONFIG_FILE_SYSTEM, EXIT_CODE_VULNERABILITIES_FOUND, EXIT_CODE_OK, EXIT_CODE_FAILURE +from safety.errors import InvalidCredentialError, SafetyException, SafetyError from safety.formatter import SafetyFormatter +from safety.models import SafetyCLI from safety.output_utils import should_add_nl from safety.safety import get_packages, read_vulnerabilities, process_fixes -from safety.util import get_proxy_dict, get_packages_licenses, output_exception, \ +from safety.util import get_packages_licenses, initializate_config_dirs, output_exception, \ MutuallyExclusiveOption, DependentOption, transform_ignore, SafetyPolicyFile, active_color_if_needed, \ get_processed_options, get_safety_version, json_alias, bare_alias, html_alias, SafetyContext, is_a_remote_mirror, \ filter_announcements, get_fix_options +from safety.scan.command import scan_project_app, scan_system_app +from safety.auth.cli import auth_app +from safety_schemas.models import ConfigModel, Stage + +try: + from typing import Annotated +except ImportError: + from typing_extensions import Annotated -LOG = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) -@click.group() -@click.option('--debug/--no-debug', default=False) -@click.option('--telemetry/--disable-telemetry', default=True, hidden=True) -@click.option('--disable-optional-telemetry-data', default=False, cls=MutuallyExclusiveOption, - mutually_exclusive=["telemetry", "disable-telemetry"], is_flag=True, show_default=True) +@click.group(cls=SafetyCLILegacyGroup, help=CLI_MAIN_INTRODUCTION, epilog=DEFAULT_EPILOG) +@auth_options() +@proxy_options +@click.option('--disable-optional-telemetry', default=False, is_flag=True, show_default=True, help=CLI_DISABLE_OPTIONAL_TELEMETRY_DATA_HELP) +@click.option('--debug', default=False, help=CLI_DEBUG_HELP) @click.version_option(version=get_safety_version()) @click.pass_context -def cli(ctx, debug, telemetry, disable_optional_telemetry_data): +@inject_session +def cli(ctx, debug, disable_optional_telemetry): """ - Safety checks Python dependencies for known security vulnerabilities and suggests the proper - remediations for vulnerabilities detected. Safety can be run on developer machines, in CI/CD pipelines and - on production systems. + Scan and secure Python projects against package vulnerabilities. To get started navigate to a Python project and run `safety scan`. """ SafetyContext().safety_source = 'cli' - ctx.telemetry = telemetry and not disable_optional_telemetry_data + telemetry = not disable_optional_telemetry + ctx.obj.config = ConfigModel(telemetry_enabled=telemetry) level = logging.CRITICAL if debug: level = logging.DEBUG logging.basicConfig(format='%(asctime)s %(name)s => %(message)s', level=level) - LOG.info(f'Telemetry enabled: {ctx.telemetry}') + LOG.info(f'Telemetry enabled: {ctx.obj.config.telemetry_enabled}') - @ctx.call_on_close - def clean_up_on_close(): - LOG.debug('Calling clean up on close function.') - safety.close_session() + # Before any command make sure that the parent dirs for Safety config are present. + initializate_config_dirs() def clean_check_command(f): @@ -58,32 +81,48 @@ def clean_check_command(f): Main entry point for validation. """ @wraps(f) - def inner(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinned_requirements, output, - json, html, bare, proxy_protocol, proxy_host, proxy_port, exit_code, policy_file, save_json, save_html, - audit_and_monitor, project, apply_remediations, auto_remediation_limit, no_prompt, json_version, - *args, **kwargs): - + def inner(ctx, *args, **kwargs): + + save_json = kwargs["save_json"] + output = kwargs["output"] + authenticated: bool = ctx.obj.auth.client.is_using_auth_credentials() + files = kwargs["files"] + policy_file = kwargs["policy_file"] + auto_remediation_limit = kwargs["auto_remediation_limit"] + audit_and_monitor = kwargs["audit_and_monitor"] + exit_code = kwargs["exit_code"] + + # This is handled in the custom subgroup Click class + # TODO: Remove this soon, for now it keeps a legacy behavior + kwargs.pop("key", None) + kwargs.pop('proxy_protocol', None) + kwargs.pop('proxy_host', None) + kwargs.pop('proxy_port', None) + if ctx.get_parameter_source("json_version") != click.core.ParameterSource.DEFAULT and not ( save_json or json or output == 'json'): raise click.UsageError( - f"Illegal usage: `--json-version` only works with JSON related outputs." + "Illegal usage: `--json-version` only works with JSON related outputs." ) try: - proxy_dictionary = get_proxy_dict(proxy_protocol, proxy_host, proxy_port) if ctx.get_parameter_source("apply_remediations") != click.core.ParameterSource.DEFAULT: - if not key: - raise InvalidKeyError(message="The --apply-security-updates option needs an API-KEY. See {link}.") + if not authenticated: + raise InvalidCredentialError(message="The --apply-security-updates option needs authentication. See {link}.") if not files: raise SafetyError(message='--apply-security-updates only works with files; use the "-r" option to ' 'specify files to remediate.') auto_remediation_limit = get_fix_options(policy_file, auto_remediation_limit) - policy_file, server_audit_and_monitor = safety.get_server_policies(key=key, policy_file=policy_file, - proxy_dictionary=proxy_dictionary) + policy_file, server_audit_and_monitor = safety.get_server_policies(ctx.obj.auth.client, policy_file=policy_file, + proxy_dictionary=None) audit_and_monitor = (audit_and_monitor and server_audit_and_monitor) + kwargs.update({"auto_remediation_limit": auto_remediation_limit, + "policy_file":policy_file, + "audit_and_monitor": audit_and_monitor}) + except SafetyError as e: LOG.exception('Expected SafetyError happened: %s', e) output_exception(e, exit_code_output=exit_code) @@ -92,18 +131,14 @@ def inner(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinne exception = e if isinstance(e, SafetyException) else SafetyException(info=e) output_exception(exception, exit_code_output=exit_code) - return f(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinned_requirements, output, json, - html, bare, proxy_protocol, proxy_host, proxy_port, exit_code, policy_file, audit_and_monitor, - project, save_json, save_html, apply_remediations, auto_remediation_limit, no_prompt, json_version, - *args, **kwargs) + return f(ctx, *args, **kwargs) return inner -@cli.command() -@click.option("--key", default="", envvar="SAFETY_API_KEY", - help="API Key for pyup.io's vulnerability database. Can be set as SAFETY_API_KEY " - "environment variable. Default: empty") +@cli.command(cls=SafetyCLILegacyCommand, utility_command=True, help=CLI_CHECK_COMMAND_HELP) +@proxy_options +@auth_options(stage=False) @click.option("--db", default="", help="Path to a local or remote vulnerability database. Default: empty") @click.option("--full-report/--short-report", default=False, cls=MutuallyExclusiveOption, @@ -133,22 +168,14 @@ def inner(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinne hidden=True, is_flag=True, show_default=True) @click.option('--output', "-o", type=click.Choice(['screen', 'text', 'json', 'bare', 'html'], case_sensitive=False), default='screen', callback=active_color_if_needed, envvar='SAFETY_OUTPUT') -@click.option("--proxy-protocol", "-pr", type=click.Choice(['http', 'https']), default='https', cls=DependentOption, - required_options=['proxy_host'], - help="Proxy protocol (https or http) --proxy-protocol") -@click.option("--proxy-host", "-ph", multiple=False, type=str, default=None, - help="Proxy host IP or DNS --proxy-host") -@click.option("--proxy-port", "-pp", multiple=False, type=int, default=80, cls=DependentOption, - required_options=['proxy_host'], - help="Proxy port number --proxy-port") @click.option("--exit-code/--continue-on-error", default=True, help="Output standard exit codes. Default: --exit-code") @click.option("--policy-file", type=SafetyPolicyFile(), default='.safety-policy.yml', help="Define the policy file to be used") @click.option("--audit-and-monitor/--disable-audit-and-monitor", default=True, - help="Send results back to pyup.io for viewing on your dashboard. Requires an API key.") + help="Send results back to safetycli.com for viewing on your dashboard. Requires an API key.") @click.option("project", "--project-id", "--project", default=None, - help="Project to associate this scan with on pyup.io. " + help="Project to associate this scan with on safetycli.com. " "Defaults to a canonicalized github style name if available, otherwise unknown") @click.option("--save-json", default="", help="Path to where the output file will be placed; if the path is a" " directory, Safety will use safety-report.json as filename." @@ -169,13 +196,12 @@ def inner(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinne help="Select the JSON version to be used in the output", show_default=True) @click.pass_context @clean_check_command -def check(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinned_requirements, output, json, - html, bare, proxy_protocol, proxy_host, proxy_port, exit_code, policy_file, audit_and_monitor, project, +def check(ctx, db, full_report, stdin, files, cache, ignore, ignore_unpinned_requirements, output, json, + html, bare, exit_code, policy_file, audit_and_monitor, project, save_json, save_html, apply_remediations, auto_remediation_limit, no_prompt, json_version): """ - Find vulnerabilities in Python dependencies at the target provided. - + [underline][DEPRECATED][/underline] `check` has been replaced by the `scan` command, and will be unsupported beyond 1 May 2024.Find vulnerabilities at a target file or enviroment. """ LOG.info('Running check command') @@ -187,7 +213,6 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinne try: packages = get_packages(files, stdin) - proxy_dictionary = get_proxy_dict(proxy_protocol, proxy_host, proxy_port) ignore_severity_rules = None ignore, ignore_severity_rules, exit_code, ignore_unpinned_requirements, project = \ @@ -203,9 +228,9 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinne 'ignore_unpinned_requirements': ignore_unpinned_requirements} LOG.info('Calling the check function') - vulns, db_full = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_vulns=ignore, - ignore_severity_rules=ignore_severity_rules, proxy=proxy_dictionary, - include_ignored=True, is_env_scan=is_env_scan, telemetry=ctx.parent.telemetry, + vulns, db_full = safety.check(session=ctx.obj.auth.client, packages=packages, db_mirror=db, cached=cache, ignore_vulns=ignore, + ignore_severity_rules=ignore_severity_rules, proxy=None, + include_ignored=True, is_env_scan=is_env_scan, telemetry=ctx.obj.config.telemetry_enabled, params=params) LOG.debug('Vulnerabilities returned: %s', vulns) LOG.debug('full database returned is None: %s', db_full is None) @@ -217,7 +242,7 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinne announcements = [] if not db or is_a_remote_mirror(db): LOG.info('Not local DB used, Getting announcements') - announcements = safety.get_announcements(key=key, proxy=proxy_dictionary, telemetry=ctx.parent.telemetry) + announcements = safety.get_announcements(ctx.obj.auth.client, telemetry=ctx.obj.config.telemetry_enabled) announcements.extend(safety.add_local_notifications(packages, ignore_unpinned_requirements)) @@ -260,7 +285,6 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinne remediations, full_report, packages, fixes) - safety.push_audit_and_monitor(key, proxy_dictionary, audit_and_monitor, json_report, policy_file) safety.save_report(save_json, 'safety-report.json', json_report) if save_html: @@ -284,102 +308,12 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, ignore_unpinne output_exception(exception, exit_code_output=exit_code) -@cli.command() -@click.option("--full-report/--short-report", default=False, cls=MutuallyExclusiveOption, mutually_exclusive=["output"], with_values={"output": ['json', 'bare']}, - help='Full reports include a security advisory (if available). Default: ' - '--short-report') -@click.option('--output', "-o", type=click.Choice(['screen', 'text', 'json', 'bare'], case_sensitive=False), - default='screen', callback=active_color_if_needed) -@click.option("file", "--file", "-f", type=click.File(), required=True, - help="Read input from an insecure report file. Default: empty") -@click.pass_context -def review(ctx, full_report, output, file): - """ - Show an output from a previous exported JSON report. - """ - LOG.info('Running check command') - report = {} - - try: - report = read_vulnerabilities(file) - except SafetyError as e: - LOG.exception('Expected SafetyError happened: %s', e) - output_exception(e, exit_code_output=True) - except Exception as e: - LOG.exception('Unexpected Exception happened: %s', e) - exception = e if isinstance(e, SafetyException) else SafetyException(info=e) - output_exception(exception, exit_code_output=True) - - params = {'file': file} - vulns, remediations, packages = safety.review(report, params=params) - - announcements = safety.get_announcements(key=None, proxy=None, telemetry=ctx.parent.telemetry) - output_report = SafetyFormatter(output=output).render_vulnerabilities(announcements, vulns, remediations, - full_report, packages) - - found_vulns = list(filter(lambda v: not v.ignored, vulns)) - click.secho(output_report, nl=should_add_nl(output, found_vulns), file=sys.stdout) - sys.exit(EXIT_CODE_OK) - - -@cli.command() -@click.option("--key", envvar="SAFETY_API_KEY", - help="API Key for pyup.io's vulnerability database. Can be set as SAFETY_API_KEY " - "environment variable. Default: empty") -@click.option("--db", default="", - help="Path to a local license database. Default: empty") -@click.option('--output', "-o", type=click.Choice(['screen', 'text', 'json', 'bare'], case_sensitive=False), - default='screen') -@click.option("--cache", default=0, - help='Whether license database file should be cached.' - 'Default: 0 seconds') -@click.option("files", "--file", "-r", multiple=True, type=click.File(), - help="Read input from one (or multiple) requirement files. Default: empty") -@click.option("proxyhost", "--proxy-host", "-ph", multiple=False, type=str, default=None, - help="Proxy host IP or DNS --proxy-host") -@click.option("proxyport", "--proxy-port", "-pp", multiple=False, type=int, default=80, - help="Proxy port number --proxy-port") -@click.option("proxyprotocol", "--proxy-protocol", "-pr", multiple=False, type=str, default='http', - help="Proxy protocol (https or http) --proxy-protocol") -@click.pass_context -def license(ctx, key, db, output, cache, files, proxyprotocol, proxyhost, proxyport): - """ - Find the open source licenses used by your Python dependencies. - """ - LOG.info('Running license command') - packages = get_packages(files, False) - - proxy_dictionary = get_proxy_dict(proxyprotocol, proxyhost, proxyport) - licenses_db = {} - - try: - licenses_db = safety.get_licenses(key=key, db_mirror=db, cached=cache, proxy=proxy_dictionary, - telemetry=ctx.parent.telemetry) - except SafetyError as e: - LOG.exception('Expected SafetyError happened: %s', e) - output_exception(e, exit_code_output=False) - except Exception as e: - LOG.exception('Unexpected Exception happened: %s', e) - exception = e if isinstance(e, SafetyException) else SafetyException(info=e) - output_exception(exception, exit_code_output=False) - - filtered_packages_licenses = get_packages_licenses(packages=packages, licenses_db=licenses_db) - - announcements = [] - if not db: - announcements = safety.get_announcements(key=key, proxy=proxy_dictionary, telemetry=ctx.parent.telemetry) - - output_report = SafetyFormatter(output=output).render_licenses(announcements, filtered_packages_licenses) - - click.secho(output_report, nl=True) - - -@cli.command() -@click.option("--path", default=".", help="Path where the generated file will be saved. Default: current directory") -@click.argument('name') +@cli.command(cls=SafetyCLILegacyCommand, utility_command=True, help=CLI_GENERATE_HELP) +@click.option("--path", default=".", help=CLI_GENERATE_PATH) +@click.argument('name', required=True) @click.pass_context def generate(ctx, name, path): - """Create a boilerplate supported file type. + """Create a boilerplate Safety CLI policy file NAME is the name of the file type to generate. Valid values are: policy_file """ @@ -390,36 +324,38 @@ def generate(ctx, name, path): LOG.info('Running generate %s', name) - if not os.path.exists(path): + path = Path(path) + if not path.exists(): click.secho(f'The path "{path}" does not exist.', fg='red', file=sys.stderr) sys.exit(EXIT_CODE_FAILURE) - policy = os.path.join(path, '.safety-policy.yml') - ROOT = os.path.dirname(os.path.abspath(__file__)) + policy = path / '.safety-policy.yml' + + default_config = ConfigModel() try: - with open(policy, "w") as f: - f.write(open(os.path.join(ROOT, 'safety-policy-template.yml')).read()) - LOG.debug('Safety created the policy file.') - msg = f'A default Safety policy file has been generated! Review the file contents in the path {path} in the ' \ - 'file: .safety-policy.yml' - click.secho(msg, fg='green') + default_config.save_policy_file(policy) + LOG.debug('Safety created the policy file.') + msg = f'A default Safety policy file has been generated! Review the file contents in the path {path} in the ' \ + 'file: .safety-policy.yml' + click.secho(msg, fg='green') except Exception as exc: if isinstance(exc, OSError): LOG.debug('Unable to generate %s because: %s', name, exc.errno) - click.secho(f'Unable to generate {name}, because: {str(exc)} error.', fg='red', + click.secho(f'{str(exc)} error.', fg='red', file=sys.stderr) sys.exit(EXIT_CODE_FAILURE) -@cli.command() +@cli.command(cls=SafetyCLILegacyCommand, utility_command=True) @click.option("--path", default=".safety-policy.yml", help="Path where the generated file will be saved. Default: current directory") @click.argument('name') +@click.argument('version', required=False) @click.pass_context -def validate(ctx, name, path): - """Verify the validity of a supported file type. +def validate(ctx, name, version, path): + """Verify that a local policy file is valid NAME is the name of the file type to validate. Valid values are: policy_file """ @@ -433,21 +369,281 @@ def validate(ctx, name, path): if not os.path.exists(path): click.secho(f'The path "{path}" does not exist.', fg='red', file=sys.stderr) sys.exit(EXIT_CODE_FAILURE) + + if version not in ["3.0", "2.0", None]: + click.secho(f'Version "{version}" is not a valid value, allowed values are 3.0 and 2.0. Use --path to specify the target file.', fg='red', file=sys.stderr) + sys.exit(EXIT_CODE_FAILURE) + + def fail_validation(e): + click.secho(str(e).lstrip(), fg='red', file=sys.stderr) + sys.exit(EXIT_CODE_FAILURE) + + if not version: + version = "3.0" + + result = "" + + if version == "3.0": + policy = None + + try: + from .scan.main import load_policy_file + policy = load_policy_file(Path(path)) + except Exception as e: + fail_validation(e) + + click.secho(f"The Safety policy ({version}) file " \ + "(Used for scan and system-scan commands) " \ + "was successfully parsed " \ + "with the following values:", fg="green") + if policy and policy.config: + result = policy.config.as_v30().json() + else: + try: + values = SafetyPolicyFile().convert(path, None, None) + except Exception as e: + click.secho(str(e).lstrip(), fg='red', file=sys.stderr) + sys.exit(EXIT_CODE_FAILURE) + + del values['raw'] + + result = json.dumps(values, indent=4, default=str) + + click.secho("The Safety policy file " \ + "(Valid only for the check command) " \ + "was successfully parsed with the " \ + "following values:", fg="green") + + console.print_json(result) + + +@cli.command(cls=SafetyCLILegacyCommand, + help=CLI_CONFIGURE_HELP, + utility_command=True) +@click.option("--proxy-protocol", "-pr", type=click.Choice(['http', 'https']), default='https', cls=DependentOption, + required_options=['proxy_host'], + help=CLI_CONFIGURE_PROXY_PROTOCOL_HELP) +@click.option("--proxy-host", "-ph", multiple=False, type=str, default=None, + help=CLI_CONFIGURE_PROXY_HOST_HELP) +@click.option("--proxy-port", "-pp", multiple=False, type=int, default=80, + cls=DependentOption, + required_options=['proxy_host'], + help=CLI_CONFIGURE_PROXY_PORT_HELP) +@click.option("--proxy-timeout", "-pt", multiple=False, type=int, default=None, + help=CLI_CONFIGURE_PROXY_TIMEOUT) +@click.option('--proxy-required', default=False, + help=CLI_CONFIGURE_PROXY_REQUIRED) +@click.option("--organization-id", "-org-id", multiple=False, default=None, + cls=DependentOption, + required_options=['organization_name'], + help=CLI_CONFIGURE_ORGANIZATION_ID) +@click.option("--organization-name", "-org-name", multiple=False, default=None, + cls=DependentOption, + required_options=['organization_id'], + help=CLI_CONFIGURE_ORGANIZATION_NAME) +@click.option("--stage", "-stg", multiple=False, default=Stage.development.value, + type=click.Choice([stage.value for stage in Stage]), + help="The project development stage to be tied to the current device.") +@click.option("--save-to-system/--save-to-user", default=False, is_flag=True, + help=CLI_CONFIGURE_SAVE_TO_SYSTEM) +@click.pass_context +def configure(ctx, proxy_protocol, proxy_host, proxy_port, proxy_timeout, + proxy_required, organization_id, organization_name, stage, + save_to_system): + """ + Configure global settings, like proxy settings and organization details + """ + + config = configparser.ConfigParser() + if save_to_system: + if not CONFIG_FILE_SYSTEM: + click.secho( + f"Unable to determine the system wide config path. You can set the SAFETY_SYSTEM_CONFIG_PATH env var") + sys.exit(1) + + CONFIG_FILE = CONFIG_FILE_SYSTEM + else: + CONFIG_FILE = CONFIG_FILE_USER + + config.read(CONFIG_FILE) + + PROXY_SECTION_NAME: str = 'proxy' + PROXY_TIMEOUT_KEY: str = 'timeout' + PROXY_REQUIRED_KEY: str = 'required' + + if organization_id: + config['organization'] = asdict(Organization(id=organization_id, + name=organization_name)) + + DEFAULT_PROXY_TIMEOUT: int = 500 + + if not proxy_timeout: + try: + proxy_timeout = int(config['proxy']['timeout']) + except Exception: + proxy_timeout = DEFAULT_PROXY_TIMEOUT + + new_proxy_config = {} + new_proxy_config.setdefault(PROXY_TIMEOUT_KEY, str(proxy_timeout)) + new_proxy_config.setdefault(PROXY_REQUIRED_KEY, str(proxy_required)) + + if proxy_host: + new_proxy_config.update({ + 'protocol': proxy_protocol, + 'host': proxy_host, + 'port': str(proxy_port) + }) + + if not config.has_section(PROXY_SECTION_NAME): + config.add_section(PROXY_SECTION_NAME) + + proxy_config = dict(config.items(PROXY_SECTION_NAME)) + proxy_config.update(new_proxy_config) + + for key, value in proxy_config.items(): + config.set(PROXY_SECTION_NAME, key, value) + + if stage: + config['host'] = {'stage': "development" if stage == "dev" else stage} try: - values = SafetyPolicyFile().convert(path, None, None) + with open(CONFIG_FILE, 'w') as configfile: + config.write(configfile) except Exception as e: - click.secho(str(e).lstrip(), fg='red', file=sys.stderr) - sys.exit(EXIT_CODE_FAILURE) + if (isinstance(e, OSError) and e.errno == 2 or e is PermissionError) and save_to_system: + click.secho("Unable to save the configuration: writing to system-wide Safety configuration file requires admin privileges") + else: + click.secho(f"Unable to save the configuration, error: {e}") + sys.exit(1) + + +cli_app = typer.Typer(rich_markup_mode="rich", cls=SafetyCLISubGroup) +typer.rich_utils.STYLE_HELPTEXT = "" + +def print_check_updates_header(console): + VERSION = get_safety_version() + console.print( + f"Safety {VERSION} checking for Safety version and configuration updates:") + +class Output(str, Enum): + SCREEN = "screen" + JSON = "json" + +@cli_app.command( + cls=SafetyCLIUtilityCommand, + help=CLI_CHECK_UPDATES_HELP, + name="check-updates", epilog=DEFAULT_EPILOG, + context_settings={"allow_extra_args": True, + "ignore_unknown_options": True}, + ) +@handle_cmd_exception +def check_updates(ctx: typer.Context, + version: Annotated[ + int, + typer.Option(min=1), + ] = 1, + output: Annotated[Output, + typer.Option( + help="The main output generated by Safety CLI.") + ] = Output.SCREEN): + """ + Check for Safety CLI version updates + """ - del values['raw'] + if output is Output.JSON: + console.quiet = True - click.secho(f'The Safety policy file was successfully parsed with the following values:', fg='green') - click.secho(json.dumps(values, indent=4, default=str)) + print_check_updates_header(console) + wait_msg = "Authenticating and checking for Safety CLI updates" -cli.add_command(alert) + VERSION = get_safety_version() + PYTHON_VERSION = platform.python_version() + OS_TYPE = platform.system() + authenticated = ctx.obj.auth.client.is_using_auth_credentials() + data = None + + console.print() + with console.status(wait_msg, spinner=DEFAULT_SPINNER): + try: + data = ctx.obj.auth.client.check_updates(version=1, + safety_version=VERSION, + python_version=PYTHON_VERSION, + os_type=OS_TYPE, + os_release=platform.release(), + os_description=platform.platform()) + except InvalidCredentialError as e: + authenticated = False + except Exception as e: + LOG.exception(f'Failed to check updates, reason: {e}') + raise e + + if not authenticated: + if console.quiet: + console.quiet = False + response = { + "status": 401, + "message": "Authenticated failed, please authenticate Safety and try again", + "data": {} + } + console.print_json(json.dumps(response)) + else: + console.print() + console.print("[red]Safety is not authenticated, please first authenticate and try again.[/red]") + console.print() + console.print("To authenticate, use the `auth` command: `safety auth login` Or for more help: `safety auth —help`") + sys.exit(1) + + if not data: + raise SafetyException("No data found.") + + console.print("[green]Safety CLI is authenticated:[/green]") + + from rich.padding import Padding + organization = data.get("organization", "-") + account = data.get("user_email", "-") + current_version = f"Current version: {VERSION} (Python {PYTHON_VERSION} on {OS_TYPE})" + latest_available_version = data.get("safety_updates", {}).get("stable_version", "-") + + details = [f"Organization: {organization}", + f"Account: {account}", + current_version, + f"Latest available version: {latest_available_version}" + ] + + for msg in details: + console.print(Padding(msg, (0, 0, 0, 1)), emoji=True) + + console.print() + + if latest_available_version: + console.print(f"Update available: Safety version {latest_available_version}") + console.print() + console.print( + f"If Safety was installed from a requirements file, update Safety to version {latest_available_version} in that requirements file." + ) + console.print() + # `pip -i install safety=={latest_available_version}` OR + console.print(f"Pip: To install the updated version of Safety directly via pip, run: `pip install safety=={latest_available_version}`") + + if console.quiet: + console.quiet = False + response = { + "status": 200, + "message": "", + "data": data + } + console.print_json(json.dumps(response)) + + +cli.add_command(typer.main.get_command(cli_app), "check-updates") +cli.add_command(typer.main.get_command(scan_project_app), "scan") +cli.add_command(typer.main.get_command(scan_system_app), "system-scan") + +cli.add_command(typer.main.get_command(auth_app), "auth") + +cli.add_command(alert) -if __name__ == "__main__": +if __name__ == "__main__": cli() diff --git a/safety/cli_util.py b/safety/cli_util.py new file mode 100644 index 00000000..abac0e96 --- /dev/null +++ b/safety/cli_util.py @@ -0,0 +1,589 @@ +from collections import defaultdict +import logging +import sys +from typing import Any, DefaultDict, Dict, List, Optional, Tuple, Union +import click +from functools import wraps +import typer +from typer.core import TyperGroup, TyperCommand, MarkupMode +from safety.auth.constants import CLI_AUTH, MSG_NON_AUTHENTICATED +from safety.auth.models import Auth +from safety.constants import MSG_NO_AUTHD_CICD_PROD_STG, MSG_NO_AUTHD_CICD_PROD_STG_ORG, MSG_NO_AUTHD_DEV_STG, MSG_NO_AUTHD_DEV_STG_ORG_PROMPT, MSG_NO_AUTHD_DEV_STG_PROMPT, MSG_NO_AUTHD_NOTE_CICD_PROD_STG_TPL, MSG_NO_VERIFIED_EMAIL_TPL +from safety.scan.constants import CONSOLE_HELP_THEME + +from safety.scan.models import ScanOutput + +from .util import output_exception +from .errors import SafetyError, SafetyException + +LOG = logging.getLogger(__name__) + + +def get_command_for(name:str, typer_instance: typer.Typer): + single_command = next( + (command + for command in typer_instance.registered_commands + if command.name == name), None) + + if not single_command: + raise ValueError("Unable to find the command name.") + + single_command.context_settings = typer_instance.info.context_settings + click_command = typer.main.get_command_from_info( + single_command, + pretty_exceptions_short=typer_instance.pretty_exceptions_short, + rich_markup_mode=typer_instance.rich_markup_mode, + ) + if typer_instance._add_completion: + click_install_param, click_show_param = \ + typer.main.get_install_completion_arguments() + click_command.params.append(click_install_param) + click_command.params.append(click_show_param) + return click_command + + +def pass_safety_cli_obj(func): + """ + Make sure the SafetyCLI object exists for a command. + """ + @wraps(func) + def inner(ctx, *args, **kwargs): + + if not ctx.obj: + from .models import SafetyCLI + ctx.obj = SafetyCLI() + + return func(ctx, *args, **kwargs) + + return inner + + +def pretty_format_help(obj: Union[click.Command, click.Group], + ctx: click.Context, markup_mode: MarkupMode) -> None: + from typer.rich_utils import _print_options_panel, _get_rich_console, \ + _get_help_text, highlighter, STYLE_HELPTEXT, STYLE_USAGE_COMMAND, _print_commands_panel, \ + _RICH_HELP_PANEL_NAME, ARGUMENTS_PANEL_TITLE, OPTIONS_PANEL_TITLE, \ + COMMANDS_PANEL_TITLE, _make_rich_rext + from rich.align import Align + from rich.padding import Padding + from rich.console import Console + from rich.theme import Theme + + typer_console = _get_rich_console() + + with typer_console.use_theme(Theme(styles=CONSOLE_HELP_THEME)) as theme_context: + console = theme_context.console + # Print command / group help if we have some + if obj.help: + console.print() + + # Print with some padding + console.print( + Padding( + Align(_get_help_text(obj=obj, markup_mode=markup_mode), pad=False), + (0, 1, 0, 1) + ) + ) + + # Print usage + console.print( + Padding(highlighter(obj.get_usage(ctx)), 1), style=STYLE_USAGE_COMMAND + ) + + if isinstance(obj, click.MultiCommand): + panel_to_commands: DefaultDict[str, List[click.Command]] = defaultdict(list) + for command_name in obj.list_commands(ctx): + command = obj.get_command(ctx, command_name) + if command and not command.hidden: + panel_name = ( + getattr(command, _RICH_HELP_PANEL_NAME, None) + or COMMANDS_PANEL_TITLE + ) + panel_to_commands[panel_name].append(command) + + # Print each command group panel + default_commands = panel_to_commands.get(COMMANDS_PANEL_TITLE, []) + _print_commands_panel( + name=COMMANDS_PANEL_TITLE, + commands=default_commands, + markup_mode=markup_mode, + console=console, + ) + for panel_name, commands in panel_to_commands.items(): + if panel_name == COMMANDS_PANEL_TITLE: + # Already printed above + continue + _print_commands_panel( + name=panel_name, + commands=commands, + markup_mode=markup_mode, + console=console, + ) + + panel_to_arguments: DefaultDict[str, List[click.Argument]] = defaultdict(list) + panel_to_options: DefaultDict[str, List[click.Option]] = defaultdict(list) + for param in obj.get_params(ctx): + # Skip if option is hidden + if getattr(param, "hidden", False): + continue + if isinstance(param, click.Argument): + panel_name = ( + getattr(param, _RICH_HELP_PANEL_NAME, None) or ARGUMENTS_PANEL_TITLE + ) + panel_to_arguments[panel_name].append(param) + elif isinstance(param, click.Option): + panel_name = ( + getattr(param, _RICH_HELP_PANEL_NAME, None) or OPTIONS_PANEL_TITLE + ) + panel_to_options[panel_name].append(param) + + default_options = panel_to_options.get(OPTIONS_PANEL_TITLE, []) + _print_options_panel( + name=OPTIONS_PANEL_TITLE, + params=default_options, + ctx=ctx, + markup_mode=markup_mode, + console=console, + ) + for panel_name, options in panel_to_options.items(): + if panel_name == OPTIONS_PANEL_TITLE: + # Already printed above + continue + _print_options_panel( + name=panel_name, + params=options, + ctx=ctx, + markup_mode=markup_mode, + console=console, + ) + + default_arguments = panel_to_arguments.get(ARGUMENTS_PANEL_TITLE, []) + _print_options_panel( + name=ARGUMENTS_PANEL_TITLE, + params=default_arguments, + ctx=ctx, + markup_mode=markup_mode, + console=console, + ) + for panel_name, arguments in panel_to_arguments.items(): + if panel_name == ARGUMENTS_PANEL_TITLE: + # Already printed above + continue + _print_options_panel( + name=panel_name, + params=arguments, + ctx=ctx, + markup_mode=markup_mode, + console=console, + ) + + if ctx.parent: + params = [] + for param in ctx.parent.command.params: + if isinstance(param, click.Option): + params.append(param) + + _print_options_panel( + name="Global-Options", + params=params, + ctx=ctx.parent, + markup_mode=markup_mode, + console=console, + ) + + # Epilogue if we have it + if obj.epilog: + # Remove single linebreaks, replace double with single + lines = obj.epilog.split("\n\n") + epilogue = "\n".join([x.replace("\n", " ").strip() for x in lines]) + epilogue_text = _make_rich_rext(text=epilogue, markup_mode=markup_mode) + console.print(Padding(Align(epilogue_text, pad=False), 1)) + + + +def print_main_command_panels(*, + name: str, + commands: List[click.Command], + markup_mode: MarkupMode, + console): + from rich import box + from rich.table import Table + from rich.text import Text + from rich.panel import Panel + from typer.rich_utils import STYLE_COMMANDS_TABLE_SHOW_LINES, STYLE_COMMANDS_TABLE_LEADING, \ + STYLE_COMMANDS_TABLE_BOX, STYLE_COMMANDS_TABLE_BORDER_STYLE, STYLE_COMMANDS_TABLE_ROW_STYLES, \ + STYLE_COMMANDS_TABLE_PAD_EDGE, STYLE_COMMANDS_TABLE_PADDING, STYLE_COMMANDS_PANEL_BORDER, \ + ALIGN_COMMANDS_PANEL, _make_command_help + + t_styles: Dict[str, Any] = { + "show_lines": STYLE_COMMANDS_TABLE_SHOW_LINES, + "leading": STYLE_COMMANDS_TABLE_LEADING, + "box": STYLE_COMMANDS_TABLE_BOX, + "border_style": STYLE_COMMANDS_TABLE_BORDER_STYLE, + "row_styles": STYLE_COMMANDS_TABLE_ROW_STYLES, + "pad_edge": STYLE_COMMANDS_TABLE_PAD_EDGE, + "padding": STYLE_COMMANDS_TABLE_PADDING, + } + box_style = getattr(box, t_styles.pop("box"), None) + + commands_table = Table( + highlight=False, + show_header=False, + expand=True, + box=box_style, + **t_styles, + ) + + console_width = 80 + column_width = 25 + + if console.size and console.size[0] > 80: + console_width = console.size[0] + + commands_table.add_column(style="bold cyan", no_wrap=True, width=column_width, max_width=column_width) + commands_table.add_column(width=console_width - column_width) + + rows = [] + + for command in commands: + helptext = command.short_help or command.help or "" + command_name = command.name or "" + command_name_text = Text(command_name) + rows.append( + [ + command_name_text, + _make_command_help( + help_text=helptext, + markup_mode=markup_mode, + ), + ] + ) + rows.append([]) + for row in rows: + commands_table.add_row(*row) + if commands_table.row_count: + console.print( + Panel( + commands_table, + border_style=STYLE_COMMANDS_PANEL_BORDER, + title=name, + title_align=ALIGN_COMMANDS_PANEL, + ) + ) + +# The help output for the main safety root command: `safety --help` +def format_main_help(obj: Union[click.Command, click.Group], + ctx: click.Context, markup_mode: MarkupMode) -> None: + from typer.rich_utils import _print_options_panel, _get_rich_console, \ + _get_help_text, highlighter, STYLE_USAGE_COMMAND, _print_commands_panel, \ + _RICH_HELP_PANEL_NAME, ARGUMENTS_PANEL_TITLE, OPTIONS_PANEL_TITLE, \ + COMMANDS_PANEL_TITLE, _make_rich_rext + from rich.align import Align + from rich.padding import Padding + from rich.console import Console + from rich.theme import Theme + + typer_console = _get_rich_console() + + with typer_console.use_theme(Theme(styles=CONSOLE_HELP_THEME)) as theme_context: + console = theme_context.console + + # Print command / group help if we have some + if obj.help: + console.print() + # Print with some padding + console.print( + Padding( + Align(_get_help_text(obj=obj, markup_mode=markup_mode, ), + pad=False, + ), + (0, 1, 0, 1), + ) + ) + + # Print usage + console.print( + Padding(highlighter(obj.get_usage(ctx)), 1), style=STYLE_USAGE_COMMAND + ) + + if isinstance(obj, click.MultiCommand): + UTILITY_COMMANDS_PANEL_TITLE = "Commands cont." + + panel_to_commands: DefaultDict[str, List[click.Command]] = defaultdict(list) + for command_name in obj.list_commands(ctx): + command = obj.get_command(ctx, command_name) + if command and not command.hidden: + panel_name = ( + UTILITY_COMMANDS_PANEL_TITLE if command.utility_command else COMMANDS_PANEL_TITLE + ) + panel_to_commands[panel_name].append(command) + + # Print each command group panel + default_commands = panel_to_commands.get(COMMANDS_PANEL_TITLE, []) + print_main_command_panels( + name=COMMANDS_PANEL_TITLE, + commands=default_commands, + markup_mode=markup_mode, + console=console, + ) + for panel_name, commands in panel_to_commands.items(): + if panel_name == COMMANDS_PANEL_TITLE: + # Already printed above + continue + print_main_command_panels( + name=panel_name, + commands=commands, + markup_mode=markup_mode, + console=console, + ) + + panel_to_arguments: DefaultDict[str, List[click.Argument]] = defaultdict(list) + panel_to_options: DefaultDict[str, List[click.Option]] = defaultdict(list) + for param in obj.get_params(ctx): + # Skip if option is hidden + if getattr(param, "hidden", False): + continue + if isinstance(param, click.Argument): + panel_name = ( + getattr(param, _RICH_HELP_PANEL_NAME, None) or ARGUMENTS_PANEL_TITLE + ) + panel_to_arguments[panel_name].append(param) + elif isinstance(param, click.Option): + panel_name = ( + getattr(param, _RICH_HELP_PANEL_NAME, None) or OPTIONS_PANEL_TITLE + ) + panel_to_options[panel_name].append(param) + default_arguments = panel_to_arguments.get(ARGUMENTS_PANEL_TITLE, []) + _print_options_panel( + name=ARGUMENTS_PANEL_TITLE, + params=default_arguments, + ctx=ctx, + markup_mode=markup_mode, + console=console, + ) + for panel_name, arguments in panel_to_arguments.items(): + if panel_name == ARGUMENTS_PANEL_TITLE: + # Already printed above + continue + _print_options_panel( + name=panel_name, + params=arguments, + ctx=ctx, + markup_mode=markup_mode, + console=console, + ) + default_options = panel_to_options.get(OPTIONS_PANEL_TITLE, []) + _print_options_panel( + name=OPTIONS_PANEL_TITLE, + params=default_options, + ctx=ctx, + markup_mode=markup_mode, + console=console, + ) + for panel_name, options in panel_to_options.items(): + if panel_name == OPTIONS_PANEL_TITLE: + # Already printed above + continue + _print_options_panel( + name=panel_name, + params=options, + ctx=ctx, + markup_mode=markup_mode, + console=console, + ) + + # Epilogue if we have it + if obj.epilog: + # Remove single linebreaks, replace double with single + lines = obj.epilog.split("\n\n") + epilogue = "\n".join([x.replace("\n", " ").strip() for x in lines]) + epilogue_text = _make_rich_rext(text=epilogue, markup_mode=markup_mode) + console.print(Padding(Align(epilogue_text, pad=False), 1)) + + +def process_auth_status_not_ready(console, auth: Auth, ctx: typer.Context): + from safety_schemas.models import Stage + from rich.prompt import Confirm, Prompt + + if not auth.client or not auth.client.is_using_auth_credentials(): + + if auth.stage is Stage.development: + console.print() + if auth.org: + confirmed = Confirm.ask(MSG_NO_AUTHD_DEV_STG_ORG_PROMPT, choices=["Y", "N", "y", "n"], + show_choices=False, show_default=False, + default=True, console=console) + + if not confirmed: + sys.exit(0) + + from safety.auth.cli import auth_app + login_command = get_command_for(name='login', + typer_instance=auth_app) + ctx.invoke(login_command) + else: + console.print(MSG_NO_AUTHD_DEV_STG) + console.print() + choices = ["L", "R", "l", "r"] + next_command = Prompt.ask(MSG_NO_AUTHD_DEV_STG_PROMPT, default=None, + choices=choices, show_choices=False, + console=console) + + from safety.auth.cli import auth_app + login_command = get_command_for(name='login', + typer_instance=auth_app) + register_command = get_command_for(name='register', + typer_instance=auth_app) + if next_command is None or next_command.lower() not in choices: + sys.exit(0) + + console.print() + if next_command.lower() == "r": + ctx.invoke(register_command) + else: + ctx.invoke(login_command) + + if not ctx.obj.auth.email_verified: + sys.exit(1) + else: + if not auth.org: + console.print(MSG_NO_AUTHD_CICD_PROD_STG_ORG.format(LOGIN_URL=CLI_AUTH)) + + else: + console.print(MSG_NO_AUTHD_CICD_PROD_STG) + console.print( + MSG_NO_AUTHD_NOTE_CICD_PROD_STG_TPL.format( + LOGIN_URL=CLI_AUTH, + SIGNUP_URL=f"{CLI_AUTH}/?sign_up=True")) + sys.exit(1) + + elif not auth.email_verified: + console.print() + console.print(MSG_NO_VERIFIED_EMAIL_TPL.format(email=auth.email if auth.email else "Missing email")) + sys.exit(1) + else: + console.print(MSG_NON_AUTHENTICATED) + sys.exit(1) + +class UtilityCommandMixin: + def __init__(self, *args, **kwargs): + self.utility_command = kwargs.pop('utility_command', False) + super().__init__(*args, **kwargs) + +class SafetyCLISubGroup(UtilityCommandMixin, TyperGroup): + + def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + pretty_format_help(self, ctx, markup_mode=self.rich_markup_mode) + + def format_usage(self, ctx, formatter) -> None: + command_path = ctx.command_path + pieces = self.collect_usage_pieces(ctx) + main_group = ctx.parent + if main_group: + command_path = f"{main_group.command_path} [GLOBAL-OPTIONS] {ctx.command.name}" + + formatter.write_usage(command_path, " ".join(pieces)) + + def command( + self, + *args: Any, + **kwargs: Any, + ): + super().command(*args, **kwargs) + +class SafetyCLICommand(UtilityCommandMixin, TyperCommand): + + def format_help(self, ctx: click.Context, formatter: click.HelpFormatter) -> None: + pretty_format_help(self, ctx, markup_mode=self.rich_markup_mode) + + def format_usage(self, ctx, formatter) -> None: + command_path = ctx.command_path + pieces = self.collect_usage_pieces(ctx) + main_group = ctx.parent + if main_group: + command_path = f"{main_group.command_path} [GLOBAL-OPTIONS] {ctx.command.name}" + + formatter.write_usage(command_path, " ".join(pieces)) + + +class SafetyCLIUtilityCommand(TyperCommand): + def __init__(self, *args, **kwargs): + self.utility_command = True + super().__init__(*args, **kwargs) + +class SafetyCLILegacyGroup(UtilityCommandMixin, click.Group): + + def parse_legacy_args(self, args: List[str]) -> Tuple[Optional[Dict[str, str]], Optional[str]]: + options = { + 'proxy_protocol': 'https', + 'proxy_port': 80, + 'proxy_host': None + } + key = None + + for i, arg in enumerate(args): + if arg in ['--proxy-protocol', '-pr'] and i + 1 < len(args): + options['proxy_protocol'] = args[i + 1] + elif arg in ['--proxy-port', '-pp'] and i + 1 < len(args): + options['proxy_port'] = int(args[i + 1]) + elif arg in ['--proxy-host', '-ph'] and i + 1 < len(args): + options['proxy_host'] = args[i + 1] + elif arg in ['--key'] and i + 1 < len(args): + key = args[i + 1] + + proxy = options if options['proxy_host'] else None + return proxy, key + + def invoke(self, ctx): + args = ctx.args + + # Workaround for legacy check options, that now are global options + subcommand_args = set(args) + PROXY_HOST_OPTIONS = set(["--proxy-host", "-ph"]) + if "check" in ctx.protected_args and (bool(PROXY_HOST_OPTIONS.intersection(subcommand_args) or "--key" in subcommand_args)) : + proxy_options, key = self.parse_legacy_args(args) + if proxy_options: + ctx.params.update(proxy_options) + + if key: + ctx.params.update({"key": key}) + + # Now, invoke the original behavior + super(SafetyCLILegacyGroup, self).invoke(ctx) + + + def format_help(self, ctx, formatter) -> None: + # The main `safety --help` + if self.name == "cli": + format_main_help(self, ctx, markup_mode="rich") + # All other help outputs + else: + pretty_format_help(self, ctx, markup_mode="rich") + +class SafetyCLILegacyCommand(UtilityCommandMixin, click.Command): + def format_help(self, ctx, formatter) -> None: + pretty_format_help(self, ctx, markup_mode="rich") + + +def handle_cmd_exception(func): + @wraps(func) + def inner(ctx, output: Optional[ScanOutput], *args, **kwargs): + if output: + kwargs.update({"output": output}) + + if output is ScanOutput.NONE: + return func(ctx, *args, **kwargs) + + try: + return func(ctx, *args, **kwargs) + except click.ClickException as e: + raise e + except SafetyError as e: + LOG.exception('Expected SafetyError happened: %s', e) + output_exception(e, exit_code_output=True) + except Exception as e: + LOG.exception('Unexpected Exception happened: %s', e) + exception = e if isinstance(e, SafetyException) else SafetyException(info=e) + output_exception(exception, exit_code_output=True) + + return inner \ No newline at end of file diff --git a/safety/cli_utils.py b/safety/cli_utils.py new file mode 100644 index 00000000..e69de29b diff --git a/safety/console.py b/safety/console.py new file mode 100644 index 00000000..35d914ae --- /dev/null +++ b/safety/console.py @@ -0,0 +1,35 @@ +import logging +import os +from rich.console import Console +from rich.theme import Theme + +LOG = logging.getLogger(__name__) + +SAFETY_THEME = { + "file_title": "bold default on default", + "dep_name": "bold yellow on default", + "scan_meta_title": "bold default on default", + "vuln_brief": "red on default", + "rem_brief": "bold green on default", + "rem_severity": "bold red on default", + "brief_severity": "bold default on default", + "status.spinner": "green", + "recommended_ver": "bold cyan on default", + "vuln_id": "bold default on default", + "number": "bold cyan on default", + "link": "underline bright_blue on default", + "tip": "bold default on default", + "specifier": "bold cyan on default", + "vulns_found_number": "red on default", +} + +non_interactive = os.getenv('NON_INTERACTIVE') == '1' + +console_kwargs = {"theme": Theme(SAFETY_THEME, inherit=False)} + +if non_interactive: + LOG.info("NON_INTERACTIVE environment variable is set, forcing non-interactive mode") + console_kwargs.update({"force_terminal": True, "force_interactive": False}) + + +main_console = Console(**console_kwargs) diff --git a/safety/constants.py b/safety/constants.py index 86625eab..9154a05a 100644 --- a/safety/constants.py +++ b/safety/constants.py @@ -1,29 +1,105 @@ # -*- coding: utf-8 -*- -import os +import configparser +from enum import Enum +from pathlib import Path +from typing import Optional + JSON_SCHEMA_VERSION = '2.0.0' +# TODO fix this OPEN_MIRRORS = [ f"https://pyup.io/aws/safety/free/{JSON_SCHEMA_VERSION}/", ] -API_VERSION = 'v1/' -SAFETY_ENDPOINT = 'safety/' -API_BASE_URL = 'https://pyup.io/api/' + API_VERSION + SAFETY_ENDPOINT +DIR_NAME = ".safety" + +def get_system_dir() -> Path: + import sys + import os + raw_dir = os.getenv("SAFETY_SYSTEM_CONFIG_PATH") + app_data = os.environ.get('ALLUSERSPROFILE', None) + + if not raw_dir: + if sys.platform.startswith('win') and app_data: + raw_dir = app_data + elif sys.platform.startswith("darwin"): + raw_dir = "/Library/Application Support" + elif sys.platform.startswith("linux"): + raw_dir = "/etc" + else: + raw_dir = "/" + + return Path(raw_dir, DIR_NAME) + + +def get_user_dir() -> Path: + path = Path("~", DIR_NAME).expanduser() + return path + +USER_CONFIG_DIR = get_user_dir() +SYSTEM_CONFIG_DIR = get_system_dir() + +CACHE_FILE_DIR = USER_CONFIG_DIR / f"{JSON_SCHEMA_VERSION.replace('.', '')}" +DB_CACHE_FILE = CACHE_FILE_DIR / "cache.json" + +CONFIG_FILE_NAME = "config.ini" +CONFIG_FILE_SYSTEM = SYSTEM_CONFIG_DIR / CONFIG_FILE_NAME if SYSTEM_CONFIG_DIR else None +CONFIG_FILE_USER = USER_CONFIG_DIR / CONFIG_FILE_NAME + +CONFIG = CONFIG_FILE_SYSTEM if CONFIG_FILE_SYSTEM and CONFIG_FILE_SYSTEM.exists() \ + else CONFIG_FILE_USER + +SAFETY_POLICY_FILE_NAME = ".safety-policy.yml" +SYSTEM_POLICY_FILE = SYSTEM_CONFIG_DIR / SAFETY_POLICY_FILE_NAME +USER_POLICY_FILE = USER_CONFIG_DIR / SAFETY_POLICY_FILE_NAME + +DEFAULT_DOMAIN = "safetycli.com" +DEFAULT_EMAIL = f"support@{DEFAULT_DOMAIN}" + +class URLSettings(Enum): + PLATFORM_API_BASE_URL = f"https://platform.{DEFAULT_DOMAIN}/cli/api/v1" + DATA_API_BASE_URL = f"https://data.{DEFAULT_DOMAIN}/api/v1/safety/" + CLIENT_ID = 'AWnwFBMr9DdZbxbDwYxjm4Gb24pFTnMp' + AUTH_SERVER_URL = f'https://auth.{DEFAULT_DOMAIN}' + SAFETY_PLATFORM_URL = f"https://platform.{DEFAULT_DOMAIN}" + + +def get_config_setting(name: str) -> Optional[str]: + config = configparser.ConfigParser() + config.read(CONFIG) + + default = None + + if name in [setting.name for setting in URLSettings]: + default = URLSettings[name] + + if 'settings' in config.sections() and name in config['settings']: + value = config['settings'][name] + if value: + return value + + return default.value if default else default + + +DATA_API_BASE_URL = get_config_setting("DATA_API_BASE_URL") +PLATFORM_API_BASE_URL = get_config_setting("PLATFORM_API_BASE_URL") + +PLATFORM_API_PROJECT_ENDPOINT = f"{PLATFORM_API_BASE_URL}/project" +PLATFORM_API_PROJECT_CHECK_ENDPOINT = f"{PLATFORM_API_BASE_URL}/project-check" +PLATFORM_API_POLICY_ENDPOINT = f"{PLATFORM_API_BASE_URL}/policy" +PLATFORM_API_PROJECT_SCAN_REQUEST_ENDPOINT = f"{PLATFORM_API_BASE_URL}/project-scan-request" +PLATFORM_API_PROJECT_UPLOAD_SCAN_ENDPOINT = f"{PLATFORM_API_BASE_URL}/scan" +PLATFORM_API_CHECK_UPDATES_ENDPOINT = f"{PLATFORM_API_BASE_URL}/versions-and-configs" +PLATFORM_API_INITIALIZE_SCAN_ENDPOINT = f"{PLATFORM_API_BASE_URL}/initialize-scan" + API_MIRRORS = [ - API_BASE_URL + DATA_API_BASE_URL ] REQUEST_TIMEOUT = 5 -CACHE_FILE = os.path.join( - os.path.expanduser("~"), - ".safety", - f"{JSON_SCHEMA_VERSION.replace('.', '')}", - "cache.json" -) - # Colors YELLOW = 'yellow' RED = 'red' @@ -38,6 +114,41 @@ # REGEXES HASH_REGEX_GROUPS = r"--hash[=| ](\w+):(\w+)" +DOCS_API_KEY_URL = "https://docs.safetycli.com/cli/api-keys" +MSG_NO_AUTHD_DEV_STG = "Please login or register Safety CLI [bold](free forever)[/bold] to scan and secure your projects with Safety" +MSG_NO_AUTHD_DEV_STG_PROMPT = "(R)egister for a free account in 30 seconds, or (L)ogin with an existing account to continue (R/L)" +MSG_NO_AUTHD_DEV_STG_ORG_PROMPT = "Please log in to secure your projects with Safety. Press enter to continue to log in (Y/N)" +MSG_NO_AUTHD_CICD_PROD_STG = "Enter your Safety API key to scan projects in CI/CD using the --key argument or setting your API key in the SAFETY_API_KEY environment variable." +MSG_NO_AUTHD_CICD_PROD_STG_ORG = \ +f""" +Login to get your API key + +To log in: [link]{{LOGIN_URL}}[/link] + +Read more at: [link]{DOCS_API_KEY_URL}[/link] +""" + +MSG_NO_AUTHD_NOTE_CICD_PROD_STG_TPL = \ +f""" +Login or register for a free account to get your API key + +To log in: [link]{{LOGIN_URL}}[/link] +To register: [link]{{SIGNUP_URL}}[/link] + +Read more at: [link]{DOCS_API_KEY_URL}[/link] +""" + +MSG_FINISH_REGISTRATION_TPL = "To complete your account open the “verify your email” email sent to {email}" + +MSG_VERIFICATION_HINT = "Can’t find the verification email? Login at [link]`https://platform.safetycli.com/login/`[/link] to resend the verification email" + +MSG_NO_VERIFIED_EMAIL_TPL = \ + f"""Email verification is required for {{email}} + + {MSG_FINISH_REGISTRATION_TPL} + + {MSG_VERIFICATION_HINT}""" + # Exit codes EXIT_CODE_OK = 0 EXIT_CODE_FAILURE = 1 @@ -49,3 +160,4 @@ EXIT_CODE_MALFORMED_DB = 69 EXIT_CODE_INVALID_PROVIDED_REPORT = 70 EXIT_CODE_INVALID_REQUIREMENT = 71 +EXIT_CODE_EMAIL_NOT_VERIFIED = 72 \ No newline at end of file diff --git a/safety/errors.py b/safety/errors.py index 8febd32a..c50d789f 100644 --- a/safety/errors.py +++ b/safety/errors.py @@ -1,4 +1,5 @@ -from safety.constants import EXIT_CODE_FAILURE, EXIT_CODE_INVALID_API_KEY, EXIT_CODE_TOO_MANY_REQUESTS, \ +from typing import Optional +from safety.constants import EXIT_CODE_EMAIL_NOT_VERIFIED, EXIT_CODE_FAILURE, EXIT_CODE_INVALID_API_KEY, EXIT_CODE_TOO_MANY_REQUESTS, \ EXIT_CODE_UNABLE_TO_FETCH_VULNERABILITY_DB, EXIT_CODE_UNABLE_TO_LOAD_LOCAL_VULNERABILITY_DB, EXIT_CODE_MALFORMED_DB, \ EXIT_CODE_INVALID_PROVIDED_REPORT, EXIT_CODE_INVALID_REQUIREMENT @@ -15,8 +16,9 @@ def get_exit_code(self): class SafetyError(Exception): - def __init__(self, message="Unhandled Safety generic error"): + def __init__(self, message="Unhandled Safety generic error", error_code=None): self.message = message + self.error_code = error_code super().__init__(self.message) def get_exit_code(self): @@ -77,12 +79,12 @@ def get_exit_code(self): return EXIT_CODE_UNABLE_TO_LOAD_LOCAL_VULNERABILITY_DB -class InvalidKeyError(DatabaseFetchError): +class InvalidCredentialError(DatabaseFetchError): - def __init__(self, key=None, message="Your API Key '{key}' is invalid. See {link}.", reason=None): - self.key = key + def __init__(self, credential: Optional[str] = None, message="Your authentication credential '{credential}' is invalid. See {link}.", reason=None): + self.credential = credential self.link = 'https://bit.ly/3OY2wEI' - self.message = message.format(key=key, link=self.link) if key else message.format(link=self.link) + self.message = message.format(credential=self.credential, link=self.link) if self.credential else message.format(link=self.link) info = f" Reason: {reason}" self.message = self.message + (info if reason else "") super().__init__(self.message) @@ -90,6 +92,13 @@ def __init__(self, key=None, message="Your API Key '{key}' is invalid. See {link def get_exit_code(self): return EXIT_CODE_INVALID_API_KEY +class NotVerifiedEmailError(SafetyError): + def __init__(self, message="email is not verified"): + self.message = message + super().__init__(self.message) + + def get_exit_code(self): + return EXIT_CODE_EMAIL_NOT_VERIFIED class TooManyRequestsError(DatabaseFetchError): diff --git a/safety/formatters/html.py b/safety/formatters/html.py index 64914640..ade31056 100644 --- a/safety/formatters/html.py +++ b/safety/formatters/html.py @@ -17,7 +17,7 @@ def render_vulnerabilities(self, announcements, vulnerabilities, remediations, f f'remediations with full_report: {full}') report = build_json_report(announcements, vulnerabilities, remediations, packages) - return parse_html(report) + return parse_html(kwargs={"json_data": report}) def render_licenses(self, announcements, licenses): pass diff --git a/safety/formatters/schemas/3_0.json b/safety/formatters/schemas/3_0.json new file mode 100644 index 00000000..44039568 --- /dev/null +++ b/safety/formatters/schemas/3_0.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "array", + "items": [ + { + "type": "array", + "items": [ + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "string" + }, + { + "type": "null" + }, + { + "type": "null" + } + ] + } + ] + } \ No newline at end of file diff --git a/safety/formatters/schemas/common.py b/safety/formatters/schemas/common.py new file mode 100644 index 00000000..5220323e --- /dev/null +++ b/safety/formatters/schemas/common.py @@ -0,0 +1,36 @@ +from typing import Dict, Generic, TypeVar + +from pydantic import BaseModel as PydanticBaseModel +from pydantic import Extra +from pydantic.validators import dict_validator + +from common.const import SCHEMA_DICT_ITEMS_COUNT_LIMIT +from common.exceptions import DictMaxLengthError + + +class BaseModel(PydanticBaseModel): + class Config: + arbitrary_types_allowed = True + max_anystr_length = 50 + validate_assignment = True + extra = Extra.forbid + + +KeyType = TypeVar("KeyType") +ValueType = TypeVar("ValueType") + + +class ConstrainedDict(Generic[KeyType, ValueType]): + def __init__(self, v: Dict[KeyType, ValueType]): + super().__init__() + + @classmethod + def __get_validators__(cls): + yield cls.dict_length_validator + + @classmethod + def dict_length_validator(cls, v): + v = dict_validator(v) + if len(v) > SCHEMA_DICT_ITEMS_COUNT_LIMIT: + raise DictMaxLengthError(limit_value=SCHEMA_DICT_ITEMS_COUNT_LIMIT) + return v diff --git a/safety/formatters/schemas/v0_5.py b/safety/formatters/schemas/v0_5.py new file mode 100644 index 00000000..e69de29b diff --git a/safety/formatters/schemas/v3_0.json b/safety/formatters/schemas/v3_0.json new file mode 100644 index 00000000..e69de29b diff --git a/safety/formatters/schemas/v3_0.py b/safety/formatters/schemas/v3_0.py new file mode 100644 index 00000000..87e0af94 --- /dev/null +++ b/safety/formatters/schemas/v3_0.py @@ -0,0 +1,105 @@ +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Literal, Optional, Union + +from pydantic import Field, HttpUrl + +from common.schemas import BaseModel, ConstrainedDict +from scans.schemas.base import ( + GitInfo, + NoGit, + PackageShort, + RemediationsResults, + RequirementInfo, + Telemetry, + Vulnerability, +) + + +class Meta(BaseModel): + scan_type: Literal["system-scan", "scan", "check"] + scan_location: Path + logged_to_dashboard: bool + authenticated: bool + authentication_method: Literal["token", "api_key"] + local_database_path: Optional[Path] + safety_version: str + timestamp: datetime + telemetry: Telemetry + schema_version: str + + +class Package(BaseModel): + requirements: ConstrainedDict[str, RequirementInfo] + current_version: Optional[str] + vulnerabilities_found: Optional[int] + recommended_version: Optional[str] + other_recommended_versions: List[str] = Field([], max_items=100, unique_items=True) + more_info_url: Optional[HttpUrl] + + +class OSVulnerabilities(BaseModel): + packages: ConstrainedDict[str, Package] + vulnerabilities: List[Vulnerability] = Field(..., max_items=100, unique_items=True) + + +class EnvironmentFindings(BaseModel): + configuration: ConstrainedDict + packages: ConstrainedDict[str, Package] + os_vulnerabilities: OSVulnerabilities + + +class Environment(BaseModel): + full_location: Path + type: Literal["environment"] + findings: EnvironmentFindings + + +class DependencyVulnerabilities(BaseModel): + packages: List[PackageShort] = Field(..., max_items=500, unique_items=True) + vulnerabilities: List[Vulnerability] = Field(..., max_items=100, unique_items=True) + + +class FileFindings(BaseModel): + configuration: ConstrainedDict + packages: List[PackageShort] = Field(..., max_items=500, unique_items=True) + dependency_vulnerabilities: DependencyVulnerabilities + + +class Remediations(BaseModel): + configuration: ConstrainedDict + packages: ConstrainedDict[str, Package] + dependency_vulnerabilities: ConstrainedDict[str, Package] + remediations_results: RemediationsResults + + +class File(BaseModel): + full_location: Path + type: str + language: Literal["python"] + format: str + findings: FileFindings + remediations: Remediations + + +class Results(BaseModel): + environments: List[ConstrainedDict[Path, Environment]] = Field( + [], max_items=100, unique_items=True + ) + files: List[ConstrainedDict[str, File]] = Field( + [], max_items=100, unique_items=True + ) + + +class Project(Results): + id: Optional[int] + location: Path + policy: Optional[Path] + policy_source: Optional[Literal["local", "cloud"]] + git: Union[GitInfo, NoGit] + + +class ScanReportV30(BaseModel): + meta: Meta + results: Results | Dict = {} + projects: Project | Dict = {} \ No newline at end of file diff --git a/safety/formatters/screen.py b/safety/formatters/screen.py index 96191149..2d6a1848 100644 --- a/safety/formatters/screen.py +++ b/safety/formatters/screen.py @@ -23,7 +23,7 @@ class ScreenReport(FormatterAPI): |_______/ \_______/|__/ \_______/ \___/ \____ $$ /$$ | $$ | $$$$$$/ - by pyup.io \______/ + by safetycli.com \______/ """ + DIVIDER_SECTIONS diff --git a/safety/formatters/text.py b/safety/formatters/text.py index 89b9a406..51625809 100644 --- a/safety/formatters/text.py +++ b/safety/formatters/text.py @@ -26,7 +26,7 @@ class TextReport(FormatterAPI): |_______/ \_______/|__/ \_______/ \___/ \____ $$ /$$ | $$ | $$$$$$/ - by pyup.io \______/ + by safetycli.com \______/ """ + SMALL_DIVIDER_SECTIONS diff --git a/safety/models.py b/safety/models.py index b933dcf0..64c1932d 100644 --- a/safety/models.py +++ b/safety/models.py @@ -12,8 +12,11 @@ from packaging.version import parse as parse_version from packaging.version import Version +from safety_schemas.models import ConfigModel from safety.errors import InvalidRequirementError +from safety_schemas.models import MetadataModel, ReportSchemaVersion, TelemetryModel, \ + PolicyFileModel try: from packaging.version import LegacyVersion as legacyType @@ -74,6 +77,21 @@ def __init__(self, requirement: [str, Dependency], found: Optional[str] = None) def __eq__(self, other): return str(self) == str(other) + + def to_dict(self, **kwargs): + specifier_obj = self.specifier + if not "specifier_obj" in kwargs: + specifier_obj = str(self.specifier) + + return { + 'raw': self.raw, + 'extras': list(self.extras), + 'marker': str(self.marker) if self.marker else None, + 'name': self.name, + 'specifier': specifier_obj, + 'url': self.url, + 'found': self.found + } def is_pinned_requirement(spec: SpecifierSet) -> bool: @@ -89,7 +107,7 @@ def is_pinned_requirement(spec: SpecifierSet) -> bool: class Package(DictConverter): name: str version: Optional[str] - requirements: [SafetyRequirement] + requirements: List[SafetyRequirement] found: Optional[str] = None absolute_path: Optional[str] = None insecure_versions: List[str] = field(default_factory=lambda: []) @@ -211,15 +229,7 @@ def to_dict(self): class SafetyEncoder(json.JSONEncoder): def default(self, value): if isinstance(value, SafetyRequirement): - return { - 'raw': value.raw, - 'extras': list(value.extras), - 'marker': str(value.marker) if value.marker else None, - 'name': value.name, - 'specifier': str(value.specifier), - 'url': value.url, - 'found': value.found - } + return value.to_dict() elif isinstance(value, Version) or (legacyType and isinstance(value, legacyType)): return str(value) else: @@ -260,3 +270,44 @@ def to_dict(self): def get_advisory(self): return self.advisory.replace('\r', '') if self.advisory else "No advisory found for this vulnerability." + + def to_model_dict(self): + try: + affected_spec = next(iter(self.vulnerable_spec)) + except Exception: + affected_spec = "" + + repr = { + "id": self.vulnerability_id, + "package_name": self.package_name, + "vulnerable_spec": affected_spec, + "analyzed_specification": self.analyzed_requirement.raw + } + + if self.ignored: + repr["ignored"] = {"reason": self.ignored_reason, + "expires": self.ignored_expires} + + return repr + + +@dataclass +class Safety: + client: Any + keys: Any + + +from safety.auth.models import Auth +from rich.console import Console + +@dataclass +class SafetyCLI: + auth: Optional[Auth] = None + telemetry: Optional[TelemetryModel] = None + metadata: Optional[MetadataModel] = None + schema: Optional[ReportSchemaVersion] = None + project = None + config: Optional[ConfigModel] = None + console: Optional[Console] = None + system_scan_policy: Optional[PolicyFileModel] = None + platform_enabled: bool = False diff --git a/safety/output_utils.py b/safety/output_utils.py index e690e60a..5ad25c8e 100644 --- a/safety/output_utils.py +++ b/safety/output_utils.py @@ -1,3 +1,4 @@ +from dataclasses import asdict import json import logging import os @@ -598,7 +599,8 @@ def build_report_for_review_vuln_report(as_dict=False): scanned_items.append([{'styled': False, 'value': '-> ' + name}]) nl = [{'style': False, 'value': ''}] - using_sentence = build_using_sentence(report_from_file.get('api_key', None), + using_sentence = build_using_sentence(None, + report_from_file.get('api_key', None), report_from_file.get('local_database_path_used', None)) scanned_count_sentence = build_scanned_count_sentence(packages) old_timestamp = report_from_file.get('timestamp', None) @@ -618,15 +620,18 @@ def build_report_for_review_vuln_report(as_dict=False): return brief_info -def build_using_sentence(key, db): +def build_using_sentence(account, key, db): key_sentence = [] custom_integration = os.environ.get('SAFETY_CUSTOM_INTEGRATION', 'false').lower() == 'true' - if key: - key_sentence = [{'style': True, 'value': 'an API KEY'}, + if key or account: + t = {'style': True, 'value': 'an API KEY'} + if not key: + t = {'style': True, 'value': f'the account {account}'} + key_sentence = [t, {'style': False, 'value': ' and the '}] - db_name = 'PyUp Commercial' + db_name = 'Safety Commercial' elif db: if is_a_remote_mirror(db): if custom_integration: @@ -681,6 +686,7 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs): review = build_report_for_review_vuln_report(as_dict) return review + account = context.account key = context.key db = context.db_mirror @@ -724,7 +730,7 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs): audit_and_monitor = [] if context.params.get('audit_and_monitor'): - logged_url = context.params.get('audit_and_monitor_url') if context.params.get('audit_and_monitor_url') else "https://pyup.io" + logged_url = context.params.get('audit_and_monitor_url') if context.params.get('audit_and_monitor_url') else "https://safetycli.com" audit_and_monitor = [ {'style': False, 'value': '\nLogging scan results to'}, {'style': True, 'value': ' {0}'.format(logged_url)}, @@ -737,6 +743,7 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs): current_time = str(datetime.now().strftime('%Y-%m-%d %H:%M:%S')) brief_data['api_key'] = bool(key) + brief_data['account'] = account brief_data['local_database_path'] = db if db else None brief_data['safety_version'] = get_safety_version() brief_data['timestamp'] = current_time @@ -769,14 +776,14 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs): {'style': True, 'value': f' license {"type" if brief_data["licenses_found"] == 1 else "types"} found'}], ] - brief_data['telemetry'] = build_telemetry_data() + brief_data['telemetry'] = asdict(build_telemetry_data()) brief_data['git'] = build_git_data() brief_data['project'] = context.params.get('project', None) brief_data['json_version'] = "1.1" - using_sentence = build_using_sentence(key, db) + using_sentence = build_using_sentence(account, key, db) sentence_array = [] for section in using_sentence: sentence_array.append(section['value']) @@ -823,7 +830,7 @@ def build_primary_announcement(primary_announcement, columns=None, only_text=Fal def is_using_api_key(): - return bool(SafetyContext().key) + return bool(SafetyContext().key) or bool(SafetyContext().account) def is_using_a_safety_policy_file(): @@ -853,15 +860,18 @@ def get_skip_reason(fix: Fix) -> str: def get_applied_msg(fix, mode="auto") -> str: - return f"Applied {mode} fix for {fix.package} from {fix.previous_spec} to =={fix.updated_version}." + return f"{fix.package}{fix.previous_spec} has a {fix.update_type} version fix available: {mode} updating to =={fix.updated_version}." def get_skipped_msg(fix) -> str: return f'{fix.package} remediation was skipped because {get_skip_reason(fix)}' -def get_fix_opt_used_msg() -> str: - fix_options = SafetyContext().params.get('auto_remediation_limit', []) +def get_fix_opt_used_msg(fix_options=None) -> str: + + if not fix_options: + fix_options = SafetyContext().params.get('auto_remediation_limit', []) + msg = "no automatic" if fix_options: @@ -914,11 +924,11 @@ def prompt_service(output: Tuple[str, Dict], out_format: str, format_text: Optio return click.prompt(msg) -def parse_html(json_data): +def parse_html(*, kwargs, template='index.html'): file_loader = PackageLoader('safety', 'templates') env = Environment(loader=file_loader) - template = env.get_template("index.html") - return template.render(json_data=json_data) + template = env.get_template(template) + return template.render(**kwargs) def format_unpinned_vulnerabilities(unpinned_packages, columns=None): diff --git a/safety/reqs_scan.py b/safety/reqs_scan.py new file mode 100644 index 00000000..8e5ba1ed --- /dev/null +++ b/safety/reqs_scan.py @@ -0,0 +1,69 @@ +# Python code to search for requirements files +import os +import random + +def scan_file(root, filename): + if filename.endswith('requirements.txt'): + file_name_randomizer = round(random.random()*100000000) + filepath = f"{root}/{str(filename)}" + print(f" Scanning {filepath} using safety check") + os.system(f"safety check -r {filepath} --cache 100 --output json >> {save_results_path}/{file_name_randomizer}-scan.json") + +def check_file_name(root, filename): + if filename.endswith('requirements.txt') or filename.endswith('pyproject.toml') or filename.endswith('poetry.lock') or filename.endswith('Pipfile') or filename.endswith('Pipfile.lock'): + print (f"found requirements file: {root}/{str(filename)}") + scan_file(root, filename) + +# From https://gist.github.com/TheMatt2/faf5ca760c61a267412c46bb977718fa +def walklevel(path, depth = 1, deny_list = []): + """It works just like os.walk, but you can pass it a level parameter + that indicates how deep the recursion will go. + If depth is 1, the current directory is listed. + If depth is 0, nothing is returned. + If depth is -1 (or less than 0), the full depth is walked. + """ + + # If depth is negative, just walk + # Not using yield from for python2 compat + # and copy dirs to keep consistant behavior for depth = -1 and depth = inf + if depth < 0: + for root, dirs, files in os.walk(path): + yield root, dirs[:], files + return + elif depth == 0: + return + + # path.count(os.path.sep) is safe because + # - On Windows "\\" is never allowed in the name of a file or directory + # - On UNIX "/" is never allowed in the name of a file or directory + # - On MacOS a literal "/" is quitely translated to a ":" so it is still + # safe to count "/". + base_depth = path.rstrip(os.path.sep).count(os.path.sep) + for root, dirs, files in os.walk(path): + for idx, directory in enumerate(dirs): + if f"{root}{directory}" in deny_list: + print(f"Not scanning {root}{directory}") + del dirs[idx] + yield root, dirs[:], files + cur_depth = root.count(os.path.sep) + if base_depth + depth <= cur_depth: + del dirs[:] + + + +# This is to get the directory that the program +# is currently running in. +current_path = os.path.dirname(os.path.realpath(__file__)) +save_results_path = f"{current_path}/.demo_safety_scan_results" +starting_path = "/" +print(f"Scanning from {starting_path}") + +folder_depth_limit = 8 +paths_excluded_list = ["/System"] + +# Create folder for scan results +os.system(f"mkdir {save_results_path}") + +for root, dirs, files in walklevel(starting_path, folder_depth_limit, paths_excluded_list): + for file in files: + check_file_name(root, file) diff --git a/safety/safety-policy-template.yml b/safety/safety-policy-template.yml index d8b7ec5b..e3369f93 100644 --- a/safety/safety-policy-template.yml +++ b/safety/safety-policy-template.yml @@ -5,6 +5,9 @@ # To validate and review your policy file, run the validate command: `safety validate policy_file --path ` project: # Project to associate the scans with on pyup.io. id: '' +organization: + id: '' + name: '' security: # configuration for the `safety check` command ignore-unpinned-requirements: True # This will ignore dependencies found in requirement files without a pinned specification. Like requests or requests>=0 or django>=2.2 ignore-cvss-severity-below: 0 # A severity number between 0 and 10. Some helpful reference points: 9=ignore all vulnerabilities except CRITICAL severity. 7=ignore all vulnerabilities except CRITICAL & HIGH severity. 4=ignore all vulnerabilities except CRITICAL, HIGH & MEDIUM severity. diff --git a/safety/safety.py b/safety/safety.py index a0b473e6..72543bd9 100644 --- a/safety/safety.py +++ b/safety/safety.py @@ -1,15 +1,18 @@ # -*- coding: utf-8 -*- +from dataclasses import asdict import errno import itertools import json import logging import os +from pathlib import Path +import random import sys import tempfile import time from collections import defaultdict from datetime import datetime -from typing import Dict, Optional, List, Union +from typing import Dict, Optional, List import click import requests @@ -17,40 +20,44 @@ from packaging.specifiers import SpecifierSet from packaging.utils import canonicalize_name from packaging.version import parse as parse_version, Version +from pydantic.json import pydantic_encoder -from .constants import (API_MIRRORS, CACHE_FILE, OPEN_MIRRORS, REQUEST_TIMEOUT, API_BASE_URL, JSON_SCHEMA_VERSION, +from safety_schemas.models import Ecosystem, FileType + + + +from .constants import (API_MIRRORS, DB_CACHE_FILE, OPEN_MIRRORS, REQUEST_TIMEOUT, DATA_API_BASE_URL, JSON_SCHEMA_VERSION, IGNORE_UNPINNED_REQ_REASON) -from .errors import (DatabaseFetchError, DatabaseFileNotFoundError, - InvalidKeyError, TooManyRequestsError, NetworkConnectionError, +from .errors import (DatabaseFetchError, DatabaseFileNotFoundError, InvalidCredentialError, + TooManyRequestsError, NetworkConnectionError, RequestTimeoutError, ServerError, MalformedDatabase) from .models import Vulnerability, CVE, Severity, Fix, is_pinned_requirement, SafetyRequirement from .output_utils import print_service, get_applied_msg, prompt_service, get_skipped_msg, get_fix_opt_used_msg, \ is_using_api_key, get_specifier_range_info -from .util import RequirementFile, read_requirements, Package, build_telemetry_data, sync_safety_context, \ +from .util import build_remediation_info_url, pluralize, read_requirements, Package, build_telemetry_data, sync_safety_context, \ SafetyContext, validate_expiration_date, is_a_remote_mirror, get_requirements_content, SafetyPolicyFile, \ get_terminal_size, is_ignore_unpinned_mode, get_hashes -session = requests.session() - LOG = logging.getLogger(__name__) -def get_from_cache(db_name, cache_valid_seconds=0): - LOG.debug('Trying to get from cache...') - if os.path.exists(CACHE_FILE): - LOG.info('Cache file path: %s', CACHE_FILE) - with open(CACHE_FILE) as f: +def get_from_cache(db_name, cache_valid_seconds=0, skip_time_verification=False): + if os.path.exists(DB_CACHE_FILE): + with open(DB_CACHE_FILE) as f: try: data = json.loads(f.read()) - LOG.debug('Trying to get the %s from the cache file', db_name) - LOG.debug('Databases in CACHE file: %s', ', '.join(data)) if db_name in data: - LOG.debug('db_name %s', db_name) if "cached_at" in data[db_name]: - if data[db_name]["cached_at"] + cache_valid_seconds > time.time(): + if data[db_name]["cached_at"] + cache_valid_seconds > time.time() or skip_time_verification: LOG.debug('Getting the database from cache at %s, cache setting: %s', data[db_name]["cached_at"], cache_valid_seconds) + + try: + data[db_name]["db"]["meta"]["base_domain"] = "https://data.safetycli.com" + except KeyError as e: + pass + return data[db_name]["db"] LOG.debug('Cached file is too old, it was cached at %s', data[db_name]["cached_at"]) @@ -77,10 +84,10 @@ def write_to_cache(db_name, data): # "db": {} # }, # } - if not os.path.exists(os.path.dirname(CACHE_FILE)): + if not os.path.exists(os.path.dirname(DB_CACHE_FILE)): try: - os.makedirs(os.path.dirname(CACHE_FILE)) - with open(CACHE_FILE, "w") as _: + os.makedirs(os.path.dirname(DB_CACHE_FILE)) + with open(DB_CACHE_FILE, "w") as _: _.write(json.dumps({})) LOG.debug('Cache file created') except OSError as exc: # Guard against race condition @@ -88,14 +95,14 @@ def write_to_cache(db_name, data): if exc.errno != errno.EEXIST: raise - with open(CACHE_FILE, "r") as f: + with open(DB_CACHE_FILE, "r") as f: try: cache = json.loads(f.read()) except json.JSONDecodeError: LOG.debug('JSONDecodeError in the local cache, dumping the full cache file.') cache = {} - with open(CACHE_FILE, "w") as f: + with open(DB_CACHE_FILE, "w") as f: cache[db_name] = { "cached_at": time.time(), "db": data @@ -104,26 +111,25 @@ def write_to_cache(db_name, data): LOG.debug('Safety updated the cache file for %s database.', db_name) -def fetch_database_url(mirror, db_name, key, cached, proxy, telemetry=True): - headers = {'schema-version': JSON_SCHEMA_VERSION} - - if key: - headers["X-Api-Key"] = key +def fetch_database_url(session, mirror, db_name, cached, telemetry=True, + ecosystem: Ecosystem = Ecosystem.PYTHON, from_cache=True): + headers = {'schema-version': JSON_SCHEMA_VERSION, 'ecosystem': ecosystem.value} - if not proxy: - proxy = {} - - if cached: + if cached and from_cache: cached_data = get_from_cache(db_name=db_name, cache_valid_seconds=cached) if cached_data: LOG.info('Database %s returned from cache.', db_name) return cached_data url = mirror + db_name - telemetry_data = {'telemetry': json.dumps(build_telemetry_data(telemetry=telemetry))} + + telemetry_data = { + 'telemetry': json.dumps(build_telemetry_data(telemetry=telemetry), + default=pydantic_encoder)} try: - r = session.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy, params=telemetry_data) + r = session.get(url=url, timeout=REQUEST_TIMEOUT, + headers=headers, params=telemetry_data) except requests.exceptions.ConnectionError: raise NetworkConnectionError() except requests.exceptions.Timeout: @@ -132,7 +138,7 @@ def fetch_database_url(mirror, db_name, key, cached, proxy, telemetry=True): raise DatabaseFetchError() if r.status_code == 403: - raise InvalidKeyError(key=key, reason=r.text) + raise InvalidCredentialError(credential=session.get_credential(), reason=r.text) if r.status_code == 429: raise TooManyRequestsError(reason=r.text) @@ -152,35 +158,22 @@ def fetch_database_url(mirror, db_name, key, cached, proxy, telemetry=True): return data -def fetch_policy(key, proxy): - url = f"{API_BASE_URL}policy/" - headers = {"X-Api-Key": key} - - if not proxy: - proxy = {} +def fetch_policy(session): + url = f"{DATA_API_BASE_URL}policy/" try: LOG.debug(f'Getting policy') - r = session.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy) + r = session.get(url=url, timeout=REQUEST_TIMEOUT) LOG.debug(r.text) return r.json() except Exception: LOG.exception("Error fetching policy") - click.secho( - "Warning: couldn't fetch policy from pyup.io.", - fg="yellow", - file=sys.stderr - ) return {"safety_policy": "", "audit_and_monitor": False} -def post_results(key, proxy, safety_json, policy_file): - url = f"{API_BASE_URL}result/" - headers = {"X-Api-Key": key} - - if not proxy: - proxy = {} +def post_results(session, safety_json, policy_file): + url = f"{DATA_API_BASE_URL}result/" # safety_json is in text form already. policy_file is a text YAML audit_report = { @@ -191,14 +184,14 @@ def post_results(key, proxy, safety_json, policy_file): try: LOG.debug(f'Posting results to: {url}') LOG.debug(f'Posting results: {audit_report}') - r = session.post(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy, json=audit_report) + r = session.post(url=url, timeout=REQUEST_TIMEOUT, json=audit_report) LOG.debug(r.text) return r.json() except: LOG.exception("Error posting results") click.secho( - "Warning: couldn't upload results to pyup.io.", + "Warning: couldn't upload results to safetycli.com.", fg="yellow", file=sys.stderr ) @@ -206,7 +199,7 @@ def post_results(key, proxy, safety_json, policy_file): return {} -def fetch_database_file(path, db_name): +def fetch_database_file(path, db_name, ecosystem: Ecosystem = Ecosystem.PYTHON): full_path = os.path.join(path, db_name) if not os.path.exists(full_path): raise DatabaseFileNotFoundError(db=path) @@ -224,8 +217,9 @@ def is_valid_database(db) -> bool: return False -def fetch_database(full=False, key=False, db=False, cached=0, proxy=None, telemetry=True): - if key: +def fetch_database(session, full=False, db=False, cached=0, telemetry=True, + ecosystem: Ecosystem = Ecosystem.PYTHON, from_cache=True): + if session.is_using_auth_credentials(): mirrors = API_MIRRORS elif db: mirrors = [db] @@ -236,9 +230,10 @@ def fetch_database(full=False, key=False, db=False, cached=0, proxy=None, teleme for mirror in mirrors: # mirror can either be a local path or a URL if is_a_remote_mirror(mirror): - data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy, telemetry=telemetry) + data = fetch_database_url(session, mirror, db_name=db_name, cached=cached, + telemetry=telemetry, ecosystem=ecosystem, from_cache=from_cache) else: - data = fetch_database_file(mirror, db_name=db_name) + data = fetch_database_file(mirror, db_name=db_name, ecosystem=ecosystem) if data: if is_valid_database(data): return data @@ -357,17 +352,26 @@ def ignore_vuln_if_needed(pkg: Package, vuln_id, cve, ignore_vulns, ignore_sever def is_vulnerable(vulnerable_spec: SpecifierSet, requirement, package): if is_pinned_requirement(requirement.specifier): - return vulnerable_spec.contains(next(iter(requirement.specifier)).version) + try: + return vulnerable_spec.contains(next(iter(requirement.specifier)).version) + except Exception: + # Ugly for now... + message = f'Version {requirement.specifier} for {package.name} is invalid and is ignored by Safety. Please See PEP 440.' + if message not in [a['message'] for a in SafetyContext.local_announcements]: + SafetyContext.local_announcements.append( + {'message': message, + 'type': 'warning', 'local': True}) + return False return any(requirement.specifier.filter(vulnerable_spec.filter(package.insecure_versions, prereleases=True), prereleases=True)) @sync_safety_context -def check(packages, key=False, db_mirror=False, cached=0, ignore_vulns=None, ignore_severity_rules=None, proxy=None, +def check(*, session=None, packages=[], db_mirror=False, cached=0, ignore_vulns=None, ignore_severity_rules=None, proxy=None, include_ignored=False, is_env_scan=True, telemetry=True, params=None, project=None): SafetyContext().command = 'check' - db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy, telemetry=telemetry) + db = fetch_database(session, db=db_mirror, cached=cached, telemetry=telemetry) db_full = None vulnerable_packages = frozenset(db.get('vulnerable_packages', [])) vulnerabilities = [] @@ -387,7 +391,7 @@ def check(packages, key=False, db_mirror=False, cached=0, ignore_vulns=None, ign if not pkg.version: if not db_full: - db_full = fetch_database(full=True, key=key, db=db_mirror, cached=cached, proxy=proxy, + db_full = fetch_database(session, full=True, db=db_mirror, cached=cached, telemetry=telemetry) pkg.refresh_from(db_full) @@ -398,7 +402,7 @@ def check(packages, key=False, db_mirror=False, cached=0, ignore_vulns=None, ign if is_vulnerable(spec_set, req, pkg): if not db_full: - db_full = fetch_database(full=True, key=key, db=db_mirror, cached=cached, proxy=proxy, + db_full = fetch_database(session, full=True, db=db_mirror, cached=cached, telemetry=telemetry) if not pkg.latest_version: pkg.refresh_from(db_full) @@ -516,24 +520,6 @@ def compute_sec_ver_for_user(package: Package, secure_vulns_by_user, db_full): return sorted(sec_ver_for_user, key=lambda ver: parse_version(ver), reverse=True) -def build_remediation_info_url(base_url: str, version: Optional[str], spec: str, - target_version: Optional[str] = ''): - - if not is_using_api_key(): - return base_url - - params = {'from': version, 'to': target_version} - - # No pinned version - if not version: - params = {'spec': spec} - - req = PreparedRequest() - req.prepare_url(base_url, params) - - return req.url - - def compute_sec_ver(remediations, packages: Dict[str, Package], secure_vulns_by_user, db_full): """ Compute the secure_versions and the closest_secure_version for each remediation using the affected_versions @@ -580,9 +566,9 @@ def compute_sec_ver(remediations, packages: Dict[str, Package], secure_vulns_by_ # Refresh the URL with the recommended version. spec = str(rem['requirement'].specifier) - base_url = rem['more_info_url'] - rem['more_info_url'] = \ - build_remediation_info_url(base_url=base_url, version=version, + if is_using_api_key(): + rem['more_info_url'] = \ + build_remediation_info_url(base_url=rem['more_info_url'], version=version, spec=spec, target_version=recommended_version) @@ -644,6 +630,59 @@ def process_fixes(files, remediations, auto_remediation_limit, output, no_output return fixes +def process_fixes_scan(file_to_fix, to_fix_spec, auto_remediation_limit, output, no_output=True, prompt=False): + to_fix_remediations = [] + + def get_remmediation_from(spec): + upper = None + lower = None + recommended = None + + try: + upper = Version(spec.remediation.closest_secure.upper) if spec.remediation.closest_secure.upper else None + except Exception as e: + LOG.error(f'Error getting upper remediation version, ignoring', exc_info=True) + + try: + lower = Version(spec.remediation.closest_secure.lower) if spec.remediation.closest_secure.lower else None + except Exception as e: + LOG.error(f'Error getting lower remediation version, ignoring', exc_info=True) + + try: + recommended = Version(spec.remediation.recommended) + except Exception as e: + LOG.error(f'Error getting recommended version for remediation, ignoring', exc_info=True) + + return { + "vulnerabilities_found": spec.remediation.vulnerabilities_found, + "version": next(iter(spec.specifier)).version if spec.is_pinned() else None, + "requirement": spec, + "more_info_url": spec.remediation.more_info_url, + "closest_secure_version": { + 'upper': upper, + 'lower': lower + }, + "recommended_version": recommended, + "other_recommended_versions": spec.remediation.other_recommended + } + + req_remediations = iter(get_remmediation_from(spec) for spec in to_fix_spec) + SUPPORTED_FILE_TYPES = [FileType.REQUIREMENTS_TXT] + + if file_to_fix.file_type in SUPPORTED_FILE_TYPES: + files = (open(file_to_fix.location),) + requirements = compute_fixes_per_requirements(files, req_remediations, auto_remediation_limit, prompt=prompt) + else: + requirements = { + 'files': {str(file_to_fix.location): {'content': None, 'fixes': {'TO_SKIP': [], 'TO_APPLY': [], 'TO_CONFIRM': []}, 'supported': False, 'filename': file_to_fix.location.name}}, + 'dependencies': defaultdict(dict), + } + + fixes = apply_fixes(requirements, output, no_output, prompt, scan_flow=True, auto_remediation_limit=auto_remediation_limit) + + return fixes + + def compute_fixes_per_requirements(files, req_remediations, auto_remediation_limit, prompt=False): requirements_files = get_requirements_content(files) @@ -749,7 +788,7 @@ def compute_fixes_per_requirements(files, req_remediations, auto_remediation_lim return requirements -def apply_fixes(requirements, out_type, no_output, prompt): +def apply_fixes(requirements, out_type, no_output, prompt, scan_flow=False, auto_remediation_limit=None): from dparse.updater import RequirementsTXTUpdater @@ -760,84 +799,98 @@ def apply_fixes(requirements, out_type, no_output, prompt): brief = [] if not no_output: - brief.append((f"Safety fix running with {get_fix_opt_used_msg()} fix policy.", {})) + style_kwargs = {} + + if not scan_flow: + brief.append(('', {})) + brief.append((f"Safety fix running", style_kwargs)) print_service(brief, out_type) for name, data in requirements['files'].items(): output = [('', {}), - (f"Analyzing {name}...", {'styling': {'bold': True}, 'start_line_decorator': '->', 'indent': ' '})] - - new_content = data['content'] - + (f"Analyzing {name}... [{get_fix_opt_used_msg(auto_remediation_limit)} limit]", {'styling': {'bold': True}, 'start_line_decorator': '->', 'indent': ' '})] + r_skip = data['fixes']['TO_SKIP'] r_apply = data['fixes']['TO_APPLY'] r_confirm = data['fixes']['TO_CONFIRM'] - updated: bool = False - - for f in r_apply: - new_content = RequirementsTXTUpdater.update(content=new_content, version=f.updated_version, - dependency=f.dependency, hashes=get_hashes(f.dependency)) - f.status = 'APPLIED' - updated = True - output.append((f'- {get_applied_msg(f, mode="auto")}', {})) - - for f in r_skip: - output.append((f'- {get_skipped_msg(f)}', {})) - - if not no_output: - print_service(output, out_type) - - if prompt and not no_output: - for f in r_confirm: - changelog_detail = f'Changelogs notes: {f.more_info_url}' - options = [f"({index}) =={option}" for index, option in enumerate(f.other_options)] - other_options = '' - input_hint = '[y/n]' - - if len(options) > 0: - other_options = f' Other secure options: {", ".join(options)}.' - input_hint = '[y/n/index]' - - confirmed: str = prompt_service( - (f'- Do you want to update {f.package} from {f.previous_spec} to =={f.updated_version}? ' - f'({changelog_detail}).{other_options} {input_hint}', {}), - out_type - ).lower() - - try: - index: int = int(confirmed) - if index <= len(f.other_options): - confirmed = 'y' - except ValueError: - index = -1 - - if confirmed == 'y' or index > -1: - f.status = 'APPLIED' - updated = True - - if index > -1: - f.updated_version = f.other_options[index] - - new_content = RequirementsTXTUpdater.update(content=new_content, version=f.updated_version, - dependency=f.dependency, - hashes=get_hashes(f.dependency)) - output.append((get_applied_msg(f, mode="manual"), {'indent': ' ' * 5})) - else: - f.status = 'MANUALLY_SKIPPED' - output.append((get_skipped_msg(f), {'indent': ' ' * 5})) - - if not no_output: - print_service(output, out_type) - - if updated: - output.append((f"Updating {name}...", {})) - with open(name, mode="w") as r_file: - r_file.write(new_content) - - output.append((f"Changes applied to {name}.", {})) + if data.get('supported', True): + new_content = data['content'] + + updated: bool = False + + for f in r_apply: + new_content = RequirementsTXTUpdater.update(content=new_content, version=f.updated_version, + dependency=f.dependency, hashes=get_hashes(f.dependency)) + f.status = 'APPLIED' + updated = True + output.append(('', {})) + output.append((f'- {get_applied_msg(f, mode="auto")}', {})) + + for f in r_skip: + output.append(('', {})) + output.append((f'- {get_skipped_msg(f)}', {})) + + if not no_output: + print_service(output, out_type) + + if prompt and not no_output: + for f in r_confirm: + options = [f"({index}) =={option}" for index, option in enumerate(f.other_options)] + input_hint = f'Enter “y” to update to {f.package}=={f.updated_version}, “n” to skip this package upgrade' + + if len(options) > 0: + input_hint += f', or enter the index from these secure versions to upgrade to that version: {", ".join(options)}' + + print_service([('', {})], out_type) + confirmed: str = prompt_service( + (f'- {f.package}{f.previous_spec} requires at least a {f.update_type} version update. Do you want to update {f.package} from {f.previous_spec} to =={f.updated_version}, which is the closest secure version? {input_hint}', {}), + out_type + ).lower() + + try: + index: int = int(confirmed) + if index <= len(f.other_options): + confirmed = 'y' + except ValueError: + index = -1 + + if confirmed == 'y' or index > -1: + f.status = 'APPLIED' + updated = True + + if index > -1: + f.updated_version = f.other_options[index] + + new_content = RequirementsTXTUpdater.update(content=new_content, version=f.updated_version, + dependency=f.dependency, + hashes=get_hashes(f.dependency)) + output.append((get_applied_msg(f, mode="manual"), {'indent': ' ' * 5})) + else: + f.status = 'MANUALLY_SKIPPED' + output.append((get_skipped_msg(f), {'indent': ' ' * 5})) + + if not no_output: + print_service(output, out_type) + + if updated: + output.append(('', {})) + output.append((f"Updating {name}...", {})) + with open(name, mode="w") as r_file: + r_file.write(new_content) + output.append((f"Changes applied to {name}.", {})) + count = len(r_apply) + len([1 for fix in r_confirm if fix.status == 'APPLIED']) + output.append((f"{count} package {pluralize('version', count)} {pluralize('has', count)} been updated to secure versions in {Path(name).name}", {})) + output.append(("Always check for breaking changes after updating packages.", {})) + else: + output.append((f"No fixes to be made in {name}.", {})) + output.append(('', {})) else: - output.append((f"No fixes to be made in {name}.", {})) + not_supported_filename = data.get('filename', name) + output.append( + (f"{not_supported_filename} updates not supported: Please update these dependencies using your package manager.", + {'start_line_decorator': ' -', 'indent': ' '})) + output.append(('', {})) if not no_output: print_service(output, out_type) @@ -846,7 +899,8 @@ def apply_fixes(requirements, out_type, no_output, prompt): apply.extend(r_apply) confirm.extend(r_confirm) - if not no_output: + # The scan flow will handle the header and divider, because the scan flow can be called multiple times. + if not no_output and not scan_flow: divider = f'{"=" * 78}' if out_type == 'text' else f'{"=" * (get_terminal_size().columns - 2)}' format_text = {'start_line_decorator': '+', 'end_line_decorator': '+', 'indent': ''} print_service([(divider, {})], out_type, format_text=format_text) @@ -854,15 +908,18 @@ def apply_fixes(requirements, out_type, no_output, prompt): return skip + apply + confirm -def find_vulnerabilities_fixed(vulnerabilities: Dict, fixes): +def find_vulnerabilities_fixed(vulnerabilities: Dict, fixes) -> List[Vulnerability]: fixed_specs = set(fix.previous_spec for fix in fixes) + if not fixed_specs: + return [] + return [vulnerability for vulnerability in vulnerabilities if str(vulnerability['analyzed_requirement'].specifier) in fixed_specs] @sync_safety_context -def review(report=None, params=None): +def review(*, report=None, params=None): SafetyContext().command = 'review' vulnerable = [] vulnerabilities = report.get('vulnerabilities', []) + report.get('ignored_vulnerabilities', []) @@ -928,11 +985,12 @@ def review(report=None, params=None): @sync_safety_context -def get_licenses(key=False, db_mirror=False, cached=0, proxy=None, telemetry=True): - key = key if key else os.environ.get("SAFETY_API_KEY", False) +def get_licenses(*, session=None, db_mirror=False, cached=0, telemetry=True): + # key = key if key else os.environ.get("SAFETY_API_KEY", False) + key = session.api_key if not key and not db_mirror: - raise InvalidKeyError(message="The API-KEY was not provided.") + raise InvalidCredentialError(message="The API-KEY was not provided.") if db_mirror: mirrors = [db_mirror] else: @@ -943,7 +1001,7 @@ def get_licenses(key=False, db_mirror=False, cached=0, proxy=None, telemetry=Tru for mirror in mirrors: # mirror can either be a local path or a URL if is_a_remote_mirror(mirror): - licenses = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy, + licenses = fetch_database_url(session, mirror, db_name=db_name, cached=cached, telemetry=telemetry) else: licenses = fetch_database_file(mirror, db_name=db_name) @@ -982,19 +1040,16 @@ def add_local_notifications(packages: List[Package], return announcements -def get_announcements(key, proxy, telemetry=True): +def get_announcements(session, telemetry=True, with_telemetry=None): LOG.info('Getting announcements') announcements = [] - headers = {} - if key: - headers["X-Api-Key"] = key - - url = f"{API_BASE_URL}announcements/" + url = f"{DATA_API_BASE_URL}announcements/" method = 'post' - data = build_telemetry_data(telemetry=telemetry) - request_kwargs = {'headers': headers, 'proxies': proxy, 'timeout': 3} + telemetry_data = with_telemetry if with_telemetry else build_telemetry_data(telemetry=telemetry) + data = asdict(telemetry_data) + request_kwargs = {'timeout': 3} data_keyword = 'json' source = os.environ.get('SAFETY_ANNOUNCEMENTS_URL', None) @@ -1084,9 +1139,9 @@ def read_vulnerabilities(fh): return data -def get_server_policies(key: str, policy_file, proxy_dictionary: Dict): - if key: - server_policies = fetch_policy(key=key, proxy=proxy_dictionary) +def get_server_policies(session, policy_file, proxy_dictionary: Dict): + if session.api_key: + server_policies = fetch_policy(session) server_audit_and_monitor = server_policies["audit_and_monitor"] server_safety_policy = server_policies["safety_policy"] else: @@ -1122,16 +1177,170 @@ def save_report(path: str, default_name: str, report: str): report_file.write(report) -def push_audit_and_monitor(key, proxy, audit_and_monitor, json_report, policy_file): - if audit_and_monitor: - policy_contents = '' - if policy_file: - policy_contents = policy_file.get('raw', '') +import os +import subprocess + +def walklevel(path, depth = 1, deny_list = []): + """It works just like os.walk, but you can pass it a level parameter + that indicates how deep the recursion will go. + If depth is 1, the current directory is listed. + If depth is 0, nothing is returned. + If depth is -1 (or less than 0), the full depth is walked. + """ + + # If depth is negative, just walk + # Not using yield from for python2 compat + # and copy dirs to keep consistant behavior for depth = -1 and depth = inf + if depth < 0: + for root, dirs, files in os.walk(path): + yield root, dirs[:], files + return + elif depth == 0: + return + + # path.count(os.path.sep) is safe because + # - On Windows "\\" is never allowed in the name of a file or directory + # - On UNIX "/" is never allowed in the name of a file or directory + # - On MacOS a literal "/" is quitely translated to a ":" so it is still + # safe to count "/". + base_depth = path.rstrip(os.path.sep).count(os.path.sep) + for root, dirs, files in os.walk(path): + for idx, directory in enumerate(dirs): + if f"{root}{directory}" in deny_list: + # print(f"Not scanning {root}{directory}") + del dirs[idx] + yield root, dirs[:], files + cur_depth = root.count(os.path.sep) + if base_depth + depth <= cur_depth: + del dirs[:] + +def scan_directory(directory, timeout=None, max_depth=0, current_path=None): + virtual_envs = [] + python_interpreters = [] + requirements_files = [] + + deny_list = { + # Windows directories + "C:\\Windows", + "C:\\Program Files", + "C:\\Program Files (x86)", + "C:\\ProgramData", + + # Linux and macOS directories + "/usr", + "/usr/local", + "/opt", + "/var", + "/etc", + "/Library", + "/System", + "/Applications", + "~/Library", + "/proc", + "/dev" + } + + go_up = '' + + for root, dirs, files in walklevel(directory, max_depth, deny_list): + # Skip symbolic links, /proc, and /dev directories + # if os.path.islink(root) or root.startswith('/proc') or root.startswith('/dev'): + # continue + + status = f'Scanning: {root.strip()}...' + found = f'Python items found: {len(python_interpreters) + len(requirements_files)}' + status_pad = ' ' * (get_terminal_size().columns - len(status)) + found_pad = ' ' * (get_terminal_size().columns - len(found)) + + click.echo('{}{}\r{}'.format(go_up, f"{status}{status_pad}\n", f"{found}{found_pad}"), nl=False, err=True) + + if not go_up: + go_up = "\033[F" + + # Look for Python interpreters and requirements + for file_name in files: + file_path = os.path.join(root, file_name) + + if file_name.endswith('requirements.txt'): + file_name_randomizer = round(random.random()*100000000) + filepath = f"{root}/{str(file_name)}" + requirements_files.append(file_path) + os.system(f"safety check -r {filepath} --cache 100 --output json >> {current_path}/requirements/{file_name_randomizer}-scan.json") + + if file_name.startswith('python'): + try: + + p = file_path + + # Check if the path is a symbolic link + # if os.path.islink(file_path): + # p = os.path.realpath(file_path) + # if os.path.islink(p): + # p = os.path.realpath(p) + # if os.path.islink(p): + # p = os.path.realpath(p) + + output = subprocess.check_output( + [file_path, '--version', '--version'], + stderr=subprocess.STDOUT, timeout=timeout) + result = output.decode('utf-8') + + if result.startswith('Python'): + output = subprocess.check_output([p, '-m', 'pip', 'freeze'], stderr=subprocess.STDOUT, timeout=timeout) + requirements += output.decode('utf-8') + '\n' + + # Command to run + cmd = ['safety', 'check', '--cache', '100', '--output', 'json' '--stdin'] + + if not requirements: + continue + + # click.secho("Safety check is running...") + # Run the command and pass the string as stdin + result = subprocess.run(cmd, input=requirements, text=True, capture_output=True) + file_name_randomizer = round(random.random()*100000000) + with open(f"{current_path}/environments/{file_name_randomizer}-scan.json", "w") as outfile: + outfile.write(result.stdout) + + + # output = subprocess.check_output( + # [file_path, '--version', '--version'], + # stderr=subprocess.STDOUT, timeout=timeout) + # result = output.decode('utf-8') + + # if result.startswith('Python'): + # parts = result.split() + # # Extract the version number + # version = parts[1] + + # # Extract the date and time + # date = re.findall(r'\(.*?\)', result)[0].strip('()') + + # # Extract the compiler information + # compiler = re.findall(r'\[.*?\]', result)[0].strip('[]') + + python_interpreters.append(file_path) + except Exception: + pass + + requirements = "" + + click.secho(f"Results are save in {current_path}/...") + + # for interp in python_interpreters: + # output = subprocess.check_output( + # [interp, '-m', 'pip', 'freeze'], stderr=subprocess.STDOUT, timeout=timeout) + # requirements += output.decode('utf-8') + '\n' + + # # Command to run + # cmd = ['safety', 'check', '--stdin'] - r = post_results(key=key, proxy=proxy, safety_json=json_report, policy_file=policy_contents) - SafetyContext().params['audit_and_monitor_url'] = r.get('url') + # if not requirements: + # click.secho("No packages found.") + # click.secho("Safety check is running...") + # # Run the command and pass the string as stdin + # result = subprocess.run(cmd, input=requirements, text=True, capture_output=True) -def close_session(): - LOG.debug('Closing requests session.') - session.close() + # click.secho(result.stdout) + # click.secho(result.stderr) diff --git a/safety/scan/__init__.py b/safety/scan/__init__.py new file mode 100644 index 00000000..f2991c48 --- /dev/null +++ b/safety/scan/__init__.py @@ -0,0 +1,7 @@ +from rich.console import Console + +from safety_schemas.models import Vulnerability, RemediationModel +from safety.scan.render import get_render_console +console = Console() + +Vulnerability.__render__ = get_render_console(Vulnerability) diff --git a/safety/scan/command.py b/safety/scan/command.py new file mode 100644 index 00000000..433e8742 --- /dev/null +++ b/safety/scan/command.py @@ -0,0 +1,764 @@ + +from datetime import datetime +from enum import Enum +import itertools +import logging +from pathlib import Path +import sys +from typing import Any, List, Optional, Set, Tuple +from typing_extensions import Annotated + +from safety.constants import EXIT_CODE_VULNERABILITIES_FOUND +from safety.safety import process_fixes, process_fixes_scan +from safety.scan.finder.handlers import ECOSYSTEM_HANDLER_MAPPING, FileHandler +from safety.scan.validators import output_callback, save_as_callback +from safety.util import pluralize +from ..cli_util import SafetyCLICommand, SafetyCLISubGroup, handle_cmd_exception +from rich.padding import Padding +import typer +from safety.auth.constants import SAFETY_PLATFORM_URL +from safety.cli_util import get_command_for +from rich.console import Console +from safety.errors import SafetyError + +from safety.scan.finder import FileFinder +from safety.scan.constants import CMD_PROJECT_NAME, CMD_SYSTEM_NAME, DEFAULT_SPINNER, \ + SCAN_OUTPUT_HELP, DEFAULT_EPILOG, SCAN_POLICY_FILE_HELP, SCAN_SAVE_AS_HELP, \ + SCAN_TARGET_HELP, SYSTEM_SCAN_OUTPUT_HELP, SYSTEM_SCAN_POLICY_FILE_HELP, SYSTEM_SCAN_SAVE_AS_HELP, \ + SYSTEM_SCAN_TARGET_HELP, SCAN_APPLY_FIXES, SCAN_DETAILED_OUTPUT, CLI_SCAN_COMMAND_HELP, CLI_SYSTEM_SCAN_COMMAND_HELP +from safety.scan.decorators import inject_metadata, scan_project_command_init, scan_system_command_init +from safety.scan.finder.file_finder import should_exclude +from safety.scan.main import load_policy_file, load_unverified_project_from_config, process_files, save_report_as +from safety.scan.models import ScanExport, ScanOutput, SystemScanExport, SystemScanOutput +from safety.scan.render import print_brief, print_detected_ecosystems_section, print_fixes_section, print_ignore_details, render_scan_html, render_scan_spdx, render_to_console +from safety.scan.util import Stage +from safety_schemas.models import Ecosystem, FileModel, FileType, ProjectModel, \ + ReportModel, ScanType, VulnerabilitySeverityLabels, SecurityUpdates, Vulnerability + +LOG = logging.getLogger(__name__) + + +cli_apps_opts = {"rich_markup_mode": "rich", "cls": SafetyCLISubGroup} + +scan_project_app = typer.Typer(**cli_apps_opts) +scan_system_app = typer.Typer(**cli_apps_opts) + + +class ScannableEcosystems(Enum): + PYTHON = Ecosystem.PYTHON.value + + +def process_report(obj: Any, console: Console, report: ReportModel, output: str, + save_as: Optional[Tuple[str, Path]], **kwargs): + + wait_msg = "Processing report" + with console.status(wait_msg, spinner=DEFAULT_SPINNER) as status: + json_format = report.as_v30().json() + + export_type, export_path = None, None + + if save_as: + export_type, export_path = save_as + export_type = ScanExport(export_type) + + output = ScanOutput(output) + + report_to_export = None + report_to_output = None + + with console.status(wait_msg, spinner=DEFAULT_SPINNER) as status: + + spdx_format, html_format = None, None + + if ScanExport.is_format(export_type, ScanExport.SPDX) or ScanOutput.is_format(output, ScanOutput.SPDX): + spdx_version = None + if export_type: + spdx_version = export_type.version if export_type.version and ScanExport.is_format(export_type, ScanExport.SPDX) else None + + if not spdx_version and output: + spdx_version = output.version if output.version and ScanOutput.is_format(output, ScanOutput.SPDX) else None + + spdx_format = render_scan_spdx(report, obj, spdx_version=spdx_version) + + if export_type is ScanExport.HTML or output is ScanOutput.HTML: + html_format = render_scan_html(report, obj) + + save_as_format_mapping = { + ScanExport.JSON: json_format, + ScanExport.HTML: html_format, + ScanExport.SPDX: spdx_format, + ScanExport.SPDX_2_3: spdx_format, + ScanExport.SPDX_2_2: spdx_format, + } + + output_format_mapping = { + ScanOutput.JSON: json_format, + ScanOutput.HTML: html_format, + ScanOutput.SPDX: spdx_format, + ScanOutput.SPDX_2_3: spdx_format, + ScanOutput.SPDX_2_2: spdx_format, + } + + report_to_export = save_as_format_mapping.get(export_type, None) + report_to_output = output_format_mapping.get(output, None) + + if report_to_export: + msg = f"Saving {export_type} report at: {export_path}" + status.update(msg) + LOG.debug(msg) + save_report_as(report.metadata.scan_type, export_type, Path(export_path), + report_to_export) + report_url = None + + if obj.platform_enabled: + status.update(f"Uploading report to: {SAFETY_PLATFORM_URL}") + try: + result = obj.auth.client.upload_report(json_format) + status.update("Report uploaded") + report_url = f"{SAFETY_PLATFORM_URL}{result['url']}" + except Exception as e: + raise e + + if output is ScanOutput.SCREEN: + console.print() + lines = [] + + if obj.platform_enabled and report_url: + if report.metadata.scan_type is ScanType.scan: + project_url = f"{SAFETY_PLATFORM_URL}{obj.project.url_path}" + lines.append(f"Scan report: [link]{report_url}[/link]") + lines.append("Project dashboard: " \ + f"[link]{project_url}[/link]") + elif report.metadata.scan_type is ScanType.system_scan: + lines.append(f"System scan report: [link]{report_url}[/link]") + + for line in lines: + console.print(line, emoji=True) + + if output.is_silent(): + console.quiet = False + + if output is ScanOutput.JSON or ScanOutput.is_format(output, ScanOutput.SPDX): + if output is ScanOutput.JSON: + kwargs = {"json": report_to_output} + else: + kwargs = {"data": report_to_output} + console.print_json(**kwargs) + + else: + console.print(report_to_output) + + console.quiet = True + + return report_url + + +def generate_updates_arguments() -> list: + """Generates a list of file types and update limits for apply fixes.""" + fixes = [] + limit_type = SecurityUpdates.UpdateLevel.PATCH + DEFAULT_FILE_TYPES = [FileType.REQUIREMENTS_TXT, FileType.PIPENV_LOCK, + FileType.POETRY_LOCK, FileType.VIRTUAL_ENVIRONMENT] + fixes.extend([(default_file_type, limit_type) for default_file_type in DEFAULT_FILE_TYPES]) + + return fixes + + +@scan_project_app.command( + cls=SafetyCLICommand, + help=CLI_SCAN_COMMAND_HELP, + name=CMD_PROJECT_NAME, epilog=DEFAULT_EPILOG, + options_metavar="[OPTIONS]", + context_settings={"allow_extra_args": True, + "ignore_unknown_options": True}, + ) +@handle_cmd_exception +@inject_metadata +@scan_project_command_init +def scan(ctx: typer.Context, + target: Annotated[ + Path, + typer.Option( + exists=True, + file_okay=False, + dir_okay=True, + writable=False, + readable=True, + resolve_path=True, + show_default=False, + help=SCAN_TARGET_HELP + ), + ] = Path("."), + output: Annotated[ScanOutput, + typer.Option( + help=SCAN_OUTPUT_HELP, + show_default=False, + callback=output_callback) + ] = ScanOutput.SCREEN, + detailed_output: Annotated[bool, + typer.Option("--detailed-output", + help=SCAN_DETAILED_OUTPUT, + show_default=False) + ] = False, + save_as: Annotated[Optional[Tuple[ScanExport, Path]], + typer.Option( + help=SCAN_SAVE_AS_HELP, + show_default=False, + callback=save_as_callback) + ] = (None, None), + policy_file_path: Annotated[ + Optional[Path], + typer.Option( + "--policy-file", + exists=False, + file_okay=True, + dir_okay=False, + writable=True, + readable=True, + resolve_path=True, + help=SCAN_POLICY_FILE_HELP, + show_default=False + )] = None, + apply_updates: Annotated[bool, + typer.Option("--apply-fixes", + help=SCAN_APPLY_FIXES, + show_default=False) + ] = False + ): + """ + Scans a project (defaulted to the current directory) for supply-chain security and configuration issues + """ + + fixes_target = [] + if apply_updates: + fixes_target = generate_updates_arguments() + + if not all(save_as): + ctx.params["save_as"] = None + + console = ctx.obj.console + ecosystems = [Ecosystem(member.value) for member in list(ScannableEcosystems)] + to_include = {file_type: paths for file_type, paths in ctx.obj.config.scan.include_files.items() if file_type.ecosystem in ecosystems} + + file_finder = FileFinder(target=target, ecosystems=ecosystems, + max_level=ctx.obj.config.scan.max_depth, + exclude=ctx.obj.config.scan.ignore, + include_files=to_include, + console=console) + + for handler in file_finder.handlers: + if handler.ecosystem: + wait_msg = "Fetching Safety's vulnerability database..." + with console.status(wait_msg, spinner=DEFAULT_SPINNER): + handler.download_required_assets(ctx.obj.auth.client) + + + wait_msg = "Scanning project directory" + + path = None + file_paths = {} + + with console.status(wait_msg, spinner=DEFAULT_SPINNER): + path, file_paths = file_finder.search() + print_detected_ecosystems_section(console, file_paths, + include_safety_prjs=True) + + target_ecosystems = ", ".join([member.value for member in ecosystems]) + wait_msg = f"Analyzing {target_ecosystems} files and environments for security findings" + + import time + + files: List[FileModel] = [] + + config = ctx.obj.config + + count = 0 + ignored = set() + + affected_count = 0 + dependency_vuln_detected = False + + ignored_vulns_data = iter([]) + + exit_code = 0 + fixes_count = 0 + to_fix_files = [] + fix_file_types = [fix_target[0] if isinstance(fix_target[0], str) else fix_target[0].value for fix_target in fixes_target] + requirements_txt_found = False + display_apply_fix_suggestion = False + + with console.status(wait_msg, spinner=DEFAULT_SPINNER) as status: + for path, analyzed_file in process_files(paths=file_paths, + config=config): + count += len(analyzed_file.dependency_results.dependencies) + + if exit_code == 0 and analyzed_file.dependency_results.failed: + exit_code = EXIT_CODE_VULNERABILITIES_FOUND + + if detailed_output: + vulns_ignored = analyzed_file.dependency_results.ignored_vulns_data \ + .values() + ignored_vulns_data = itertools.chain(vulns_ignored, + ignored_vulns_data) + + ignored.update(analyzed_file.dependency_results.ignored_vulns.keys()) + + affected_specifications = analyzed_file.dependency_results.get_affected_specifications() + affected_count += len(affected_specifications) + + def sort_vulns_by_score(vuln: Vulnerability) -> int: + if vuln.severity and vuln.severity.cvssv3: + return vuln.severity.cvssv3.get("base_score", 0) + + return 0 + + to_fix_spec = [] + file_matched_for_fix = analyzed_file.file_type.value in fix_file_types + + if any(affected_specifications): + if not dependency_vuln_detected: + console.print() + console.print("Dependency vulnerabilities detected:") + dependency_vuln_detected = True + + console.print() + msg = f":pencil: [file_title]{path.relative_to(target)}:[/file_title]" + console.print(msg, emoji=True) + for spec in affected_specifications: + if file_matched_for_fix: + to_fix_spec.append(spec) + + console.print() + vulns_to_report = sorted( + [vuln for vuln in spec.vulnerabilities if not vuln.ignored], + key=sort_vulns_by_score, + reverse=True) + + critical_vulns_count = sum(1 for vuln in vulns_to_report if vuln.severity and vuln.severity.cvssv3 and vuln.severity.cvssv3.get("base_severity", "none").lower() == VulnerabilitySeverityLabels.CRITICAL.value.lower()) + + vulns_found = len(vulns_to_report) + vuln_word = pluralize("vulnerability", vulns_found) + + msg = f"[dep_name]{spec.name}[/dep_name][specifier]{spec.raw.replace(spec.name, '')}[/specifier] [{vulns_found} {vuln_word} found" + + if vulns_found > 3 and critical_vulns_count > 0: + msg += f", [brief_severity]including {critical_vulns_count} critical severity {pluralize('vulnerability', critical_vulns_count)}[/brief_severity]" + + console.print(Padding(f"{msg}]", (0, 0, 0, 1)), emoji=True, + overflow="crop") + + if detailed_output or vulns_found < 3: + for vuln in vulns_to_report: + render_to_console(vuln, console, + rich_kwargs={"emoji": True, + "overflow": "crop"}, + detailed_output=detailed_output) + + lines = [] + + # Put remediation here + if not spec.remediation.recommended: + lines.append(f"No known fix for [dep_name]{spec.name}[/dep_name][specifier]{spec.raw.replace(spec.name, '')}[/specifier] to fix " \ + f"[number]{spec.remediation.vulnerabilities_found}[/number] " \ + f"{vuln_word}") + else: + msg = f"[rem_brief]Update {spec.raw} to " \ + f"{spec.name}=={spec.remediation.recommended}[/rem_brief] to fix " \ + f"[number]{spec.remediation.vulnerabilities_found}[/number] " \ + f"{vuln_word}" + + if spec.remediation.vulnerabilities_found > 3 and critical_vulns_count > 0: + msg += f", [rem_severity]including {critical_vulns_count} critical severity {pluralize('vulnerability', critical_vulns_count)}[/rem_severity] :stop_sign:" + + fixes_count += 1 + lines.append(f"{msg}") + if spec.remediation.other_recommended: + other = "[/recommended_ver], [recommended_ver]".join(spec.remediation.other_recommended) + lines.append(f"Versions of {spec.name} with no known vulnerabilities: " \ + f"[recommended_ver]{other}[/recommended_ver]") + + for line in lines: + console.print(Padding(line, (0, 0, 0, 1)), emoji=True) + + console.print( + Padding(f"Learn more: [link]{spec.remediation.more_info_url}[/link]", + (0, 0, 0, 1)), emoji=True) + else: + console.print() + console.print(f":white_check_mark: [file_title]{path.relative_to(target)}: No issues found.[/file_title]", + emoji=True) + + if(ctx.obj.auth.stage == Stage.development + and analyzed_file.ecosystem == Ecosystem.PYTHON + and analyzed_file.file_type == FileType.REQUIREMENTS_TXT + and any(affected_specifications) + and not apply_updates): + display_apply_fix_suggestion = True + + if not requirements_txt_found and analyzed_file.file_type is FileType.REQUIREMENTS_TXT: + requirements_txt_found = True + + file = FileModel(location=path, + file_type=analyzed_file.file_type, + results=analyzed_file.dependency_results) + + if file_matched_for_fix: + to_fix_files.append((file, to_fix_spec)) + + files.append(file) + + if display_apply_fix_suggestion: + console.print() + print_fixes_section(console, requirements_txt_found, detailed_output) + + console.print() + print_brief(console, ctx.obj.project, count, affected_count, + fixes_count) + print_ignore_details(console, ctx.obj.project, ignored, + is_detailed_output=detailed_output, + ignored_vulns_data=ignored_vulns_data) + + + version = ctx.obj.schema + metadata = ctx.obj.metadata + telemetry = ctx.obj.telemetry + ctx.obj.project.files = files + + report = ReportModel(version=version, + metadata=metadata, + telemetry=telemetry, + files=[], + projects=[ctx.obj.project]) + + report_url = process_report(ctx.obj, console, report, **{**ctx.params}) + project_url = f"{SAFETY_PLATFORM_URL}{ctx.obj.project.url_path}" + + if apply_updates: + options = dict(fixes_target) + update_limits = [] + policy_limits = ctx.obj.config.depedendency_vulnerability.security_updates.auto_security_updates_limit + + no_output = output is not ScanOutput.SCREEN + prompt = output is ScanOutput.SCREEN + + # TODO: rename that 'no_output' confusing name + if not no_output: + console.print() + console.print("-" * console.size.width) + console.print("Safety updates running") + console.print("-" * console.size.width) + + for file_to_fix, specs_to_fix in to_fix_files: + try: + limit = options[file_to_fix.file_type] + except KeyError: + try: + limit = options[file_to_fix.file_type.value] + except KeyError: + limit = SecurityUpdates.UpdateLevel("patch") + + # Set defaults + update_limits = [limit.value] + + if any(policy_limits): + update_limits = [policy_limit.value for policy_limit in policy_limits] + + fixes = process_fixes_scan(file_to_fix, + specs_to_fix, update_limits, output, no_output=no_output, + prompt=prompt) + + if not no_output: + console.print("-" * console.size.width) + + if output is not ScanOutput.NONE: + if detailed_output: + if exit_code > 0: + console.print(f":stop_sign: Scan-failing vulnerabilities were found, returning non-zero exit code: {exit_code}") + else: + console.print("No scan-failing vulnerabilities were matched, returning success exit code: 0") + sys.exit(exit_code) + + return project_url, report, report_url + + +@scan_system_app.command( + cls=SafetyCLICommand, + help=CLI_SYSTEM_SCAN_COMMAND_HELP, + options_metavar="[COMMAND-OPTIONS]", + name=CMD_SYSTEM_NAME, epilog=DEFAULT_EPILOG) +@handle_cmd_exception +@inject_metadata +@scan_system_command_init +def system_scan(ctx: typer.Context, + policy_file_path: Annotated[ + Optional[Path], + typer.Option( + "--policy-file", + exists=False, + file_okay=True, + dir_okay=False, + writable=True, + readable=True, + resolve_path=True, + help=SYSTEM_SCAN_POLICY_FILE_HELP, + show_default=False + )] = None, + targets: Annotated[ + List[Path], + typer.Option( + "--target", + exists=True, + file_okay=False, + dir_okay=True, + writable=False, + readable=True, + resolve_path=True, + help=SYSTEM_SCAN_TARGET_HELP, + show_default=False + ), + ] = [], + output: Annotated[SystemScanOutput, + typer.Option( + help=SYSTEM_SCAN_OUTPUT_HELP, + show_default=False) + ] = SystemScanOutput.SCREEN, + save_as: Annotated[Optional[Tuple[SystemScanExport, Path]], + typer.Option( + help=SYSTEM_SCAN_SAVE_AS_HELP, + show_default=False) + ] = (None, None)): + """ + Scans a system (machine) for supply-chain security and configuration issues\n + This will search for projects, requirements files and environment variables + """ + if not all(save_as): + ctx.params["save_as"] = None + + console = ctx.obj.console + version = ctx.obj.schema + metadata = ctx.obj.metadata + telemetry = ctx.obj.telemetry + + ecosystems = [Ecosystem(member.value) for member in list(ScannableEcosystems)] + ecosystems.append(Ecosystem.SAFETY_PROJECT) + + config = ctx.obj.config + + console.print("Searching for Python projects, requirements files and virtual environments across this machine.") + console.print("If necessary, please grant Safety permission to access folders you want scanned.") + console.print() + + with console.status("...", spinner=DEFAULT_SPINNER) as status: + handlers : Set[FileHandler] = set(ECOSYSTEM_HANDLER_MAPPING[ecosystem]() + for ecosystem in ecosystems) + for handler in handlers: + if handler.ecosystem: + wait_msg = "Fetching Safety's proprietary vulnerability database..." + status.update(wait_msg) + handler.download_required_assets(ctx.obj.auth.client) + + file_paths = {} + file_finders = [] + to_include = {file_type: paths for file_type, paths in config.scan.include_files.items() if file_type.ecosystem in ecosystems} + + for target in targets: + file_finder = FileFinder(target=target, + ecosystems=ecosystems, + max_level=config.scan.max_depth, + exclude=config.scan.ignore, console=console, + include_files=to_include, + live_status=status, handlers=handlers) + file_finders.append(file_finder) + + _, target_paths = file_finder.search() + + for file_type, paths in target_paths.items(): + current = file_paths.get(file_type, set()) + current.update(paths) + file_paths[file_type] = current + + scan_project_command = get_command_for(name=CMD_PROJECT_NAME, + typer_instance=scan_project_app) + + projects_dirs = set() + projects: List[ProjectModel] = [] + + project_data = {} + with console.status(":mag:", spinner=DEFAULT_SPINNER) as status: + # Handle projects first + if FileType.SAFETY_PROJECT.value in file_paths.keys(): + projects_file_paths = file_paths[FileType.SAFETY_PROJECT.value] + basic_params = ctx.params.copy() + basic_params.pop("targets", None) + + prjs_console = Console(quiet=True) + + for project_path in projects_file_paths: + projects_dirs.add(project_path.parent) + project_dir = str(project_path.parent) + try: + project = load_unverified_project_from_config(project_path.parent) + local_policy_file = load_policy_file(project_path.parent / ".safety-policy.yml") + except Exception as e: + LOG.exception(f"Unable to load project from {project_path}. Reason {e}") + console.print(f"{project_dir}: unable to load project found, skipped, use --debug for more details.") + continue + + if not project or not project.id: + LOG.warn(f"{project_path} parsed but project id is not defined or valid.") + continue + + if not ctx.obj.platform_enabled: + msg = f"project found and skipped, navigate to `{project.project_path}` and scan this project with ‘safety scan’" + console.print(f"{project.id}: {msg}") + continue + + msg = f"Existing project found at {project_dir}" + console.print(f"{project.id}: {msg}") + project_data[project.id] = {"path": project_dir, + "report_url": None, + "project_url": None, + "failed_exception": None} + + upload_request_id = None + try: + result = ctx.obj.auth.client.project_scan_request(project_id=project.id) + if "scan_upload_request_id" in result: + upload_request_id = result["scan_upload_request_id"] + else: + raise SafetyError(message=str(result)) + except Exception as e: + project_data[project.id]["failed_exception"] = e + LOG.exception(f"Unable to get a valid scan request id. Reason {e}") + console.print( + Padding(f":no_entry_sign: Unable to start project scan for {project.id}, reason: {e}", + (0, 0, 0, 1)), emoji=True) + continue + + projects.append(ProjectModel(id=project.id, + upload_request_id=upload_request_id)) + + kwargs = {"target": project_dir, "output": str(ScanOutput.NONE.value), + "save_as": (None, None), "upload_request_id": upload_request_id, + "local_policy": local_policy_file, "console": prjs_console} + try: + # TODO: Refactor to avoid calling invoke, also, launch + # this on background. + console.print( + Padding(f"Running safety scan for {project.id} project", + (0, 0, 0, 1)), emoji=True) + status.update(f":mag: Processing project scan for {project.id}") + + project_url, report, report_url = ctx.invoke(scan_project_command, **{**basic_params, **kwargs}) + project_data[project.id]["project_url"] = project_url + project_data[project.id]["report_url"] = report_url + + except Exception as e: + project_data[project.id]["failed_exception"] = e + console.print( + Padding(f":cross_mark: Failed project scan for {project.id}, reason: {e}", + (0, 0, 0, 1)), emoji=True) + LOG.exception(f"Failed to run scan on project {project.id}, " \ + f"Upload request ID: {upload_request_id}. Reason {e}") + + console.print() + + file_paths.pop(FileType.SAFETY_PROJECT.value, None) + + files: List[FileModel] = [] + + status.update(":mag: Finishing projects processing.") + + for k, f_paths in file_paths.items(): + file_paths[k] = {fp for fp in f_paths + if not should_exclude(excludes=projects_dirs, + to_analyze=fp)} + + pkgs_count = 0 + file_count = 0 + venv_count = 0 + + for path, analyzed_file in process_files(paths=file_paths, config=config): + status.update(f":mag: {path}") + files.append(FileModel(location=path, + file_type=analyzed_file.file_type, + results=analyzed_file.dependency_results)) + file_pkg_count = len(analyzed_file.dependency_results.dependencies) + + affected_dependencies = analyzed_file.dependency_results.get_affected_dependencies() + + # Per file + affected_pkgs_count = 0 + critical_vulns_count = 0 + other_vulns_count = 0 + + if any(affected_dependencies): + affected_pkgs_count = len(affected_dependencies) + + for dep in affected_dependencies: + for spec in dep.specifications: + for vuln in spec.vulnerabilities: + if vuln.ignored: + continue + if vuln.CVE and vuln.CVE.cvssv3 \ + and VulnerabilitySeverityLabels( + vuln.CVE.cvssv3.get( + "base_severity", "none") + .lower()) is VulnerabilitySeverityLabels.CRITICAL: + critical_vulns_count += 1 + else: + other_vulns_count += 1 + + msg = pluralize("package", file_pkg_count) + if analyzed_file.file_type is FileType.VIRTUAL_ENVIRONMENT: + msg = f"installed {msg} found" + venv_count += 1 + else: + file_count += 1 + + pkgs_count += file_pkg_count + console.print(f":package: {file_pkg_count} {msg} in {path}", emoji=True) + + if affected_pkgs_count <= 0: + msg = "No vulnerabilities found" + else: + msg = f"{affected_pkgs_count} vulnerable {pluralize('package', affected_pkgs_count)}" + if critical_vulns_count > 0: + msg += f", {critical_vulns_count} critical" + if other_vulns_count > 0: + msg += f" and {other_vulns_count} other {pluralize('vulnerability', other_vulns_count)} found" + + console.print( + Padding(msg, + (0, 0, 0, 1)), emoji=True) + console.print() + + report = ReportModel(version=version, + metadata=metadata, + telemetry=telemetry, + files=files, + projects=projects) + + console.print() + total_count = sum([finder.file_count for finder in file_finders], 0) + console.print(f"Searched {total_count:,} files for dependency security issues") + packages_msg = f"{pkgs_count:,} {pluralize('package', pkgs_count)} found across" + files_msg = f"{file_count:,} {pluralize('file', file_count)}" + venv_msg = f"{venv_count:,} virtual {pluralize('environment', venv_count)}" + console.print(f":package: Python files and environments: {packages_msg} {files_msg} and {venv_msg}", emoji=True) + console.print() + + proccessed = dict(filter( + lambda item: item[1]["report_url"] and item[1]["project_url"], + project_data.items())) + + if proccessed: + run_word = "runs" if len(proccessed) == 1 else "run" + console.print(f"Project {pluralize('scan', len(proccessed))} {run_word} on {len(proccessed)} existing {pluralize('project', len(proccessed))}:") + + for prj, data in proccessed.items(): + console.print(f"[bold]{prj}[/bold] at {data['path']}") + for detail in [f"{prj} dashboard: {data['project_url']}"]: + console.print(Padding(detail, (0, 0, 0, 1)), emoji=True, overflow="crop") + + process_report(ctx.obj, console, report, **{**ctx.params}) diff --git a/safety/scan/constants.py b/safety/scan/constants.py new file mode 100644 index 00000000..e499f7cd --- /dev/null +++ b/safety/scan/constants.py @@ -0,0 +1,143 @@ +from safety.util import get_safety_version + +# Console Help Theme +CONSOLE_HELP_THEME = { + "nhc": "grey82" +} + +CLI_VERSION = get_safety_version() +CLI_WEBSITE_URL="https://safetycli.com" +CLI_DOCUMENTATION_URL="https://docs.safetycli.com" +CLI_SUPPORT_EMAIL="support@safetycli.com" + +# Main Safety --help data: +CLI_MAIN_INTRODUCTION = f"[bold]Safety CLI 3 - Vulnerability Scanning for Secure Python Development[/bold]\n\n" \ +"Leverage the most comprehensive vulnerability data available to secure your projects against vulnerable and malicious packages. Safety CLI is a Python dependency vulnerability scanner that enhances software supply chain security at every stage of development.\n\n" \ +f"Documentation: [underline]{CLI_DOCUMENTATION_URL}[/underline]\n"\ +f"Contact: {CLI_SUPPORT_EMAIL}\n\n" + +CLI_AUTH_COMMAND_HELP = "Authenticate Safety CLI to perform scans. Your default browser will automatically open to https://platform.safetycli.com."\ +"\n[bold]Example: safety auth login[/bold]" +CLI_SCAN_COMMAND_HELP = "Scans a Python project directory."\ +"\n[bold]Example: safety scan[/bold] to scan the current directory" +CLI_SYSTEM_SCAN_COMMAND_HELP = "\\[beta] Run a comprehensive scan for packages and vulnerabilities across your entire machine/environment."\ +"\n[bold]Example: safety system-scan[/bold]" + +CLI_CHECK_COMMAND_HELP = "\\[deprecated] Find vulnerabilities at target files or enviroments. Now replaced by [bold]safety scan[/bold], and will be unsupported beyond 1 May 2024." \ +"\n[bold]Example: safety check -r requirements.txt[/bold]" + + +CLI_ALERT_COMMAND_HELP = "\\[deprecated] Create GitHub pull requests or GitHub issues using a `safety check` json report file. Being replaced by newer features." \ +"\n[bold]Example: safety alert --check-report your-report.json --key API_KEY github-pr --repo my-org/my-repo --token github-token[/bold]" + +CLI_CHECK_UPDATES_HELP = "Check for version updates to Safety CLI."\ +"\n[bold]Example: safety check-updates[/bold]" + +CLI_CONFIGURE_HELP = "Set up global configurations for Safety CLI, including proxy settings and organization details."\ +"\n[bold]Example: safety configure --proxy-host 192.168.0.1[/bold]" + +CLI_GENERATE_HELP = "Generate a boilerplate Safety CLI policy file for customized security policies."\ +"\nNote: Safety Platform policies will override any local policy files found"\ +"\n[bold]Example: safety generate policy_file[/bold]" + +CLI_VALIDATE_HELP = "Check if your local Safety CLI policy file is valid."\ +"\n[bold]Example: Example: safety validate --path /path/to/policy.yml[/bold]" + +# Global options help +_CLI_PROXY_TIP_HELP = f"[nhc]Note: proxy details can be set globally in a config file.[/nhc]\n\nSee [bold]safety configure --help[/bold]\n\n" + +CLI_PROXY_HOST_HELP = "Specify a proxy host for network communications. \n\n" + \ + _CLI_PROXY_TIP_HELP + +CLI_PROXY_PORT_HELP = "Set the proxy port (default: 80).\n\n" + \ +_CLI_PROXY_TIP_HELP + +CLI_PROXY_PROTOCOL_HELP = "Choose the proxy protocol (default: https).\n\n" + \ +_CLI_PROXY_TIP_HELP + +CLI_KEY_HELP = "The API key required for cicd stage or production stage scans.\n\n" \ +"[nhc]For development stage scans unset the API key and authenticate using [bold]safety auth[/bold].[/nhc]\n\n" \ +"[nhc]Tip: the API key can also be set using the environment variable: SAFETY_API_KEY[/nhc]\n\n"\ +"[bold]Example: safety --key API_KEY --stage cicd scan[/bold]" + +CLI_STAGE_HELP = "Assign a development lifecycle stage to your scan (default: development).\n\n" \ +"[nhc]This labels the scan and its findings in Safety Platform with this stage.[/nhc]\n\n" \ +"[bold]Example: safety --stage production scan[/bold]" + +CLI_DEBUG_HELP = "Enable debug mode for detailed output.\n\n" \ +"[bold]Example: safety --debug scan[/bold]" + +CLI_DISABLE_OPTIONAL_TELEMETRY_DATA_HELP = "Opt-out of sending optional telemetry data. Anonymized telemetry data will remain.\n\n" \ +"[bold]Example: safety --disable-optional-telemetry scan[/bold]" + + +# Scan Help options +SCAN_POLICY_FILE_HELP = "Use a local policy file to configure the scan.\n\n" \ + "[nhc]Note: Project scan policies defined in Safety Platform will override local policy files[/nhc]\n\n" \ + "[bold]Example: safety scan --policy-file /path/to/policy.yml[/bold]" +SCAN_TARGET_HELP = "Define a specific project path to scan. (default: current directory)\n\n" \ + "[bold]Example: safety scan --target /path/to/project[/bold]" +SCAN_OUTPUT_HELP = "Set the output format for scan results (default: screen)\n\n" \ + "[bold]Example: safety scan --output json[/bold]" +SCAN_SAVE_AS_HELP = "In addition to regular output save the scan results to a json, html, text, or spdx file using: FORMAT FILE_PATH\n\n" \ + "[bold]Example: safety scan --save-as json results.json[/bold]" +SCAN_DETAILED_OUTPUT = "Enable a verbose scan report for detailed insights (only for screen output)\n\n" \ + "[bold]Example: safety scan --detailed-output[/bold]" +SCAN_APPLY_FIXES = "[bold]Update packages listed in requirements.txt files to secure versions where possible[/bold]\n\n"\ + "[nhc]Currently supports: requirements.txt files[/nhc]\n\n"\ + "Note: this will update your requirements.txt file " + +# System Scan options +SYSTEM_SCAN_POLICY_FILE_HELP = "Use a local policy file to configure the scan.\n\n" \ + "[nhc]Note: Scan policies defined in Safety Platform will override local policy files[/nhc]\n\n" \ + "[bold]Example: safety scan --policy-file /path/to/policy.yml[/bold]" +SYSTEM_SCAN_TARGET_HELP = "Define a specific location to start the system scan. (default: current directory)\n\n" \ + "[bold]Example: safety scan --target /path/to/project[/bold]" +SYSTEM_SCAN_OUTPUT_HELP = "Set the output format for scan results (default: screen)\n\n" \ + "[bold]Example: safety scan --output json[/bold]" +SYSTEM_SCAN_SAVE_AS_HELP = "In addition to the terminal/console output (set by --output), save system-scan results to a screen (text) or json file.\n\n" \ + """[nhc]Use [bold]--save-as [/bold]. For example: [bold]--save-as json my-machine-scan.json[/bold] to save the system-scan results to `my-machine-scan.json` in the current directory[/nhc]\n\n""" \ + "[nhc][Default: json .][/nhc]" + +# Auth options +CLI_AUTH_LOGIN_HELP = "Authenticate with Safety CLI to perform scans. Your default browser will automatically open to https://platform.safetycli.com unless already authenticated.\n\n" \ + "[bold]Example: safety auth login[/bold]" +CLI_AUTH_LOGOUT_HELP = "Log out from the current Safety CLI session.\n\n" \ + "[bold]Example: safety auth logout[/bold]" +CLI_AUTH_STATUS_HELP = "Show the current authentication status.\n\n" \ + "[bold]Example: safety auth status[/bold]" + +DEFAULT_EPILOG = f"\nSafety CLI version: {CLI_VERSION}\n" \ + f"\nDocumentation: [underline]{CLI_DOCUMENTATION_URL}[/underline]\n\n\n\n" \ + "Made with :heart: by [purple]Safety Cybersecurity[/purple]\n\n" \ + f"{CLI_WEBSITE_URL}\n\n"\ + f"{CLI_SUPPORT_EMAIL}\n\n" + +# Configure options +CLI_CONFIGURE_PROXY_HOST_HELP = "Specify a proxy host for network communications to be saved into Safety's configuration. \n\n" +CLI_CONFIGURE_PROXY_PORT_HELP = "Set the proxy port to be saved into Safety's configuration file (default: 80).\n\n" +CLI_CONFIGURE_PROXY_PROTOCOL_HELP = "Choose the proxy protocol to be saved into Safety's configuration file (default: https).\n\n" +CLI_CONFIGURE_PROXY_TIMEOUT = "Set the timeout duration for proxy network calls.\n\n" + \ +"[bold]Example: safety configure --proxy-timeout 30[/bold]" +CLI_CONFIGURE_PROXY_REQUIRED = "Enable or disable the requirement for a proxy in network communications\n\n" + \ +"[bold]Example: safety configure --proxy-required[/bold]" +CLI_CONFIGURE_ORGANIZATION_ID = "Set the current device with an organization ID." \ +" - see your Safety Platform Organization page\n\n" + \ +"[bold]Example: safety configure --organization-id your_org_unique_id[/bold]" +CLI_CONFIGURE_ORGANIZATION_NAME = "Set the current device with an organization name." \ +" - see your Safety Platform Organization page.\n\n" + \ +"[bold]Example: safety configure --organization-name \"Your Org Name\"[/bold]" +CLI_CONFIGURE_SAVE_TO_SYSTEM = "Save the configuration to a system config file.\n" \ +"This will configure Safety CLI for all users on this machine. Use --save-to-user to " \ +"configure Safety CLI for only your user.\n\n" \ +"[bold]Example: safety configure --save-to-system[/bold]" + +# Generate options +CLI_GENERATE_PATH = "The path where the generated file will be saved (default: current directory).\n\n" \ +"[bold]Example: safety generate policy_file --path .my-project-safety-policy.yml[/bold]" + +# Command default settings +CMD_PROJECT_NAME = "scan" +CMD_SYSTEM_NAME = "system-scan" +DEFAULT_CMD = CMD_PROJECT_NAME +DEFAULT_SPINNER = "bouncingBar" \ No newline at end of file diff --git a/safety/scan/decorators.py b/safety/scan/decorators.py new file mode 100644 index 00000000..519c4023 --- /dev/null +++ b/safety/scan/decorators.py @@ -0,0 +1,324 @@ +from functools import wraps +import logging +from pathlib import Path +from random import randint +import sys +from typing import List, Optional + +from rich.padding import Padding +from safety_schemas.models import ConfigModel, ProjectModel +from rich.console import Console +from safety.auth.cli import render_email_note +from safety.cli_util import process_auth_status_not_ready +from safety.console import main_console +from safety.constants import SAFETY_POLICY_FILE_NAME, SYSTEM_CONFIG_DIR, SYSTEM_POLICY_FILE, USER_POLICY_FILE +from safety.errors import SafetyError, SafetyException, ServerError +from safety.scan.constants import DEFAULT_SPINNER +from safety.scan.main import PROJECT_CONFIG, download_policy, load_policy_file, \ + load_unverified_project_from_config, resolve_policy +from safety.scan.models import ScanOutput, SystemScanOutput +from safety.scan.render import print_announcements, print_header, print_project_info, print_wait_policy_download +from safety.scan.util import GIT + +from safety.scan.validators import fail_if_not_allowed_stage, verify_project +from safety.util import build_telemetry_data, pluralize +from safety_schemas.models import MetadataModel, ScanType, ReportSchemaVersion, \ + PolicySource + +LOG = logging.getLogger(__name__) + + +def initialize_scan(ctx, console): + data = None + + try: + data = ctx.obj.auth.client.initialize_scan() + except SafetyException as e: + LOG.error("Unable to initialize scan", exc_info=True) + except SafetyError as e: + if e.error_code: + raise e + except Exception as e: + LOG.exception("Exception trying to initialize scan", exc_info=True) + + if data: + ctx.obj.platform_enabled = data.get("platform-enabled", False) + + +def scan_project_command_init(func): + """ + Make general verifications before each scan command. + """ + @wraps(func) + def inner(ctx, policy_file_path: Optional[Path], target: Path, + output: ScanOutput, + console: Console = main_console, + *args, **kwargs): + ctx.obj.console = console + ctx.params.pop("console", None) + + if output.is_silent(): + console.quiet = True + + if not ctx.obj.auth.is_valid(): + process_auth_status_not_ready(console=console, + auth=ctx.obj.auth, ctx=ctx) + + upload_request_id = kwargs.pop("upload_request_id", None) + + fail_if_not_allowed_stage(ctx=ctx) + + # Run the initialize if it was not fired by a system-scan + if not upload_request_id: + initialize_scan(ctx, console) + + # Load .safety-project.ini + unverified_project = load_unverified_project_from_config(project_root=target) + + print_header(console=console, targets=[target]) + + stage = ctx.obj.auth.stage + session = ctx.obj.auth.client + git_data = GIT(root=target).build_git_data() + origin = None + branch = None + + if git_data: + origin = git_data.origin + branch = git_data.branch + + if ctx.obj.platform_enabled: + verify_project(console, ctx, session, unverified_project, stage, origin) + else: + ctx.obj.project = ProjectModel( + id="", + name="Undefined project", + project_path=unverified_project.project_path + ) + + ctx.obj.project.upload_request_id = upload_request_id + ctx.obj.project.git = git_data + + if not policy_file_path: + policy_file_path = target / Path(".safety-policy.yml") + + # Load Policy file and pull it from CLOUD + local_policy = kwargs.pop("local_policy", + load_policy_file(policy_file_path)) + + cloud_policy = None + if ctx.obj.platform_enabled: + cloud_policy = print_wait_policy_download(console, (download_policy, + {"session": session, + "project_id": ctx.obj.project.id, + "stage": stage, + "branch": branch})) + + ctx.obj.project.policy = resolve_policy(local_policy, cloud_policy) + config = ctx.obj.project.policy.config \ + if ctx.obj.project.policy and ctx.obj.project.policy.config \ + else ConfigModel() + + # Preserve global telemetry preference. + if ctx.obj.config: + if ctx.obj.config.telemetry_enabled is not None: + config.telemetry_enabled = ctx.obj.config.telemetry_enabled + + ctx.obj.config = config + + console.print() + + if ctx.obj.auth.org and ctx.obj.auth.org.name: + console.print(f"[bold]Organization[/bold]: {ctx.obj.auth.org.name}") + + # Check if an API key is set + if ctx.obj.auth.client.get_authentication_type() == "api_key": + details = {"Account": f"API key used"} + else: + content = ctx.obj.auth.email + if ctx.obj.auth.name != ctx.obj.auth.email: + content = f"{ctx.obj.auth.name}, {ctx.obj.auth.email}" + + details = {"Account": f"{content} {render_email_note(ctx.obj.auth)}"} + + if ctx.obj.project.id: + details["Project"] = ctx.obj.project.id + + if ctx.obj.project.git: + details[" Git branch"] = ctx.obj.project.git.branch + + details[" Environment"] = ctx.obj.auth.stage + + msg = "None, using Safety CLI default policies" + + if ctx.obj.project.policy: + if ctx.obj.project.policy.source is PolicySource.cloud: + msg = f"fetched from Safety Platform, " \ + "ignoring any local Safety CLI policy files" + else: + if ctx.obj.project.id: + msg = f"local {ctx.obj.project.id} project scan policy" + else: + msg = f"local scan policy file" + + details[" Scan policy"] = msg + + for k,v in details.items(): + console.print(f"[scan_meta_title]{k}[/scan_meta_title]: {v}") + + print_announcements(console=console, ctx=ctx) + + console.print() + + result = func(ctx, target=target, output=output, *args, **kwargs) + + + return result + + return inner + + +def scan_system_command_init(func): + """ + Make general verifications before each system scan command. + """ + @wraps(func) + def inner(ctx, policy_file_path: Optional[Path], targets: List[Path], + output: SystemScanOutput, + console: Console = main_console, *args, **kwargs): + ctx.obj.console = console + ctx.params.pop("console", None) + + if output.is_silent(): + console.quiet = True + + if not ctx.obj.auth.is_valid(): + process_auth_status_not_ready(console=console, + auth=ctx.obj.auth, ctx=ctx) + + initialize_scan(ctx, console) + + console.print() + print_header(console=console, targets=targets, is_system_scan=True) + wait_msg = "Checking authentication and system scan policies" + + with console.status(wait_msg, spinner=DEFAULT_SPINNER): + fail_if_not_allowed_stage(ctx=ctx) + + if not policy_file_path: + if SYSTEM_POLICY_FILE.exists(): + policy_file_path = SYSTEM_POLICY_FILE + elif USER_POLICY_FILE.exists(): + policy_file_path = USER_POLICY_FILE + + # Load Policy file + ctx.obj.system_scan_policy = load_policy_file(policy_file_path) if policy_file_path else None + config = ctx.obj.system_scan_policy.config \ + if ctx.obj.system_scan_policy and ctx.obj.system_scan_policy.config \ + else ConfigModel() + + # Preserve global telemetry preference. + if ctx.obj.config: + if ctx.obj.config.telemetry_enabled is not None: + config.telemetry_enabled = ctx.obj.config.telemetry_enabled + + ctx.obj.config = config + + if not any(targets): + if any(config.scan.system_targets): + targets = [Path(t).expanduser().absolute() for t in config.scan.system_targets] + else: + targets = [Path("/")] + + ctx.obj.metadata.scan_locations = targets + + console.print() + + if ctx.obj.auth.org and ctx.obj.auth.org.name: + console.print(f"[bold]Organization[/bold]: {ctx.obj.auth.org.name}") + + details = {"Account": f"{ctx.obj.auth.name}, {ctx.obj.auth.email}", + "Scan stage": ctx.obj.auth.stage} + + if ctx.obj.system_scan_policy: + if ctx.obj.system_scan_policy.source is PolicySource.cloud: + policy_type = "remote" + else: + policy_type = f'local ("{ctx.obj.system_scan_policy.id}")' + + org_name = " " + if ctx.obj.auth.org and ctx.obj.auth.org.name: + org_name = f" {ctx.obj.auth.org.name} " + + details["System scan policy"] = f"{policy_type}{org_name}organization policy:" + + for k,v in details.items(): + console.print(f"[bold]{k}[/bold]: {v}") + + if ctx.obj.system_scan_policy: + + dirs = [ign for ign in ctx.obj.config.scan.ignore if Path(ign).is_dir()] + + policy_details = [ + f"-> scanning from root {', '.join([str(t) for t in targets])} to a max folder depth of {ctx.obj.config.scan.max_depth}", + f"-> excluding {len(dirs)} {pluralize('directory', len(dirs))} and their sub-directories", + "-> target ecosystems: Python" + ] + for policy_detail in policy_details: + console.print( + Padding(policy_detail, + (0, 0, 0, 1)), emoji=True) + + print_announcements(console=console, ctx=ctx) + + console.print() + + kwargs.update({"targets": targets}) + result = func(ctx, *args, **kwargs) + return result + + return inner + + +def inject_metadata(func): + """ + Build metadata per subcommand. A system scan can trigger a project scan, + the project scan will need to build its own metadata. + """ + @wraps(func) + def inner(ctx, *args, **kwargs): + telemetry = build_telemetry_data(telemetry=ctx.obj.config.telemetry_enabled, + command=ctx.command.name, + subcommand=ctx.invoked_subcommand) + + auth_type = ctx.obj.auth.client.get_authentication_type() + + scan_type = ScanType(ctx.command.name) + target = kwargs.get("target", None) + targets = kwargs.get("targets", None) + + if not scan_type: + raise SafetyException("Missing scan_type.") + + if scan_type is ScanType.scan: + if not target: + raise SafetyException("Missing target.") + targets = [target] + + metadata = MetadataModel( + scan_type=scan_type, + stage=ctx.obj.auth.stage, + scan_locations=targets, + authenticated=ctx.obj.auth.client.is_using_auth_credentials(), + authentication_type=auth_type, + telemetry=telemetry, + schema_version=ReportSchemaVersion.v3_0 + ) + + ctx.obj.schema = ReportSchemaVersion.v3_0 + ctx.obj.metadata = metadata + ctx.obj.telemetry = telemetry + + return func(ctx, *args, **kwargs) + + return inner diff --git a/safety/scan/ecosystems/__init__.py b/safety/scan/ecosystems/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/safety/scan/ecosystems/base.py b/safety/scan/ecosystems/base.py new file mode 100644 index 00000000..2d4e4820 --- /dev/null +++ b/safety/scan/ecosystems/base.py @@ -0,0 +1,35 @@ +from abc import ABC, abstractmethod +from typing import List + +from safety_schemas.models import Ecosystem, FileType, ConfigModel, \ + DependencyResultModel +from typer import FileTextWrite + +NOT_IMPLEMENTED = "Not implemented funtion" + + +class Inspectable(ABC): + + @abstractmethod + def inspect(self, config: ConfigModel) -> DependencyResultModel: + return NotImplementedError(NOT_IMPLEMENTED) + + +class Remediable(ABC): + + @abstractmethod + def remediate(self): + return NotImplementedError(NOT_IMPLEMENTED) + + +class InspectableFile(Inspectable): + + def __init__(self, file: FileTextWrite): + self.file = file + self.ecosystem: Ecosystem + self.file_type: FileType + self.dependency_results: DependencyResultModel = \ + DependencyResultModel(dependencies=[]) + + + diff --git a/safety/scan/ecosystems/python/__init__.py b/safety/scan/ecosystems/python/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/safety/scan/ecosystems/python/__init__.py @@ -0,0 +1 @@ + diff --git a/safety/scan/ecosystems/python/dependencies.py b/safety/scan/ecosystems/python/dependencies.py new file mode 100644 index 00000000..51dfccf5 --- /dev/null +++ b/safety/scan/ecosystems/python/dependencies.py @@ -0,0 +1,215 @@ +from collections import defaultdict +from pathlib import Path +import sys +from typing import Generator, List, Optional + +from safety_schemas.models import FileType, PythonDependency +from safety_schemas.models.package import PythonSpecification +from ..base import InspectableFile +from dparse import parse, filetypes + +from packaging.specifiers import SpecifierSet +from packaging.version import parse as parse_version +from packaging.utils import canonicalize_name + + +def get_closest_ver(versions, version, spec: SpecifierSet): + results = {'upper': None, 'lower': None} + + if (not version and not spec) or not versions: + return results + + sorted_versions = sorted(versions, key=lambda ver: parse_version(ver), reverse=True) + + if not version: + sorted_versions = spec.filter(sorted_versions, prereleases=False) + + upper = None + lower = None + + try: + sorted_versions = list(sorted_versions) + upper = sorted_versions[0] + lower = sorted_versions[-1] + results['upper'] = upper + results['lower'] = lower if upper != lower else None + except IndexError: + pass + + return results + + current_v = parse_version(version) + + for v in sorted_versions: + index = parse_version(v) + + if index > current_v: + results['upper'] = index + + if index < current_v: + results['lower'] = index + break + + return results + + +def is_pinned_requirement(spec: SpecifierSet) -> bool: + if not spec or len(spec) != 1: + return False + + specifier = next(iter(spec)) + + return (specifier.operator == '==' and '*' != specifier.version[-1]) \ + or specifier.operator == '===' + + +def find_version(requirements): + ver = None + + if len(requirements) != 1: + return ver + + specs = requirements[0].specifier + + if is_pinned_requirement(specs): + ver = next(iter(requirements[0].specifier)).version + + return ver + + +def is_supported_by_parser(path): + supported_types = (".txt", ".in", ".yml", ".ini", "Pipfile", + "Pipfile.lock", "setup.cfg", "poetry.lock") + return path.endswith(supported_types) + + +def parse_requirement(dep, found: Optional[str]) -> PythonSpecification: + req = PythonSpecification(dep) + req.found = Path(found).resolve() if found else None + + if req.specifier == SpecifierSet(''): + req.specifier = SpecifierSet('>=0') + + return req + + +def read_requirements(fh, resolve=True): + """ + Reads requirements from a file like object and (optionally) from referenced files. + :param fh: file like object to read from + :param resolve: boolean. resolves referenced files. + :return: generator + """ + is_temp_file = not hasattr(fh, 'name') + path = None + found = Path('temp_file') + file_type = filetypes.requirements_txt + absolute_path: Optional[Path] = None + + if not is_temp_file and is_supported_by_parser(fh.name): + path = fh.name + absolute_path = Path(path).resolve() + found = absolute_path + file_type = None + + content = fh.read() + dependency_file = parse(content, path=path, resolve=resolve, + file_type=file_type) + + reqs_pkg = defaultdict(list) + + for req in dependency_file.resolved_dependencies: + reqs_pkg[canonicalize_name(req.name)].append(req) + + for pkg, reqs in reqs_pkg.items(): + specifications = list( + map(lambda req: parse_requirement(req, str(absolute_path)), reqs)) + version = find_version(specifications) + + yield PythonDependency(name=pkg, version=version, + specifications=specifications, + found=found, + absolute_path=absolute_path, + insecure_versions=[], + secure_versions=[], latest_version=None, + latest_version_without_known_vulnerabilities=None, + more_info_url=None) + + +def read_dependencies(fh, resolve=True): + path = fh.name + absolute_path = Path(path).resolve() + found = absolute_path + + content = fh.read() + dependency_file = parse(content, path=path, resolve=resolve) + + reqs_pkg = defaultdict(list) + + for req in dependency_file.resolved_dependencies: + reqs_pkg[canonicalize_name(req.name)].append(req) + + for pkg, reqs in reqs_pkg.items(): + specifications = list( + map(lambda req: parse_requirement(req, str(absolute_path)), reqs)) + version = find_version(specifications) + + yield PythonDependency(name=pkg, version=version, + specifications=specifications, + found=found, + absolute_path=absolute_path, + insecure_versions=[], + secure_versions=[], latest_version=None, + latest_version_without_known_vulnerabilities=None, + more_info_url=None) + +def read_virtual_environment_dependencies(f: InspectableFile) \ + -> Generator[PythonDependency, None, None]: + + env_path = Path(f.file.name).resolve().parent + + if sys.platform.startswith('win'): + site_pkgs_path = env_path / Path("Lib/site-packages/") + else: + site_pkgs_path = Path('lib/') + try: + site_pkgs_path = next((env_path / site_pkgs_path).glob("*/site-packages/")) + except StopIteration: + # Unable to find packages for foo env + return + + if not site_pkgs_path.resolve().exists(): + # Unable to find packages for foo env + return + + dep_paths = site_pkgs_path.glob("*/METADATA") + + for path in dep_paths: + if not path.is_file(): + continue + + dist_info_folder = path.parent + dep_name, dep_version = dist_info_folder.name.replace(".dist-info", "").split("-") + + yield PythonDependency(name=dep_name, version=dep_version, + specifications=[ + PythonSpecification(f"{dep_name}=={dep_version}", + found=site_pkgs_path)], + found=site_pkgs_path, insecure_versions=[], + secure_versions=[], latest_version=None, + latest_version_without_known_vulnerabilities=None, + more_info_url=None) + + +def get_dependencies(f: InspectableFile) -> List[PythonDependency]: + if not f.file_type: + return [] + + if f.file_type in [FileType.REQUIREMENTS_TXT, FileType.POETRY_LOCK, + FileType.PIPENV_LOCK]: + return list(read_dependencies(f.file, resolve=True)) + + if f.file_type == FileType.VIRTUAL_ENVIRONMENT: + return list(read_virtual_environment_dependencies(f)) + + return [] \ No newline at end of file diff --git a/safety/scan/ecosystems/python/main.py b/safety/scan/ecosystems/python/main.py new file mode 100644 index 00000000..592c7728 --- /dev/null +++ b/safety/scan/ecosystems/python/main.py @@ -0,0 +1,339 @@ + +from datetime import datetime +import itertools +from typing import List +from safety_schemas.models import FileType, PythonDependency, ClosestSecureVersion, \ + ConfigModel, PythonSpecification, RemediationModel, DependencyResultModel, \ + Vulnerability +from safety_schemas.models import VulnerabilitySeverityLabels, IgnoredItemDetail, \ + IgnoredItems, IgnoreCodes +from typer import FileTextWrite + +from safety.models import Severity +from safety.util import build_remediation_info_url + +from ....constants import IGNORE_UNPINNED_REQ_REASON + +from ....safety import get_cve_from, get_from_cache, get_vulnerabilities + + +from ..python.dependencies import get_closest_ver, get_dependencies, \ + is_pinned_requirement +from ..base import InspectableFile, Remediable + +from packaging.version import parse as parse_version +from packaging.utils import canonicalize_name +from packaging.specifiers import SpecifierSet + + +def ignore_vuln_if_needed(dependency: PythonDependency, file_type: FileType, + vuln_id: str, cve, ignore_vulns, + ignore_unpinned: bool, ignore_environment: bool, + specification: PythonSpecification, + ignore_severity: List[VulnerabilitySeverityLabels] = []): + + vuln_ignored: bool = vuln_id in ignore_vulns + + if vuln_ignored and ignore_vulns[vuln_id].code is IgnoreCodes.manual: + if (not ignore_vulns[vuln_id].expires + or ignore_vulns[vuln_id].expires > datetime.utcnow().date()): + return + + del ignore_vulns[vuln_id] + + if ignore_environment and file_type is FileType.VIRTUAL_ENVIRONMENT: + reason = "Ignored environment by rule in policy file." + ignore_vulns[vuln_id] = IgnoredItemDetail( + code=IgnoreCodes.environment_dependency, reason=reason) + return + + severity_label = VulnerabilitySeverityLabels.UNKNOWN + + if cve: + if cve.cvssv3 and cve.cvssv3.get("base_severity", None): + severity_label = VulnerabilitySeverityLabels( + cve.cvssv3["base_severity"].lower()) + + if severity_label in ignore_severity: + reason = f"{severity_label.value.capitalize()} severity ignored by rule in policy file." + ignore_vulns[vuln_id] = IgnoredItemDetail( + code=IgnoreCodes.cvss_severity, reason=reason) + return + + spec_ignored: bool = False + + if vuln_id in ignore_vulns.keys() and str(specification.specifier) in ignore_vulns[vuln_id].specifications: + spec_ignored = True + + if (not spec_ignored) and \ + (ignore_unpinned and not specification.is_pinned()): + + reason = IGNORE_UNPINNED_REQ_REASON + specifications = set() + specifications.add(str(specification.specifier)) + ignore_vulns[vuln_id] = IgnoredItemDetail( + code=IgnoreCodes.unpinned_specification, reason=reason, + specifications=specifications) + +def should_fail(config: ConfigModel, vulnerability: Vulnerability) -> bool: + if config.depedendency_vulnerability.fail_on.enabled and vulnerability.severity: + if vulnerability.severity.cvssv3 and vulnerability.severity.cvssv3.get("base_severity", None): + severity_label = VulnerabilitySeverityLabels( + vulnerability.severity.cvssv3["base_severity"].lower()) + if severity_label in config.depedendency_vulnerability.fail_on.cvss_severity: + return True + + return False + +def get_vulnerability(vuln_id: str, cve, + data, specifier, + db, name, ignore_vulns: IgnoredItems, + affected: PythonSpecification) -> Vulnerability: + base_domain = db.get('meta', {}).get('base_domain') + unpinned_ignored = ignore_vulns[vuln_id].specifications \ + if vuln_id in ignore_vulns.keys() else None + should_ignore = not unpinned_ignored or str(affected.specifier) in unpinned_ignored + ignored: bool = bool(ignore_vulns and + vuln_id in ignore_vulns and + should_ignore) + more_info_url = f"{base_domain}{data.get('more_info_path', '')}" + severity = None + + if cve and (cve.cvssv2 or cve.cvssv3): + severity = Severity(source=cve.name, cvssv2=cve.cvssv2, cvssv3=cve.cvssv3) + + analyzed_requirement = affected + analyzed_version = next(iter(analyzed_requirement.specifier)).version if affected.is_pinned() else None + + vulnerable_spec = set() + vulnerable_spec.add(specifier) + + reason = None + expires = None + ignore_code = None + + if ignored: + reason = ignore_vulns[vuln_id].reason + expires = str(ignore_vulns[vuln_id].expires) if ignore_vulns[vuln_id].expires else None + ignore_code = ignore_vulns[vuln_id].code.value + + return Vulnerability( + vulnerability_id=vuln_id, + package_name=name, + ignored=ignored, + ignored_reason=reason, + ignored_expires=expires, + ignored_code=ignore_code, + vulnerable_spec=vulnerable_spec, + all_vulnerable_specs=data.get("specs", []), + analyzed_version=analyzed_version, + analyzed_requirement=str(analyzed_requirement), + advisory=data.get("advisory"), + is_transitive=data.get("transitive", False), + published_date=data.get("published_date"), + fixed_versions=[ver for ver in data.get("fixed_versions", []) if ver], + closest_versions_without_known_vulnerabilities=data.get("closest_secure_versions", []), + resources=data.get("vulnerability_resources"), + CVE=cve, + severity=severity, + affected_versions=data.get("affected_versions", []), + more_info_url=more_info_url + ) + +class PythonFile(InspectableFile, Remediable): + + def __init__(self, file_type: FileType, file: FileTextWrite) -> None: + super().__init__(file=file) + self.ecosystem = file_type.ecosystem + self.file_type = file_type + + def __find_dependency_vulnerabilities__(self, dependencies: List[PythonDependency], + config: ConfigModel): + ignored_vulns_data = {} + ignore_vulns = {} \ + if not config.depedendency_vulnerability.ignore_vulnerabilities \ + else config.depedendency_vulnerability.ignore_vulnerabilities + + ignore_severity = config.depedendency_vulnerability.ignore_cvss_severity + ignore_unpinned = config.depedendency_vulnerability.python_ignore.unpinned_specifications + ignore_environment = config.depedendency_vulnerability.python_ignore.environment_results + + db = get_from_cache(db_name="insecure.json", skip_time_verification=True) + db_full = None + vulnerable_packages = frozenset(db.get('vulnerable_packages', [])) + found_dependencies = {} + specifications = iter([]) + + for dependency in dependencies: + specifications = itertools.chain(dependency.specifications, specifications) + found_dependencies[ + canonicalize_name(dependency.name) + ] = dependency + + # Let's report by req, pinned in environment will be ==version + for spec in specifications: + vuln_per_req = {} + name = canonicalize_name(spec.name) + dependency: PythonDependency = found_dependencies.get(name, None) + if not dependency: + continue + + if not dependency.version: + if not db_full: + db_full = get_from_cache(db_name="insecure_full.json", + skip_time_verification=True) + dependency.refresh_from(db_full) + + if name in vulnerable_packages: + # we have a candidate here, build the spec set + for specifier in db['vulnerable_packages'][name]: + spec_set = SpecifierSet(specifiers=specifier) + + if spec.is_vulnerable(spec_set, dependency.insecure_versions): + if not db_full: + db_full = get_from_cache(db_name="insecure_full.json", + skip_time_verification=True) + if not dependency.latest_version: + dependency.refresh_from(db_full) + + for data in get_vulnerabilities(pkg=name, spec=specifier, db=db_full): + try: + vuln_id: str = str(next(filter(lambda i: i.get('type', None) == 'pyup', data.get('ids', []))).get('id', '')) + except StopIteration: + vuln_id: str = '' + + if vuln_id in vuln_per_req: + vuln_per_req[vuln_id].vulnerable_spec.add(specifier) + continue + + cve = get_cve_from(data, db_full) + + ignore_vuln_if_needed(dependency=dependency, + file_type=self.file_type, + vuln_id=vuln_id, cve=cve, + ignore_vulns=ignore_vulns, + ignore_severity=ignore_severity, + ignore_unpinned=ignore_unpinned, + ignore_environment=ignore_environment, + specification=spec) + + include_ignored = True + vulnerability = get_vulnerability(vuln_id, cve, data, + specifier, db_full, + name, ignore_vulns, spec) + + should_add_vuln = not (vulnerability.is_transitive and + dependency.found and + dependency.found.parts[-1] == FileType.VIRTUAL_ENVIRONMENT.value) + + if vulnerability.ignored: + ignored_vulns_data[ + vulnerability.vulnerability_id] = vulnerability + + if not self.dependency_results.failed and not vulnerability.ignored: + self.dependency_results.failed = should_fail(config, vulnerability) + + + if (include_ignored or vulnerability.vulnerability_id not in ignore_vulns) and should_add_vuln: + vuln_per_req[vulnerability.vulnerability_id] = vulnerability + spec.vulnerabilities.append(vulnerability) + + # TODO: dep_result Save if it should fail the JOB + + self.dependency_results.dependencies = [dep for _, dep in found_dependencies.items()] + self.dependency_results.ignored_vulns = ignore_vulns + self.dependency_results.ignored_vulns_data = ignored_vulns_data + + def inspect(self, config: ConfigModel): + + # We only support vulnerability checking for now + dependencies = get_dependencies(self) + + if not dependencies: + self.results = [] + + self.__find_dependency_vulnerabilities__(dependencies=dependencies, + config=config) + + def __get_secure_specifications_for_user__(self, dependency: PythonDependency, db_full, + secure_vulns_by_user=None): + if not db_full: + return + + if not secure_vulns_by_user: + secure_vulns_by_user = set() + + versions = dependency.get_versions(db_full) + affected_versions = [] + + for vuln in db_full.get('vulnerable_packages', {}).get(dependency.name, []): + vuln_id: str = str(next(filter(lambda i: i.get('type', None) == 'pyup', vuln.get('ids', []))).get('id', '')) + if vuln_id and vuln_id not in secure_vulns_by_user: + affected_versions += vuln.get('affected_versions', []) + + affected_v = set(affected_versions) + sec_ver_for_user = list(versions.difference(affected_v)) + + return sorted(sec_ver_for_user, key=lambda ver: parse_version(ver), reverse=True) + + def remediate(self): + db_full = get_from_cache(db_name="insecure_full.json", + skip_time_verification=True) + if not db_full: + return + + for dependency in self.dependency_results.get_affected_dependencies(): + secure_versions = dependency.secure_versions + + if not secure_versions: + secure_versions = [] + + secure_vulns_by_user = set(self.dependency_results.ignored_vulns.keys()) + if not secure_vulns_by_user: + secure_v = sorted(secure_versions, key=lambda ver: parse_version(ver), + reverse=True) + else: + secure_v = self.__get_secure_specifications_for_user__( + dependency=dependency, db_full=db_full, + secure_vulns_by_user=secure_vulns_by_user) + + for specification in dependency.specifications: + if len(specification.vulnerabilities) <= 0: + continue + + version = None + if is_pinned_requirement(specification.specifier): + version = next(iter(specification.specifier)).version + closest_secure = {key: str(value) if value else None for key, value in + get_closest_ver(secure_v, + version, + specification.specifier).items()} + closest_secure = ClosestSecureVersion(**closest_secure) + recommended = None + + if closest_secure.upper: + recommended = closest_secure.upper + elif closest_secure.lower: + recommended = closest_secure.lower + + other_recommended = [other_v for other_v in secure_v if other_v != str(recommended)] + + remed_more_info_url = dependency.more_info_url + + if remed_more_info_url: + remed_more_info_url = build_remediation_info_url( + base_url=remed_more_info_url, version=version, + spec=str(specification.specifier), + target_version=recommended) + + if not remed_more_info_url: + remed_more_info_url = "-" + + vulns_found = sum(1 for vuln in specification.vulnerabilities if not vuln.ignored) + + specification.remediation = RemediationModel(vulnerabilities_found=vulns_found, + more_info_url=remed_more_info_url, + closest_secure=closest_secure if recommended else None, + recommended=recommended, + other_recommended=other_recommended) + diff --git a/safety/scan/ecosystems/target.py b/safety/scan/ecosystems/target.py new file mode 100644 index 00000000..8a95d421 --- /dev/null +++ b/safety/scan/ecosystems/target.py @@ -0,0 +1,39 @@ + + +from pathlib import Path +from safety_schemas.models import Ecosystem, FileType +from typer import FileTextWrite + +from .python.main import PythonFile + + +class InspectableFileContext: + def __init__(self, file_path: Path, + file_type: FileType) -> None: + self.file_path = file_path + self.inspectable_file = None + self.file_type = file_type + + def __enter__(self): # TODO: Handle permission issue /Applications/... + try: + file: FileTextWrite = open(self.file_path, mode='r+') # type: ignore + self.inspectable_file = TargetFile.create(file_type=self.file_type, file=file) + except Exception as e: + # TODO: Report this + pass + + return self.inspectable_file + + def __exit__(self, exc_type, exc_value, traceback): + if self.inspectable_file: + self.inspectable_file.file.close() + +class TargetFile(): + + @classmethod + def create(cls, file_type: FileType, file: FileTextWrite): + if file_type.ecosystem == Ecosystem.PYTHON: + return PythonFile(file=file, file_type=file_type) + + raise ValueError("Unsupported ecosystem or file type: " \ + f"{file_type.ecosystem}:{file_type.value}") diff --git a/safety/scan/finder/__init__.py b/safety/scan/finder/__init__.py new file mode 100644 index 00000000..7300903d --- /dev/null +++ b/safety/scan/finder/__init__.py @@ -0,0 +1,7 @@ +from .file_finder import FileFinder +from .handlers import PythonFileHandler + +__all__ = [ + "FileFinder", + "PythonFileHandler" +] \ No newline at end of file diff --git a/safety/scan/finder/file_finder.py b/safety/scan/finder/file_finder.py new file mode 100644 index 00000000..aaacd7ea --- /dev/null +++ b/safety/scan/finder/file_finder.py @@ -0,0 +1,114 @@ + +import logging +import os +from pathlib import Path +import re +from typing import Dict, List, Optional, Set, Tuple, Union +from safety_schemas.models import Ecosystem, FileType + +from safety.errors import SafetyException + +from .handlers import FileHandler, ECOSYSTEM_HANDLER_MAPPING + +LOG = logging.getLogger(__name__) + +def should_exclude(excludes: Set[Path], to_analyze: Path) -> bool: + + if not to_analyze.is_absolute(): + to_analyze = to_analyze.resolve() + + for exclude in excludes: + if not exclude.is_absolute(): + exclude = exclude.resolve() + + try: + if to_analyze == exclude or \ + to_analyze.relative_to(exclude): + return True + except ValueError: + pass + + return False + + +class FileFinder(): + """" + Defines a common interface to agree in what type of components Safety is trying to + find depending on the language type. + """ + + def __init__(self, max_level: int, ecosystems: List[Ecosystem], target: Path, + console, live_status=None, + exclude: Optional[List[str]] = None, + include_files: Optional[Dict[FileType, List[Path]]] = None, + handlers: Optional[Set[FileHandler]] = None) -> None: + self.max_level = max_level + self.target = target + self.include_files = include_files + + if not handlers: + handlers = set(ECOSYSTEM_HANDLER_MAPPING[ecosystem]() + for ecosystem in ecosystems) + + self.handlers = handlers + self.file_count = 0 + self.exclude_dirs: Set[Path] = set() + self.exclude_files: Set[Path] = set() + exclude = [] if not exclude else exclude + + for pattern in exclude: + for path in Path(target).glob(pattern): + if path.is_dir(): + self.exclude_dirs.add(path) + else: + self.exclude_files.add(path) + + self.console = console + self.live_status = live_status + + def process_directory(self, dir_path, max_deep: Optional[int]=None) -> Tuple[str, Dict[str, Set[Path]]]: + files: Dict[str, Set[Path]] = {} + level : int = 0 + initial_depth = len(Path(dir_path).parts) - 1 + skip_dirs = set() + skip_files = set() + + for root, dirs, filenames in os.walk(dir_path): + root_path = Path(root) + current_depth = len(root_path.parts) - initial_depth + + dirs[:] = [d for d in dirs if not should_exclude(excludes=self.exclude_dirs, + to_analyze=(root_path / Path(d)))] + + if dirs: + LOG.info(f"Directories to inspect -> {', '.join(dirs)}") + + LOG.info(f"Current -> {root}") + if self.live_status: + self.live_status.update(f":mag: Scanning {root}") + + if max_deep is not None and current_depth > max_deep: + # Don't go deeper + del dirs[:] + + filenames[:] = [f for f in filenames if not should_exclude( + excludes=self.exclude_files, + to_analyze=Path(f))] + + self.file_count += len(filenames) + + for file_name in filenames: + for handler in self.handlers: + file_type = handler.can_handle(root, file_name, self.include_files) + if file_type: + inspectable_file: Path = Path(root, file_name) + if file_type.value not in files or not files[file_type.value]: + files[file_type.value] = set() + files[file_type.value].add(inspectable_file) + break + level += 1 + + return dir_path, files + + def search(self) -> Tuple[str, Dict[str, Set[Path]]]: + return self.process_directory(self.target, self.max_level) diff --git a/safety/scan/finder/handlers.py b/safety/scan/finder/handlers.py new file mode 100644 index 00000000..395d9e0b --- /dev/null +++ b/safety/scan/finder/handlers.py @@ -0,0 +1,76 @@ +from abc import ABC, abstractmethod +from pathlib import Path +from types import MappingProxyType +from typing import Dict, List, Optional, Tuple + +from safety_schemas.models import Ecosystem, FileType + + +NOT_IMPLEMENTED = "You should implement this." + +class FileHandler(ABC): + + def __init__(self) -> None: + self.ecosystem: Optional[Ecosystem] = None + + def can_handle(self, root: str, file_name: str, include_files: Dict[FileType, List[Path]]) -> Optional[FileType]: + # Keeping it simple for now + + if not self.ecosystem: + return None + + for f_type in self.ecosystem.file_types: + if f_type in include_files: + current = Path(root, file_name).resolve() + paths = [p.resolve() if p.is_absolute() else (root / p).resolve() for p in include_files[f_type]] + if current in paths: + return f_type + + # Let's compare by name only for now + # We can put heavier logic here, but for speed reasons, + # right now is very basic, we will improve this later. + # Custom matching per File Type + if file_name.lower().endswith(f_type.value.lower()): + return f_type + + return None + + @abstractmethod + def download_required_assets(self, session) -> Dict[str, str]: + return NotImplementedError(NOT_IMPLEMENTED) + + +class PythonFileHandler(FileHandler): + # Example of a Python File Handler + + def __init__(self) -> None: + super().__init__() + self.ecosystem = Ecosystem.PYTHON + + def download_required_assets(self, session): + from safety.safety import fetch_database + + fetch_database(session=session, full=False, db=False, cached=True, + telemetry=True, ecosystem=Ecosystem.PYTHON, + from_cache=False) + + fetch_database(session=session, full=True, db=False, cached=True, + telemetry=True, ecosystem=Ecosystem.PYTHON, + from_cache=False) + + +class SafetyProjectFileHandler(FileHandler): + # Example of a Python File Handler + + def __init__(self) -> None: + super().__init__() + self.ecosystem = Ecosystem.SAFETY_PROJECT + + def download_required_assets(self, session): + pass + + +ECOSYSTEM_HANDLER_MAPPING = MappingProxyType({ + Ecosystem.PYTHON: PythonFileHandler, + Ecosystem.SAFETY_PROJECT: SafetyProjectFileHandler, +}) diff --git a/safety/scan/main.py b/safety/scan/main.py new file mode 100644 index 00000000..52508276 --- /dev/null +++ b/safety/scan/main.py @@ -0,0 +1,164 @@ +import configparser +import logging +from pathlib import Path +import re +import time +from typing import Any, Dict, Generator, Optional, Set, Tuple, Union +from pydantic import ValidationError +import typer +from ..auth.utils import SafetyAuthSession +from ..errors import SafetyError +from .ecosystems.base import InspectableFile +from .ecosystems.target import InspectableFileContext +from .models import ScanExport, UnverifiedProjectModel + +from safety_schemas.models import FileType, PolicyFileModel, PolicySource, \ + ConfigModel, Stage, ProjectModel, ScanType + + +LOG = logging.getLogger(__name__) + +PROJECT_CONFIG = ".safety-project.ini" +PROJECT_CONFIG_SECTION = "project" +PROJECT_CONFIG_ID = "id" +PROJECT_CONFIG_URL = "url" +PROJECT_CONFIG_NAME = "name" + + +def download_policy(session: SafetyAuthSession, + project_id: str, + stage: Stage, + branch: Optional[str]) -> Optional[PolicyFileModel]: + result = session.download_policy(project_id=project_id, stage=stage, + branch=branch) + + if result and "uuid" in result and result["uuid"]: + LOG.debug(f"Loading CLOUD policy file {result['uuid']} from cloud.") + LOG.debug(result) + uuid = result["uuid"] + err = f'Unable to load the Safety Policy file ("{uuid}"), from cloud.' + config = None + + try: + yml_raw = result["settings"] + # TODO: Move this to safety_schemas + parse = "parse_obj" + import importlib + module_name = ( + "safety_schemas." "config.schemas." f"v3_0.main" + ) + module = importlib.import_module(module_name) + config_model = module.Config + validated_policy_file = getattr(config_model, parse)(yml_raw) + config = ConfigModel.from_v30(obj=validated_policy_file) + except ValidationError as e: + LOG.error(f"Failed to parse policy file {uuid}.", exc_info=True) + raise SafetyError(f"{err}, details: {e}") + except ValueError as e: + LOG.error(f"Wrong YML file for policy file {uuid}.", exc_info=True) + raise SafetyError(f"{err}, details: {e}") + + return PolicyFileModel(id=result["uuid"], + source=PolicySource.cloud, + location=None, + config=config) + + return None + + +def load_unverified_project_from_config(project_root: Path) -> UnverifiedProjectModel: + config = configparser.ConfigParser() + project_path = project_root / PROJECT_CONFIG + config.read(project_path) + id = config.get(PROJECT_CONFIG_SECTION, PROJECT_CONFIG_ID, fallback=None) + id = config.get(PROJECT_CONFIG_SECTION, PROJECT_CONFIG_ID, fallback=None) + url = config.get(PROJECT_CONFIG_SECTION, PROJECT_CONFIG_URL, fallback=None) + name = config.get(PROJECT_CONFIG_SECTION, PROJECT_CONFIG_NAME, fallback=None) + created = True + if id: + created = False + + return UnverifiedProjectModel(id=id, url_path=url, + name=name, project_path=project_path, + created=created) + + +def save_project_info(project: ProjectModel, project_path: Path): + config = configparser.ConfigParser() + config.read(project_path) + + if PROJECT_CONFIG_SECTION not in config.sections(): + config[PROJECT_CONFIG_SECTION] = {} + + config[PROJECT_CONFIG_SECTION][PROJECT_CONFIG_ID] = project.id + if project.url_path: + config[PROJECT_CONFIG_SECTION][PROJECT_CONFIG_URL] = project.url_path + if project.name: + config[PROJECT_CONFIG_SECTION][PROJECT_CONFIG_NAME] = project.name + + with open(project_path, 'w') as configfile: + config.write(configfile) + + +def load_policy_file(path: Path) -> Optional[PolicyFileModel]: + config = None + + if not path or not path.exists(): + return None + + err = f'Unable to load the Safety Policy file ("{path}"), this command ' \ + "only supports version 3.0" + + try: + config = ConfigModel.parse_policy_file(raw_report=path) + except ValidationError as e: + LOG.error(f"Failed to parse policy file {path}.", exc_info=True) + raise SafetyError(f"{err}, details: {e}") + except ValueError as e: + LOG.error(f"Wrong YML file for policy file {path}.", exc_info=True) + raise SafetyError(f"{err}, details: {e}") + + return PolicyFileModel(id=str(path), source=PolicySource.local, + location=path, config=config) + + +def resolve_policy(local_policy: Optional[PolicyFileModel], + cloud_policy: Optional[PolicyFileModel]) \ + -> Optional[PolicyFileModel]: + policy = None + + if cloud_policy: + policy = cloud_policy + elif local_policy: + policy = local_policy + + return policy + + +def save_report_as(scan_type: ScanType, export_type: ScanExport, at: Path, report: Any): + tag = int(time.time()) + + if at.is_dir(): + at = at / Path( + f"{scan_type.value}-{export_type.get_default_file_name(tag=tag)}") + + with open(at, 'w+') as report_file: + report_file.write(report) + + +def process_files(paths: Dict[str, Set[Path]], + config: Optional[ConfigModel] = None) -> \ + Generator[Tuple[Path, InspectableFile], None, None]: + if not config: + config = ConfigModel() + + for file_type_key, f_paths in paths.items(): + file_type = FileType(file_type_key) + if not file_type or not file_type.ecosystem: + continue + for f_path in f_paths: + with InspectableFileContext(f_path, file_type=file_type) as inspectable_file: + if inspectable_file and inspectable_file.file_type: + inspectable_file.inspect(config=config) + inspectable_file.remediate() + yield f_path, inspectable_file diff --git a/safety/scan/models.py b/safety/scan/models.py new file mode 100644 index 00000000..54a32895 --- /dev/null +++ b/safety/scan/models.py @@ -0,0 +1,81 @@ +from enum import Enum +from pathlib import Path +from typing import Optional + +from pydantic.dataclasses import dataclass + +class FormatMixin: + + @classmethod + def is_format(cls, format_sub: Optional[Enum], format_instance: Enum): + """ Check if the value is a variant of the specified format. """ + if not format_sub: + return False + + if format_sub is format_instance: + return True + + prefix = format_sub.value.split('@')[0] + return prefix == format_instance.value + + @property + def version(self): + """ Return the version of the format. """ + result = self.value.split('@') + + if len(result) == 2: + return result[1] + + return None + + +class ScanOutput(FormatMixin, str, Enum): + JSON = "json" + SPDX = "spdx" + SPDX_2_3 = "spdx@2.3" + SPDX_2_2 = "spdx@2.2" + HTML = "html" + + SCREEN = "screen" + NONE = "none" + + def is_silent(self): + return self in (ScanOutput.JSON, ScanOutput.SPDX, ScanOutput.SPDX_2_3, ScanOutput.SPDX_2_2, ScanOutput.HTML) + + +class ScanExport(FormatMixin, str, Enum): + JSON = "json" + SPDX = "spdx" + SPDX_2_3 = "spdx@2.3" + SPDX_2_2 = "spdx@2.2" + HTML = "html" + + def get_default_file_name(self, tag: int): + + if self is ScanExport.JSON: + return f"safety-report-{tag}.json" + elif self in [ScanExport.SPDX, ScanExport.SPDX_2_3, ScanExport.SPDX_2_2]: + return f"safety-report-spdx-{tag}.json" + elif self is ScanExport.HTML: + return f"safety-report-{tag}.html" + else: + raise ValueError("Unsupported scan export type") + + +class SystemScanOutput(str, Enum): + JSON = "json" + SCREEN = "screen" + + def is_silent(self): + return self in (SystemScanOutput.JSON,) + +class SystemScanExport(str, Enum): + JSON = "json" + +@dataclass +class UnverifiedProjectModel(): + id: Optional[str] + project_path: Path + created: bool + name: Optional[str] = None + url_path: Optional[str] = None diff --git a/safety/scan/render.py b/safety/scan/render.py new file mode 100644 index 00000000..f5f1da2f --- /dev/null +++ b/safety/scan/render.py @@ -0,0 +1,554 @@ +from collections import defaultdict +from datetime import datetime +import itertools +import json +import logging +from pathlib import Path +import time +from typing import Any, Dict, List, Optional, Set +from rich.prompt import Prompt +from rich.text import Text +from rich.console import Console +from rich.padding import Padding +from safety_schemas.models import Vulnerability, ReportModel +import typer +from safety import safety +from safety.auth.constants import SAFETY_PLATFORM_URL +from safety.errors import SafetyException +from safety.output_utils import parse_html +from safety.scan.constants import DEFAULT_SPINNER + +from safety_schemas.models import Ecosystem, FileType, PolicyFileModel, \ + PolicySource, ProjectModel, IgnoreCodes, Stage, PythonDependency + +from safety.util import get_basic_announcements, get_safety_version + +LOG = logging.getLogger(__name__) + +import datetime + +def render_header(targets: List[Path], is_system_scan: bool) -> Text: + version = get_safety_version() + scan_datetime = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") + + action = f"scanning {', '.join([str(t) for t in targets])}" + if is_system_scan: + action = "running [bold]system scan[/bold]" + + return Text.from_markup( + f"[bold]Safety[/bold] {version} {action}\n{scan_datetime}") + +def print_header(console, targets: List[Path], is_system_scan: bool = False): + console.print(render_header(targets, is_system_scan), markup=True) + +def print_announcements(console, ctx): + colors = {"error": "red", "warning": "yellow", "info": "default"} + + announcements = safety.get_announcements(ctx.obj.auth.client, + telemetry=ctx.obj.config.telemetry_enabled, + with_telemetry=ctx.obj.telemetry) + basic_announcements = get_basic_announcements(announcements, False) + + if any(basic_announcements): + console.print() + console.print("[bold]Safety Announcements:[/bold]") + console.print() + for announcement in announcements: + color = colors.get(announcement.get('type', "info"), "default") + console.print(f"[{color}]* {announcement.get('message')}[/{color}]") + +def print_detected_ecosystems_section(console, file_paths: Dict[str, Set[Path]], + include_safety_prjs: bool = True): + detected: Dict[Ecosystem, Dict[FileType, int]] = {} + + for file_type_key, f_paths in file_paths.items(): + file_type = FileType(file_type_key) + if file_type.ecosystem: + if file_type.ecosystem not in detected: + detected[file_type.ecosystem] = {} + detected[file_type.ecosystem][file_type] = len(f_paths) + + for ecosystem, f_type_count in detected.items(): + + if not include_safety_prjs and ecosystem is Ecosystem.SAFETY_PROJECT: + continue + + brief = "Found " + file_types = [] + + for f_type, count in f_type_count.items(): + file_types.append(f"{count} {f_type.human_name(plural=count>1)}") + + if len(file_types) > 1: + brief += ", ".join(file_types[:-1]) + " and " + file_types[-1] + else: + brief += file_types[0] + + msg = f"{ecosystem.name.replace('_', ' ').title()} detected. {brief}" + + console.print(msg) + +def print_brief(console, project: ProjectModel, dependencies_count: int = 0, + affected_count: int = 0, fixes_count: int = 0): + from ..util import pluralize + + if project.policy: + if project.policy.source is PolicySource.cloud: + policy_msg = f"policy fetched from Safety Platform" + else: + if project.id: + policy_msg = f"local {project.id} project scan policy" + else: + policy_msg = f"local scan policy file" + else: + policy_msg = "default Safety CLI policies" + + console.print(f"Tested [number]{dependencies_count}[/number] {pluralize('dependency', dependencies_count)} for known security " \ + f"issues using {policy_msg}") + console.print( + f"[number]{affected_count}[/number] security {pluralize('issue', affected_count)} found, [number]{fixes_count}[/number] {pluralize('fix', fixes_count)} suggested") + +def print_fixes_section(console, requirements_txt_found: bool = False, is_detailed_output: bool = False): + console.print("-" * console.size.width) + console.print("Apply Fixes") + console.print("-" * console.size.width) + + console.print() + + if requirements_txt_found: + console.print("[green]Run `safety scan --apply-fixes`[/green] to update these packages and fix these vulnerabilities. " + "Documentation, limitations, and configurations for applying automated fixes: [link]https://docs.safetycli.com/safety-docs/vulnerability-remediation/applying-fixes[/link]") + console.print() + console.print("Alternatively, use your package manager to update packages to their secure versions. Always check for breaking changes when updating packages.") + else: + msg = "Use your package manager to update packages to their secure versions. Always check for breaking changes when updating packages." + console.print(msg) + + if not is_detailed_output: + console.print("[tip]Tip[/tip]: For more detailed output on each vulnerability, add the `--detailed-output` flag to safety scan.") + + console.print() + console.print("-" * console.size.width) + + +def print_ignore_details(console, project: ProjectModel, ignored, + is_detailed_output: bool = False, ignored_vulns_data = None): + from ..util import pluralize + + if is_detailed_output: + if not ignored_vulns_data: + ignored_vulns_data = iter([]) + + + manual_ignored = {} + cvss_severity_ignored = {} + cvss_severity_ignored_pkgs = set() + unpinned_ignored = {} + unpinned_ignored_pkgs = set() + environment_ignored = {} + environment_ignored_pkgs = set() + + for vuln_data in ignored_vulns_data: + code = IgnoreCodes(vuln_data.ignored_code) + if code is IgnoreCodes.manual: + manual_ignored[vuln_data.vulnerability_id] = vuln_data + elif code is IgnoreCodes.cvss_severity: + cvss_severity_ignored[vuln_data.vulnerability_id] = vuln_data + cvss_severity_ignored_pkgs.add(vuln_data.package_name) + elif code is IgnoreCodes.unpinned_specification: + unpinned_ignored[vuln_data.vulnerability_id] = vuln_data + unpinned_ignored_pkgs.add(vuln_data.package_name) + elif code is IgnoreCodes.environment_dependency: + environment_ignored[vuln_data.vulnerability_id] = vuln_data + environment_ignored_pkgs.add(vuln_data.package_name) + + if manual_ignored: + count = len(manual_ignored) + console.print( + f"[number]{count}[/number] were manually ignored due to the project policy:") + for vuln in manual_ignored.values(): + render_to_console(vuln, console, + rich_kwargs={"emoji": True, "overflow": "crop"}, + detailed_output=is_detailed_output) + if cvss_severity_ignored: + count = len(cvss_severity_ignored) + console.print( + f"[number]{count}[/number] {pluralize('vulnerability', count)} {pluralize('was', count)} ignored because " \ + "of their severity or exploitability impacted the following" \ + f" {pluralize('package', len(cvss_severity_ignored_pkgs))}: {', '.join(cvss_severity_ignored_pkgs)}" + ) + if environment_ignored: + count = len(environment_ignored) + console.print( + f"[number]{count}[/number] {pluralize('vulnerability', count)} {pluralize('was', count)} ignored because " \ + "they are inside an environment dependency." + ) + if unpinned_ignored: + count = len(unpinned_ignored) + console.print( + f"[number]{count}[/number] {pluralize('vulnerability', count)} {pluralize('was', count)} ignored because " \ + f"{pluralize('this', len(unpinned_ignored_pkgs))} {pluralize('package', len(unpinned_ignored_pkgs))} {pluralize('has', len(unpinned_ignored_pkgs))} unpinned specs: " \ + f"{', '.join(unpinned_ignored_pkgs)}" + ) + + else: + if len(ignored) > 0: + console.print(f"([number]{len(ignored)}[/number] {pluralize('vulnerability', len(ignored))} {pluralize('was', len(ignored))} ignored due to " \ + "project policy)") + + +def print_wait_project_verification(console, project_id, closure, on_error_delay=1): + status = None + wait_msg = f"Verifying project {project_id} with Safety Platform." + + with console.status(wait_msg, spinner=DEFAULT_SPINNER): + try: + f, kwargs = closure + status = f(**kwargs) + except Exception as e: + LOG.exception(f'Unable to verify the project, reason: {e}') + reason = "We are currently unable to verify the project, " \ + "and it is necessary to link the scan to a specific " \ + f"project. Reason: {e}" + raise SafetyException(message=reason) + + if not status: + wait_msg = f'Unable to verify "{project_id}". Starting again...' + time.sleep(on_error_delay) + + return status + +def print_project_info(console, project: ProjectModel): + config_msg = "loaded without policies or custom configuration." + + if project.policy: + if project.policy.source is PolicySource.local: + rel_location = project.policy.location.name if project.policy.location else "" + config_msg = "configuration and policies fetched " \ + f"from {rel_location}." + else: + config_msg = " policies fetched " \ + "from Safety Platform." + + msg = f"[bold]{project.id} project found[/bold] - {config_msg}" + console.print(msg) + +def print_wait_policy_download(console, closure) -> Optional[PolicyFileModel]: + policy = None + wait_msg = "Looking for a policy from cloud..." + + with console.status(wait_msg, spinner=DEFAULT_SPINNER): + try: + f, kwargs = closure + policy = f(**kwargs) + except Exception as e: + LOG.exception(f'Policy download failed, reason: {e}') + console.print("Not using cloud policy file.") + + if policy: + wait_msg = "Policy fetched from Safety Platform." + else: + # TODO: Send a log + pass + return policy + + +def prompt_project_id(console, stage: Stage, + prj_root_name: Optional[str], + do_not_exit=True) -> str: + from safety.util import clean_project_id + default_prj_id = clean_project_id(prj_root_name) if prj_root_name else None + + non_interactive_mode = console.quiet or not console.is_interactive + if stage is not Stage.development and non_interactive_mode: + # Fail here + console.print("The scan needs to be linked to a project.") + raise typer.Exit(code=1) + + hint = "" + if default_prj_id: + hint = f" If empty Safety will use [bold]{default_prj_id}[/bold]" + prompt_text = f"Set a project id for this scan (no spaces).{hint}" + + def ask(): + prj_id = None + + result = Prompt.ask(prompt_text, default=None, console=console) + + if result: + prj_id = clean_project_id(result) + elif default_prj_id: + prj_id = default_prj_id + + return prj_id + + project_id = ask() + + while not project_id and do_not_exit: + project_id = ask() + + return project_id + + +def prompt_link_project(console, prj_name: str, prj_admin_email: str) -> bool: + console.print("[bold]Safety found an existing project with this name in your organization:[/bold]") + + for detail in (f"[bold]Project name:[/bold] {prj_name}", + f"[bold]Project admin:[/bold] {prj_admin_email}"): + console.print(Padding(detail, (0, 0, 0, 2)), emoji=True) + + prompt_question = "Do you want to link this scan with this existing project?" + + answer = Prompt.ask(prompt=prompt_question, choices=["y", "n"], + default="y", show_default=True, console=console).lower() + + return answer == "y" + + +def render_to_console(cls: Vulnerability, console: Console, rich_kwargs, + detailed_output: bool = False): + cls.__render__(console, detailed_output, rich_kwargs) + + +def get_render_console(entity_type): + + if entity_type is Vulnerability: + def __render__(self, console: Console, detailed_output: bool, rich_kwargs): + if not rich_kwargs: + rich_kwargs = {} + + pre = " Ignored:" if self.ignored else "" + severity_detail = None + + if self.severity and self.severity.source: + severity_detail = self.severity.source + + if self.severity.cvssv3 and "base_severity" in self.severity.cvssv3: + severity_detail += f", CVSS Severity {self.severity.cvssv3['base_severity'].upper()}" + + advisory_length = 200 if detailed_output else 110 + + console.print( + Padding( + f"->{pre} Vuln ID [vuln_id]{self.vulnerability_id}[/vuln_id]: {severity_detail if severity_detail else ''}", + (0, 0, 0, 2) + ), **rich_kwargs) + console.print( + Padding( + f"{self.advisory[:advisory_length]}{'...' if len(self.advisory) > advisory_length else ''}", + (0, 0, 0, 5) + ), **rich_kwargs) + + if detailed_output: + console.print( + Padding(f"For more information: [link]{self.more_info_url}[/link]", (0, 0, 0, 5)), + **rich_kwargs) + + return __render__ + + +def render_scan_html(report: ReportModel, obj) -> str: + from safety.scan.command import ScannableEcosystems + + project = report.projects[0] if any(report.projects) else None + + scanned_packages = 0 + affected_packages = 0 + ignored_packages = 0 + remediations_recommended = 0 + ignored_vulnerabilities = 0 + vulnerabilities = 0 + vulns_per_file = defaultdict(int) + remed_per_file = defaultdict(int) + + for file in project.files: + scanned_packages += len(file.results.dependencies) + affected_packages += len(file.results.get_affected_dependencies()) + ignored_vulnerabilities += len(file.results.ignored_vulns) + + for spec in file.results.get_affected_specifications(): + vulnerabilities += len(spec.vulnerabilities) + vulns_per_file[file.location] += len(spec.vulnerabilities) + if spec.remediation: + remed_per_file[file.location] += 1 + remediations_recommended += 1 + + ignored_packages += len(file.results.ignored_vulns) + + # TODO: Get this information for the report model (?) + summary = {"scanned_packages": scanned_packages, + "affected_packages": affected_packages, + "remediations_recommended": remediations_recommended, + "ignored_vulnerabilities": ignored_vulnerabilities, "vulnerabilities": vulnerabilities} + + vulnerabilities = [] + + + # TODO: This should be based on the configs per command + ecosystems = [(f"{ecosystem.name.title()}", + [file_type.human_name(plural=True) for file_type in ecosystem.file_types]) for ecosystem in [Ecosystem(member.value) for member in list(ScannableEcosystems)]] + + settings ={"audit_and_monitor": True, "platform_url": SAFETY_PLATFORM_URL, "ecosystems": ecosystems} + template_context = {"report": report, "summary": summary, "announcements": [], + "project": project, + "platform_enabled": obj.platform_enabled, + "settings": settings, + "vulns_per_file": vulns_per_file, + "remed_per_file": remed_per_file} + + return parse_html(kwargs=template_context, template="scan/index.html") + + +def generate_spdx_creation_info(*, spdx_version: str, project_identifier: str) -> Any: + from spdx_tools.spdx.model import ( + Actor, + ActorType, + CreationInfo, + ) + + version = int(time.time()) + SPDX_ID_TYPE = "SPDXRef-DOCUMENT" + DOC_NAME = f"{project_identifier}-{version}" + + DOC_NAMESPACE = f"https://spdx.org/spdxdocs/{DOC_NAME}" + # DOC_NAMESPACE = f"urn:safety:{project_identifier}:{version}" + + DOC_COMMENT = f"This document was created using SPDX {spdx_version}" + CREATOR_COMMENT = "Safety CLI automatically created this SPDX document from a scan report." + + from ..util import get_safety_version + TOOL_ID = "safety" + TOOL_VERSION = get_safety_version() + + doc_creator = Actor( + actor_type=ActorType.TOOL, + name=f"{TOOL_ID}-{TOOL_VERSION}", + email=None + ) + + creation_info = CreationInfo( + spdx_version=f"SPDX-{spdx_version}", + spdx_id=SPDX_ID_TYPE, + name=DOC_NAME, + document_namespace=DOC_NAMESPACE, + creators=[doc_creator], + created=datetime.datetime.now(), + document_comment=DOC_COMMENT, + creator_comment=CREATOR_COMMENT + ) + return creation_info + + +def create_pkg_ext_ref(*, package: PythonDependency, version: Optional[str]): + from spdx_tools.spdx.model import ( + ExternalPackageRef, + ExternalPackageRefCategory, + ) + + version_detail = f'@{version}' if version else '' + pkg_ref = ExternalPackageRef( + ExternalPackageRefCategory.PACKAGE_MANAGER, + "purl", + f"pkg:pypi/{package.name}{version_detail}", + ) + return pkg_ref + + +def create_packages(dependencies: List[PythonDependency]) -> List[Any]: + from spdx_tools.spdx.model.spdx_no_assertion import SpdxNoAssertion + + from spdx_tools.spdx.model import ( + Package, + ) + + doc_pkgs = [] + pkgs_added = set([]) + for dependency in dependencies: + for spec in dependency.specifications: + pkg_version = next(iter(spec.specifier)).version if spec.is_pinned() else f"{spec.specifier}" + dep_name = dependency.name.replace('_', '-') + pkg_id = f"SPDXRef-pip-{dep_name}-{pkg_version}" if spec.is_pinned() else f"SPDXRef-pip-{dep_name}" + if pkg_id in pkgs_added: + continue + pkg_ref = create_pkg_ext_ref(package=dependency, version=pkg_version) + + pkg = Package( + spdx_id=pkg_id, + name=f"pip:{dep_name}", + download_location=SpdxNoAssertion(), + version=pkg_version, + file_name="", + supplier=SpdxNoAssertion(), + files_analyzed=False, + license_concluded=SpdxNoAssertion(), + license_declared=SpdxNoAssertion(), + copyright_text=SpdxNoAssertion(), + external_references=[pkg_ref], + ) + pkgs_added.add(pkg_id) + doc_pkgs.append(pkg) + return doc_pkgs + + +def create_spdx_document(*, report: ReportModel, spdx_version: str) -> Optional[Any]: + from spdx_tools.spdx.model import ( + Document, + Relationship, + RelationshipType, + ) + + project = report.projects[0] if any(report.projects) else None + + if not project: + return None + + prj_id = project.id + + if not prj_id: + parent_name = project.project_path.parent.name + prj_id = parent_name if parent_name else str(int(time.time())) + + creation_info = generate_spdx_creation_info(spdx_version=spdx_version, project_identifier=prj_id) + + depedencies = iter([]) + for file in project.files: + depedencies = itertools.chain(depedencies, file.results.dependencies) + + packages = create_packages(depedencies) + + # Requirement for document to have atleast one relationship + relationship = Relationship( + "SPDXRef-DOCUMENT", + RelationshipType.DESCRIBES, + "SPDXRef-DOCUMENT" + ) + spdx_doc = Document( + creation_info, + packages, + [], + [], + [], + [relationship], + [] + ) + return spdx_doc + + +def render_scan_spdx(report: ReportModel, obj, spdx_version: Optional[str]) -> Optional[Any]: + from spdx_tools.spdx.writer.write_utils import ( + convert, + validate_and_deduplicate + ) + + # Set to latest supported if a version is not specified + if not spdx_version: + spdx_version = "2.3" + + document_obj = create_spdx_document(report=report, spdx_version=spdx_version) + document_obj = validate_and_deduplicate(document=document_obj, validate=True, drop_duplicates=True) + doc = None + + if document_obj: + doc = convert(document=document_obj, converter=None) + + return json.dumps(doc) if doc else None diff --git a/safety/scan/util.py b/safety/scan/util.py new file mode 100644 index 00000000..388b3f98 --- /dev/null +++ b/safety/scan/util.py @@ -0,0 +1,121 @@ +from enum import Enum +import logging +import os +from pathlib import Path +import subprocess +from typing import Optional, Tuple + +import typer + +from safety.scan.finder.handlers import FileHandler, PythonFileHandler, SafetyProjectFileHandler +from safety_schemas.models import Stage + +LOG = logging.getLogger(__name__) + +class Language(str, Enum): + python = "python" + javascript = "javascript" + safety_project = "safety_project" + + def handler(self) -> FileHandler: + if self is Language.python: + return PythonFileHandler() + if self is Language.safety_project: + return SafetyProjectFileHandler() + + return PythonFileHandler() + +class Output(Enum): + json = "json" + +class AuthenticationType(str, Enum): + token = "token" + api_key = "api_key" + none = "unauthenticated" + + def is_allowed_in(self, stage: Stage = Stage.development) -> bool: + if self is AuthenticationType.none: + return False + + if stage == Stage.development and self is AuthenticationType.api_key: + return False + + if (not stage == Stage.development) and self is AuthenticationType.token: + return False + + return True + + +class GIT: + ORIGIN_CMD: Tuple[str, ...] = ("remote", "get-url", "origin") + BRANCH_CMD: Tuple[str, ...] = ("symbolic-ref", "--short", "-q", "HEAD") + TAG_CMD: Tuple[str, ...] = ("describe", "--tags", "--exact-match") + DESCRIBE_CMD: Tuple[str, ...] = ("describe", '--match=""', '--always', + '--abbrev=40', '--dirty') + GIT_CHECK_CMD: Tuple[str, ...] = ("rev-parse", "--is-inside-work-tree") + + def __init__(self, root: Path = Path(".")) -> None: + self.git = ("git", "-C", root.resolve()) + + def __run__(self, cmd: Tuple[str, ...], env_var: Optional[str] = None) -> Optional[str]: + if env_var and os.environ.get(env_var): + return os.environ.get(env_var) + + try: + return subprocess.run(self.git + cmd, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL).stdout.decode('utf-8').strip() + except Exception as e: + LOG.exception(e) + + return None + + def origin(self) -> Optional[str]: + return self.__run__(self.ORIGIN_CMD, env_var="SAFETY_GIT_ORIGIN") + + def branch(self) -> Optional[str]: + return self.__run__(self.BRANCH_CMD, env_var="SAFETY_GIT_BRANCH") + + def tag(self) -> Optional[str]: + return self.__run__(self.TAG_CMD, env_var="SAFETY_GIT_TAG") + + def describe(self) -> Optional[str]: + return self.__run__(self.DESCRIBE_CMD) + + def dirty(self, raw_describe: str) -> bool: + if os.environ.get("SAFETY_GIT_DIRTY") in ["0", "1"]: + return bool(int(os.environ.get("SAFETY_GIT_DIRTY"))) + + return raw_describe.endswith('-dirty') + + def commit(self, raw_describe: str) -> Optional[str]: + if os.environ.get("SAFETY_GIT_COMMIT"): + return os.environ.get("SAFETY_GIT_COMMIT") + + try: + return raw_describe.split("-dirty")[0] + except Exception: + pass + + def is_git(self) -> bool: + result = self.__run__(self.GIT_CHECK_CMD) + + if result == "true": + return True + + return False + + def build_git_data(self): + from safety_schemas.models import GITModel + + if self.is_git(): + raw_describe = self.describe() + commit = None + dirty = None + if raw_describe: + commit = self.commit(raw_describe) + dirty = self.dirty(raw_describe) + return GITModel(branch=self.branch(), + tag=self.tag(), commit=commit, dirty=dirty, + origin=self.origin()) + + return None diff --git a/safety/scan/validators.py b/safety/scan/validators.py new file mode 100644 index 00000000..b9e3aa18 --- /dev/null +++ b/safety/scan/validators.py @@ -0,0 +1,134 @@ + +from pathlib import Path +from typing import Optional, Tuple +import typer +from safety.scan.main import save_project_info +from safety.scan.models import ScanExport, ScanOutput, UnverifiedProjectModel +from safety.scan.render import print_wait_project_verification, prompt_project_id, prompt_link_project + +from safety_schemas.models import AuthenticationType, ProjectModel, Stage + + +MISSING_SPDX_EXTENSION_MSG = "spdx extra is not installed, please install it with: pip install safety[spdx]" + + +def raise_if_not_spdx_extension_installed(): + try: + import spdx_tools.spdx + except Exception as e: + raise typer.BadParameter(MISSING_SPDX_EXTENSION_MSG) + + +def save_as_callback(save_as: Optional[Tuple[ScanExport, Path]]): + export_type, export_path = save_as if save_as else (None, None) + + if ScanExport.is_format(export_type, ScanExport.SPDX): + raise_if_not_spdx_extension_installed() + + return (export_type.value, export_path) if export_type and export_path else (export_type, export_path) + +def output_callback(output: ScanOutput): + + if ScanOutput.is_format(output, ScanExport.SPDX): + raise_if_not_spdx_extension_installed() + + return output.value + + +def fail_if_not_allowed_stage(ctx: typer.Context): + if ctx.resilient_parsing: + return + + stage = ctx.obj.auth.stage + auth_type: AuthenticationType = ctx.obj.auth.client.get_authentication_type() + + if not auth_type.is_allowed_in(stage): + raise typer.BadParameter(f"'{auth_type.value}' auth type isn't allowed with " \ + f"the '{stage}' stage.") + + +def save_verified_project(ctx, slug, name, project_path, url_path): + ctx.obj.project = ProjectModel( + id=slug, + name=name, + project_path=project_path, + url_path=url_path + ) + if ctx.obj.auth.stage is Stage.development: + save_project_info(project=ctx.obj.project, + project_path=project_path) + + +def check_project(console, ctx, session, + unverified_project: UnverifiedProjectModel, + stage, + git_origin, ask_project_id=False): + stage = ctx.obj.auth.stage + source = ctx.obj.telemetry.safety_source if ctx.obj.telemetry else None + data = {"scan_stage": stage, "safety_source": source} + + PRJ_SLUG_KEY = "project_slug" + PRJ_SLUG_SOURCE_KEY = "project_slug_source" + PRJ_GIT_ORIGIN_KEY = "git_origin" + + if git_origin: + data[PRJ_GIT_ORIGIN_KEY] = git_origin + + if unverified_project.id: + data[PRJ_SLUG_KEY] = unverified_project.id + data[PRJ_SLUG_SOURCE_KEY] = ".safety-project.ini" + elif not git_origin or ask_project_id: + # Set a project id for this scan (no spaces). If empty Safety will use: pyupio: + parent_root_name = None + if unverified_project.project_path.parent.name: + parent_root_name = unverified_project.project_path.parent.name + + unverified_project.id = prompt_project_id(console, stage, parent_root_name) + data[PRJ_SLUG_KEY] = unverified_project.id + data[PRJ_SLUG_SOURCE_KEY] = "user" + + status = print_wait_project_verification(console, data[PRJ_SLUG_KEY] if data.get(PRJ_SLUG_KEY, None) else "-", + (session.check_project, data), on_error_delay=1) + + return status + + +def verify_project(console, ctx, session, + unverified_project: UnverifiedProjectModel, + stage, + git_origin): + + verified_prj = False + + link_prj = True + + while not verified_prj: + result = check_project(console, ctx, session, unverified_project, stage, git_origin, ask_project_id=not link_prj) + + unverified_slug = result.get("slug") + + project = result.get("project", None) + user_confirm = result.get("user_confirm", False) + + if user_confirm: + if project and link_prj: + prj_name = project.get("name", None) + prj_admin_email = project.get("admin", None) + + link_prj = prompt_link_project(prj_name=prj_name, + prj_admin_email=prj_admin_email, + console=console) + + if not link_prj: + continue + + verified_prj = print_wait_project_verification( + console, unverified_slug, (session.project, + {"project_id": unverified_slug}), + on_error_delay=1) + + if verified_prj and isinstance(verified_prj, dict) and verified_prj.get("slug", None): + save_verified_project(ctx, verified_prj["slug"], verified_prj.get("name", None), + unverified_project.project_path, verified_prj.get("url", None)) + else: + verified_prj = False diff --git a/safety/templates/.DS_Store b/safety/templates/.DS_Store new file mode 100644 index 00000000..0f035963 Binary files /dev/null and b/safety/templates/.DS_Store differ diff --git a/safety/templates/index.html b/safety/templates/index.html index 930f853d..ab4dd4c4 100644 --- a/safety/templates/index.html +++ b/safety/templates/index.html @@ -367,14 +367,14 @@

Remediations suggested [
-

Use API Key: Running Safety using an API Key uses a more comprehensive commercial vulnerability database and adds other features such as remediation suggestions and enhanced vulnerability and package information. Learn more and get a free API Key

+

Use API Key: Running Safety using an API Key uses a more comprehensive commercial vulnerability database and adds other features such as remediation suggestions and enhanced vulnerability and package information. Learn more and get a free API Key

{% endif %}
-

Safety Scanner and vulnerability data proudly maintained by PyUp Cybersecurity

+

Safety Scanner and vulnerability data proudly maintained by Safety Cybersecurity

diff --git a/safety/templates/scan/index.html b/safety/templates/scan/index.html new file mode 100644 index 00000000..3261979c --- /dev/null +++ b/safety/templates/scan/index.html @@ -0,0 +1,396 @@ + + + + + Safety Check Report + + +
+ + {% if announcements|length > 0 %} +
+
+
+
+

Announcements

+
    + {% for announcement in announcements %} + {% set color = "#DC3545" if announcement.type == "error" else "#8B4000" if + announcement.type == "warning" else "#6C757D" %} +
  • {{ announcement.message }}
    • + {% endfor %} +
+
+
+
+
+ {% endif %} + +

Safety Scan Report

+ +
+
+
+
+

Scan Summary

+
+
+
+
+

Packages Found (details ↓)

+
{{ summary.scanned_packages }}
+
+
+
+
+
+
+

Vulnerabilities Reported (details ↓)

+
{{ summary.vulnerabilities }}
+ {% if summary.ignored_vulnerabilities > 0 %} +

Found vulnerabilities that were ignored: {{ summary.ignored_vulnerabilities }}

+ {% endif %} +
+
+
+ {% if summary.remediations_recommended > 0 %} +
+
+
+

Remediations Suggested (details ↓)

+
{{ summary.remediations_recommended }}
+
+
+
+ {% endif %} +
+ +
+
+
Meta-data
+
+
+

Time: {{report.metadata.timestamp}}

+

Safety version: {{report.metadata.telemetry.safety_version}}

+

+ {% if report.metadata.authenticated %} + {{report.metadata.authentication_type|title}} authentication using the Safety's proprietary vulnerability database + {% else %} + No authenticated using the Safety's free vulnerability database + {% endif %} +

+

+ Configuration file: + {% if project and project.policy %} + {{ project.policy.id }} (source: {{project.policy.source.value|title}}) + {% else %} + None + {% endif%} +

+ {% if settings.audit_and_monitor %} +

Audit and monitor: Enabled. Logging scan results to Safety Platform →

+ {% endif %} +
+
+

Scan ecosystems:

+
    + {% for ecosystem, file_types in settings.ecosystems %} +
  • {{ ecosystem }}: {{file_types | join (', ')}}
    • + {% endfor %} +
+

Scan paths:

+
    + {% for location in report.metadata.scan_locations %} +
  • {{location}}
    • + {% endfor %} +
+ {% if project and project.git %} +

Scan git context

+

  origin: {{ project.git.origin }}

+

  branch: {{ project.git.branch }}

+ {% endif %} +
+
+
+
+ +
+
+
+
+ + {% for file in project.files %} +
+

{{ file.file_type.human_name() }}: {{ file.location }}

+
+
+
+

Scanned Packages [ # ]

+
+ + + + + + + + + {% for dependency in file.results.dependencies %} + + + + + {% endfor %} + +
Package nameFound requirements
{{dependency.name}} +
    + {% for spec in dependency.specifications %} +
  • {{ spec }}
    • + {% endfor %} +
+
+
+
+
+ +
+
+

Vulnerabilities Reported [ # ]

+ {% if vulns_per_file[file.location] > 0 %} +
+ + + + + + + + + + + + + + + + {% for dependency in file.results.dependencies %} + {% for spec in dependency.specifications %} + {% for vulnerability in spec.vulnerabilities %} + + + + + + + + + + + + {% endfor %} + {% endfor %} + {% endfor %} + +
Vulnerability IDPackage nameAnalyzed requirementVulnerable specAll vulnerable specsAdvisoryPublished dateCVEseverity
{{ vulnerability.vulnerability_id }}{{vulnerability.package_name}}{{vulnerability.analyzed_requirement}}{{vulnerability.vulnerable_spec}}{{vulnerability.all_vulnerable_specs}}{{vulnerability.advisory}}{{vulnerability.published_date}}{% if vulnerability.CVE %}{{vulnerability.CVE.name}}{% else %}No CVE{% endif %} + {% if not report.metadata.authenticated and not vulnerability.severity %} + Use a Safety account (?) + {% else %} + {{vulnerability.severity}} + {% endif %} +
+
+ {% else %} +

+ No known security vulnerabilities were found. + {% if not report.metadata.authenticated %} + Vulnerabilities may be missing. For comprehensive vulnerability scanning, use a Safety account + {% endif %} +

+ {% endif %} +
+
+ + {% if file.results.ignored_vulns_data|length > 0 %} +
+
+

Vulnerabilities ignored [ # ]

+
+ + + + + + + + + + + + {% for vuln_id, data in file.results.ignored_vulns_data.items() %} + + + + + + + + {% endfor %} + +
Vulnerability IDPackage nameVersion/SpecNotesExpires
{{ data.vulnerability_id }}{{ data.package_name }} + {% if data.analyzed_version %} + {{ data.analyzed_version }} + {% else %} + {{ data.analyzed_requirement }} + {% endif %} + {{data.ignored_reason|default("-", true)}}{{data.ignored_expires|default("-", true)}}
+
+
+
+ {% endif %} + + {% if vulns_per_file[file.location] > 0 %} +
+
+

Affected Packages [ # ]

+
+ + + + + + + + + + + + + {% for affected_dep in file.results.get_affected_dependencies() %} + + + + + + + + + {% endfor %} + +
Package nameVersion/RequirementsLocationInsecure versionsLatest version without known vulnerabilitiesMore info
{{affected_dep.name}} + {% if affected_dep.version %} + {{ affected_dep.version }} + {% else %} +
    + {% for spec in affected_dep.specifications %} +
  • {{ spec }}
    • + {% endfor %} +
+ {% endif %} + 		+
    + {% for spec in affected_dep.specifications %} +
  • {{ spec.found }}
    • + {% endfor %} +
+ 		{{affected_dep.insecure_versions}}{{affected_dep.latest_version_without_known_vulnerabilities}} + More Info +
+
+
+
+ {% endif %} + + {% if vulns_per_file[file.location] > 0 %} +
+
+

Remediations suggested [ # ]

+ + {% if remed_per_file[file.location] > 0 %} +
+ + + + + + + + + + + + + + {% for affected_spec in file.results.get_affected_specifications() %} + {% with remediation = affected_spec.remediation %} + + + + + + + + + {% endwith %} + {% endfor %} + +
Package nameVersion/RequirementVulnerabilities reportedRecommended versionsOther recommended versionsMore info
+ {{ affected_spec.name }} + + {{ affected_spec.specifier }} + + {{ remediation.vulnerabilities_found }} + + {% if not report.metadata.authenticated and not remediation.recommended %} + Use an account or API key (?) + {% else %} + {{ remediation.recommended }} + {% endif %} + + {% if not report.metadata.authenticated and remediation.other_recommended|length==0 %} + Use an account or API key (?) + {% else %} + {{ remediation.other_recommended }} + {% endif %} + + {% if remediation.more_info_url %} + {{ remediation.more_info_url }} + {% else %} + Use an account or API key (?) + {% endif %} +
+
+ {% else %} +

Use an account or API key to get remediation recommendations (?)

+ {% endif %} +
+
+ {% endif %} + +
+
+ + + {% endfor %} + + {% if not report.metadata.authenticated %} +
+
+
+

Use an account or API Key: Running Safety using an account or API Key uses a more comprehensive commercial vulnerability database and adds other features such as remediation suggestions and enhanced vulnerability and package information. Learn more and get a free account or API Key

+
+
+
+ {% endif %} + +
+

Safety Scanner and vulnerability data proudly maintained by Safety Cybersecurity

+
+ +
+   +
+ +
+ + \ No newline at end of file diff --git a/safety/util.py b/safety/util.py index c5d6cec4..b720b044 100644 --- a/safety/util.py +++ b/safety/util.py @@ -1,4 +1,3 @@ -import itertools import logging import os import platform @@ -8,7 +7,7 @@ from datetime import datetime from difflib import SequenceMatcher from threading import Lock -from typing import List, Dict, Optional +from typing import List, Optional import click from click import BadParameter @@ -16,12 +15,14 @@ from packaging.utils import canonicalize_name from packaging.version import parse as parse_version from packaging.specifiers import SpecifierSet +from requests import PreparedRequest from ruamel.yaml import YAML from ruamel.yaml.error import MarkedYAMLError -from safety.constants import EXIT_CODE_FAILURE, EXIT_CODE_OK, HASH_REGEX_GROUPS +from safety.constants import EXIT_CODE_FAILURE, EXIT_CODE_OK, HASH_REGEX_GROUPS, SYSTEM_CONFIG_DIR, USER_CONFIG_DIR from safety.errors import InvalidProvidedReportError from safety.models import Package, RequirementFile, is_pinned_requirement, SafetyRequirement +from safety_schemas.models import TelemetryModel LOG = logging.getLogger(__name__) @@ -186,7 +187,9 @@ def filter_announcements(announcements, by_type='error'): announcement.get('type', '').lower() == by_type] -def build_telemetry_data(telemetry=True): +def build_telemetry_data(telemetry = True, + command: Optional[str] = None, + subcommand: Optional[str] = None) -> TelemetryModel: context = SafetyContext() body = { @@ -194,16 +197,19 @@ def build_telemetry_data(telemetry=True): 'os_release': os.environ.get("SAFETY_OS_RELEASE", None) or platform.release(), 'os_description': os.environ.get("SAFETY_OS_DESCRIPTION", None) or platform.platform(), 'python_version': platform.python_version(), - 'safety_command': context.command, + 'safety_command': command if command else context.command, 'safety_options': get_used_options() } if telemetry else {} body['safety_version'] = get_safety_version() body['safety_source'] = os.environ.get("SAFETY_SOURCE", None) or context.safety_source + if not 'safety_options' in body: + body['safety_options'] = {} + LOG.debug(f'Telemetry body built: {body}') - return body + return TelemetryModel(**body) def build_git_data(): @@ -257,6 +263,19 @@ def output_exception(exception, exit_code_output=True): sys.exit(exit_code) +def build_remediation_info_url(base_url: str, version: Optional[str], spec: str, + target_version: Optional[str] = ''): + + params = {'from': version, 'to': target_version} + + # No pinned version + if not version: + params = {'spec': spec} + + req = PreparedRequest() + req.prepare_url(base_url, params) + + return req.url def get_processed_options(policy_file, ignore, ignore_severity_rules, exit_code, ignore_unpinned_requirements=None, project=None): @@ -350,8 +369,7 @@ def __init__(self, *args, **kwargs): if self.required_options: ex_str = ', '.join(self.required_options) kwargs['help'] = help + ( - ' NOTE: This argument requires the following flags ' - ' [' + ex_str + '].' + f" Requires: [ {ex_str} ]" ) super(DependentOption, self).__init__(*args, **kwargs) @@ -379,7 +397,7 @@ def handle_parse_result(self, ctx, opts, args): def transform_ignore(ctx, param, value): ignored_default_dict = {'reason': '', 'expires': None} - if isinstance(value, tuple): + if isinstance(value, tuple) and any(value): # Following code is required to support the 2 ways of providing 'ignore' # --ignore=1234,567,789 # or, the historical way (supported for backward compatibility) @@ -435,6 +453,14 @@ def get_terminal_size(): return os.terminal_size((columns, lines)) +def clean_project_id(input_string): + input_string = re.sub(r'[^a-zA-Z0-9]+', '-', input_string) + input_string = input_string.strip('-') + input_string = input_string.lower() + + return input_string + + def validate_expiration_date(expiration_date): d = None @@ -528,8 +554,17 @@ def convert(self, value, param, ctx): self.fail(msg.format(name=value, hint=hint), param, ctx) if not safety_policy or not isinstance(safety_policy, dict) or not safety_policy.get('security', None): + hint = "you are missing the security root tag" + try: + version = safety_policy["version"] + if version: + hint = f"{filename} is a policy file version {version}. " \ + "Legacy policy file parser only accepts versions minor than 3.0" \ + "\nNote: `safety check` command accepts policy file versions <= 2.0. Versions >= 2.0 are not supported." + except Exception: + pass self.fail( - msg.format(hint='you are missing the security root tag'), param, ctx) + msg.format(hint=hint), param, ctx) security_config = safety_policy.get('security', {}) security_keys = ['ignore-cvss-severity-below', 'ignore-cvss-unknown-severity', 'ignore-vulnerabilities', @@ -625,7 +660,8 @@ def convert(self, value, param, ctx): except Exception as e: # Don't fail in the default case if ctx and isinstance(e, OSError): - source = ctx.get_parameter_source("policy_file") + default = ctx.get_parameter_source + source = default("policy_file") if default("policy_file") else default("policy_file_path") if e.errno == 2 and source == click.core.ParameterSource.DEFAULT and value == '.safety-policy.yml': return None @@ -677,29 +713,40 @@ class SafetyContext(metaclass=SingletonMeta): files = None stdin = None is_env_scan = None - command = None + command: Optional[str] = None + subcommand: Optional[str] = None review = None params = {} safety_source = 'code' local_announcements = [] scanned_full_path = [] + account = None + def sync_safety_context(f): def new_func(*args, **kwargs): ctx = SafetyContext() + legacy_key_added = False + if "session" in kwargs: + legacy_key_added = True + kwargs["key"] = kwargs.get("session").api_key + for attr in dir(ctx): if attr in kwargs: setattr(ctx, attr, kwargs.get(attr)) + if legacy_key_added: + kwargs.pop("key") + return f(*args, **kwargs) return new_func @sync_safety_context -def get_packages_licenses(packages=None, licenses_db=None): +def get_packages_licenses(*, packages=None, licenses_db=None): """Get the licenses for the specified packages based on their version. :param packages: packages list @@ -779,3 +826,34 @@ def get_hashes(dependency): return [{'method': method, 'hash': hsh} for method, hsh in (pattern.match(d_hash).groups() for d_hash in dependency.hashes)] + + +def pluralize(word: str, count: int = 0) -> str: + if count == 1: + return word + + default = {"was": "were", "this": "these", "has": "have"} + + if word in default: + return default[word] + + if word.endswith("s") or word.endswith("x") or word.endswith("z") \ + or word.endswith("ch") or word.endswith("sh"): + return word + "es" + + if word.endswith("y"): + if word[-2] in "aeiou": + return word + "s" + else: + return word[:-1] + "ies" + + return word + "s" + + +def initializate_config_dirs(): + USER_CONFIG_DIR.mkdir(parents=True, exist_ok=True) + + try: + SYSTEM_CONFIG_DIR.mkdir(parents=True, exist_ok=True) + except Exception as e: + pass \ No newline at end of file diff --git a/safety_ci.png b/safety_ci.png deleted file mode 100644 index ae03bbc2..00000000 Binary files a/safety_ci.png and /dev/null differ diff --git a/setup.cfg b/setup.cfg index 2dcfddb9..6d40b73e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,8 +4,8 @@ version = file: safety/VERSION description = Checks installed dependencies for known vulnerabilities and licenses. long_description = file: README.md, CHANGELOG.md, LICENSE long_description_content_type = text/markdown -author = pyup.io -author_email = support@pyup.io +author = safetycli.com +author_email = support@safetycli.com url = https://github.com/pyupio/safety project_urls = Bug Tracker = https://github.com/pyupio/safety/issues @@ -32,33 +32,36 @@ classifiers = [options] zip_safe = False include_package_data = True -packages = safety, safety.formatters,safety.formatters.schemas, safety.alerts +packages = safety, safety.formatters,safety.formatters.schemas, safety.alerts, safety.auth, safety.scan, safety.scan.finder, safety.scan.ecosystems, safety.scan.ecosystems.python, safety.alerts.templates, safety.templates +python_requires = >=3.7 package_dir = safety = safety install_requires = - setuptools>=65.5.1; python_version>="3.7" - setuptools; python_version=="3.6" + setuptools>=65.5.1 Click>=8.0.2 urllib3>=1.26.5 requests packaging>=21.0 - dparse>=0.6.2 + dparse>=0.6.4b0 ruamel.yaml>=0.17.21 - dataclasses==0.8; python_version=="3.6" - jinja2; python_version=="3.6" - jinja2>=3.1.0; python_version>="3.7" - marshmallow; python_version=="3.6" - marshmallow>=3.15.0; python_version>="3.7" + jinja2>=3.1.0 + marshmallow>=3.15.0 + Authlib==1.2.0 + jwt==1.3.1 + rich + typer + pydantic>=1.10.12,<2.0 + safety_schemas>=0.0.1 + typing-extensions>=4.7.1 [options.entry_points] console_scripts = safety = safety.cli:cli -python_requires = >=3.6 - [options.extras_require] github = pygithub>=1.43.3 gitlab = python-gitlab>=1.3.0 +spdx = spdx-tools>=0.8.2 [flake8] exclude = docs diff --git a/test_requirements.txt b/test_requirements.txt index fd797301..0c9cad6f 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -12,4 +12,11 @@ dataclasses==0.8; python_version=="3.6" jinja2; python_version=="3.6" jinja2>=3.1.0; python_version>="3.7" marshmallow; python_version=="3.6" -marshmallow>=3.15.0; python_version>="3.7" \ No newline at end of file +marshmallow>=3.15.0; python_version>="3.7" +Authlib==1.2.0 +jwt==1.3.1 +rich +typer +pydantic>=1.10.12,<2.0 +safety_schemas>=0.0.1 +typing-extensions>=4.7.1 diff --git a/tests/auth/test_assets/config.ini b/tests/auth/test_assets/config.ini new file mode 100644 index 00000000..b7801c77 --- /dev/null +++ b/tests/auth/test_assets/config.ini @@ -0,0 +1,3 @@ +[organization] +id = "org_id23423ds" +name = "Safety CLI Org" \ No newline at end of file diff --git a/tests/auth/test_assets/config_empty.ini b/tests/auth/test_assets/config_empty.ini new file mode 100644 index 00000000..b7801c77 --- /dev/null +++ b/tests/auth/test_assets/config_empty.ini @@ -0,0 +1,3 @@ +[organization] +id = "org_id23423ds" +name = "Safety CLI Org" \ No newline at end of file diff --git a/tests/auth/test_assets/config_no_id.ini b/tests/auth/test_assets/config_no_id.ini new file mode 100644 index 00000000..8d15637c --- /dev/null +++ b/tests/auth/test_assets/config_no_id.ini @@ -0,0 +1,2 @@ +[organization] +name = "Safety CLI Org" \ No newline at end of file diff --git a/tests/auth/test_cli.py b/tests/auth/test_cli.py new file mode 100644 index 00000000..b968cfe7 --- /dev/null +++ b/tests/auth/test_cli.py @@ -0,0 +1,49 @@ + +from unittest.mock import Mock, PropertyMock, patch, ANY +import click +from click.testing import CliRunner +import unittest + +from safety.cli import cli +from safety.cli_util import get_command_for + + +class TestSafetyAuthCLI(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + self.runner = CliRunner(mix_stderr=False) + + @patch("safety.auth.cli.fail_if_authenticated") + @patch("safety.auth.cli.get_authorization_data") + @patch("safety.auth.cli.process_browser_callback") + def test_auth_calls_login(self, process_browser_callback, + get_authorization_data, fail_if_authenticated): + auth_data = "https://safetycli.com", "initialState" + get_authorization_data.return_value = auth_data + process_browser_callback.return_value = {"email": "user@safetycli.com", "name": "Safety User"} + result = self.runner.invoke(cli, ['auth']) + + fail_if_authenticated.assert_called_once() + get_authorization_data.assert_called_once() + process_browser_callback.assert_called_once_with(auth_data[0], + initial_state=auth_data[1], + ctx=ANY) + + expected = [ + "", + "Redirecting your browser to log in; once authenticated, return here to start using Safety", + "", + "You're authenticated", + " Account: Safety User, user@safetycli.com (email verification required)", + "", + "To complete your account open the “verify your email” email sent to", + "user@safetycli.com", + "", + "Can’t find the verification email? Login at", + "`https://platform.safetycli.com/login/` to resend the verification email", + "" + ] + + for res_line, exp_line in zip(result.stdout.splitlines(), expected): + self.assertIn(exp_line, res_line) diff --git a/tests/auth/test_main.py b/tests/auth/test_main.py new file mode 100644 index 00000000..f634abc6 --- /dev/null +++ b/tests/auth/test_main.py @@ -0,0 +1,70 @@ +from pathlib import Path +import unittest +from unittest.mock import Mock, patch +from safety.auth.constants import CLI_AUTH, CLI_AUTH_LOGOUT, CLI_CALLBACK + +from safety.auth import main +from safety.auth.main import get_authorization_data, \ + get_logout_url, get_organization, get_redirect_url + + + +class TestMain(unittest.TestCase): + + def setUp(self): + self.assets = Path(__file__).parent / Path("test_assets/") + + def tearDown(self): + pass + + def test_get_authorization_data(self): + org_id = "org_id3dasdasd" + client = Mock() + code_verifier = "test_code_verifier" + organization = Mock(id=org_id) + + client.create_authorization_url = Mock() + _ = get_authorization_data(client, code_verifier, organization) + + kwargs = { + "sign_up": False, + "locale": "en", + "ensure_auth": False, + "organization": org_id + } + + client.create_authorization_url.assert_called_once_with( + CLI_AUTH, code_verifier=code_verifier, **kwargs) + + client.create_authorization_url = Mock() + _ = get_authorization_data(client, code_verifier, organization=None) + + kwargs = { + "sign_up": False, + "locale": "en", + "ensure_auth":False + } + + client.create_authorization_url.assert_called_once_with( + CLI_AUTH, code_verifier=code_verifier, **kwargs) + + def get_logout_url(self, id_token): + return f'{CLI_AUTH_LOGOUT}?id_token={id_token}' + + def test_get_logout_url(self): + id_token = "test_id_token" + result = get_logout_url(id_token) + expected_result = f'{CLI_AUTH_LOGOUT}?id_token={id_token}' + self.assertEqual(result, expected_result) + + def test_get_redirect_url(self): + self.assertEqual(get_redirect_url(), CLI_CALLBACK) + + def test_get_organization(self): + with patch.object(main, "CONFIG", + (self.assets / Path("config.ini")).absolute()): + result = get_organization() + self.assertIsNotNone(result) + self.assertEqual(result.id, "org_id23423ds") + self.assertEqual(result.name, "Safety CLI Org") + \ No newline at end of file diff --git a/tests/formatters/__init__.py b/tests/formatters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/scan/test_command.py b/tests/scan/test_command.py new file mode 100644 index 00000000..48df61c8 --- /dev/null +++ b/tests/scan/test_command.py @@ -0,0 +1,25 @@ +import os +import unittest +from unittest.mock import patch, Mock +from click.testing import CliRunner +from safety.cli import cli + +from safety.scan.command import scan +from safety.scan.command import scan_project_app + +class TestScanCommand(unittest.TestCase): + + def setUp(self): + self.runner = CliRunner(mix_stderr=False) + self.dirname = os.path.dirname(__file__) + + def test_scan(self): + result = self.runner.invoke(cli, ["--stage", "cicd", "scan", "--target", self.dirname, "--output", "json"]) + self.assertEqual(result.exit_code, 1) + + result = self.runner.invoke(cli, ["--stage", "production", "scan", "--target", self.dirname, "--output", "json"]) + self.assertEqual(result.exit_code, 1) + + result = self.runner.invoke(cli, ["--stage", "cicd", "scan", "--target", self.dirname, "--output", "screen"]) + self.assertEqual(result.exit_code, 1) + diff --git a/tests/scan/test_file_finder.py b/tests/scan/test_file_finder.py new file mode 100644 index 00000000..38658264 --- /dev/null +++ b/tests/scan/test_file_finder.py @@ -0,0 +1,67 @@ +import unittest +from unittest.mock import MagicMock, patch +from pathlib import Path +from typing import Set + +from safety.scan.finder.file_finder import FileFinder, should_exclude + +class TestShouldExclude(unittest.TestCase): + @patch.object(Path, 'resolve') + @patch.object(Path, 'relative_to') + def test_should_exclude(self, mock_relative_to, mock_resolve): + mock_resolve.return_value = Path('/exclude/path') + + mock_relative_to.return_value = Path('/relative-path') + + excludes = {Path('/exclude'), Path('/file.py')} + self.assertTrue(should_exclude(excludes, Path('/exclude/path'))) + + # Target Path matches an exclude path + mock_relative_to.side_effect = ValueError + self.assertTrue(should_exclude(excludes, Path('/file.py'))) + + mock_relative_to.side_effect = ValueError + self.assertFalse(should_exclude(excludes, Path('/absolute/path'))) + + +class TestFileFinder(unittest.TestCase): + @patch.object(Path, 'glob') + @patch('os.walk') + def test_process_directory(self, mock_os_walk, mock_glob): + # Mock the os.walk function to return a fake directory structure + mock_os_walk.return_value = [ + ('/root', ['dir1', 'dir2'], ['file1', 'file2']), + ('/root/dir1', [], ['file3', 'file4']), + ('/root/dir2', [], ['file5', 'file6']), + ] + + # Mock the Path.glob method to simulate the exclusion patterns + mock_glob.return_value = [Path('/root/dir1')] + + file_finder = FileFinder(max_level=1, ecosystems=[], target=Path('/root'), console=None) + dir_path, files = file_finder.process_directory('/root') + + self.assertEqual(dir_path, '/root') + self.assertEqual(len(files), 0) # No files should be found as we didn't mock the handlers + + @patch.object(Path, 'glob') + @patch('os.walk') + def test_search(self, mock_os_walk, mock_glob): + # Mock the os.walk function to return a fake directory structure + mock_os_walk.return_value = [ + ('/root', ['dir1', 'dir2'], ['file1', 'file2']), + ('/root/dir1', [], ['file3', 'file4']), + ('/root/dir2', [], ['file5', 'file6']), + ] + + # Mock the Path.glob method to simulate the exclusion patterns + mock_glob.return_value = [Path('/root/dir1')] + + file_finder = FileFinder(max_level=1, ecosystems=[], target=Path('/root'), console=None) + dir_path, files = file_finder.search() + + self.assertEqual(dir_path, Path('/root')) + self.assertEqual(len(files), 0) # No files should be found as we didn't mock the handlers + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/scan/test_render.py b/tests/scan/test_render.py new file mode 100644 index 00000000..ad4547d5 --- /dev/null +++ b/tests/scan/test_render.py @@ -0,0 +1,84 @@ +import unittest +from unittest import mock +from unittest.mock import MagicMock, Mock, patch +from pathlib import Path +import datetime + +from safety.scan.render import print_announcements, print_ignore_details, render_header +from safety_schemas.models import ProjectModel, IgnoreCodes + +class TestRender(unittest.TestCase): + @patch('safety.scan.render.get_safety_version') + def test_render_header(self, mock_get_safety_version): + mock_get_safety_version.return_value = '3.0.0' + + datetime_mock = Mock(wraps=datetime.datetime) + datetime_mock.now.return_value = datetime.datetime(2025, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc) + + with patch('datetime.datetime', new=datetime_mock) as mock_now: + mock_now.return_value = datetime.datetime(2025, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc) + + # Project scan + targets = [Path('/target1'), Path('/target2')] + expected_result = 'Safety 3.0.0 scanning /target1, /target2\n2025-01-01 00:00:00 UTC' + self.assertEqual(str(render_header(targets, False)), expected_result) + + # System Scan + expected_result = 'Safety 3.0.0 running system scan\n2025-01-01 00:00:00 UTC' + self.assertEqual(str(render_header(targets, True)), expected_result) + + @patch('safety.scan.render.safety.get_announcements') + @patch('safety.scan.render.get_basic_announcements') + @patch('safety.scan.render.Console') + def test_print_announcements(self, mock_console, mock_get_basic_announcements, mock_get_announcements): + + mock_get_announcements.return_value = [ + {'type': 'info', 'message': 'Info message'}, + {'type': 'warning', 'message': 'Warning message'}, + {'type': 'error', 'message': 'Error message'}, + ] + + mock_get_basic_announcements.return_value = mock_get_announcements.return_value + console = mock_console.return_value + + ctx = MagicMock() + ctx.obj.auth.client = MagicMock() + ctx.obj.config.telemetry_enabled = False + ctx.obj.telemetry = MagicMock() + + print_announcements(console, ctx) + + console.print.assert_any_call() + console.print.assert_any_call("[bold]Safety Announcements:[/bold]") + console.print.assert_any_call() + console.print.assert_any_call("[default]* Info message[/default]") + console.print.assert_any_call("[yellow]* Warning message[/yellow]") + console.print.assert_any_call("[red]* Error message[/red]") + + + @patch('safety.scan.render.render_to_console') + def test_print_ignore_details(self, render_to_console_mocked): + render_to_console_mocked.return_value = "render_to_console_mocked" + from safety.console import main_console + console = MagicMock(wraps=main_console) + console.print = MagicMock() + + # Create a fake project + project = ProjectModel(id='prj-id') + + # Create a fake ignored vulnerabilities data + ignored_vulns_data = [ + MagicMock(ignored_code=IgnoreCodes.manual.value, vulnerability_id='v1', package_name='p1'), + MagicMock(ignored_code=IgnoreCodes.cvss_severity.value, vulnerability_id='v2', package_name='p2'), + MagicMock(ignored_code=IgnoreCodes.unpinned_specification.value, vulnerability_id='v3', package_name='p3'), + MagicMock(ignored_code=IgnoreCodes.environment_dependency.value, vulnerability_id='v4', package_name='p4'), + ] + + # Call the function + print_ignore_details(console, project, [], True, ignored_vulns_data) + + # Check that the console.print method was called with the expected arguments + console.print.assert_any_call("[number]1[/number] were manually ignored due to the project policy:") + console.print.assert_any_call("[number]1[/number] vulnerability was ignored because of their severity or exploitability impacted the following package: p2") + console.print.assert_any_call("[number]1[/number] vulnerability was ignored because they are inside an environment dependency.") + console.print.assert_any_call("[number]1[/number] vulnerability was ignored because this package has unpinned specs: p3") diff --git a/tests/test_cli.py b/tests/test_cli.py index cfe53ef7..2161cf73 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -69,15 +69,15 @@ def setUp(self): self.output_options = ['screen', 'text', 'json', 'bare'] self.dirname = os.path.dirname(__file__) - def test_command_line_interface(self): + def test_command_line_interface(self): runner = CliRunner() result = runner.invoke(cli.cli) - assert result.exit_code == 0 - assert 'Usage:' in result.output + expected = "Usage: cli [OPTIONS] COMMAND [ARGS]..." - help_result = runner.invoke(cli.cli, ['--help']) - assert help_result.exit_code == 0 - assert '--help' in help_result.output + for option in [[], ["--help"]]: + result = runner.invoke(cli.cli, option) + self.assertEqual(result.exit_code, 0) + self.assertIn(expected, click.unstyle(result.output)) @patch("safety.safety.check") def test_check_vulnerabilities_found_default(self, check_func): @@ -135,74 +135,6 @@ def test_announcements_if_is_not_tty(self, get_announcements_func): self.assertTrue('ANNOUNCEMENTS' in result.stderr) self.assertTrue(message in result.stderr) - @patch("safety.safety.get_announcements") - def test_review_pass(self, mocked_announcements): - mocked_announcements.return_value = [] - runner = CliRunner() - dirname = os.path.dirname(__file__) - path_to_report = os.path.join(dirname, "test_db", "report.json") - result = runner.invoke(cli.cli, ['review', '--output', 'bare', '--file', path_to_report]) - self.assertEqual(result.exit_code, 0) - self.assertEqual(result.output, u'insecure-package\n') - - @patch("safety.util.SafetyContext") - @patch("safety.safety.check") - @patch("safety.cli.get_packages") - def test_chained_review_pass(self, get_packages, check_func, ctx): - expires = datetime.strptime('2022-10-21', '%Y-%m-%d') - vulns = [get_vulnerability(), get_vulnerability(vuln_kwargs={'vulnerability_id': '25853', 'ignored': True, - 'ignored_reason': 'A basic reason', - 'ignored_expires': expires})] - packages = [pkg for pkg in {vuln.pkg.name: vuln.pkg for vuln in vulns}.values()] - get_packages.return_value = packages - provided_context = SafetyContext() - provided_context.command = 'check' - provided_context.packages = packages - ctx.return_value = provided_context - check_func.return_value = vulns, None - - with tempfile.TemporaryDirectory() as tempdir: - for output in self.output_options: - path_to_report = os.path.join(tempdir, f'report_{output}.json') - - pre_result = self.runner.invoke(cli.cli, [ - 'check', '--key', 'foo', '-o', output, - '--save-json', path_to_report]) - - self.assertEqual(pre_result.exit_code, 64) - - for output in self.output_options: - filename = f'report_{output}.json' - path_to_report = os.path.join(tempdir, filename) - result = self.runner.invoke(cli.cli, ['review', '--output', output, '--file', path_to_report]) - self.assertEqual(result.exit_code, 0, f'Unable to load the previous saved report: {filename}') - - @patch("safety.safety.session") - def test_license_with_file(self, requests_session): - licenses_db = { - "licenses": { - "BSD-3-Clause": 2 - }, - "packages": { - "django": [ - { - "start_version": "0.0", - "license_id": 2 - } - ] - } - } - - mock = Mock() - mock.json.return_value = licenses_db - mock.status_code = 200 - requests_session.get.return_value = mock - - dirname = os.path.dirname(__file__) - test_filename = os.path.join(dirname, "reqs_4.txt") - result = self.runner.invoke(cli.cli, ['license', '--key', 'foo', '--file', test_filename]) - # TODO: Add test for the screen formatter, this only test that the command doesn't crash - self.assertEqual(result.exit_code, 0) @patch("safety.safety.check") def test_check_ignore_format_backward_compatible(self, check): @@ -241,9 +173,9 @@ def test_validate_with_wrong_path(self): def test_validate_with_basic_policy_file(self): dirname = os.path.dirname(__file__) path = os.path.join(dirname, "test_policy_file", "default_policy_file.yml") - result = self.runner.invoke(cli.cli, ['validate', 'policy_file', '--path', path]) + result = self.runner.invoke(cli.cli, ['validate', 'policy_file', '2.0', '--path', path]) cleaned_stdout = click.unstyle(result.stdout) - msg = 'The Safety policy file was successfully parsed with the following values:\n' + msg = 'The Safety policy file (Valid only for the check command) was successfully parsed with the following values:\n' parsed = json.dumps( { "project-id": '', @@ -260,17 +192,77 @@ def test_validate_with_basic_policy_file(self): }, "filename": path }, - indent=4 + indent=2 ) + '\n' self.assertEqual(msg + parsed, cleaned_stdout) self.assertEqual(result.exit_code, 0) + path = os.path.join(dirname, "test_policy_file", "v3_0", "default_policy_file.yml") + result = self.runner.invoke(cli.cli, ['validate', 'policy_file', '3.0', '--path', path]) + cleaned_stdout = click.unstyle(result.stdout) + msg = 'The Safety policy (3.0) file (Used for scan and system-scan commands) was successfully parsed with the following values:\n' + parsed = json.dumps( + { + "version": "3.0", + "scan": { + "max_depth": 6, + "exclude": [], + "include_files": [], + "system": { + "targets": [] + } + }, + "report": { + "dependency_vulnerabilities": { + "enabled": True, + "auto_ignore": { + "python": { + "ignore_environment_results": True, + "ignore_unpinned_requirements": True + }, + "vulnerabilities": None, + "cvss_severity": [] + } + } + }, + "fail_scan": { + "dependency_vulnerabilities": { + "enabled": True, + "fail_on_any_of": { + "cvss_severity": [ + "critical", + "high", + "medium" + ], + "exploitability": [ + "critical", + "high", + "medium" + ] + } + } + }, + "security_updates": { + "dependency_vulnerabilities": { + "auto_security_updates_limit": [ + "patch" + ] + } + } + }, + indent=2 + ) + '\n' + + self.assertEqual(msg + parsed, cleaned_stdout) + self.assertEqual(result.exit_code, 0) + + def test_validate_with_policy_file_using_invalid_keyword(self): dirname = os.path.dirname(__file__) filename = 'default_policy_file_using_invalid_keyword.yml' path = os.path.join(dirname, "test_policy_file", filename) - result = self.runner.invoke(cli.cli, ['validate', 'policy_file', '--path', path]) + result = self.runner.invoke(cli.cli, ['validate', 'policy_file', '2.0', '--path', path]) cleaned_stdout = click.unstyle(result.stderr) msg_hint = 'HINT: "security" -> "transitive" is not a valid keyword. Valid keywords in this level are: ' \ 'ignore-cvss-severity-below, ignore-cvss-unknown-severity, ignore-vulnerabilities, ' \ @@ -280,11 +272,22 @@ def test_validate_with_policy_file_using_invalid_keyword(self): self.assertEqual(msg, cleaned_stdout) self.assertEqual(result.exit_code, 1) + path = os.path.join(dirname, "test_policy_file", "v3_0", filename) + result = self.runner.invoke(cli.cli, ['validate', 'policy_file', '3.0', '--path', path]) + cleaned_stdout = click.unstyle(result.stderr) + msg_hint = 'report -> dependency-vulnerabilities -> transitive\n' \ + ' extra fields not permitted (type=value_error.extra)\n' + msg = f'Unable to load the Safety Policy file ("{path}"), this command only supports version 3.0, details: 1 validation error for Config\n{msg_hint}' + + self.assertEqual(msg, cleaned_stdout) + self.assertEqual(result.exit_code, 1) + + def test_validate_with_policy_file_using_invalid_typo_keyword(self): dirname = os.path.dirname(__file__) filename = 'default_policy_file_using_invalid_typo_keyword.yml' path = os.path.join(dirname, "test_policy_file", filename) - result = self.runner.invoke(cli.cli, ['validate', 'policy_file', '--path', path]) + result = self.runner.invoke(cli.cli, ['validate', 'policy_file', '2.0', '--path', path]) cleaned_stdout = click.unstyle(result.stderr) msg_hint = 'HINT: "security" -> "ignore-vunerabilities" is not a valid keyword. Maybe you meant: ' \ 'ignore-vulnerabilities\n' @@ -318,7 +321,7 @@ def test_check_with_fix_does_verify_api_key(self): req_file = os.path.join(dirname, "test_fix", "basic", "reqs_simple.txt") result = self.runner.invoke(cli.cli, ['check', '-r', req_file, '--apply-security-updates']) self.assertEqual(click.unstyle(result.stderr), - "The --apply-security-updates option needs an API-KEY. See https://bit.ly/3OY2wEI.\n") + "The --apply-security-updates option needs authentication. See https://bit.ly/3OY2wEI.\n") self.assertEqual(result.exit_code, 65) def test_check_with_fix_only_works_with_files(self): @@ -477,4 +480,3 @@ def test_basic_html_output_pass(self): self.assertIn("remediations-suggested", result.stdout) self.assertIn("Use API Key", result.stdout) - diff --git a/tests/test_policy_file/v3_0/default_policy_file.yml b/tests/test_policy_file/v3_0/default_policy_file.yml new file mode 100644 index 00000000..86882b32 --- /dev/null +++ b/tests/test_policy_file/v3_0/default_policy_file.yml @@ -0,0 +1,37 @@ +version: '3.0' + +scanning-settings: + max-depth: 6 + exclude: [] + include-files: [] + system: + targets: [] + + +report: + dependency-vulnerabilities: + enabled: true + auto-ignore-in-report: + python: + environment-results: true + unpinned-requirements: true + cvss-severity: [] + + +fail-scan-with-exit-code: + dependency-vulnerabilities: + enabled: true + fail-on-any-of: + cvss-severity: + - critical + - high + - medium + exploitability: + - critical + - high + - medium + +security-updates: + dependency-vulnerabilities: + auto-security-updates-limit: + - patch diff --git a/tests/test_policy_file/v3_0/default_policy_file_using_invalid_keyword.yml b/tests/test_policy_file/v3_0/default_policy_file_using_invalid_keyword.yml new file mode 100644 index 00000000..b1f556c2 --- /dev/null +++ b/tests/test_policy_file/v3_0/default_policy_file_using_invalid_keyword.yml @@ -0,0 +1,38 @@ +version: '3.0' + +scanning-settings: + max-depth: 6 + exclude: [] + include-files: [] + system: + targets: [] + + +report: + dependency-vulnerabilities: + enabled: true + auto-ignore-in-report: + python: + environment-results: true + unpinned-requirements: true + cvss-severity: [] + transitive: True + + +fail-scan-with-exit-code: + dependency-vulnerabilities: + enabled: true + fail-on-any-of: + cvss-severity: + - critical + - high + - medium + exploitability: + - critical + - high + - medium + +security-updates: + dependency-vulnerabilities: + auto-security-updates-limit: + - patch diff --git a/tests/test_safety.py b/tests/test_safety.py index 52b7616d..086c5e81 100644 --- a/tests/test_safety.py +++ b/tests/test_safety.py @@ -22,13 +22,15 @@ from packaging.version import parse from packaging.specifiers import SpecifierSet from requests.exceptions import RequestException +from safety.auth import build_client_session +from safety.constants import DB_CACHE_FILE -from safety import util, safety -from safety.errors import MalformedDatabase +from safety.errors import DatabaseFetchError, DatabaseFileNotFoundError, MalformedDatabase, InvalidCredentialError, TooManyRequestsError from safety.formatter import SafetyFormatter from safety.models import CVE, Package, SafetyRequirement -from safety.safety import ignore_vuln_if_needed, get_closest_ver, precompute_remediations, compute_sec_ver, \ - calculate_remediations, read_vulnerabilities +from safety.safety import get_announcements, ignore_vuln_if_needed, get_closest_ver, precompute_remediations, compute_sec_ver, \ + calculate_remediations, read_vulnerabilities, check, get_licenses, review +from safety.util import get_packages_licenses, read_requirements from tests.resources import VALID_REPORT, VULNS, SCANNED_PACKAGES, REMEDIATIONS from tests.test_cli import get_vulnerability @@ -36,6 +38,7 @@ class TestSafety(unittest.TestCase): def setUp(self) -> None: + self.session, _ = build_client_session() self.maxDiff = None self.dirname = os.path.dirname(__file__) self.report = VALID_REPORT @@ -52,11 +55,11 @@ def setUp(self) -> None: def test_check_from_file(self): reqs = StringIO("Django==1.8.1") - packages = util.read_requirements(reqs) + packages = read_requirements(reqs) - vulns, _ = safety.check( + vulns, _ = check( + session=self.session, packages=packages, - key=None, db_mirror=os.path.join( os.path.dirname(os.path.realpath(__file__)), "test_db" @@ -71,14 +74,14 @@ def test_check_from_file(self): def test_check_ignores(self): reqs = StringIO("Django==1.8.1") - packages = util.read_requirements(reqs) + packages = read_requirements(reqs) # Second that ignore works ignored_vulns = {'some id': {'expires': None, 'reason': ''}} - vulns, _ = safety.check( + vulns, _ = check( + session=self.session, packages=packages, - key=None, db_mirror=os.path.join( os.path.dirname(os.path.realpath(__file__)), "test_db" @@ -95,11 +98,11 @@ def test_check_ignores(self): def test_check_from_file_with_hash_pins(self): reqs = StringIO(("Django==1.8.1 " "--hash=sha256:c6c7e7a961e2847d050d214ca96dc3167bb5f2b25cd5c6cb2eea96e1717f4ade")) - packages = util.read_requirements(reqs) + packages = read_requirements(reqs) - vulns, _ = safety.check( + vulns, _ = check( + session=self.session, packages=packages, - key=None, db_mirror=os.path.join( os.path.dirname(os.path.realpath(__file__)), "test_db" @@ -113,13 +116,13 @@ def test_check_from_file_with_hash_pins(self): self.assertEqual(len(vulns), 2) def test_multiple_versions(self): - # Probably used for external tools using safety.check directly + # Probably used for external tools using check directly reqs = StringIO("Django==1.8.1\n\rDjango==1.7.0") - packages = util.read_requirements(reqs) + packages = read_requirements(reqs) - vulns, _ = safety.check( + vulns, _ = check( + session=self.session, packages=packages, - key=None, db_mirror=os.path.join( os.path.dirname(os.path.realpath(__file__)), "test_db" @@ -134,11 +137,11 @@ def test_multiple_versions(self): def test_check_live(self): reqs = StringIO("insecure-package==0.1") - packages = util.read_requirements(reqs) + packages = read_requirements(reqs) - vulns, _ = safety.check( + vulns, _ = check( + session=self.session, packages=packages, - key=None, db_mirror=False, cached=0, ignore_vulns={}, @@ -150,21 +153,21 @@ def test_check_live(self): self.assertEqual(len(vulns), 1) def test_check_live_cached(self): - from safety.constants import CACHE_FILE + from safety.constants import DB_CACHE_FILE # lets clear the cache first try: - with open(CACHE_FILE, 'w') as f: + with open(DB_CACHE_FILE, 'w') as f: f.write(json.dumps({})) except Exception: pass reqs = StringIO("insecure-package==0.1") - packages = util.read_requirements(reqs) + packages = read_requirements(reqs) - vulns, _ = safety.check( + vulns, _ = check( + session=self.session, packages=packages, - key=None, db_mirror=False, cached=60 * 60, ignore_vulns={}, @@ -175,11 +178,11 @@ def test_check_live_cached(self): self.assertEqual(len(vulns), 1) reqs = StringIO("insecure-package==0.1") - packages = util.read_requirements(reqs) + packages = read_requirements(reqs) # make a second call to use the cache - vulns, _ = safety.check( + vulns, _ = check( + session=self.session, packages=packages, - key=None, db_mirror=False, cached=60 * 60, ignore_vulns={}, @@ -191,15 +194,15 @@ def test_check_live_cached(self): def test_get_packages_licenses(self): reqs = StringIO("Django==1.8.1\n\rinvalid==1.0.0") - packages = util.read_requirements(reqs) - licenses_db = safety.get_licenses( + packages = read_requirements(reqs) + + licenses_db = get_licenses( + session=self.session, db_mirror=os.path.join( os.path.dirname(os.path.realpath(__file__)), "test_db" ), cached=0, - key="foobarqux", - proxy={}, telemetry=False ) self.assertIn("licenses", licenses_db) @@ -207,7 +210,7 @@ def test_get_packages_licenses(self): self.assertIn("BSD-3-Clause", licenses_db['licenses']) self.assertIn("django", licenses_db['packages']) - pkg_licenses = util.get_packages_licenses(packages, licenses_db) + pkg_licenses = get_packages_licenses(packages=packages, licenses_db=licenses_db) self.assertIsInstance(pkg_licenses, list) for pkg_license in pkg_licenses: @@ -225,89 +228,83 @@ def test_get_packages_licenses(self): ) def test_get_packages_licenses_without_api_key(self): - from safety.errors import InvalidKeyError - # without providing an API-KEY - with self.assertRaises(InvalidKeyError) as error: - safety.get_licenses( + with self.assertRaises(InvalidCredentialError) as error: + get_licenses( + session=self.session, db_mirror=False, cached=0, - proxy={}, - key=None, telemetry=False ) db_generic_exception = error.exception self.assertEqual(str(db_generic_exception), 'The API-KEY was not provided.') - @patch("safety.safety.session") - def test_get_packages_licenses_with_invalid_api_key(self, requests_session): - from safety.errors import InvalidKeyError + def test_get_packages_licenses_with_invalid_api_key(self): + session = Mock() + api_key = "INVALID" + session.api_key = api_key + session.headers = {'X-Api-Key': api_key} mock = Mock() mock.status_code = 403 - requests_session.get.return_value = mock + session.get.return_value = mock # proving an invalid API-KEY - with self.assertRaises(InvalidKeyError): - safety.get_licenses( + with self.assertRaises(InvalidCredentialError): + get_licenses( + session=session, db_mirror=False, cached=0, - proxy={}, - key="INVALID", telemetry=False ) - @patch("safety.safety.session") - def test_get_packages_licenses_db_fetch_error(self, requests_session): - from safety.errors import DatabaseFetchError - + def test_get_packages_licenses_db_fetch_error(self): + session = Mock() + api_key = "MY-VALID-KEY" + session.api_key = api_key + session.headers = {'X-Api-Key': api_key} mock = Mock() mock.status_code = 500 - requests_session.get.return_value = mock + session.get.return_value = mock with self.assertRaises(DatabaseFetchError): - safety.get_licenses( + get_licenses( + session=session, db_mirror=False, cached=0, - proxy={}, - key="MY-VALID-KEY", telemetry=False ) def test_get_packages_licenses_with_invalid_db_file(self): - from safety.errors import DatabaseFileNotFoundError with self.assertRaises(DatabaseFileNotFoundError): - safety.get_licenses( + get_licenses( + session=self.session, db_mirror='/my/invalid/path', cached=0, - proxy={}, - key=None, telemetry=False ) - @patch("safety.safety.session") - def test_get_packages_licenses_very_often(self, requests_session): - from safety.errors import TooManyRequestsError - + def test_get_packages_licenses_very_often(self): # if the request is made too often, an 429 error is raise by PyUp.io + session = Mock() + api_key = "MY-VALID-KEY" + session.api_key = api_key + session.headers = {'X-Api-Key': api_key} mock = Mock() mock.status_code = 429 - requests_session.get.return_value = mock + session.get.return_value = mock with self.assertRaises(TooManyRequestsError): - safety.get_licenses( + get_licenses( + session=session, db_mirror=False, cached=0, - proxy={}, - key="MY-VALID-KEY", telemetry=False ) - @patch("safety.safety.session") - def test_get_cached_packages_licenses(self, requests_session): + def test_get_cached_packages_licenses(self): import copy - from safety.constants import CACHE_FILE licenses_db = { "licenses": { @@ -324,24 +321,27 @@ def test_get_cached_packages_licenses(self, requests_session): } original_db = copy.deepcopy(licenses_db) + session = Mock() + api_key = "MY-VALID-KEY" + session.api_key = api_key + session.headers = {'X-Api-Key': api_key} mock = Mock() - mock.json.return_value = licenses_db mock.status_code = 200 - requests_session.get.return_value = mock + mock.json.return_value = licenses_db + session.get.return_value = mock # lets clear the cache first try: - with open(CACHE_FILE, 'w') as f: + with open(DB_CACHE_FILE, 'w') as f: f.write(json.dumps({})) except Exception: pass # In order to cache the db (and get), we must set cached as True - response = safety.get_licenses( + response = get_licenses( + session=session, db_mirror=False, cached=60 * 60, # Cached for one hour - proxy={}, - key="MY-VALID-KEY", telemetry=False ) self.assertEqual(response, licenses_db) @@ -350,11 +350,10 @@ def test_get_cached_packages_licenses(self, requests_session): # changing the "live" db to test if we are getting the cached db licenses_db['licenses']['BSD-3-Clause'] = 123 - resp = safety.get_licenses( + resp = get_licenses( + session=session, db_mirror=False, cached=60 * 60, # Cached for one hour - proxy={}, - key="MY-VALID-KEY", telemetry=False ) @@ -363,21 +362,20 @@ def test_get_cached_packages_licenses(self, requests_session): def test_report_licenses_bare(self): reqs = StringIO("Django==1.8.1\n\rinexistent==1.0.0") - packages = util.read_requirements(reqs) + packages = read_requirements(reqs) # Using DB: test.test_db.licenses.json - licenses_db = safety.get_licenses( + licenses_db = get_licenses( + session=self.session, db_mirror=os.path.join( os.path.dirname(os.path.realpath(__file__)), "test_db" ), cached=0, - key=None, - proxy={}, telemetry=False ) - pkgs_licenses = util.get_packages_licenses(packages, licenses_db) + pkgs_licenses = get_packages_licenses(packages=packages, licenses_db=licenses_db) output_report = SafetyFormatter(output='bare').render_licenses([], pkgs_licenses) self.assertEqual(output_report, "BSD-3-Clause unknown") @@ -392,21 +390,20 @@ def test_report_licenses_json(self, get_report_brief_info): 'safety_version': '2.0.0.dev6'} reqs = StringIO("Django==1.8.1\n\rinexistent==1.0.0") - packages = util.read_requirements(reqs) + packages = read_requirements(reqs) # Using DB: test.test_db.licenses.json - licenses_db = safety.get_licenses( + licenses_db = get_licenses( + session=self.session, db_mirror=os.path.join( os.path.dirname(os.path.realpath(__file__)), "test_db" ), cached=0, - key=None, - proxy={}, telemetry=False ) - pkgs_licenses = util.get_packages_licenses(packages, licenses_db) + pkgs_licenses = get_packages_licenses(packages=packages, licenses_db=licenses_db) output_report = SafetyFormatter(output='json').render_licenses([], pkgs_licenses) expected_result = json.dumps( @@ -438,17 +435,19 @@ def test_report_licenses_json(self, get_report_brief_info): self.assertEqual(output_report.rstrip(), expected_result) @patch("safety.util.get_used_options") - @patch("safety.safety.session") @patch.object(click, 'get_current_context', Mock(command=Mock(name=Mock(return_value='check')))) - def test_get_announcements_catch_request_exceptions(self, requests_session, get_used_options): + def test_get_announcements_catch_request_exceptions(self, get_used_options): get_used_options.return_value = {'key': {'--key': 1}, 'output': {'--output': 1}} - requests_session.get.side_effect = RequestException() - self.assertEqual(safety.get_announcements('somekey', {}), []) + session = Mock() + api_key = "somekey" + session.api_key = api_key + session.headers = {'X-Api-Key': api_key} + session.get.side_effect = RequestException() + self.assertEqual(get_announcements(session), []) @patch("safety.util.get_used_options") - @patch("safety.safety.session") @patch.object(click, 'get_current_context', Mock(command=Mock(name=Mock(return_value='check')))) - def test_get_announcements_catch_unhandled_http_codes(self, requests_session, get_used_options): + def test_get_announcements_catch_unhandled_http_codes(self, get_used_options): get_used_options.return_value = {'key': {'--key': 1}, 'output': {'--output': 1}} unhandled_status = [status for status in HTTPStatus if status != HTTPStatus.OK] @@ -456,23 +455,26 @@ def test_get_announcements_catch_unhandled_http_codes(self, requests_session, ge for http_status in unhandled_status: mock = Mock() mock.status_code = http_status.value - requests_session.get.return_value = mock + session = Mock() + api_key = "somekey" + session.api_key = api_key + session.headers = {'X-Api-Key': api_key} + session.get.return_value = mock - self.assertEqual(safety.get_announcements('somekey', {}), []) + self.assertEqual(get_announcements(session), []) @patch("safety.util.get_used_options") - @patch("safety.safety.session") @patch.object(click, 'get_current_context', Mock(command=Mock(name=Mock(return_value='check')))) - def test_get_announcements_http_ok(self, requests_session, get_used_options): + def test_get_announcements_http_ok(self, get_used_options): announcements = { "announcements": [{ "type": "notice", - "message": "You are using an outdated version of Safety. Please upgrade to Safety version 1.2.3" + "message": "You are using an outdated version of Please upgrade to Safety version 1.2.3" }, { "type": "error", - "message": "You are using an vulnerable version of Safety. Please upgrade now" + "message": "You are using an vulnerable version of Please upgrade now" }] } @@ -481,35 +483,47 @@ def test_get_announcements_http_ok(self, requests_session, get_used_options): mock = Mock() mock.status_code = HTTPStatus.OK.value mock.json.return_value = announcements - requests_session.post.return_value = mock + session = Mock() + api_key = "somekey" + session.api_key = api_key + session.headers = {'X-Api-Key': api_key} + session.post.return_value = mock - self.assertEqual(safety.get_announcements('somekey', {}), expected) + self.assertEqual(get_announcements(session), expected) @patch("safety.util.get_used_options") - @patch("safety.safety.session") @patch.object(click, 'get_current_context', Mock(command=Mock(name=Mock(return_value='check')))) - def test_get_announcements_wrong_json_response_handling(self, requests_session, get_used_options): + def test_get_announcements_wrong_json_response_handling(self, get_used_options): # wrong JSON structure announcements = { "type": "notice", - "message": "You are using an outdated version of Safety. Please upgrade to Safety version 1.2.3" + "message": "You are using an outdated version of Please upgrade to Safety version 1.2.3" } mock = Mock() mock.status_code = HTTPStatus.OK.value mock.json.return_value = announcements - requests_session.get.return_value = mock - self.assertEqual(safety.get_announcements('somekey', {}), []) + session = Mock() + api_key = "somekey" + session.api_key = api_key + session.headers = {'X-Api-Key': api_key} + session.get.return_value = mock + + self.assertEqual(get_announcements(session), []) # JSONDecodeError mock = Mock() mock.status_code = HTTPStatus.OK.value mock.json.side_effect = JSONDecodeError(msg='Expecting value', doc='', pos=0) - requests_session.get.return_value = mock + session = Mock() + api_key = "somekey" + session.api_key = api_key + session.headers = {'X-Api-Key': api_key} + session.get.return_value = mock - self.assertEqual(safety.get_announcements('somekey', {}), []) + self.assertEqual(get_announcements(session), []) def test_ignore_vulns_by_unknown_severity(self): cve_no_cvss = CVE(name='PYUP-123', cvssv2=None, cvssv3=None) @@ -691,7 +705,7 @@ def test_read_vulnerabilities_decode_error(self): with open(os.path.join(self.dirname, "test_db", "report_invalid_decode_error.json")) as f: self.assertRaises(MalformedDatabase, lambda: read_vulnerabilities(f)) - @patch("safety.safety.json.load") + @patch("json.load") def test_read_vulnerabilities_type_error(self, json_load): json_load.side_effect = TypeError('foobar') with open(os.path.join(self.dirname, "test_db", "report.json")) as f: @@ -702,7 +716,7 @@ def test_read_vulnerabilities(self): self.assertDictEqual(self.report, read_vulnerabilities(f)) def test_review_without_recommended_fix(self): - vulns, remediations, packages = safety.review(self.report) + vulns, remediations, packages = review(report=self.report) self.assertListEqual(packages, list(self.report_packages.values())) self.assertDictEqual(remediations, self.report_remediations) self.assertListEqual(vulns, self.report_vulns) @@ -716,5 +730,5 @@ def test_report_with_recommended_fix(self): 'more_info_url': 'https://pyup.io/packages/pypi/django/?from=4.0.1&to=4.0.4'}}} with open(os.path.join(self.dirname, "test_db", "report_with_recommended_fix.json")) as f: - vulns, remediations, packages = safety.review(read_vulnerabilities(f)) + vulns, remediations, packages = review(report=read_vulnerabilities(f)) self.assertDictEqual(remediations, REMEDIATIONS_WITH_FIX) diff --git a/tox.ini b/tox.ini index 36a63dce..e3e763f9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,13 @@ [tox] -envlist = py36, py37, py38, py39, py310, py311 +envlist = py{37,38,39,310,311,313}-packaging{21,22,23}-click{8.1.7} + isolated_build = true [testenv] deps = pytest-cov pytest + commands = pytest -rP tests/ --cov=safety/ --cov-report=html