diff --git a/.github/workflows/export-osv.yaml b/.github/workflows/export-osv.yaml new file mode 100644 index 000000000..05479fc10 --- /dev/null +++ b/.github/workflows/export-osv.yaml @@ -0,0 +1,42 @@ +name: Export to OSV format + +on: + push: + branches: + - master + +jobs: + publish-web: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + # Required in order to extract dates from commit history + fetch-depth: 0 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: "8.0" + coverage: none + tools: composer + + - name: Install dependencies + run: composer install --prefer-dist --no-progress + + - name: Export to OSV format + run: | + git config user.name github-actions + git config user.email github-actions@github.com + php export-osv.php export + git add packagist + git stash + git checkout osv + echo `date` > published + git add published + git rm -r --ignore-unmatch packagist + git commit -m "Update OSV data export" + git stash pop + git commit --amend --no-edit --allow-empty + git push diff --git a/.github/workflows/php.yaml b/.github/workflows/php.yaml index 718cf7f2e..b7091a409 100644 --- a/.github/workflows/php.yaml +++ b/.github/workflows/php.yaml @@ -1,27 +1,27 @@ name: Validation on: - push: - pull_request: + push: + pull_request: jobs: - run: - runs-on: ubuntu-latest + run: + runs-on: ubuntu-latest - name: Validation - steps: - - name: Checkout - uses: actions/checkout@v2 + name: Validation + steps: + - name: Checkout + uses: actions/checkout@v2 - - name: Setup PHP - uses: shivammathur/setup-php@v2 - with: - php-version: "8.0" - coverage: none - tools: composer + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: "8.0" + coverage: none + tools: composer - - name: Install dependencies - run: composer install --prefer-dist --no-progress + - name: Install dependencies + run: composer install --prefer-dist --no-progress - - name: Run tests - run: php -d memory_limit=-1 validator.php + - name: Run tests + run: php -d memory_limit=-1 validator.php diff --git a/README.md b/README.md index 3729f30bf..73963506b 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ not** serve as the primary source of information for security issues, it is not authoritative for any referenced software, but it allows to centralize information for convenience and easy consumption. +We also export advisory data to the [OSV](https://github.com/ossf/osv-schema) format, +see the [`osv`](https://github.com/FriendsOfPHP/security-advisories/tree/osv) branch. + License ------- diff --git a/composer.json b/composer.json index 86f623e2d..00012a65c 100644 --- a/composer.json +++ b/composer.json @@ -3,7 +3,9 @@ "description": "Database of known security vulnerabilities in various PHP projects and libraries", "require-dev": { "composer/composer": "~1.0", + "symfony/cache": "^5.3", "symfony/console": "^4.0", + "symfony/http-client": "^5.3", "symfony/yaml": "^4.0" }, "license": "Unlicense", diff --git a/export-osv.php b/export-osv.php new file mode 100644 index 000000000..1f70da516 --- /dev/null +++ b/export-osv.php @@ -0,0 +1,206 @@ +setName('export') + ->setDescription('Export advisories in OSV format') + ->addArgument('target',InputArgument::OPTIONAL, 'Target folder', 'packagist'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + mkdir($targetFolder = $input->getArgument('target')); + + $cache = new FilesystemAdapter(); + + $namespaceIterator = new DirectoryIterator(__DIR__); + + // Package namespaces + foreach ($namespaceIterator as $namespaceInfo) { + if ($namespaceInfo->isDot() || !$namespaceInfo->isDir() || $namespaceInfo->getFilename() === 'vendor' || strpos($namespaceInfo->getFilename() , '.') === 0) continue; + + $namespace = $namespaceInfo->getFilename(); + $packageIterator = new DirectoryIterator($namespaceInfo->getPathname()); + + // Packages inside namespace + foreach ($packageIterator as $packageInfo) { + if ($packageIterator->isDot() || !$packageInfo->isDir()) continue; + + $package = [ + 'name' => $namespace . '/' . $packageInfo->getFilename(), + 'data' => $this->getPackageData($namespace . '/' . $packageInfo->getFilename(), $cache), + ]; + + if (is_null($package['data'])) { + $output->writeln('Skipped "' . $package['name'] . '" because it was not found on Packagist'); + continue; + } + + $fileSystemIterator = new FilesystemIterator($packageInfo->getPathname()); + + $output->write('Converting "' . $package['name'] . '" ...' . str_repeat(' ', 20) . "\r"); + + foreach ($fileSystemIterator as $fileInfo) { + $osv = $this->convertToOsv($fileInfo, $package); + + if (is_null($osv)) { + $output->writeln('Skipped "' . $package['name'] . '/' . $fileInfo->getFilename() . '" because package is not on Packagist'); + continue; + } + + if (count($osv['affected']['versions']) === 0) { + $output->writeln('Skipped "' . $package['name'] . '/' . $fileInfo->getFilename() . '" because no affected versions are available on Packagist'); + continue; + } + + $path = $targetFolder . DIRECTORY_SEPARATOR . $osv['id'] . '.json'; + + file_put_contents($path, json_encode($osv, JSON_PRETTY_PRINT)); + } + } + } + + $output->writeln(''); + + // Command::SUCCESS and Command::FAILURE constants were introduced in Symfony 5.1 + return 0; + } + + private function convertToOsv(SplFileInfo $fileInfo, array $package): ?array + { + $advisory = Yaml::parseFile($fileInfo->getPathname()); + + // Advisories with custom repositories are currently not supported + if (isset($advisory['composer-repository'])) { + return null; + } + + return [ + 'id' => $advisory['cve'] ?? self::OSV_PREFIX . '-' . $fileInfo->getBasename('.yaml'), + 'modified' => self::getDateFromGitLog($fileInfo), + 'published' => self::getDateFromGitLog($fileInfo, true), + 'aliases' => [], + 'related' => [], + 'summary' => $advisory['title'] ?? '', + 'details' => '', + 'affected' => self::getAffected($advisory, $package), + 'references' => self::getReferences($advisory, $package['name']), + ]; + } + + private function getPackageData(string $packageName, CacheInterface $cache): ?array + { + return $cache->get($packageName, function () use ($packageName) { + $response = HttpClient::create()->request( + 'GET', + 'https://repo.packagist.org/p2/' . $packageName . '.json' + ); + + try { + return $response->toArray(); + } catch (HttpExceptionInterface $httpException) { + return null; + } + }); + } + + private static function getAffected(array $advisory, array $package): array + { + return [ + 'package' => [ + 'ecosystem' => self::OSV_ECOSYSTEM, + 'name' => $package['name'], + 'purl' => sprintf('pkg:packagist/%s', $package['name']), + ], + 'versions' => self::getVersions($advisory['branches'], $package), + ]; + } + + private static function getDateFromGitLog(SplFileInfo $fileInfo, bool $created = false): string + { + $timestamp = shell_exec(sprintf( + 'git log --format="%%at" %s %s %s %s', + $created ? '' : '--max-count 1', + $created ? '--reverse' : '', + escapeshellarg($fileInfo->getPathname()), + $created ? '| head -1' : '' + )); + + return date('Y-m-d\TH:i:s\Z', (int) trim($timestamp)); + } + + private static function getVersions(array $branches, array $package): array + { + $branchConstraints = array_column($branches, 'versions'); + + $versions = array_column($package['data']['packages'][$package['name']], 'version'); + $versionsAffected = []; + + foreach ($branchConstraints as $constraints) { + foreach (array_reverse($versions) as $version) { + if (Semver::satisfies($version, implode(' ', $constraints))) { + array_push($versionsAffected, $version); + } + } + } + + return $versionsAffected; + } + + private static function getReferences(array $advisory, string $packageName): array + { + return [ + [ + 'type' => 'ADVISORY', + 'url' => $advisory['link'], + ], + [ + 'type' => 'PACKAGE', + 'url' => self::OSV_PACKAGE_URL . $packageName, + ], + ]; + } +} + +$application = new Application(); +$application->add(new ExportOsv()); +$application->run();