Remove nodesecurity and allow manually added vulnerabilities

rtfpessoa · Oct 21, 2018 · fbf325b · fbf325b
1 parent 2c1348e
commit fbf325b
Show file tree

Hide file tree

Showing 13 changed files with 58 additions and 229 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    yavdb (0.2.1)
+    yavdb (0.3.0)
       json (~> 2.1)
       kramdown (~> 1.17)
       oga (~> 2.15)
@@ -82,4 +82,4 @@ DEPENDENCIES
   yavdb!
 
 BUNDLED WITH
-   1.16.3
+   1.16.6
diff --git a/README.md b/README.md
@@ -12,8 +12,8 @@ developers identify and fix know vulnerabilities in their apps.
 The sources for this database include 
 [Rubysec](https://rubysec.com/),
 [snyk](https://snyk.io/),
-[OSSIndex](https://ossindex.net/),
-[NodeSecurity](https://nodesecurity.io/),
+[OSSIndex (deprecated)](https://ossindex.net/),
+[NodeSecurity (deprecated)](https://nodesecurity.io/),
 [Friends of PHP](https://github.com/FriendsOfPHP/security-advisories),
 [Magento Related Security Advisories](https://github.com/victims/victims-cve-db),
 [Victims CVE Database](https://github.com/victims/victims-cve-db)
@@ -25,11 +25,9 @@ The sources for this database include
 ## Installation
 
 ```sh
-gem install yavdb --pre
+gem install yavdb
 ```
 
-> Notice the `--pre` in the end
-
 ## TODO:
 
 #### Tests
@@ -38,7 +36,6 @@ gem install yavdb --pre
     - [ ] [Rubysec](lib/yavdb/sources/ruby_advisory.rb)
     - [X] [snyk](lib/yavdb/sources/snyk_io.rb)
     - [ ] [OSSIndex](lib/yavdb/sources/ossindex.rb)
-    - [X] [NodeSecurity](lib/yavdb/sources/nodesecurity_io.rb)
     - [ ] [Friends of PHP and Magento Related Security Advisories](lib/yavdb/sources/friends_of_php.rb)
     - [ ] [Victims CVE Database](lib/yavdb/sources/victims.rb)
 * Others

diff --git a/lib/yavdb/database.rb b/lib/yavdb/database.rb
@@ -60,12 +60,18 @@ def group_by_package_manager(vulns)
       def save_to_file(database_path, vulns)
         vulns.map do |package_manager, vunerabilities_by_pm|
           vunerabilities_by_pm.map do |package, vunerabilities_by_p|
+            previous_vulnerabilities = search(database_path, package_manager, package)
+
             package_path           = package_path(database_path, package_manager, package)
             package_path_directory = File.dirname(package_path)
             FileUtils.mkdir_p(package_path_directory) unless File.exist?(package_path_directory)
 
+            uniq_vunerabilities_by_p = Hash[previous_vulnerabilities.concat(vunerabilities_by_p).map { |vuln| [vuln.id, vuln] }].values
+
+            next unless uniq_vunerabilities_by_p.any?
+
             File.open(package_path, 'wb') do |file|
-              package_vulns_yml_str = vunerabilities_by_p
+              package_vulns_yml_str = uniq_vunerabilities_by_p
                                         .sort_by(&:id)
                                         .map(&:to_map)
                                         .to_yaml(

diff --git a/lib/yavdb/dtos/advisory.rb b/lib/yavdb/dtos/advisory.rb
@@ -79,9 +79,9 @@ def self.load(path)
     def to_map
       map = {}
       members.each do |m|
-        next unless self[m] && (
-        (self[m].is_a?(String) && !self[m].empty?) ||
-          (self[m].is_a?(Array) && self[m].any?))
+        next if !self[m] ||
+                (self[m].is_a?(String) && self[m].empty?) ||
+                (self[m].is_a?(Array) && self[m].none?)
 
         map[m.to_s] = self[m] if self[m]
       end

diff --git a/lib/yavdb/sources/friends_of_php.rb b/lib/yavdb/sources/friends_of_php.rb
@@ -54,9 +54,15 @@ def self.create(url, advisory_hash)
             info['versions'].join(' ')
           end.flatten
 
+          cves = [advisory_hash['cve']].reject { |cve| cve == '~' }
+
           package_name = advisory_hash['reference'].gsub(%r{composer:\/\/(.*)}, '\1')
+
+          vuln_id_stamp = (cves && cves[0]) || date
+          vuln_id       = "friendsofphp:packagist:#{package_name}:#{vuln_id_stamp}"
+
           YAVDB::Advisory.new(
-            "friendsofphp:packagist:#{package_name}:#{date}",
+            vuln_id,
             advisory_hash['title'],
             nil, #:description
             package_name,
@@ -65,7 +71,7 @@ def self.create(url, advisory_hash)
             nil, #:patched_versions
             nil, #:severity
             PACKAGE_MANAGER,
-            [advisory_hash['cve']].reject { |cve| cve == '~' },
+            cves,
             nil, #:cwe
             nil, #:osvdb
             nil, #:cvss_v2_vector

diff --git a/lib/yavdb/sources/nodesecurity_io.rb b/lib/yavdb/sources/nodesecurity_io.rb
diff --git a/lib/yavdb/sources/ossindex.rb b/lib/yavdb/sources/ossindex.rb
@@ -95,8 +95,11 @@ def create(package_manager, package, advisory)
                          .reject { |v| v == '-' }
             versions = ['*'] unless versions.any?
 
+            vuln_id_stamp = (cve && cve[0]) || published_date
+            vuln_id       = "ossindex:#{package_manager}:#{package_name}:#{vuln_id_stamp}"
+
             YAVDB::Advisory.new(
-              "ossindex:#{package_manager}:#{package_name}:#{published_date}",
+              vuln_id,
               advisory['title'],
               advisory['description'],
               package_name,

diff --git a/lib/yavdb/sources/ruby_advisory.rb b/lib/yavdb/sources/ruby_advisory.rb
@@ -54,8 +54,11 @@ def create(advisory_hash)
                                     ['*']
                                   end
 
+            vuln_id_stamp = (cve && cve[0]) || date
+            vuln_id       = "rubyadvisory:rubygems:#{advisory_hash['gem']}:#{vuln_id_stamp}"
+
             YAVDB::Advisory.new(
-              "rubyadvisory:rubygems:#{advisory_hash['gem']}:#{date}",
+              vuln_id,
               advisory_hash['title'],
               advisory_hash['description'],
               advisory_hash['gem'],

diff --git a/lib/yavdb/sources/snyk_io.rb b/lib/yavdb/sources/snyk_io.rb
@@ -119,8 +119,13 @@ def create(advisory_url, advisory_page)
                                    published_date
                                  end
 
+            vuln_id_stamp = (sidebar_data[:cve] && sidebar_data[:cve][0]) ||
+                            sidebar_data[:id].split(%r{-|:}).last ||
+                            disclosed_date
+            vuln_id       = "snykio:#{package_manager}:#{affected_package}:#{vuln_id_stamp}"
+
             YAVDB::Advisory.new(
-              "snykio:#{package_manager}:#{affected_package}:#{disclosed_date}",
+              vuln_id,
               title,
               body_data[:description],
               affected_package,
@@ -157,7 +162,6 @@ def parse_body(advisory_page)
                 last_elem        = description_sections.last
                 new_body         = last_elem[:body].push(field)
                 last_elem[:body] = new_body
-                description_sections.push(last_elem)
               end
             end
 

diff --git a/lib/yavdb/sources/victims.rb b/lib/yavdb/sources/victims.rb
@@ -59,8 +59,11 @@ def language_glob(language)
 
           def create(advisory_hash, language, url)
             advisory_hash['affected'].map do |affected_package|
+              vuln_id_stamp = advisory_hash['cve'] || 'date'
+              vuln_id       = "victims:#{language.package_manager}:#{language.name_parser[affected_package]}:#{vuln_id_stamp}"
+
               YAVDB::Advisory.new(
-                "victims:#{language.package_manager}:#{language.name_parser[affected_package]}:date",
+                vuln_id,
                 advisory_hash['title'],
                 advisory_hash['description'],
                 language.name_parser[affected_package],

diff --git a/lib/yavdb/version.rb b/lib/yavdb/version.rb
@@ -16,6 +16,6 @@
 
 module YAVDB
 
-  VERSION = '0.2.1'
+  VERSION = '0.3.0'
 
 end
diff --git a/spec/nodesecurity_io_spec.rb b/spec/nodesecurity_io_spec.rb