From 3f90451b201f1587c7d601d0c8094274b7954358 Mon Sep 17 00:00:00 2001 From: Slava Aseev Date: Wed, 19 Jun 2019 16:53:14 +0300 Subject: [PATCH] Migrate to NVD JSON Feed 1.0 --- configure.ac | 7 +- src/core.c | 484 +++++++++++++++++-------------------------- src/library/common.h | 6 +- src/library/util.c | 8 +- src/library/util.h | 7 +- src/update.c | 16 +- 6 files changed, 217 insertions(+), 311 deletions(-) diff --git a/configure.ac b/configure.ac index 9dab298..7ff7240 100644 --- a/configure.ac +++ b/configure.ac @@ -17,6 +17,7 @@ m4_define([gobject_required_version], [2.0]) m4_define([check_required_version], [0.9]) m4_define([json_required_version], [0.16.0]) m4_define([openssl_required_version],[1.0.0]) +m4_define([jansson_required_version], [2.6]) # TODO: Set minimum sqlite AC_CHECK_FUNCS_ONCE(malloc_trim) @@ -29,13 +30,15 @@ PKG_CHECK_MODULES(CVE_CHECK_TOOL, libcurl >= curl_required_version, gobject-2.0 >= gobject_required_version, sqlite3, - openssl >= openssl_required_version + openssl >= openssl_required_version, + jansson >= jansson_required_version ]) PKG_CHECK_MODULES(MODULE_COMMON, [ libxml-2.0 >= libxml2_required_version, - glib-2.0 >= glib_required_version + glib-2.0 >= glib_required_version, + jansson >= jansson_required_version ]) PKG_CHECK_MODULES(CHECK, diff --git a/src/core.c b/src/core.c index cfdda55..6c58785 100644 --- a/src/core.c +++ b/src/core.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include @@ -39,25 +39,7 @@ const char *nvd_dir = "NVDS"; bool use_frac_compare = false; struct CveDB { - /** XML traversal state */ - bool in_list; - bool in_entry; - bool in_product; - bool in_summary; - - bool in_link; - bool in_vuln_cvss; - bool in_base_metrics; - bool in_score; - bool in_vector; - bool in_date; - - xmlChar *cur_id; - xmlChar *summary; - xmlChar *score; - xmlChar *vector; - xmlChar *modified; - GList *uris; + const char *cur_id; /* SQL usage */ sqlite3 *db; @@ -89,7 +71,7 @@ static bool ensure_table(CveDB *self) query = "CREATE TABLE IF NOT EXISTS " TABLE_NAME " " - "(ID TEXT UNIQUE, SUMMARY TEXT, SCORE TEXT, MODIFIED INTEGER, VECTOR TEXT);"; + "(ID TEXT UNIQUE, SUMMARY TEXT, SCORE DOUBLE, MODIFIED INTEGER, VECTOR TEXT);"; rc = sqlite3_exec(self->db, query, NULL, NULL, &err); if (rc != SQLITE_OK) { fprintf(stderr, "ensure_table(): %s\n", err); @@ -144,7 +126,7 @@ struct cve_entry_t *cve_db_get_cve(CveDB *self, char *id) t->id = g_strdup((const char *)sqlite3_column_text(self->get_cve, 0)); t->summary = g_strdup((const char *)sqlite3_column_text(self->get_cve, 1)); - t->score = g_strdup((const char *)sqlite3_column_text(self->get_cve, 2)); + t->score = g_strdup_printf("%f", sqlite3_column_double(self->get_cve, 2)); t->modified = sqlite3_column_int64(self->get_cve, 3); t->vector = g_strdup((const char *)sqlite3_column_text(self->get_cve, 4)); @@ -227,293 +209,230 @@ GList *cve_db_get_issues(CveDB *self, char *product, char *version) return list; } -static inline void free_vuln(struct vulnerability_t *t) -{ - if (!t) { - return; - } - if (t->vendor) { - g_free(t->vendor); - } - if (t->product) { - g_free(t->product); - } - if (t->version) { - g_free(t->version); - } + +enum SqliteOrder { + SqliteOrderID = 1, + SqliteOrderSummary, + SqliteOrderScore, + SqliteOrderModified, + SqliteOrderVector +}; + +enum SqliteOrderVuln { + SqliteOrderVulnHash = 1, + SqliteOrderVulnID, + SqliteOrderVulnVendor, + SqliteOrderVulnProduct, + SqliteOrderVulnVersion +}; + +static void SqliteBindJsonText(sqlite3* db, sqlite3_stmt *stmt, size_t ordinal, json_t* str) { + if (sqlite3_bind_text(stmt, ordinal, json_string_value(str), -1, SQLITE_STATIC) != SQLITE_OK) + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(db)); } -/** - * Parse a CPE line into a consumable form - * - * @param inp cpe:/ identifier string - * @param vuln Where to store the resulting vulnerability data - * @return a boolean value, true if the operation succeeded - */ -static bool parse_vuln(char *cve_id, const xmlChar *inp, struct vulnerability_t *vuln) -{ - gchar *product = NULL; - gchar *vendor = NULL; - gchar *version = NULL; - autofree(gchar) *hash = NULL; +static void SqliteBindInt64(sqlite3* db, sqlite3_stmt *stmt, size_t ordinal, int64_t num) { + if (sqlite3_bind_int64(stmt, ordinal, num) != SQLITE_OK) + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(db)); +} - int len = 0; - /* Example: cpe:/a:oracle:siebel_crm:8.1.1 */ - gchar **splits = g_strsplit((const gchar *)inp, ":", 10); - if ((len = g_strv_length(splits)) < 4) { - g_strfreev(splits); - return false; - } +static void SqliteBindJsonDouble(sqlite3* db, sqlite3_stmt *stmt, size_t ordinal, json_t* num) { + if (sqlite3_bind_double(stmt, ordinal, json_real_value(num)) != SQLITE_OK) + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(db)); +} + +static void SqliteBindJsonDescription(sqlite3* db, sqlite3_stmt *stmt, size_t ordinal, json_t* descr) { + json_t *data = json_object_get(descr, "description_data"); + size_t size = json_array_size(data); - vendor = g_strdup(splits[2]); - product = g_strdup(splits[3]); - if (len > 4) { - version = g_strdup(splits[4]); + for (size_t i = 0; i < size; ++i) { + json_t *item = json_array_get(data, i); + json_t *lang = json_object_get(item, "lang"); + + if (!strcmp(json_string_value(lang), "en")) { + json_t *value = json_object_get(item, "value"); + + SqliteBindJsonText(db, stmt, ordinal, value); + break; + } } - g_strfreev(splits); +} - vuln->vendor = vendor; - vuln->product = product; - vuln->version = version; +static void SqliteBindJsonCveMetadata(CveDB *self, size_t ordinal, json_t* descr) { + json_t* data = json_object_get(descr, "ID"); + const char* str = json_string_value(data); - hash = g_strdup_printf("%s:%s:%s:%s", cve_id, vendor, product, version); - if (!hash) { - fprintf(stderr, "parse_vuln(): Out of memory\n"); - free_vuln(vuln); - return false; + if (sqlite3_bind_text(self->insert, ordinal, str, -1, SQLITE_STATIC) != SQLITE_OK) { + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); } - vuln->hash = g_str_hash(hash); - return true; + /* Set current cve id for later use */ + self->cur_id = str; } -static inline void _cve_db_clean(CveDB *self) -{ - if (self->uris) { - g_list_free_full(self->uris, xmlFree); - self->uris = NULL; - } +static void SqliteBindVuln(CveDB *self, struct vulnerability_t* vuln) { + autofree(gchar) *hash = NULL; - if (self->score) { - xmlFree(self->score); - self->score = NULL; - } - if (self->vector) { - xmlFree(self->vector); - self->vector = NULL; + if (vuln->version) { + hash = g_strdup_printf("%s:%s:%s:%s", self->cur_id, vuln->vendor, vuln->product, vuln->version); } - if (self->cur_id) { - xmlFree(self->cur_id); - self->cur_id = NULL; + else { + hash = g_strdup_printf("%s:%s:%s:*", self->cur_id, vuln->vendor, vuln->product); } - if (self->summary) { - xmlFree(self->summary); - self->summary = NULL; - } - if (self->modified) { - xmlFree(self->modified); - self->modified = NULL; - } -} + vuln->hash = g_str_hash(hash); -/** - * Main iterator for XML parsing - * - * @param r A valid xmlTextReaderPtr (open) - */ -static void process_node(CveDB *self, xmlTextReaderPtr r) -{ - const xmlChar *name = NULL; - const xmlChar *value = NULL; - struct vulnerability_t vuln = {.vendor = 0 }; - xmlChar *uri = NULL; - int64_t last_mod = -1; - - name = xmlTextReaderConstName(r); - if (!name) { - return; - } - /* New entry */ - if (xmlStrEqual(name, BAD_CAST "entry")) { - self->in_entry = !self->in_entry; - if (!self->in_entry) { - int rc = 0; - last_mod = parse_xml_date((char *)self->modified); - - sqlite3_reset(self->insert); - - /* ID */ - if (sqlite3_bind_text(self->insert, 1, (const char *)self->cur_id, -1, SQLITE_STATIC) != - SQLITE_OK) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto next; - } - /* SUMMARY */ - if (sqlite3_bind_text(self->insert, 2, (const char *)self->summary, -1, SQLITE_STATIC) != - SQLITE_OK) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto next; - } - /* SCORE */ - if (sqlite3_bind_text(self->insert, 3, (const char *)self->score, -1, SQLITE_STATIC) != - SQLITE_OK) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto next; - } - /* MODIFIED */ - if (sqlite3_bind_int64(self->insert, 4, last_mod)) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto next; - } - /* VECTOR */ - if (sqlite3_bind_text(self->insert, 5, (const char *)self->vector, -1, SQLITE_STATIC) != - SQLITE_OK) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto next; - } - - rc = sqlite3_step(self->insert); - if (rc != SQLITE_DONE) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - } - next: - _cve_db_clean(self); - return; - } - if (self->cur_id) { - xmlFree(self->cur_id); - } - self->cur_id = xmlTextReaderGetAttribute(r, BAD_CAST "id"); - return; + sqlite3_reset(self->insert_product); + + if (sqlite3_bind_int(self->insert_product, SqliteOrderVulnHash, vuln->hash) != SQLITE_OK) { + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); } - if (xmlStrEqual(name, BAD_CAST "vuln:references")) { - self->in_link = !self->in_link; - return; + if (sqlite3_bind_text(self->insert_product, SqliteOrderVulnID, + self->cur_id, -1, SQLITE_STATIC) != SQLITE_OK) + { + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); } - if (self->in_link && xmlStrEqual(name, BAD_CAST "vuln:reference")) { - uri = xmlTextReaderGetAttribute(r, BAD_CAST "href"); - if (!uri) { - return; - } - self->uris = g_list_append(self->uris, uri); - uri = NULL; + if (sqlite3_bind_text(self->insert_product, SqliteOrderVulnVendor, + vuln->vendor, -1, SQLITE_STATIC) != SQLITE_OK) + { + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); } - if (xmlStrEqual(name, BAD_CAST "vuln:vulnerable-software-list")) { - self->in_list = !self->in_list; - return; + if (sqlite3_bind_text(self->insert_product, SqliteOrderVulnProduct, + vuln->product, -1, SQLITE_STATIC) != SQLITE_OK) + { + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); } - if (self->in_list && xmlStrEqual(name, BAD_CAST "vuln:product")) { - self->in_product = !self->in_product; - return; + if (sqlite3_bind_text(self->insert_product, SqliteOrderVulnVersion, + vuln->version, -1, SQLITE_STATIC) != SQLITE_OK) + { + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); } - /* Score checking */ - if (xmlStrEqual(name, BAD_CAST "vuln:cvss")) { - self->in_vuln_cvss = !self->in_vuln_cvss; - return; + + if (sqlite3_step(self->insert_product) != SQLITE_DONE) { + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); } - if (self->in_vuln_cvss && xmlStrEqual(name, BAD_CAST "cvss:base_metrics")) { - self->in_base_metrics = !self->in_base_metrics; +} + +static void process_versions(CveDB *self, json_t *r, struct vulnerability_t* vuln) { + size_t size = json_array_size(r); + + if (size == 0) { + vuln->version = NULL; + SqliteBindVuln(self, vuln); return; } - if (self->in_base_metrics && xmlStrEqual(name, BAD_CAST "cvss:score")) { - self->in_score = !self->in_score; - } - if (self->in_base_metrics && xmlStrEqual(name, BAD_CAST "cvss:access-vector")) { - self->in_vector = !self->in_vector; - } - if (self->in_base_metrics && self->in_vector) { - value = xmlTextReaderConstValue(r); - if (!value) { - return; - } - self->vector = xmlStrdup(value); - } - if (self->in_base_metrics && self->in_score) { - value = xmlTextReaderConstValue(r); - if (!value) { - return; - } - self->score = xmlStrdup(value); - } - /* Get last modified */ - if (xmlStrEqual(name, BAD_CAST "vuln:last-modified-datetime")) { - self->in_date = !self->in_date; + + for (size_t i = 0; i < size; ++i) { + json_t* version_node = json_array_get(r, i); + json_t* version_value = json_object_get(version_node, "version_value"); + + vuln->version = json_string_value(version_value); + + SqliteBindVuln(self, vuln); } - if (self->in_date) { - value = xmlTextReaderConstValue(r); - if (!value) { - return; +} + +static void process_affects(CveDB *self, json_t *r) { + json_t *vendor = json_object_get(r, "vendor"); + json_t *vendor_data = json_object_get(vendor, "vendor_data"); + size_t size = json_array_size(vendor_data); + + struct vulnerability_t vuln; + + for (size_t v = 0; v < size; ++v) { + json_t *node = json_array_get(vendor_data, v); + json_t *vendor_name = json_object_get(node, "vendor_name"); + + vuln.vendor = json_string_value(vendor_name); + + json_t *product = json_object_get(node, "product"); + json_t *product_data = json_object_get(product, "product_data"); + size_t products_size = json_array_size(product_data); + + for (size_t p = 0; p < products_size; ++p) { + json_t *product_node = json_array_get(product_data, p); + json_t *product_name = json_object_get(product_node, "product_name"); + + vuln.product = json_string_value(product_name); + + json_t *version = json_object_get(product_node, "version"); + json_t *version_data = json_object_get(version, "version_data"); + + process_versions(self, version_data, &vuln); } - self->modified = xmlStrdup(value); } - /* Product checking */ - if (self->in_list && self->in_product) { - value = xmlTextReaderConstValue(r); - int rc = 0; +} - if (!value) { - return; - } - if (!parse_vuln((char *)self->cur_id, value, &vuln)) { - return; - } +static void process_cve(CveDB *self, json_t *r) { + json_t *cve_data_meta = json_object_get(r, "CVE_data_meta"); + json_t *description = json_object_get(r, "description"); + json_t *affects = json_object_get(r, "affects"); - sqlite3_reset(self->insert_product); + SqliteBindJsonCveMetadata(self, SqliteOrderID, cve_data_meta); + SqliteBindJsonDescription(self->db, self->insert, SqliteOrderSummary, description); - /* HASH */ - if (sqlite3_bind_int(self->insert_product, 1, vuln.hash) != SQLITE_OK) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto end_product; - } + process_affects(self, affects); +} - /* ID */ - if (sqlite3_bind_text(self->insert_product, 2, (const char *)self->cur_id, -1, SQLITE_STATIC) != - SQLITE_OK) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto end_product; - } - /* VENDOR */ - if (sqlite3_bind_text(self->insert_product, 3, vuln.vendor, -1, SQLITE_STATIC) != SQLITE_OK) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto end_product; - } - /* PRODUCT */ - if (sqlite3_bind_text(self->insert_product, 4, vuln.product, -1, SQLITE_STATIC) != SQLITE_OK) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto end_product; - } - /* VERSION */ - if (sqlite3_bind_text(self->insert_product, 5, vuln.version, -1, SQLITE_STATIC) != SQLITE_OK) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - goto end_product; - } +static void process_impact(CveDB *self, json_t *r) { + json_t *metric = json_object_get(r, "baseMetricV2"); + json_t *cvss = json_object_get(metric, "cvssV2"); + json_t *score = json_object_get(cvss, "baseScore"); + json_t *vector = json_object_get(cvss, "accessVector"); - /* Commit product. */ - rc = sqlite3_step(self->insert_product); - if (rc != SQLITE_DONE) { - fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); - } + SqliteBindJsonDouble(self->db, self->insert, SqliteOrderScore, score); + SqliteBindJsonText (self->db, self->insert, SqliteOrderVector, vector); +} - end_product: - free_vuln(&vuln); - return; +static void process_last_modified(CveDB *self, json_t *r) { + const char *date = json_string_value(r); + int64_t last_mod = parse_date(date); + + SqliteBindInt64(self->db, self->insert, SqliteOrderModified, last_mod); +} + +static bool process_json(CveDB *self, json_t *r) { + json_t *cve_items = json_object_get(r, "CVE_Items"); + + size_t size = json_array_size(cve_items); + + if (size == 0) { + fprintf(stderr, "process_json(): 'CVE_Items array' not found or empty\n"); + return false; } - if (self->in_entry && xmlStrEqual(name, BAD_CAST "vuln:summary")) { - self->in_summary = !self->in_summary; - if (self->in_summary && self->summary) { - xmlFree(self->summary); - self->summary = NULL; + + for (size_t i = 0; i < size; ++i) { + sqlite3_reset(self->insert); + + json_t *node = json_array_get(cve_items, i); + + process_cve(self, json_object_get(node, "cve")); + process_last_modified(self, json_object_get(node, "lastModifiedDate")); + process_impact(self, json_object_get(node, "impact")); + + if (sqlite3_step(self->insert) != SQLITE_DONE) { + fprintf(stderr, "process_node(): %s\n", sqlite3_errmsg(self->db)); } - return; } - if (self->in_summary) { - self->summary = xmlTextReaderValue(r); - return; + + return true; +} + +static json_t *load_json(const char *filename) { + json_t *root; + json_error_t error; + + root = json_load_file(filename, 0, &error); + + if (root) { + return root; + } else { + fprintf(stderr, "Json error: %s:%d: %s\n", filename, error.line, error.text); + return (json_t*)0; } } /** - * Parse an NVD xml database + * Parse an NVD json database * * @param fname Path to the nvd db * @return a boolean value, true if the operation succeeded @@ -526,34 +445,20 @@ bool cve_db_load(CveDB *self, const char *fname) if (!self || !fname) { return false; } - int fd = 0; - fd = open(fname, O_RDONLY); - if (fd < 0) { - return false; - } - /* If it fails, it fails */ - rc = posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL); + json_t* r = load_json(fname); - xmlTextReaderPtr r = xmlReaderForFd(fd, fname, NULL, 0); - if (!r) { - close(fd); + if (r == (json_t*)0) { return false; } - int ret; - while ((ret = xmlTextReaderRead(r)) > 0) { - process_node(self, r); - } + b = process_json(self, r); + + json_decref(r); - b = true; #ifdef HAVE_MALLOC_TRIM malloc_trim(0); #endif - xmlFreeTextReader(r); - if (fd) { - close(fd); - } return b; } @@ -691,7 +596,6 @@ void cve_db_free(CveDB *self) if (self->db) { sqlite3_close(self->db); } - _cve_db_clean(self); free(self); } diff --git a/src/library/common.h b/src/library/common.h index eeedabf..754971e 100644 --- a/src/library/common.h +++ b/src/library/common.h @@ -33,9 +33,9 @@ typedef enum { * Maps vulnerabilities into a consumable format */ struct vulnerability_t { - gchar *product; /**str, 'T', tmp->len))) { return -1; } - if (!(c = memchr(c, '-', tmp->len - (tmp->str - c)))) { + if (!(c = strpbrk(c, "+-Z"))) { return -1; } - gint y, m, d, h, min, s; - if (sscanf(date, "%4d-%2d-%2dT%2d:%2d:%2d", &y, &m, &d, &h, &min, &s) != 6) { + gint y, m, d, h = 0, min = 0, s = 0; + if (sscanf(date, "%4d-%2d-%2dT%2d:%2d:%2d", &y, &m, &d, &h, &min, &s) < 3) { return -1; } tz = g_time_zone_new(c); diff --git a/src/library/util.h b/src/library/util.h index 9f7e8fa..05d8d7f 100644 --- a/src/library/util.h +++ b/src/library/util.h @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -53,12 +52,12 @@ typedef bool (*package_match_func)(const gchar *); gchar *demacro(CveHashmap *macros, gchar *str); /** - * Convert an XML formatted date into unix seconds + * Convert date into unix seconds * - * @param date XML input + * @param date input * @return int64_t unix timestamp, or -1 if it doesn't parse */ -int64_t parse_xml_date(const char *date); +int64_t parse_date(const char *date); /** * Search the repo source directory for all matching sources diff --git a/src/update.c b/src/update.c index 3cc6b67..26ee61e 100644 --- a/src/update.c +++ b/src/update.c @@ -39,7 +39,7 @@ #include "update.h" #define YEAR_START 2002 -#define URI_PREFIX "https://static.nvd.nist.gov/feeds/xml/cve" +#define URI_PREFIX "https://static.nvd.nist.gov/feeds/json/cve/1.0" #include "fetch.h" #define UPDATE_THRESHOLD 7200 @@ -104,10 +104,10 @@ cve_string *get_db_path(const char *path) static cve_string *nvdcve_make_fname(int year, const char *fext) { - if (year < 0) { - return cve_string_dup_printf("nvdcve-2.0-Modified.%s", fext); - } else { - return cve_string_dup_printf("nvdcve-2.0-%d.%s", year, fext); + switch(year) { + case -2: return cve_string_dup_printf("nvdcve-1.0-recent.%s", fext); + case -1: return cve_string_dup_printf("nvdcve-1.0-modified.%s", fext); + default: return cve_string_dup_printf("nvdcve-1.0-%d.%s", year, fext); } } @@ -295,7 +295,7 @@ static int do_fetch_update(int year, const char *db_dir, CveDB *cve_db, bool db_ } /* Prepare NVD XML file/uri pathes */ - nvd_xml = nvdcve_make_fname(year, "xml"); + nvd_xml = nvdcve_make_fname(year, "json"); if (!nvd_xml) { return ENOMEM; } @@ -454,8 +454,8 @@ bool update_db(bool quiet, const char *db_file, const char *cacert_file) goto end; } - for (int i = YEAR_START; i <= year + 1; i++) { - int y = i > year ? -1 : i; + for (int i = YEAR_START; i <= year + 2; i++) { + int y = i > year ? year - i : i; int rc; rc = do_fetch_update(y, db_dir, cve_db, db_exist, !quiet, cacert_file);