1#!/usr/bin/env python3 2 3import xml.etree.ElementTree as ET 4from xml.etree.ElementTree import Element, SubElement 5import gzip 6import os 7import requests 8import time 9from xml.dom import minidom 10 11VALID_REFS = ['VENDOR', 'VERSION', 'CHANGE_LOG', 'PRODUCT', 'PROJECT', 'ADVISORY'] 12 13CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz" 14 15ns = { 16 '': 'http://cpe.mitre.org/dictionary/2.0', 17 'cpe-23': 'http://scap.nist.gov/schema/cpe-extension/2.3', 18 'xml': 'http://www.w3.org/XML/1998/namespace' 19} 20 21 22class CPE: 23 def __init__(self, cpe_str, titles, refs): 24 self.cpe_str = cpe_str 25 self.titles = titles 26 self.references = refs 27 self.cpe_cur_ver = "".join(self.cpe_str.split(":")[5:6]) 28 29 def update_xml_dict(self): 30 ET.register_namespace('', 'http://cpe.mitre.org/dictionary/2.0') 31 cpes = Element('cpe-list') 32 cpes.set('xmlns:cpe-23', "http://scap.nist.gov/schema/cpe-extension/2.3") 33 cpes.set('xmlns:ns6', "http://scap.nist.gov/schema/scap-core/0.1") 34 cpes.set('xmlns:scap-core', "http://scap.nist.gov/schema/scap-core/0.3") 35 cpes.set('xmlns:config', "http://scap.nist.gov/schema/configuration/0.1") 36 cpes.set('xmlns:xsi', "http://www.w3.org/2001/XMLSchema-instance") 37 cpes.set('xmlns:meta', "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2") 38 cpes.set('xsi:schemaLocation', " ".join(["http://scap.nist.gov/schema/cpe-extension/2.3", 39 "https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary-extension_2.3.xsd", 40 "http://cpe.mitre.org/dictionary/2.0", 41 "https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary_2.3.xsd", 42 "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2", 43 "https://scap.nist.gov/schema/cpe/2.1/cpe-dictionary-metadata_0.2.xsd", 44 "http://scap.nist.gov/schema/scap-core/0.3", 45 "https://scap.nist.gov/schema/nvd/scap-core_0.3.xsd", 46 "http://scap.nist.gov/schema/configuration/0.1", 47 "https://scap.nist.gov/schema/nvd/configuration_0.1.xsd", 48 "http://scap.nist.gov/schema/scap-core/0.1", 49 "https://scap.nist.gov/schema/nvd/scap-core_0.1.xsd"])) 50 item = SubElement(cpes, 'cpe-item') 51 cpe_short_name = CPE.short_name(self.cpe_str) 52 cpe_new_ver = CPE.version_update(self.cpe_str) 53 54 item.set('name', 'cpe:/' + cpe_short_name) 55 self.titles[0].text.replace(self.cpe_cur_ver, cpe_new_ver) 56 for title in self.titles: 57 item.append(title) 58 if self.references: 59 item.append(self.references) 60 cpe23item = SubElement(item, 'cpe-23:cpe23-item') 61 cpe23item.set('name', self.cpe_str) 62 63 # Generate the XML as a string 64 xmlstr = ET.tostring(cpes) 65 66 # And use minidom to pretty print the XML 67 return minidom.parseString(xmlstr).toprettyxml(encoding="utf-8").decode("utf-8") 68 69 @staticmethod 70 def version(cpe): 71 return cpe.split(":")[5] 72 73 @staticmethod 74 def product(cpe): 75 return cpe.split(":")[4] 76 77 @staticmethod 78 def short_name(cpe): 79 return ":".join(cpe.split(":")[2:6]) 80 81 @staticmethod 82 def version_update(cpe): 83 return ":".join(cpe.split(":")[5:6]) 84 85 @staticmethod 86 def no_version(cpe): 87 return ":".join(cpe.split(":")[:5]) 88 89 90class CPEDB: 91 def __init__(self, nvd_path): 92 self.all_cpes = dict() 93 self.all_cpes_no_version = dict() 94 self.nvd_path = nvd_path 95 96 def get_xml_dict(self): 97 print("CPE: Setting up NIST dictionary") 98 if not os.path.exists(os.path.join(self.nvd_path, "cpe")): 99 os.makedirs(os.path.join(self.nvd_path, "cpe")) 100 101 cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(CPEDB_URL)) 102 if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400: 103 print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]") 104 cpe_dict = requests.get(CPEDB_URL) 105 open(cpe_dict_local, "wb").write(cpe_dict.content) 106 107 print("CPE: Unzipping xml manifest...") 108 nist_cpe_file = gzip.GzipFile(fileobj=open(cpe_dict_local, 'rb')) 109 print("CPE: Converting xml manifest to dict...") 110 tree = ET.parse(nist_cpe_file) 111 all_cpedb = tree.getroot() 112 self.parse_dict(all_cpedb) 113 114 def parse_dict(self, all_cpedb): 115 # Cycle through the dict and build two dict to be used for custom 116 # lookups of partial and complete CPE objects 117 # The objects are then used to create new proposed XML updates if 118 # if is determined one is required 119 # Out of the different language titles, select English 120 for cpe in all_cpedb.findall(".//{http://cpe.mitre.org/dictionary/2.0}cpe-item"): 121 cpe_titles = [] 122 for title in cpe.findall('.//{http://cpe.mitre.org/dictionary/2.0}title[@xml:lang="en-US"]', ns): 123 title.tail = None 124 cpe_titles.append(title) 125 126 # Some older CPE don't include references, if they do, make 127 # sure we handle the case of one ref needing to be packed 128 # in a list 129 cpe_ref = cpe.find(".//{http://cpe.mitre.org/dictionary/2.0}references") 130 if cpe_ref: 131 for ref in cpe_ref.findall(".//{http://cpe.mitre.org/dictionary/2.0}reference"): 132 ref.tail = None 133 ref.text = ref.text.upper() 134 if ref.text not in VALID_REFS: 135 ref.text = ref.text + "-- UPDATE this entry, here are some examples and just one word should be used -- " + ' '.join(VALID_REFS) # noqa E501 136 cpe_ref.tail = None 137 cpe_ref.text = None 138 139 cpe_str = cpe.find(".//{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item").get('name') 140 item = CPE(cpe_str, cpe_titles, cpe_ref) 141 cpe_str_no_version = CPE.no_version(cpe_str) 142 # This dict must have a unique key for every CPE version 143 # which allows matching to the specific obj data of that 144 # NIST dict entry 145 self.all_cpes.update({cpe_str: item}) 146 # This dict has one entry for every CPE (w/o version) to allow 147 # partial match (no valid version) check (the obj is saved and 148 # used as seed for suggested xml updates. By updating the same 149 # non-version'd entry, it assumes the last update here is the 150 # latest version in the NIST dict) 151 self.all_cpes_no_version.update({cpe_str_no_version: item}) 152 153 def find_partial(self, cpe_str): 154 cpe_str_no_version = CPE.no_version(cpe_str) 155 if cpe_str_no_version in self.all_cpes_no_version: 156 return cpe_str_no_version 157 158 def find_partial_obj(self, cpe_str): 159 cpe_str_no_version = CPE.no_version(cpe_str) 160 if cpe_str_no_version in self.all_cpes_no_version: 161 return self.all_cpes_no_version[cpe_str_no_version] 162 163 def find_partial_latest_version(self, cpe_str_partial): 164 cpe_obj = self.find_partial_obj(cpe_str_partial) 165 return cpe_obj.cpe_cur_ver 166 167 def find(self, cpe_str): 168 if self.find_partial(cpe_str): 169 if cpe_str in self.all_cpes: 170 return cpe_str 171 172 def gen_update_xml(self, cpe_str): 173 cpe = self.find_partial_obj(cpe_str) 174 return cpe.update_xml_dict() 175