%s

- -class Document(object): - title = str() - abstract = str() - docversion = str() - encoding = str() - doctype = str() - date = str() - version = str() - license = str() - link = str() - lang = str() - - chapters = list() - authors = list() - - - def __init__(self, title, version, abstract, lang, link, docversion='1.0', - encoding='UTF-8', doctype='/dtd/guide.dtd', date=None, license=None): - - if type(title) is not str: - raise TypeError - if type(abstract) is not str: - raise TypeError - if type(docversion) is not str: - raise TypeError - if type(encoding) is not str: - raise TypeError - if type(doctype) is not str: - raise TypeError - if type(version) is not str: - raise TypeError - if type(date) is not str and date is not None: - raise TypeError - if type(license) is not str and license is not None: - raise TypeError - if type(link) is not str: - raise TypeError - if type(lang) is not str: - raise TypeError - if lang not in LANGUAGES: - raise ValueError - - self.title = title - self.abstract = abstract - self.docversion = docversion - self.encoding = encoding - self.doctype = doctype - self.version = version - self.link = link - self.lang = lang - - if date is None: - self.date = datetime.datetime.now().isoformat()[:10] - else: - self.date = date #TODO check YYYY-MM-DD - - if license is None: - self.license = LICENSE - else: - self.license = license - - - def addAuthor(self, title, name, email=None): - self.authors.append(Author(title, name, email)) - - - def append(self, chapter): - if type(chapter) is not Chapter: - raise TypeError - else: - self.chapters.append(chapter) - - - def create(self, path, filename): - xmlfile = os.path.join(path, filename) #TODO - - output = list() - - if len(self.authors) == 0: - raise ValueError - - output.append('\n' % (self.docversion, self.encoding)) - output.append('\n\n\n' % (self.doctype)) - output.append('\n%s\n\n' % (self.title)) - - for item in self.authors: - output.append('\n' % (item.title)) - if item.email is not None: - output.append(' %s\n' % (item.email, item.name)) - else: - output.append(' %s\n' % (item.name)) - - output.append('\n\n\n%s\n' % (linebreak(self.abstract))) - output.append('\n\n%s\n\n%s\n' % (self.license, self.version)) - output.append('%s\n\n' % (self.date)) - - if len(self.chapters) > 0: - for chapter in self.chapters: - output.append('\n') - if len(chapter.sections) > 0: - for section in chapter.sections: - output.append('

\n') - output.append('\n') - for tag in section.tags: - if tag.tagtype in ['pre', 'p']: - output.append('\n%s\n' % repr(tag)) - else: - output.append(repr(tag)) - - output.append('\n\n') - output.append('

\n') - output.append('\n') - - output.append('') - - print ''.join(output) - - -#TODO -def linebreak(text): - if len(text) <= MAXLENGTH: - return text - - linebreak = str() - i = 0; - while i < len(text): - if i + MAXLENGTH < len(text): - linebreak += ''.join([text[i:i + MAXLENGTH], '\n']) - else: - linebreak += text[i:i + MAXLENGTH] - i += MAXLENGTH - return linebreak - - -class Author(object): - title = str() - name = str() - email = str() - - def __init__(self, title, name, email=None): - if type(title) is not str: - raise TypeError - if type(name) is not str: - raise TypeError - if type(email) is not str and email is not None: - raise TypeError - - self.title = title - self.name = name - self.email = email - - -class Chapter(object): - title = None - sections = list() - - def __init__(self, title): - if type(title) is not str: - raise TypeError - - self.title = title - - - def append(self, section): - if type(section) is not Section: - raise TypeError - else: - self.sections.append(section) - - -class Section(object): - title = None - bodys = list() - - def __init__(self, title): - if type(title) is not str: - raise TypeError - - self.title = title - - - tags = list() - - def append(self, tag): - if type(tag) is Tag: - self.tags.append(tag) - elif type(tag) is list: - for item in tag: - self.append(item) - elif type(tag) is str: - self.tags.append(Tag('text', tag)) - else: - raise TypeError - - -class Tag(object): - tagtype = str() - text = str() - - def __init__(self, tagtype, text): - if tagtype not in TAGTYPES: - raise TypeError - if type(text) is not str: - raise TypeError - - self.tagtype = tagtype - self.text = text - - def __repr__(self): - if (len(self.tagtype) * 2 + 5 + len(self.text)) > MAXLENGTH: - return '<%s>\n%s\n' % (self.tagtype, self.text, self.tagtype) - else: - return '<%s>%s' % (self.tagtype, self.text, self.tagtype) - -def preserve(text): - return Tag('pre', text) - - -def paragraph(text): - return Tag('p', text) - - -def warning(text): - return Tag('warn', text) - - -def important(text): - return Tag('impo', text) - - -def note(text): - return Tag('note', text) - - -def comment(text): - return Tag('comment', text) - - -def path(text): - return Tag('path', text) - - -def command(text): - return Tag('c', text) - - -def userinput(text): - return Tag('i', text) - - -def keyword(text): - return Tag('keyword', text) - - -def identifier(text): - return Tag('ident', text) - - -def constant(text): - return Tag('const', text) - - -def statement(text): - return Tag('stmt', text) - - -def variable(text): - return Tag('var', text) - - -def bold(text): - return Tag('b', text) - - -def emphasize(text): - return Tag('e', text) - - -def subscript(text): - return Tag('sub', text) - - -def superscript(text): - return Tag('sup', text) - - -def uri(text): - return Tag('uri', text) - diff --git a/kernel-check.py b/kernel-check.py index 9c7e563..a6d6051 100755 --- a/kernel-check.py +++ b/kernel-check.py @@ -9,7 +9,7 @@ import sys import textwrap import os -import kernellib as lib +import lib.kernellib as lib info = portage.output.EOutput().einfo warn = portage.output.EOutput().ewarn @@ -61,8 +61,10 @@ def main(argv): (kernel.version, kernel.revision)))) info('Kernel source : %s' % color('GOOD', kernel.source)) - kernel.genpatch = lib.get_genpatch(lib.read_genpatch_file(lib.DIR['out']), - kernel) + + genpatches = lib.parse_genpatch_list(lib.PORTDIR) + + kernel.genpatch = lib.get_genpatch(genpatches, kernel) if kernel.genpatch is not None: info('Gen(too)patch : %s' % color('GOOD', '%s %s' % @@ -79,6 +81,7 @@ def main(argv): print '\n>>> Reading all kernel vulnerabilities' + """ supported = list() for item in lib.SUPPORTED: best = (lib.all_version(item)) @@ -88,6 +91,7 @@ def main(argv): i.genpatch = lib.get_genpatch(lib.read_genpatch_file( lib.DIR['out']), i) supported.append(i) + """ kernel_eval = lib.eval_cve_files(lib.DIR['out'], kernel, arch) if not kernel_eval: @@ -109,6 +113,7 @@ def main(argv): color('BAD', str(len(kernel_eval.affected)))) print_summary(kernel_eval.affected) + """ info('You have the following choices: ') print '' @@ -148,6 +153,7 @@ def main(argv): color('BAD', str(len(kernel_eval.affected))), color('BAD', str(len(comparison.new))))) print '' + """ print_information() print_beta() @@ -244,17 +250,15 @@ def print_beta(): 'Prints a beta warning message' print('') - error('%s You are using a very early version of kernel-check.' % + error('%s You are using a early version of kernel-check.' % color('BAD', 'IMPORTANT')) error('Please note that this tool might not operate as expected.') - error('Moreover the given information are most likely incorrect.') def print_information(): 'Prints an information message' info('To print more information about a vulnerability try:') - info('') info(' $ %s -s [bugid|cve]' % sys.argv[0]) diff --git a/kernellib.py b/kernellib.py deleted file mode 100755 index fe63a3e..0000000 --- a/kernellib.py +++ /dev/null @@ -1,1068 +0,0 @@ -#!/usr/bin/env python -# kernel-check -- Kernel security information -# Copyright 2009-2009 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -from __future__ import with_statement -from contextlib import closing -import xml.etree.cElementTree as et -import cStringIO -import datetime -import inspect -import logging -import mmap -import os -import portage -import re -import urllib - - -ARCHES = [ - 'all', 'alpha', 'amd64', 'amd64-fbsd', 'arm', 'hppa', 'ia64', 'm68k', - 'mips', 'ppc', 'ppc64', 's390', 'sh', 'sparc', 'sparc-fbsd', 'x86', - 'x86-fbsd' -] - -BUGORDER = ['bugid', 'reporter', 'reported', 'status', 'arch', 'affected'] -CVEORDER = ['cve', 'published', 'desc', 'severity', 'vector', 'score', 'refs'] - -REGEX = { - 'bugzilla' : re.compile(r'(?<=bug.cgi\?id=)\d*'), - 'gp_version' : re.compile(r'(?<=K_GENPATCHES_VER\=\").+(?=\")'), - 'gp_want' : re.compile(r'(?<=K_WANT_GENPATCHES\=\").+(?=\")'), - 'grp_all' : re.compile(r'(?<=$)[ (]*CVE-(\d{4})' \ - r'([-,(){}|, \d]+)(?=$)'), - 'grp_split' : re.compile(r'(?<=\D)(\d{4})(?=\D|$)'), - 'm_nomatch' : re.compile(r'.*GENERIC-MAP-NOMATCH.*'), - 'wb_match' : re.compile(r'\s*\[\s*([^ +<=>]+)\s*(\+?)' \ - r'\s*([<=>]{1,2})\s*([^ <=>\]' \ - r']+)\s*(?:([<=>]{1,2})\s*([^' \ - r' \]]+))?\s*\]\s*(.*)'), - 'wb_version' : re.compile(r'^(?:\d{1,2}\.){0,3}\d{1,2}' \ - r'(?:[-_](?:r|rc)?\d{1,2})*$'), - 'k_version' : re.compile(r'^((?:\d{1,2}\.){0,4}\d{1,2})(-.*)?$'), - 'rc_kernel' : re.compile(r'^rc\d{1,3}$'), - 'git_kernel' : re.compile(r'^git(\d{1,3})$'), - 'r_kernel' : re.compile(r'^r\d{1,3}$') -} - -SUPPORTED = ['gentoo', 'vanilla', 'hardened'] - -KERNEL_TYPES = [ - 'aa', 'acpi', 'ac', 'alpha', 'arm', 'as', 'cell', 'ck', 'compaq', 'crypto', - 'development', 'gaming','gentoo-dev', 'gentoo', 'gentoo-test', 'gfs', - 'git', 'grsec', 'gs', 'hardened-dev', 'hardened', 'hppa-dev', 'hppa', - 'ia64', 'kurobox', 'linux', 'lolo', 'mips-prepatch', 'mips', 'mjc', 'mm', - 'mosix', 'openblocks', 'openmosix','openvz', 'pac', 'pegasos-dev', - 'pegasos', 'pfeifer', 'planet-ccrma', 'ppc64', 'ppc-development', - 'ppc-dev', 'ppc', 'redhat', 'rsbac-dev', 'rsbac', 'selinux', 'sh', - 'sparc-dev', 'sparc', 'suspend2', 'systrace', 'tuxonice', 'uclinux', - 'usermode', 'vanilla-prepatch', 'vanilla', 'vanilla-tiny', 'vserver-dev', - 'vserver', 'win4lin', 'wolk-dev', 'wolk', 'xbox', 'xen', 'xfs' -] - -VERSION = '0.3.8' -NOCVE = 'GENERIC-MAP-NOMATCH' -NOCVEDESC = 'This GENERIC identifier is not specific to any vulnerability. '\ - 'GENERIC-MAP-NOMATCH is used by products, databases, and ' \ - 'services to specify when a particular vulnerability element ' \ - 'does not map to a corresponding CVE entry.' -CVES = dict() -PORTDIR = portage.settings['PORTDIR'] -DEBUG = False -VERBOSE = False -FORCE = False -SKIP = False -DELAY = 0 -FILEPATH = os.path.dirname(os.path.realpath(__file__)) -DIR = { - 'tmp' : os.path.join(FILEPATH, 'tmp'), - 'out' : os.path.join(FILEPATH, 'out'), - 'bug' : os.path.join(FILEPATH, 'tmp', 'bug'), - 'nvd' : os.path.join(FILEPATH, 'tmp', 'nvd') -} - -def BUG_ON(msg): - if DEBUG: - print 'DEBUG line %s in %s(): %s' % (inspect.stack()[1][2], - inspect.stack()[1][3], msg) - -class InvalidWhiteboardError(Exception): - def __init__(self, value): - self.value = value - -class InvalidCveError(Exception): - def __init__(self, value): - self.value = value - -class CveDuplicateError(Exception): - def __init__(self, value): - self.value = value - -class NvdEntryError(Exception): - def __init__(self, value): - self.value = value - -class Evaluation: - """Evaluation class - - Provides information about the vulnerability of a kernel. - """ - - read = int() - arch = int() - affected = list() - unaffected = list() - - def __init__(self): - self.affected = list() - self.unaffected = list() - - -class Comparison: - """Comparison class - """ - - fixed = int() - new = list() - #TODO add more information - - def __init__(self): - self.fixed = list() - self.new = list() - - -class Cve: - """Common vulnerabilities and exposures class - - Contains all important information about a cve. - - Attributes: - cve: a string represeting the cve number of the class. - desc: a string providing a detailed description for the cve. - published: a string representing the original cve release date. - refs: a list of external references. - severity: a string representing the cve severity. - score: a floating point representing cvss base score. - vector: a string providing the cve access vector. - """ - - cve = str() - desc = str() - published = str() - refs = list() - severity = str() - score = float() - vector = str() - - def __init__(self, cve): - self.cve = cve - - def __eq__(self, other): - return (self.cve == other.cve) #FIXME is this enough? - - def __ne__(self, other): - return not self.__eq__(other) - - -class Genpatch: - 'Genpatch class' - - base = bool() - extras = bool() - kernel = None - version = str() - - def __init__(self, version): - self.version = version - - - def __repr__(self): - if self.base and self.extras: - return 'base extras' - if self.base: - return 'base' - if self.extras: - return 'extras' - - - def __eq__(self, other): - if self.kernel == other.kernel: - return (''.join((str(self.base), str(self.extras), self.version)) - == ''.join((str(other.base), str(other.extras), other.version))) - else: - return False - - - def __ne__(self, other): - return not self.__eq__(other) - - -class Kernel: - 'Kernel class' - - revision = str() - source = str() - version = str() - genpatch = None - - def __init__(self, source): - self.source = source - - - def __repr__(self): - return str(self.version + '-' + self.source + '-' + self.revision) - - - def __eq__(self, other): - return (''.join((self.revision, self.source, self.version, - str(self.genpatch))) == ''.join((other.revision, - other.source, other.version, str(other.genpatch)))) - - - def __ne__(self, other): - return not self.__eq__(other) - - -class Vulnerability: - 'Vulnerability class' - - arch = str() - bugid = int() - cvelist = list() - cves = list() - affected = list() - reported = str() - reporter = str() - status = str() - - def __init__(self, bugid): - self.bugid = bugid - - def __eq__(self, other): - return (self.bugid == other.bugid) #FIXME is this enough? - - def __ne__(self, other): - return not self.__eq__(other) - - -class Interval: - """Interval class - - Provides one interval entry for a vulnerability - - Attributes: - name: a string representing the name of the kernel release - lower: a string representing the lower boundary of the interval - upper: a string representing the upper boundary of the interval - lower_i: a boolean indicating if the lower boundary is inclusive - upper_i: a boolean indicating if the upper boundary is inclusive - expand: a boolean indicating if the interval is shadowing other intervals - """ - - name = str() - lower = str() - upper = str() - lower_i = bool() - upper_i = bool() - expand = str() - - def __init__(self, name, lower, upper, lower_i, upper_i, expand): - if name == 'linux' or name == 'genpatches': - pass - elif name == 'gp': - name = 'genpatches' - - name = name.replace('-sources', '') - - self.name = name - self.lower_i = lower_i - self.upper_i = upper_i - if name == 'genpatches': - if lower: - self.lower = lower.replace('-','.') - else: - self.lower = lower - if upper: - self.upper = upper.replace('-','.') - else: - self.upper = upper - else: - self.lower = lower - self.upper = upper - - self.expand = expand - - - def __repr__(self): - interval = str(self.name) - if self.expand: - interval += '+' - interval += ' ' - if self.lower and self.lower_i: - interval += '>=%s ' % (self.lower) - if self.lower and not self.lower_i: - interval += '>%s ' % (self.lower) - if self.upper and self.upper_i: - interval += '<=%s' % (self.upper) - if self.upper and not self.upper_i: - interval += '<%s' % (self.upper) - - return interval - - -def interval_to_xml(interval, root): - 'Formats an interval for xml output' - - intnode = et.Element('interval') - intnode.set('source', interval.name) - - root.append(intnode) - - for item in ('lower', 'upper'): - if getattr(interval, item): - node = et.SubElement(intnode, item) - node.text = getattr(interval, item) - node.set('inclusive', str(getattr(interval, item + '_i')).lower()) - - -def interval_from_xml(root): - 'Returns an interval from xml' - - name = root.get('source') - - lower = '' - upper = '' - lower_i = False - upper_i = False - expand = '' #TODO implement - - if root.find('lower') is not None: - lower = root.find('lower').text - lower_i = (root.find('lower').get('inclusive') == 'true') - - if root.find('upper') is not None: - upper = root.find('upper').text - upper_i = (root.find('upper').get('inclusive') == 'true') - - return Interval(name, lower, upper, lower_i, upper_i, expand) - - -#TODO Use exceptions -def is_in_interval(interval, kernel, bugid=None): - 'Returns True if the given version is inside our specified interval' - - version = str() - - if interval.name == 'linux': - version = kernel.version - - elif interval.name == 'genpatches': - version = kernel.version.replace('-', '.') - - elif interval.name == 'hardened': - version = kernel.version #TODO is this correct? - - elif interval.name == 'xen': - version = kernel.version #TODO is this correct? - - elif interval.name == 'vserver': - return False - - else: - BUG_ON(interval.name + ' ' + bugid.bugid) - return False - - for item in ['lower', 'upper']: - if getattr(interval, item): - result = portage.versions.vercmp(version, getattr(interval, item)) - - if result == None: - BUG_ON('Could not compare %s and %s' % - (getattr(interval, item),version)) - - if result == 0 and not getattr(interval, item + '_i'): - return False - - if result == 0 and getattr(interval, item + '_i'): - return True - - if item == 'lower' and result < 0: - return False - - if item == 'upper' and result > 0: - return False - - return True - - -def extract_genpatch(ebuild, directory, sources): - 'Returns a genpatch from an ebuild' - - pkg = portage.versions.catpkgsplit('sys-kernel/%s' % ebuild[:-7]) - - with open(os.path.join(directory, sources, ebuild), 'r') as ebuild_file: - content = ebuild_file.read() - - try: - genpatch_v = REGEX['gp_version'].findall(content)[0] - genpatch_w = REGEX['gp_want'].findall(content)[0] - except: - return None - - kernel = Kernel(pkg[1].replace('-sources', '')) - kernel.version = pkg[2] - kernel.revision = pkg[3] - - genpatch = Genpatch(pkg[2] + '-' + genpatch_v) - genpatch.kernel = kernel - genpatch.base = ('base' in genpatch_w) - genpatch.extras = ('extras' in genpatch_w) - - return genpatch - - -def parse_genpatch_list(directory): - 'Returns a list containing all genpatches from portage' - - patches = list() - directory = os.path.join(directory, 'sys-kernel') - - for sources in os.listdir(directory): - if '-sources' in sources: - for ebuild in os.listdir(os.path.join(directory, sources)): - if '.ebuild' in ebuild: - genpatch = extract_genpatch(ebuild, directory, sources) - - if genpatch is not None: - patches.append(genpatch) - - return patches - - -def read_genpatch_file(directory): - 'Read the genpatch file created by collector' - - patches = list() - filename = os.path.join(directory, 'genpatches.xml') - - try: - with open(filename, 'r+') as xml_data: - memory_map = mmap.mmap(xml_data.fileno(), 0) - root = et.parse(memory_map).getroot() - - except SyntaxError: - return list() - - except IOError: - return list() - - for tree in root: - kernel = extract_version(tree.get('kernel')) - - if kernel is None: - continue - - genpatch = Genpatch(tree.get('version')) - genpatch.kernel = kernel - genpatch.base = (tree.get('base') == 'true') - genpatch.extras = (tree.get('extras') == 'true') - - patches.append(genpatch) - - return patches - - -def write_genpatch_file(directory, patches): - 'Write the genpatch file with all genpatches' - - filename = os.path.join(directory, 'genpatches.xml') - root = et.Element('patches') - - for item in patches: - genpatch = et.SubElement(root, 'genpatch') - genpatch.set('kernel', repr(item.kernel)) - genpatch.set('version', item.version) - genpatch.set('base', str(item.base).lower()) - genpatch.set('extras', str(item.extras).lower()) - - write_xml(root, filename) - - -def get_genpatch(patches, kernel): - 'Returns the genpatch for a specific kernel' - - for item in patches: - if item.kernel == kernel: - return item - - return None - - -def parse_bugzilla_list(directory): - 'Returns a list containing all bugzilla kernel bugs' - - filename = os.path.join(directory, 'bugzilla.xml') - - with open(filename, 'r+') as buglist_file: - memory_map = mmap.mmap(buglist_file.fileno(), 0) - - buglist = REGEX['bugzilla'].findall(memory_map.read(-1)) - - return buglist - - -def parse_bugzilla_dict(directory, bugid): - 'Returns a vulnerability class containing information about a bug' - - filename = os.path.join(directory, bugid) - - try: - with open(filename, 'r+') as xml_data: - memory_map = mmap.mmap(xml_data.fileno(), 0) - root = et.parse(memory_map).getroot()[0] - - except IOError: #FIXME Handle Exception - return - - vul = Vulnerability(bugid) - - try: - vul.cvelist = extract_cves(root.find('short_desc').text) - if not vul.cvelist: - raise InvalidCveError(root.find('short_desc').text) - - for item in vul.cvelist: - if item != NOCVE: - if item not in CVES: - CVES[item] = vul.bugid - else: - raise CveDuplicateError(CVES[item]) - - vul.arch = root.find('rep_platform').text.lower() - vul.reported = root.find('creation_ts').text - vul.reporter = root.find('reporter').text.lower() - vul.status = root.find('bug_status').text.lower() - - except AttributeError: - #TODO Error - pass - - try: - vul.affected = interval_from_whiteboard( - root.find('status_whiteboard').text) - - except AttributeError: - raise InvalidWhiteboardError('Empty') - - return vul - - -def search_nvd_dict(nvd, vul): - 'Adds all matching cves found in the nvd dicitonay to vul' - - cves = list() - for item in vul.cvelist: - if item == NOCVE: - vul.cves = list() - return vul - - try: - cves.append(nvd[item]) - except KeyError: - raise NvdEntryError(item) - - vul.cves = cves - - return vul - - -def parse_nvd_dict(directory): - 'Returns a dictionary from the National Vulnerability Database' - - nvd = dict() - - for nvdfile in os.listdir(directory): - filename = os.path.join(directory, nvdfile) - - with open(filename, 'r+') as xml_data: - memory_map = mmap.mmap(xml_data.fileno(), 0) - root = et.parse(memory_map).getroot() - - namespace = root.tag[:-3] - - for tree in root: - cve = Cve(tree.get('name')) - cve.published = tree.get('published') - cve.severity = tree.get('severity') - cve.vector = tree.get('CVSS_vector') - cve.score = tree.get('CVSS_score') - - #FIXME - desc = tree.find('%sdesc/%sdescript/' % (namespace, namespace)) - if desc is not None: - cve.desc = desc.text - - #TODO Rework! - reftree = tree.find(namespace + 'refs') - reftree.tag = reftree.tag.replace(namespace, '') - for elem in reftree.findall('.//*'): - elem.tag = elem.tag.replace(namespace, '') - bugref = et.SubElement(reftree, 'ref') - bugref.set('source', 'GENTOO') - bugref.set('url', 'https://bugs.gentoo.org/show_bug.cgi?id=%s' % - cve.cve) - bugref.text = 'Gentoo %s' % cve.cve - - cve.refs = reftree - - nvd[cve.cve] = cve - - return nvd - - -def extract_cves(string): - 'Returns a list containing all CVEs of a particular string' - - cves = list() - string = string.replace('CAN', 'CVE') - - if string in REGEX['m_nomatch'].findall(string): - return [NOCVE] - - for (year, split_cves) in REGEX['grp_all'].findall(string): - for cve in REGEX['grp_split'].findall(split_cves): - cves.append('CVE-%s-%s' % (year, cve)) - - return cves - - -def parse_cve_files(directory): - 'Returns all bug files as list' - - files = list() - - if (os.path.exists(directory)): - for item in os.listdir(directory): - try: - cve_file = read_cve_file(directory, item[:-4]) - if cve_file is not None: - files.append(cve_file) - - except AttributeError: - pass - - return files - - -def find_cve(cve, directory): - 'Returns a bug containing the cve' - - for item in parse_cve_files(directory): - for cves in item.cves: - if cve == cves.cve: - return item - - return None - - -def eval_cve_files(directory, kernel, arch): - 'Returns a vulnerabilty evaluation' - - files = parse_cve_files(directory) - - if not files: - return None - - evaluation = Evaluation() - - for item in files: - evaluation.read += 1 - - if item.arch not in ARCHES: - BUG_ON('[Error] Wrong architecture %s in bugid: %s' % - (item.arch, item.bugid)) - - if item.arch != arch and item.arch != 'all': - evaluation.unaffected.append(item) - else: - evaluation.arch += 1 - - if is_affected(item.affected, kernel, item): - evaluation.affected.append(item) - else: - evaluation.unaffected.append(item) - - return evaluation - - -def is_affected(interval_list, kernel, item): #TODO Remove item - 'Returns true if a kernel is affected' - - kernel_gentoo = (kernel.source == 'gentoo' and kernel.genpatch is not None) - kernel_affected = False - kernel_linux_affected = False - kernel_gp_affected = False - #TODO kernel_gp_exp_affected = False - linux_interval = False - gentoo_interval = False - - for interval in interval_list: - if interval.name == 'genpatches': - gentoo_interval = True - if kernel_gentoo: - if is_in_interval(interval, kernel.genpatch, item): - kernel_gp_affected = True - - elif interval.name == 'linux': - linux_interval = True - if is_in_interval(interval, kernel, item): - kernel_linux_affected = True - - else: - pass #TODO - - if linux_interval: - if kernel_linux_affected: - if gentoo_interval and kernel_gentoo: - if kernel_gp_affected: - kernel_affected = True - else: - kernel_affected = False - else: - kernel_affected = True - else: - kernel_affected = False - else: - if kernel_gentoo and gentoo_interval: - if kernel_gp_affected: - kernel_affected = True - else: - kernel_affected = False - #TODO Implement else for hardend/xen/expand - - return kernel_affected - - -def compare_evaluation(kernel, compare): - 'Creates a comparison out of two evaluation instances' - - comparison = Comparison() - - if kernel.read != compare.read or kernel.arch != compare.arch: - BUG_ON('Kernels do not match: %s %s' % (kernel1.read, kernel2.read)) - return - - for item in kernel.affected: - if item not in compare.affected: - comparison.fixed.append(item) - - for item in compare.affected: - if item not in kernel.affected: - comparison.new.append(item) - - return comparison - - -def read_cve_file(directory, bugid): - 'Read a bug file created by collector' - - cves = list() - affected = list() - - filename = os.path.join(directory, bugid + '.xml') - - try: - with open(filename, 'r+') as xml_data: - memory_map = mmap.mmap(xml_data.fileno(), 0) - root = et.parse(memory_map).getroot() - except IOError: - return None - - bugroot = root.find('bug') - - vul = Vulnerability(bugroot.find('bugid').text) - vul.arch = bugroot.find('arch').text - vul.reported = bugroot.find('reported').text - vul.reporter = bugroot.find('reporter').text - vul.status = bugroot.find('status').text - - affectedroot = bugroot.find('affected') - - for item in affectedroot: - interval = interval_from_xml(item) - affected.append(interval) - - vul.affected = affected - - for item in root: - if item.tag == 'cve': - cve = Cve(item.find('cve').text) - cve.desc = item.find('desc').text - cve.published = item.find('published').text - cve.refs = item.find('refs').text #FIXME - cve.severity = item.find('severity').text - cve.score = item.find('score').text - cve.vector = item.find('vector').text - - cves.append(cve) - vul.cves = cves - - return vul - - -def write_cve_file(directory, vul): - 'Write a bug file containing all important information for kernel-check' - - filename = os.path.join(directory, vul.bugid + '.xml') - - root = et.Element('vulnerability') - bugroot = et.SubElement(root, 'bug') - - for element in BUGORDER: - if element == 'affected': - affectedroot = et.SubElement(bugroot, 'affected') - for item in vul.affected: - interval_to_xml(item, affectedroot) - else: - node = et.SubElement(bugroot, element) - node.text = getattr(vul, element) - - for cve in vul.cves: - cveroot = et.SubElement(root, 'cve') - for element in CVEORDER: - if element == 'refs': - cveroot.append(cve.refs) - else: - node = et.SubElement(cveroot, element) - node.text = getattr(cve, element) - - write_xml(root, filename) - - -def write_xml(root, filename): - 'Write root to a xml file' - - with open(filename, 'w') as xmlout: - __indent__(root) - doc = et.ElementTree(root) - doc.write(xmlout, encoding='utf-8') - - -def __indent__(node, level=0): - 'Indents xml layout for printing' - - i = '\n' + level * ' ' * 4 - if len(node): - if not node.text or not node.text.strip(): - node.text = i + ' ' * 4 - if not node.tail or not node.tail.strip(): - node.tail = i - for node in node: - __indent__(node, level + 1) - if not node.tail or not node.tail.strip(): - node.tail = i - else: - if level and (not node.tail or not node.tail.strip()): - node.tail = i - - -def interval_from_whiteboard(whiteboard): - 'Returns a list of intervals within a whiteboard string' - - expand = False #TODO - upper_inc = None - upper = None - lower_inc = None - lower = None - - affected = list() - - while len(whiteboard.strip()) > 0: - match = REGEX['wb_match'].match(whiteboard) - if not match: - raise InvalidWhiteboardError(whiteboard) - - name = match.group(1) - exp = match.group(2) - comp1 = match.group(3) - vers1 = match.group(4) - comp2 = match.group(5) - vers2 = match.group(6) - - if exp == '+': - expand = True - - if comp1 == '=' or comp1 == '==': - lower_inc = True - upper_inc = True - lower = vers1 - upper = vers1 - - if not REGEX['wb_version'].match(vers1): - raise InvalidWhiteboardError(whiteboard) - else: - for (char, version) in ((comp1, vers1), (comp2, vers2)): - if char == '<': - upper_inc = False - upper = version - elif char == '<=' or char == '=<': - upper_inc = True - upper = version - elif char == '>': - lower_inc = False - lower = version - elif char == '>=' or char == '=>': - lower_inc = True - lower = version - elif char: - raise InvalidWhiteboardError(whiteboard) - - if version and not REGEX['wb_version'].match(version): - raise InvalidWhiteboardError(whiteboard) - - affected.append(Interval(name, lower, upper, lower_inc, - upper_inc, expand)) - - whiteboard = match.group(7) - - return affected - - -#TODO Use Exceptions -def extract_version(release): - 'Extracts revision, source and version out of a release tag' - - match = REGEX['k_version'].match(release) - if not match: - BUG_ON('[Error] Release %s does not contain any valid information' % - release) - return None - - version, rest = match.groups() - - kernel = Kernel('vanilla') - kernel.revision = 'r0' - kernel.version = version - - for elem in (rest or '').split('-'): - if elem == 'sources': - pass - elif REGEX['rc_kernel'].match(elem): - kernel.version += '_' + elem - elif REGEX['git_kernel'].match(elem): - kernel.source = 'git' - kernel.revision = 'r' + REGEX['gitd'].match(elem).groups()[0] - elif REGEX['r_kernel'].match(elem): - kernel.revision = elem - elif elem in KERNEL_TYPES: - kernel.source = elem - elif elem != '': - BUG_ON('[Error] Dropping unknown version component \'%s\', \ - probably local tag.' % elem) - - return kernel - - -def all_version(source): - """ Given a kernel source name (e.g. vanilla), returns a Kernel object - for the latest revision in the tree, or None if none exists. """ - - versions = list() - - porttree = portage.db[portage.root]['porttree'] - matches = porttree.dbapi.xmatch('match-all', - 'sys-kernel/%s-sources' % source) - - for item in matches: - best = portage.versions.catpkgsplit(item) - if not best: - continue - - kernel = Kernel(best[1].replace('-sources', '')) - kernel.version = best[2] - kernel.revision = best[3] - - versions.append(kernel) - - return versions - - -#TODO Remove BUG_ON; use Exceptions -def receive_file(directory, path, xml_file, - max_age = datetime.timedelta(0, 59*60)): - 'Generic download function' - - filename = os.path.join(directory, xml_file) - - if not FORCE: - if os.path.exists(filename): - age = datetime.datetime.now() - \ - datetime.datetime.fromtimestamp(os.path.getmtime(filename)) - if age < max_age: - BUG_ON('File %s - %sKB is recent enough [%s]' % - (filename, os.path.getsize(filename)/1024, - str(age)[:-7])) - return - try: - with closing(cStringIO.StringIO()) as data: - with closing(urllib.urlopen(path + xml_file)) as resource: - data.write(resource.read()) - - with open(filename, 'w') as output: - output.write(data.getvalue()) - - except IOError: - BUG_ON('Download failed!') #FIXME Handle exception - - BUG_ON('File %s - %sKB received' % - (filename, os.path.getsize(filename)/1024)) - - -def receive_nvd_recent(directory): - 'Download the latest CVEs file from the National Vulnerability Database' - - url = 'http://nvd.nist.gov/download/' - - receive_file(directory, url, 'nvdcve-recent.xml') - - -def receive_nvd_all(directory): - 'Download all earlier CVEs files from the National Vulnerability Database' - - url = 'http://nvd.nist.gov/download/' - year = datetime.datetime.now().year - - if year < 2002 or year > 2020: - year = 2020 - - for i in xrange(2002, year + 1): - receive_file(directory, url, 'nvdcve-%s.xml' % str(i), - max_age = datetime.timedelta(1)) - - -def receive_bugzilla_list(directory): - 'Download a list containing all Bugzilla kernel bugs' - - status = ['NEW', 'ASSIGNED', 'REOPENED', 'RESOLVED', 'VERIFIED', 'CLOSED'] - resolution = ['FIXED', 'LATER', 'TEST-REQUEST', 'UPSTREAM', '---'] - - url = ['https://bugs.gentoo.org/buglist.cgi?', - 'query_format=advanced&component=Kernel'] - - for i in status: - url.append('&bug_status=' + i) - for i in resolution: - url.append('&resolution=' + i) - url.append('#') - - receive_file(directory, ''.join(url), 'bugzilla.xml') - - -def receive_bugzilla_bug(directory, bugid): - 'Download the xml file of a particular Bugzilla kernel bug' - - url = 'https://bugs.gentoo.org/show_bug.cgi?ctype=xml&id=' - - receive_file(directory, url, bugid) - diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/guidexml.py b/lib/guidexml.py new file mode 100755 index 0000000..d6babb8 --- /dev/null +++ b/lib/guidexml.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python +# guidexml -- guidexml class for python +# Copyright 2009-2009 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import datetime +import os +import xml.etree.cElementTree + +MAXLENGTH = 80 + +LANGUAGES = ['en', 'de'] + +LICENSE = ''.join(['\n', +'\n']) + +TAGTYPES = ['p', 'c', 'b', 'e', 'i', 'pre', 'path', 'uri', 'note', 'warn', 'impo', + 'comment', 'sub', 'sup', 'keyword', 'ident', 'const', 'stmt', 'var'] + +#pre only in first body +#

+
+class Document(object):
+    title      = str()
+    abstract   = str()
+    docversion = str()
+    encoding   = str()
+    doctype    = str()
+    date       = str()
+    version    = str()
+    license    = str()
+    link       = str()
+    lang       = str()
+
+    chapters   = list()
+    authors    = list()
+
+
+    def __init__(self, title, version, abstract, lang, link, docversion='1.0',
+        encoding='UTF-8', doctype='/dtd/guide.dtd', date=None, license=None):
+
+        if type(title) is not str:
+            raise TypeError
+        if type(abstract) is not str:
+            raise TypeError
+        if type(docversion) is not str:
+            raise TypeError
+        if type(encoding) is not str:
+            raise TypeError
+        if type(doctype) is not str:
+            raise TypeError
+        if type(version) is not str:
+            raise TypeError
+        if type(date) is not str and date is not None:
+            raise TypeError
+        if type(license) is not str and license is not None:
+            raise TypeError
+        if type(link) is not str:
+            raise TypeError
+        if type(lang) is not str:
+            raise TypeError
+        if lang not in LANGUAGES:
+            raise ValueError
+
+        self.title = title
+        self.abstract = abstract
+        self.docversion = docversion
+        self.encoding = encoding
+        self.doctype = doctype
+        self.version = version
+        self.link = link
+        self.lang = lang
+
+        if date is None:
+            self.date = datetime.datetime.now().isoformat()[:10]
+        else:
+            self.date = date #TODO check YYYY-MM-DD
+
+        if license is None:
+            self.license = LICENSE
+        else:
+            self.license = license
+
+
+    def addAuthor(self, title, name, email=None):
+        self.authors.append(Author(title, name, email))
+
+
+    def append(self, chapter):
+        if type(chapter) is not Chapter:
+            raise TypeError
+        else:
+            self.chapters.append(chapter)
+
+
+    def create(self, path, filename):
+        xmlfile = os.path.join(path, filename) #TODO
+
+        output = list()
+        
+        if len(self.authors) == 0:
+            raise ValueError
+
+        output.append('\n' % (self.docversion, self.encoding))
+        output.append('\n\n\n' % (self.doctype))
+        output.append('\n%s\n\n' % (self.title))
+
+        for item in self.authors:
+            output.append('\n' % (item.title))
+            if item.email is not None:
+                output.append('  %s\n' % (item.email, item.name))
+            else:
+                output.append('  %s\n' % (item.name))
+
+        output.append('\n\n\n%s\n' % (linebreak(self.abstract)))
+        output.append('\n\n%s\n\n%s\n' % (self.license, self.version))
+        output.append('%s\n\n' % (self.date))
+
+        if len(self.chapters) > 0:
+            for chapter in self.chapters:
+                output.append('\n')
+                if len(chapter.sections) > 0:
+                    for section in chapter.sections:
+                        output.append('\n')
+                        output.append('\n')
+                        for tag in section.tags:
+                            if tag.tagtype in ['pre', 'p']:
+                                output.append('\n%s\n' % repr(tag))
+                            else:
+                                output.append(repr(tag))
+
+                        output.append('\n\n')
+                        output.append('\n')
+                output.append('\n')
+
+        output.append('')
+        
+        print ''.join(output)
+
+
+#TODO
+def linebreak(text):
+    if len(text) <= MAXLENGTH:
+        return text
+
+    linebreak = str()
+    i = 0;
+    while i < len(text):
+        if i + MAXLENGTH < len(text):
+            linebreak += ''.join([text[i:i + MAXLENGTH], '\n'])
+        else:
+            linebreak += text[i:i + MAXLENGTH]
+        i += MAXLENGTH
+    return linebreak
+
+
+class Author(object):
+    title = str()
+    name  = str()
+    email = str()
+
+    def __init__(self, title, name, email=None):
+        if type(title) is not str:
+            raise TypeError
+        if type(name) is not str:
+            raise TypeError
+        if type(email) is not str and email is not None:
+            raise TypeError
+
+        self.title = title
+        self.name = name
+        self.email = email
+
+
+class Chapter(object):
+    title    = None
+    sections = list()
+
+    def __init__(self, title):
+        if type(title) is not str:
+            raise TypeError
+
+        self.title = title
+
+
+    def append(self, section):
+        if type(section) is not Section:
+            raise TypeError
+        else:
+            self.sections.append(section)
+
+
+class Section(object):
+    title = None
+    bodys = list()
+
+    def __init__(self, title):
+        if type(title) is not str:
+            raise TypeError
+
+        self.title = title
+
+
+    tags = list()
+
+    def append(self, tag):
+        if type(tag) is Tag:
+            self.tags.append(tag)
+        elif type(tag) is list:
+            for item in tag:
+                self.append(item)
+        elif type(tag) is str:
+            self.tags.append(Tag('text', tag))
+        else:
+            raise TypeError
+
+
+class Tag(object):
+    tagtype = str()
+    text    = str()
+
+    def __init__(self, tagtype, text):
+        if tagtype not in TAGTYPES:
+            raise TypeError
+        if type(text) is not str:
+            raise TypeError
+
+        self.tagtype = tagtype
+        self.text = text
+
+    def __repr__(self):
+        if (len(self.tagtype) * 2 + 5 + len(self.text)) > MAXLENGTH:
+            return '<%s>\n%s\n' % (self.tagtype, self.text, self.tagtype)
+        else:
+            return '<%s>%s' % (self.tagtype, self.text, self.tagtype)
+
+def preserve(text):
+    return Tag('pre', text)
+
+
+def paragraph(text):
+    return Tag('p', text)
+
+
+def warning(text):
+    return Tag('warn', text)
+
+
+def important(text):
+    return Tag('impo', text)
+
+
+def note(text):
+    return Tag('note', text)
+
+
+def comment(text):
+    return Tag('comment', text)
+
+
+def path(text):
+    return Tag('path', text)
+
+
+def command(text):
+    return Tag('c', text)
+
+
+def userinput(text):
+    return Tag('i', text)
+
+
+def keyword(text):
+    return Tag('keyword', text)
+
+
+def identifier(text):
+    return Tag('ident', text)
+
+
+def constant(text):
+    return Tag('const', text)
+
+
+def statement(text):
+    return Tag('stmt', text)
+
+
+def variable(text):
+    return Tag('var', text)
+
+
+def bold(text):
+    return Tag('b', text)
+
+
+def emphasize(text):
+    return Tag('e', text)
+
+
+def subscript(text):
+    return Tag('sub', text)
+
+
+def superscript(text):
+    return Tag('sup', text)
+
+
+def uri(text):
+    return Tag('uri', text)
+
diff --git a/lib/kernellib.py b/lib/kernellib.py
new file mode 100755
index 0000000..d320c20
--- /dev/null
+++ b/lib/kernellib.py
@@ -0,0 +1,640 @@
+#!/usr/bin/env python
+# kernel-check -- Kernel security information
+# Copyright 2009-2009 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from __future__ import with_statement
+from contextlib import closing
+import xml.etree.cElementTree as et
+import cStringIO
+import datetime
+import inspect
+import logging
+import mmap
+import os
+import portage
+import re
+import urllib
+
+
+ARCHES = [
+    'all', 'alpha', 'amd64', 'amd64-fbsd', 'arm', 'hppa', 'ia64', 'm68k',
+    'mips', 'ppc', 'ppc64', 's390', 'sh', 'sparc', 'sparc-fbsd', 'x86',
+    'x86-fbsd'
+]
+
+BUGORDER = ['bugid', 'reporter', 'reported', 'status', 'arch', 'affected']
+CVEORDER = ['cve', 'published', 'desc', 'severity', 'vector', 'score', 'refs']
+
+REGEX = {
+    'gp_version' : re.compile(r'(?<=K_GENPATCHES_VER\=\").+(?=\")'),
+    'gp_want'    : re.compile(r'(?<=K_WANT_GENPATCHES\=\").+(?=\")'),
+    'k_version'  : re.compile(r'^((?:\d{1,2}\.){0,4}\d{1,2})(-.*)?$'),
+    'rc_kernel'  : re.compile(r'^rc\d{1,3}$'),
+    'git_kernel' : re.compile(r'^git(\d{1,3})$'),
+    'r_kernel'   : re.compile(r'^r\d{1,3}$')
+}
+
+SUPPORTED = ['gentoo', 'vanilla', 'hardened']
+
+KERNEL_TYPES = [
+    'aa', 'acpi', 'ac', 'alpha', 'arm', 'as', 'cell', 'ck', 'compaq', 'crypto',
+    'development', 'gaming','gentoo-dev', 'gentoo', 'gentoo-test', 'gfs',
+    'git', 'grsec', 'gs', 'hardened-dev', 'hardened', 'hppa-dev', 'hppa',
+    'ia64', 'kurobox', 'linux', 'lolo', 'mips-prepatch', 'mips', 'mjc', 'mm',
+    'mosix', 'openblocks', 'openmosix','openvz', 'pac', 'pegasos-dev',
+    'pegasos', 'pfeifer', 'planet-ccrma', 'ppc64', 'ppc-development',
+    'ppc-dev', 'ppc', 'redhat', 'rsbac-dev', 'rsbac', 'selinux', 'sh',
+    'sparc-dev', 'sparc', 'suspend2', 'systrace', 'tuxonice', 'uclinux',
+    'usermode', 'vanilla-prepatch', 'vanilla', 'vanilla-tiny', 'vserver-dev',
+    'vserver', 'win4lin', 'wolk-dev', 'wolk', 'xbox', 'xen', 'xfs'
+]
+
+VERSION = '0.3.9'
+NOCVE = 'GENERIC-MAP-NOMATCH'
+NOCVEDESC = 'This GENERIC identifier is not specific to any vulnerability. '\
+            'GENERIC-MAP-NOMATCH is used by products, databases, and ' \
+            'services to specify when a particular vulnerability element ' \
+            'does not map to a corresponding CVE entry.'
+CVES = dict()
+DEBUG = False
+VERBOSE = False
+FORCE = False
+SKIP = False
+DELAY = 0
+FILEPATH = os.path.dirname(os.path.realpath(__file__))
+PORTDIR = portage.settings['PORTDIR']
+DIR = {
+    'tmp' : os.path.join(FILEPATH, 'tmp'),
+    'out' : os.path.join(PORTDIR, 'metadata', 'kernel'),
+    'bug' : os.path.join(FILEPATH, 'tmp', 'bug'),
+    'nvd' : os.path.join(FILEPATH, 'tmp', 'nvd')
+}
+
+def BUG_ON(msg):
+    if DEBUG:
+        print 'DEBUG line %s in %s(): %s' % (inspect.stack()[1][2],
+              inspect.stack()[1][3], msg)
+
+class Evaluation:
+    """Evaluation class
+
+    Provides information about the vulnerability of a kernel.
+    """
+
+    read = int()
+    arch = int()
+    affected = list()
+    unaffected = list()
+
+    def __init__(self):
+        self.affected = list()
+        self.unaffected = list()
+
+
+class Comparison:
+    """Comparison class
+    """
+
+    fixed = int()
+    new = list()
+    #TODO add more information
+
+    def __init__(self):
+        self.fixed = list()
+        self.new = list()
+
+
+class Cve:
+    """Common vulnerabilities and exposures class
+
+    Contains all important information about a cve.
+
+    Attributes:
+    cve: a string represeting the cve number of the class.
+    desc: a string providing a detailed description for the cve.
+    published: a string representing the original cve release date.
+    refs: a list of external references.
+    severity: a string representing the cve severity.
+    score: a floating point representing cvss base score.
+    vector: a string providing the cve access vector.
+    """
+
+    cve = str()
+    desc = str()
+    published = str()
+    refs = list()
+    severity = str()
+    score = float()
+    vector = str()
+
+    def __init__(self, cve):
+        self.cve = cve
+
+    def __eq__(self, other):
+        return (self.cve == other.cve) #FIXME is this enough?
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class Genpatch:
+    'Genpatch class'
+
+    base = bool()
+    extras = bool()
+    kernel = None
+    version = str()
+
+    def __init__(self, version):
+        self.version = version
+
+
+    def __repr__(self):
+        if self.base and self.extras:
+            return 'base extras'
+        if self.base:
+            return 'base'
+        if self.extras:
+            return 'extras'
+
+
+    def __eq__(self, other):
+        if self.kernel == other.kernel:
+            return (''.join((str(self.base), str(self.extras), self.version))
+            == ''.join((str(other.base), str(other.extras), other.version)))
+        else:
+            return False
+
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class Kernel:
+    'Kernel class'
+
+    revision = str()
+    source = str()
+    version = str()
+    genpatch = None
+
+    def __init__(self, source):
+        self.source = source
+
+
+    def __repr__(self):
+        return str(self.version + '-' + self.source + '-' + self.revision)
+
+
+    def __eq__(self, other):
+        return (''.join((self.revision, self.source, self.version,
+                         str(self.genpatch))) == ''.join((other.revision,
+                         other.source, other.version, str(other.genpatch))))
+
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class Vulnerability:
+    'Vulnerability class'
+
+    arch = str()
+    bugid = int()
+    cvelist  = list()
+    cves = list()
+    affected = list()
+    reported = str()
+    reporter = str()
+    status = str()
+
+    def __init__(self, bugid):
+        self.bugid = bugid
+
+    def __eq__(self, other):
+        return (self.bugid == other.bugid) #FIXME is this enough?
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class Interval:
+    """Interval class
+
+    Provides one interval entry for a vulnerability
+
+    Attributes:
+    name: a string representing the name of the kernel release
+    lower: a string representing the lower boundary of the interval
+    upper: a string representing the upper boundary of the interval
+    lower_i: a boolean indicating if the lower boundary is inclusive
+    upper_i: a boolean indicating if the upper boundary is inclusive
+    expand: a boolean indicating if the interval is shadowing other intervals
+    """
+
+    name = str()
+    lower = str()
+    upper = str()
+    lower_i = bool()
+    upper_i = bool()
+    expand = str()
+
+    def __init__(self, name, lower, upper, lower_i, upper_i, expand):
+        if name == 'linux' or name == 'genpatches':
+            pass
+        elif name == 'gp':
+            name = 'genpatches'
+
+        name = name.replace('-sources', '')
+
+        self.name = name
+        self.lower_i = lower_i
+        self.upper_i = upper_i
+        if name == 'genpatches':
+            if lower:
+                self.lower = lower.replace('-','.')
+            else:
+                self.lower = lower
+            if upper:
+                self.upper = upper.replace('-','.')
+            else:
+                self.upper = upper
+        else:
+            self.lower = lower
+            self.upper = upper
+
+        self.expand = expand
+
+
+    def __repr__(self):
+        interval = str(self.name)
+        if self.expand:
+            interval += '+'
+        interval += ' '
+        if self.lower and self.lower_i:
+            interval += '>=%s ' % (self.lower)
+        if self.lower and not self.lower_i:
+            interval += '>%s ' % (self.lower)
+        if self.upper and self.upper_i:
+            interval += '<=%s' % (self.upper)
+        if self.upper and not self.upper_i:
+            interval += '<%s' % (self.upper)
+
+        return interval
+
+
+def interval_from_xml(root):
+    'Returns an interval from xml'
+
+    name = root.get('source')
+
+    lower = ''
+    upper = ''
+    lower_i = False
+    upper_i = False
+    expand = '' #TODO implement
+
+    if root.find('lower') is not None:
+        lower = root.find('lower').text
+        lower_i = (root.find('lower').get('inclusive') == 'true')
+
+    if root.find('upper') is not None:
+        upper = root.find('upper').text
+        upper_i = (root.find('upper').get('inclusive') == 'true')
+
+    return Interval(name, lower, upper, lower_i, upper_i, expand)
+
+
+#TODO Use exceptions
+def is_in_interval(interval, kernel, bugid=None):
+    'Returns True if the given version is inside our specified interval'
+
+    version = str()
+
+    if interval.name == 'linux':
+        version = kernel.version
+
+    elif interval.name == 'genpatches':
+        version = kernel.version.replace('-', '.')
+
+    elif interval.name == 'hardened':
+        version = kernel.version #TODO is this correct?
+
+    elif interval.name == 'xen':
+        version = kernel.version #TODO is this correct?
+
+    elif interval.name == 'vserver':
+        return False
+
+    else:
+        BUG_ON(interval.name + ' ' + bugid.bugid)
+        return False
+
+    for item in ['lower', 'upper']:
+        if getattr(interval, item):
+            result = portage.versions.vercmp(version, getattr(interval, item))
+
+            if result == None:
+                BUG_ON('Could not compare %s and %s' %
+                       (getattr(interval, item),version))
+
+            if result == 0 and not getattr(interval, item + '_i'):
+                return False
+
+            if result == 0 and getattr(interval, item + '_i'):
+                return True
+
+            if item == 'lower' and result < 0:
+                    return False
+
+            if item == 'upper' and result > 0:
+                    return False
+
+    return True
+
+#TODO Add inline get_genpatch
+def parse_genpatch_list(directory):
+    'Returns a list containing all genpatches from portage'
+
+    patches = list()
+    directory = os.path.join(directory, 'sys-kernel')
+
+    for sources in os.listdir(directory):
+        if '-sources' in sources:
+            for ebuild in os.listdir(os.path.join(directory, sources)):
+                if '.ebuild' in ebuild:
+                    genpatch = extract_genpatch(ebuild, directory, sources)
+
+                    if genpatch is not None:
+                        patches.append(genpatch)
+
+    return patches
+
+
+def get_genpatch(patches, kernel):
+    'Returns the genpatch for a specific kernel'
+
+    for item in patches:
+        if item.kernel == kernel:
+                return item
+
+    return None
+
+
+def extract_genpatch(ebuild, directory, sources):
+    'Returns a genpatch from an ebuild'
+
+    pkg = portage.versions.catpkgsplit('sys-kernel/%s' % ebuild[:-7])
+
+    with open(os.path.join(directory, sources, ebuild), 'r') as ebuild_file:
+        content = ebuild_file.read()
+
+    try:
+        genpatch_v = REGEX['gp_version'].findall(content)[0]
+        genpatch_w = REGEX['gp_want'].findall(content)[0]
+    except:
+        return None
+
+    kernel = Kernel(pkg[1].replace('-sources', ''))
+    kernel.version = pkg[2]
+    kernel.revision = pkg[3]
+
+    genpatch = Genpatch(pkg[2] + '-' + genpatch_v)
+    genpatch.kernel = kernel
+    genpatch.base = ('base' in genpatch_w)
+    genpatch.extras = ('extras' in genpatch_w)
+
+    return genpatch
+
+
+def parse_cve_files(directory):
+    'Returns all bug files as list'
+
+    files = list()
+
+    if (os.path.exists(directory)):
+        for item in os.listdir(directory):
+            try:
+                cve_file = read_cve_file(directory, item[:-4])
+                if cve_file is not None:
+                    files.append(cve_file)
+
+            except AttributeError:
+                pass
+
+    return files
+
+
+def find_cve(cve, directory):
+    'Returns a bug containing the cve'
+
+    for item in parse_cve_files(directory):
+        for cves in item.cves:
+            if cve == cves.cve:
+                return item
+
+    return None
+
+
+def eval_cve_files(directory, kernel, arch):
+    'Returns a vulnerabilty evaluation'
+
+    files = parse_cve_files(directory)
+
+    if not files:
+        return None
+
+    evaluation = Evaluation()
+
+    for item in files:
+        evaluation.read += 1
+
+        if item.arch not in ARCHES:
+            BUG_ON('[Error] Wrong architecture %s in bugid: %s' %
+                   (item.arch, item.bugid))
+
+        if item.arch != arch and item.arch != 'all':
+            evaluation.unaffected.append(item)
+        else:
+            evaluation.arch += 1
+
+            if is_affected(item.affected, kernel, item):
+                evaluation.affected.append(item)
+            else:
+                evaluation.unaffected.append(item)
+
+    return evaluation
+
+#TODO Remove item
+def is_affected(interval_list, kernel, item):
+    'Returns true if a kernel is affected'
+
+    kernel_gentoo = (kernel.source == 'gentoo' and kernel.genpatch is not None)
+    kernel_affected = False
+    kernel_linux_affected = False
+    kernel_gp_affected  = False
+    linux_interval = False
+    gentoo_interval = False
+
+    for interval in interval_list:
+        if interval.name == 'genpatches':
+            gentoo_interval = True
+            if kernel_gentoo:
+                if is_in_interval(interval, kernel.genpatch, item):
+                    kernel_gp_affected = True
+
+        elif interval.name == 'linux':
+            linux_interval = True
+            if is_in_interval(interval, kernel, item):
+               kernel_linux_affected = True
+
+        else:
+            pass #TODO
+
+    if linux_interval:
+        if kernel_linux_affected:
+            if gentoo_interval and kernel_gentoo:
+                if kernel_gp_affected:
+                    kernel_affected = True
+                else:
+                    kernel_affected = False
+            else:
+                kernel_affected = True
+        else:
+            kernel_affected = False
+    else:
+        if kernel_gentoo and gentoo_interval:
+            if kernel_gp_affected:
+                kernel_affected = True
+            else:
+                kernel_affected = False
+        #TODO Implement else for hardend/xen/expand
+
+    return kernel_affected
+
+
+def compare_evaluation(kernel, compare):
+    'Creates a comparison out of two evaluation instances'
+
+    comparison = Comparison()
+
+    if kernel.read != compare.read or kernel.arch != compare.arch:
+        BUG_ON('Kernels do not match: %s %s' % (kernel1.read, kernel2.read))
+        return
+
+    for item in kernel.affected:
+        if item not in compare.affected:
+            comparison.fixed.append(item)
+
+    for item in compare.affected:
+        if item not in kernel.affected:
+            comparison.new.append(item)
+
+    return comparison
+
+
+def read_cve_file(directory, bugid):
+    'Read a bug file created by collector'
+
+    cves = list()
+    affected = list()
+
+    filename = os.path.join(directory, bugid + '.xml')
+
+    try:
+        with open(filename, 'r+') as xml_data:
+                memory_map = mmap.mmap(xml_data.fileno(), 0)
+                root = et.parse(memory_map).getroot()
+    except IOError:
+        return None
+
+    bugroot = root.find('bug')
+
+    vul = Vulnerability(bugroot.find('bugid').text)
+    vul.arch = bugroot.find('arch').text
+    vul.reported = bugroot.find('reported').text
+    vul.reporter = bugroot.find('reporter').text
+    vul.status   = bugroot.find('status').text
+
+    affectedroot = bugroot.find('affected')
+
+    for item in affectedroot:
+        interval = interval_from_xml(item)
+        affected.append(interval)
+
+    vul.affected = affected
+
+    for item in root:
+        if item.tag == 'cve':
+            cve = Cve(item.find('cve').text)
+            cve.desc = item.find('desc').text
+            cve.published = item.find('published').text
+            cve.refs = item.find('refs').text #FIXME
+            cve.severity = item.find('severity').text
+            cve.score = item.find('score').text
+            cve.vector = item.find('vector').text
+
+            cves.append(cve)
+    vul.cves = cves
+
+    return vul
+
+#TODO Use Exceptions
+def extract_version(release):
+    'Extracts revision, source and version out of a release tag'
+
+    match = REGEX['k_version'].match(release)
+    if not match:
+        BUG_ON('[Error] Release %s does not contain any valid information' %
+               release)
+        return None
+
+    version, rest = match.groups()
+
+    kernel = Kernel('vanilla')
+    kernel.revision = 'r0'
+    kernel.version = version
+
+    for elem in (rest or '').split('-'):
+        if elem == 'sources':
+            pass
+        elif REGEX['rc_kernel'].match(elem):
+            kernel.version += '_' + elem
+        elif REGEX['git_kernel'].match(elem):
+            kernel.source = 'git'
+            kernel.revision = 'r' + REGEX['gitd'].match(elem).groups()[0]
+        elif REGEX['r_kernel'].match(elem):
+            kernel.revision = elem
+        elif elem in KERNEL_TYPES:
+            kernel.source = elem
+        elif elem != '':
+            BUG_ON('[Error] Dropping unknown version component \'%s\', \
+                   probably local tag.' % elem)
+
+    return kernel
+
+
+def all_version(source):
+    """ Given a kernel source name (e.g. vanilla), returns a Kernel object
+        for the latest revision in the tree, or None if none exists. """
+
+    versions = list()
+
+    porttree = portage.db[portage.root]['porttree']
+    matches = porttree.dbapi.xmatch('match-all',
+                                    'sys-kernel/%s-sources' % source)
+
+    for item in matches:
+        best = portage.versions.catpkgsplit(item)
+        if not best:
+            continue
+
+        kernel = Kernel(best[1].replace('-sources', ''))
+        kernel.version = best[2]
+        kernel.revision = best[3]
+
+        versions.append(kernel)
+
+    return versions
+
diff --git a/testsuite.py b/testsuite.py
deleted file mode 100755
index 1a9287c..0000000
--- a/testsuite.py
+++ /dev/null
@@ -1,212 +0,0 @@
-#!/usr/bin/env python
-
-import unittest
-import sys
-import kernellib as lib
-import guidexml
-
-"""
-import matplotlib.pyplot as pyplot
-
-
-def draw_graph():
-    'Draws a graph x=kernel releases, y=amount of vulnerabilities'
-
-LINUX26_RELEASES = [
-'0', '1', '2', '3', '4', '5', '6', '7', '8', '8.1', '9', '10', '11', '11.1',
-'11.2', '11.3', '11.4', '11.5', '11.6', '11.7', '11.8', '11.9', '11.10',
-'11.11', '11.12', '12', '12.1', '12.2', '12.3', '12.4', '12.5', '12.6',
-'13', '13.1', '13.2', '13.3', '13.4', '13.5', '14', '14.1', '14.2', '14.3',
-'14.4', '14.5', '14.6', '14.7', '15', '15.1', '15.2', '15.3', '15.4', '15.5',
-'15.6', '15.7', '16', '16.1', '16.2', '16.3', '16.4', '16.5', '16.6', '16.7',
-'16.8', '16.9', '16.10', '16.11', '16.12', '16.13', '16.14', '16.15', '16.16',
-'16.17', '16.18', '16.19', '16.20', '16.21', '16.22', '16.23', '16.24',
-'16.25', '16.26', '16.27', '16.28', '16.29', '16.30', '16.31', '16.32',
-'16.33', '16.34', '16.35', '16.36', '16.37', '16.38', '16.39', '16.40',
-'16.41', '16.42', '16.43', '16.44', '16.45', '16.46', '16.47', '16.48',
-'16.49', '16.50', '16.51', '16.52', '16.53', '16.54', '16.55', '16.56',
-'16.57', '16.58', '16.59', '16.60', '16.61', '16.62', '17', '17.1', '17.2',
-'17.3', '17.4', '17.5', '17.6', '17.7', '17.8', '17.9', '17.10', '17.11',
-'17.12', '17.13', '17.14', '18', '18.1', '18.2', '18.3', '18.4', '18.5',
-'18.6', '18.7', '18.8', '19', '19.1', '19.2', '19.3', '19.4', '19.5', '19.6',
-'19.7', '20', '20.1', '20.2', '20.3', '20.4', '20.5', '20.6', '20.7', '20.8',
-'20.9', '20.10', '20.11', '20.12', '20.13', '20.14', '20.15', '20.16', '20.17',
-'20.18', '20.19', '20.20', '20.21', '21', '21.1', '21.2', '21.3', '21.4',
-'21.5', '21.6', '21.7', '22', '22.1', '22.2', '22.3', '22.4', '22.5', '22.6',
-'22.7', '22.8', '22.9', '22.10', '22.11', '22.12', '22.13', '22.14', '22.15',
-'22.16', '22.17', '22.18', '22.19', '23', '23.1', '23.2', '23.3', '23.4',
-'23.5', '23.6', '23.7', '23.8', '23.9', '23.10', '23.11', '23.12', '23.13',
-'23.14', '23.15', '23.16', '23.17', '24', '24.1', '24.2', '24.3', '24.4',
-'24.5', '24.6', '24.7', '25', '25.1', '25.2', '25.3', '25.4', '25.5', '25.6',
-'25.7', '25.8', '25.9', '25.10', '25.11', '25.12', '25.13', '25.14', '25.15',
-'25.16', '25.17', '25.18', '25.19', '25.20', '26', '26.1', '26.2', '26.3',
-'26.4', '26.5', '26.6', '26.7', '26.8', '27', '27.1', '27.2', '27.3', '27.4',
-'27.5', '27.6', '27.7', '27.8', '27.9', '27.10', '27.11', '27.12', '27.13',
-'27.14', '27.15', '27.16', '27.17', '27.18', '27.19', '27.20', '27.21',
-'27.22', '27.23', '27.24', '27.25', '27.26', '27.27', '27.28', '28', '28.1',
-'28.2', '28.3', '28.4', '28.5', '28.6', '28.7', '28.8', '28.9', '28.10', '29',
-'29.1', '29.2', '29.3', '29.4', '29.5', '29.6', '30', '30.1', '30.2', '30.3',
-'30.4', '30.5'
-]
-
-vul, line, value = [], [], []
-
-for item in LINUX26_RELEASES:
-    kernel = lib.Kernel('gentoo-sources')
-    kernel.revision = ''
-    kernel.version = '2.6.' + item
-    best = lib.best_version(kernel.source)
-    schedule = lib.parse_cve_files('out', kernel, best, 'all') #FIXME
-    vul.append(len(schedule.canfix))
-    if '.' not in item:
-        line.append(len(schedule.canfix))
-        value.append(len(vul) - 1)
-
-pyplot.plot(vul, color='#7A5ADA', linewidth=1)
-for i in range(0, len(line)):
-    pyplot.bar(value[i], line[i], color='#7A5ADA', edgecolor='#FFFFFF')
-
-pyplot.axis('tight')
-pyplot.ylabel('Number of vulnerabilities')
-pyplot.xlabel('Kernel release')
-pyplot.show()
-
-
-def write_guidexml():
-    'Testcases for guidexml'
-
-doc = guidexml.Document('mydocument', '2.0', 'Sehr nuetzliches infos', 'en',
-                       '/doc/en/bla')
-
-doc.addAuthor('creator', 'asyme', 'asymmail@googlemail.de')
-doc.addAuthor('creator', 'rbu')
-
-chap = guidexml.Chapter('1 chapter')
-sect = guidexml.Section('1 section')
-
-taguri = guidexml.uri("http://google.de")
-tagpara = guidexml.paragraph("hello world, Visit: %s.das ist ein sehr langer' \
-                            'text mit mehr als MAXLENGTH Zeichen." %
-                            repr(taguri))
-
-doc.append(chap)
-chap.append(sect)
-sect.append(tagpara)
-
-doc.create('/', 'file')
-
-
-def test_kernel_version():
-    'Test if kernel releases are parsed correctly'
-
-    kernelcases = {
-    '2.6.29.1' :
-            {'source'   : 'vanilla',
-             'revision' : 'r0',
-             'version'  : '2.6.29.1'},
-    '2.6.18-xen-r11' :
-            {'source'   : 'xen',
-             'revision' : 'r11',
-             'version'  : '2.6.18'},
-    '2.6.27-gentoo-r2' :
-            {'source'   : 'gentoo',
-             'revision' : 'r2',
-             'version'  : '2.6.27'},
-    '2.6.28-hardened-r77' :
-            {'source'   : 'hardened',
-             'revision' : 'r77',
-             'version'  : '2.6.28'},
-    '2.6.28-r7' :
-            {'source'   : 'vanilla',
-             'revision' : 'r7',
-             'version'  : '2.6.28'},
-    '2.6.28-rc8' :
-            {'source'   : 'vanilla',
-             'revision' : 'r0',
-             'version'  : '2.6.28_rc8'},
-    '2.6.28-rc8-git6' :
-            {'source'   : 'git',
-             'revision' : 'r6',
-             'version'  : '2.6.28_rc8'},
-    '2' :
-            {'source'   : 'vanilla',
-             'revision' : 'r0',
-             'version'  : '2'},
-    '2.6.28-hardened-r7777' :
-            {'source'   : 'hardened',
-             'revision' : 'r0',
-             'version'  : '2.6.28'},
-    '2.6.2833-hardened-r7' :
-            None,
-
-}
-
-    suite = unittest.TestSuite()
-
-    for name, dict in kernelcases.iteritems():
-        newtest = KernelVersionTestCase(name, dict)
-        suite.addTest(newtest)
-
-    unittest.TextTestRunner(verbosity=2).run(suite)
-
-
-class KernelVersionTestCase(unittest.TestCase):
-
-    def __init__(self, name, dict):
-        unittest.TestCase.__init__(self, 'testFunction')
-        self.name = name
-        self.dict = dict
-
-    def testFunction(self):
-        rvalue = lib.extract_version_from(self.name)
-        self.assertEqual(self.dict, rvalue)
-"""
-
-
-def test_interval():
-    'Test if a kernel release is affected'
-
-    kernelcase = ['gentoo', '2.6.30', 'r4', '2.6.30-5']
-    intervalcases = {
-        '[linux >=2.6.27 <=2.6.30] [gp <2.6.30.3]' : False,
-        '[linux >=2.6.27 <=2.6.30] [gp <2.6.30.6]' : True,
-        '[linux >=2.6.28 <=2.6.31] [gp <=2.6.30.5]' : True,
-        '[linux >=2.6.26 <2.6.28] [linux >2.6.29 <=2.6.30]' : True,
-
-    }
-
-    suite = unittest.TestSuite()
-
-    kernel = lib.Kernel(kernelcase[0])
-    kernel.version = kernelcase[1]
-    kernel.revision = kernelcase[2]
-    kernel_gp = lib.Genpatch(kernelcase[3])
-
-    for wb, affected in intervalcases.iteritems():
-        interval_list = lib.interval_from_whiteboard(wb)
-
-        newtest = IntervalTestCase(interval_list, kernel, kernel_gp, affected)
-        suite.addTest(newtest)
-
-    unittest.TextTestRunner(verbosity=2).run(suite)
-
-
-class IntervalTestCase(unittest.TestCase):
-
-    def __init__(self, interval_list, kernel, kernel_gp, affected):
-        unittest.TestCase.__init__(self, 'testFunction')
-        self.interval_list = interval_list
-        self.kernel = kernel
-        self.kernel_gp = kernel_gp
-        self.item = lib.Vulnerability('000000')
-        self.affected = affected
-
-    def testFunction(self):
-        rvalue = lib.is_affected(self.interval_list, self.kernel,
-                                self.kernel_gp, self.item)
-        self.assertEqual(self.affected, rvalue)
-
-
-if __name__ == '__main__':
-    test_interval()
-
diff --git a/tools/cron.py b/tools/cron.py
new file mode 100755
index 0000000..01e04af
--- /dev/null
+++ b/tools/cron.py
@@ -0,0 +1,405 @@
+#!/usr/bin/env python
+# kernel-check -- Kernel security information
+# Copyright 2009-2009 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+from contextlib import closing
+import xml.etree.cElementTree as et
+import cStringIO
+import datetime
+import logging
+import mmap
+import os
+import portage
+import re
+import sys
+import time
+import urllib
+
+
+class CronError(Exception):
+    def __init__(self, value):
+        self.value = value
+
+NOCVE = 'GENERIC-MAP-NOMATCH'
+NOCVEDESC = 'This GENERIC identifier is not specific to any vulnerability. '\
+            'GENERIC-MAP-NOMATCH is used by products, databases, and ' \
+            'services to specify when a particular vulnerability element ' \
+            'does not map to a corresponding CVE entry.'
+DELAY = 0.2
+SKIP = True
+MINYEAR = 2002
+MAXYEAR = 2020
+NVDURL = 'http://nvd.nist.gov/'
+BZURL = 'https://bugs.gentoo.org/'
+STATE = ['NEW', 'ASSIGNED', 'REOPENED', 'RESOLVED', 'VERIFIED', 'CLOSED']
+RESOLUTION = ['FIXED', 'LATER', 'TEST-REQUEST', 'UPSTREAM', '---']
+BUGORDER = ['bugid', 'reporter', 'reported', 'status', 'arch', 'affected']
+CVEORDER = ['cve', 'published', 'desc', 'severity', 'vector', 'score', 'refs']
+FILEPATH = os.path.dirname(os.path.realpath(__file__))
+PORTDIR = portage.settings['PORTDIR']
+LOGFILE = None #os.path.join(FILEPATH, 'cron.log')
+DIR = {
+    'tmp' : os.path.join(FILEPATH, 'tmp'),
+    'out' : os.path.join(PORTDIR, 'metadata', 'kernel'),
+    'bug' : os.path.join(FILEPATH, 'tmp', 'bug'),
+    'nvd' : os.path.join(FILEPATH, 'tmp', 'nvd')
+}
+REGEX = {
+    'bugzilla'   : re.compile(r'(?<=bug.cgi\?id=)\d*'),
+    'grp_all'    : re.compile(r'(?<=\()[ (]*CVE-(\d{4})' \
+                              r'([-,(){}|, \d]+)(?=\))'),
+    'm_nomatch'  : re.compile(r'.*GENERIC-MAP-NOMATCH.*'),
+    'grp_split'  : re.compile(r'(?<=\D)(\d{4})(?=\D|$)'),
+    'wb_match'   : re.compile(r'\s*\[\s*([^ +<=>]+)\s*([' \
+                              r'<=>]{1,2})\s*([^ <=>\]]+' \
+                              r')\s*(?:([<=>]{1,2})\s*([' \
+                              r'^ \]]+))?\s*\]\s*(.*)'),
+    'wb_version' : re.compile(r'^(?:\d{1,2}\.){0,3}\d{1,2}' \
+                              r'(?:[-_](?:r|rc)?\d{1,2})*$')
+}
+CVES = dict()
+logging.basicConfig(format='[%(asctime)s] %(levelname)-6s : %(message)s',
+                    datefmt='%H:%M:%S', filename=LOGFILE, level=logging.DEBUG)
+
+
+def main(argv):
+    'Main function'
+
+    logging.info('Running cron...')
+
+    current_year = datetime.datetime.now().year
+    if current_year < MINYEAR or current_year > MAXYEAR:
+        current_year = MAXYEAR
+
+    for directory in DIR:
+        if not os.path.isdir(DIR[directory]):
+            os.makedirs(DIR[directory])
+
+    logging.info('Receiving the latest xml file from the nvd')
+
+    receive_file(DIR['nvd'], [NVDURL, 'download/'],'nvdcve-recent.xml')
+
+    if not SKIP:
+        logging.info('Receiving earlier xml files from the nvd')
+
+        for year in xrange(MINYEAR, current_year + 1):
+            receive_file(DIR['nvd'], [NVDURL, 'download/'],
+                         'nvdcve-%s.xml' % str(year))
+
+    logging.info('Receiving the kernel vulnerability list from bugzilla')
+
+    url = [BZURL, 'buglist.cgi?query_format=advanced&component=Kernel']
+
+    for item in STATE:
+        url.append('&bug_status=' + item)
+    for item in RESOLUTION:
+        url.append('&resolution=' + item)
+    url.append('#')
+
+    receive_file(DIR['tmp'], url, 'bugzilla.xml')
+
+    filename = os.path.join(DIR['tmp'], 'bugzilla.xml')
+    with open(filename, 'r+') as buglist_file:
+        memory_map = mmap.mmap(buglist_file.fileno(), 0)
+        buglist = REGEX['bugzilla'].findall(memory_map.read(-1))
+
+    logging.info('Found %i kernel vulnerabilities' % len(buglist))
+
+    logging.info('Creating the nvd dictionary')
+    nvd_dict = parse_nvd_dict(DIR['nvd'])
+
+    logging.info('Creating the xml files')
+
+    created_files = 0
+    for item in buglist:
+        try:
+            receive_file(DIR['bug'], [BZURL, 'show_bug.cgi?ctype=xml&id='],
+                         item)
+
+            vul = parse_bugzilla_dict(DIR['bug'], item)
+
+            for cve in vul['cvelist']:
+                if cve == NOCVE:
+                    vul['cves'] = [NOCVE]
+                    break; #TODO
+                else:
+                    try:
+                        vul['cves'].append(nvd_dict[cve])
+                    except KeyError:
+                        raise CronError('No Nvd entry: ' + cve)
+
+            write_cve_file(DIR['out'], vul)
+            created_files += 1
+            time.sleep(DELAY)
+
+        except CronError, e:
+            logging.error('[%s] %s' % (item, e.value))
+
+    logging.info('Created %i xml files' % created_files)
+
+
+def receive_file(directory, url, xml_file):
+    'Generic download function'
+
+    filename = os.path.join(directory, xml_file)
+    url.append(xml_file)
+
+    try:
+        with closing(cStringIO.StringIO()) as data:
+            with closing(urllib.urlopen(''.join(url))) as resource:
+                data.write(resource.read())
+
+            with open(filename, 'w') as output:
+                output.write(data.getvalue())
+
+    except IOError:
+        logging.error('File %s - Download failed!' % filename)
+
+    logging.debug('File %s - %sKB received' %
+          (filename, os.path.getsize(filename)/1024))
+
+
+def parse_nvd_dict(directory):
+    'Returns a dictionary from the National Vulnerability Database'
+
+    nvd = dict()
+
+    for nvdfile in os.listdir(directory):
+        filename = os.path.join(directory, nvdfile)
+        try:
+            with open(filename, 'r+') as xml_data:
+                memory_map = mmap.mmap(xml_data.fileno(), 0)
+
+        except SyntaxError:
+            continue
+
+        root = et.parse(memory_map).getroot()
+        namespace = root.tag[:-3]
+
+        for tree in root:
+            cve = {
+                'cve'      : tree.get('name'),
+                'published' : tree.get('published'),
+                'severity'  : tree.get('severity'),
+                'vector'    : tree.get('CVSS_vector'),
+                'score'     : tree.get('CVSS_score')
+            }
+
+            desc = tree.find('%sdesc/%sdescript/' % (namespace, namespace))
+            if desc is not None:
+                cve['desc'] = desc.text
+
+            reftree = tree.find(namespace + 'refs')
+            reftree.tag = reftree.tag.replace(namespace, '')
+            for elem in reftree.findall('.//*'):
+                elem.tag = elem.tag.replace(namespace, '')
+
+            bugref = et.SubElement(reftree, 'ref')
+            bugref.set('source', 'GENTOO')
+            bugref.set('url', '%sshow_bug.cgi?id=%s' % (BZURL, cve['cve']))
+            bugref.text = 'Gentoo %s' % cve['cve']
+
+            cve['refs'] = reftree
+
+            nvd[cve['cve']] = cve
+
+    return nvd
+
+def parse_bugzilla_dict(directory, bugid):
+    'Returns a vulnerability class containing information about a bug'
+
+    filename = os.path.join(directory, bugid)
+
+    try:
+        with open(filename, 'r+') as xml_data:
+            memory_map = mmap.mmap(xml_data.fileno(), 0)
+            root = et.parse(memory_map).getroot()[0]
+
+    except IOError:
+        return
+
+    string = str()
+
+    try:
+        string = root.find('short_desc').text
+    except AttributeError:
+        CronError('No Cve')
+
+    try:
+        cvelist = list()
+        string = string.replace('CAN', 'CVE')
+
+        if string in REGEX['m_nomatch'].findall(string):
+            cvelist = [NOCVE]
+
+        for (year, split_cves) in REGEX['grp_all'].findall(string):
+            for cve in REGEX['grp_split'].findall(split_cves):
+                cvelist.append('CVE-%s-%s' % (year, cve))
+
+        vul = {
+            'bugid'    : bugid,
+            'cvelist'  : cvelist,
+            'cves'     : list(),
+            'arch'     : root.find('rep_platform').text.lower(),
+            'reporter' : root.find('reporter').text.lower(),
+            'reported' : root.find('creation_ts').text,
+            'status'   : root.find('bug_status').text.lower(),
+        }
+
+        for item in vul['cvelist']:
+            if item != NOCVE:
+                if item not in CVES:
+                    CVES[item] = vul.bugid
+                else:
+                    raise CronError('Duplicate: ' + CVES[item])
+
+    except AttributeError:
+        pass
+
+    try:
+        wb = root.find('status_whiteboard').text;
+        vul['affected'] = interval_from_whiteboard(wb)
+
+        if vul['affected'] == None:
+            raise CronError('Invalid whiteboard: ' + wb)
+
+    except AttributeError:
+        raise CronError('Empty whiteboard')
+
+    return vul
+
+
+def interval_from_whiteboard(whiteboard):
+    'Returns a list of intervals within a whiteboard string'
+
+    upper_inc = None
+    upper = None
+    lower_inc = None
+    lower = None
+
+    affected = list()
+
+    while len(whiteboard.strip()) > 0:
+        match = REGEX['wb_match'].match(whiteboard)
+        if not match:
+            return None
+
+        name = match.group(1)
+        comp1 = match.group(2)
+        vers1 = match.group(3)
+        comp2 = match.group(4)
+        vers2 = match.group(5)
+        whiteboard = match.group(6)
+
+        if comp1 == '=' or comp1 == '==':
+            lower_inc = True
+            upper_inc = True
+            lower = vers1
+            upper = vers1
+
+            if not REGEX['wb_version'].match(vers1):
+                return None
+        else:
+            for (char, version) in ((comp1, vers1), (comp2, vers2)):
+
+                if char == '<':
+                    upper_inc = False
+                    upper = version
+                elif char == '<=' or char == '=<':
+                    upper_inc = True
+                    upper = version
+                elif char == '>':
+                    lower_inc = False
+                    lower = version
+                elif char == '>=' or char == '=>':
+                    lower_inc = True
+                    lower = version
+                elif char:
+                    return None
+
+                if version and not REGEX['wb_version'].match(version):
+                    return None
+
+        interval = {
+            'name'      : name,
+            'lower'     : lower,
+            'upper'     : upper,
+            'lower_inc' : lower_inc,
+            'upper_inc' : upper_inc
+        }
+
+        affected.append(interval)
+
+    return affected
+
+
+def write_cve_file(directory, vul):
+    'Write a bug file containing all important information for kernel-check'
+
+    filename = os.path.join(directory, vul['bugid'] + '.xml')
+
+    root = et.Element('vulnerability')
+    bugroot = et.SubElement(root, 'bug')
+
+    for element in BUGORDER:
+        if element == 'affected':
+                affectedroot = et.SubElement(bugroot, 'affected')
+                for item in vul['affected']:
+                        intnode = et.Element('interval')
+                        intnode.set('source', item['name'])
+
+                        affectedroot.append(intnode)
+
+                        for i in ('lower', 'upper'):
+                            if item[i]:
+                                node = et.SubElement(intnode, i)
+                                node.text = item[i]
+                                node.set('inclusive',
+                                str(item[i + '_inc']).lower())
+        else:
+            node = et.SubElement(bugroot, element)
+            node.text = vul[element]
+
+    for cve in vul['cves']:
+        cveroot = et.SubElement(root, 'cve')
+        if cve == NOCVE:
+            node = et.SubElement(cveroot, 'cve')
+            node.text = NOCVE
+            node = et.SubElement(cveroot, 'desc')
+            node.text = NOCVEDESC
+        else:
+            for element in CVEORDER:
+                if element == 'refs':
+                    cveroot.append(cve[element])
+                else:
+                    node = et.SubElement(cveroot, element)
+                    node.text = cve[element]
+
+    with open(filename, 'w') as xmlout:
+        __indent__(root)
+        doc = et.ElementTree(root)
+        doc.write(xmlout, encoding='utf-8')
+
+
+def __indent__(node, level=0):
+    'Indents xml layout for printing'
+
+    i = '\n' + level * ' ' * 4
+    if len(node):
+        if not node.text or not node.text.strip():
+            node.text = i + ' ' * 4
+        if not node.tail or not node.tail.strip():
+            node.tail = i
+        for node in node:
+            __indent__(node, level + 1)
+        if not node.tail or not node.tail.strip():
+            node.tail = i
+    else:
+        if level and (not node.tail or not node.tail.strip()):
+            node.tail = i
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
+
diff --git a/tools/findcommit.sh b/tools/findcommit.sh
new file mode 100755
index 0000000..770078b
--- /dev/null
+++ b/tools/findcommit.sh
@@ -0,0 +1,62 @@
+#!/bin/sh
+# kernel-check -- Kernel security information
+# Copyright 2009-2009 Gentoo Foundation
+# Distributed under the terms of the GNU General Public License v2
+
+TEXT=".*$@.*"
+GITPATH="./"
+
+KNOWNCOMMITS=""
+LASTFIXEDBRANCH=""
+WHITEBOARD=""
+
+GOOD=$'\e[32;01m'
+BAD=$'\e[31;01m'
+BRACKET=$'\e[34;01m'
+NORMAL=$'\e[0m'
+
+GITPATH="--git-dir=$GITPATH.git --work-tree=$GITPATH"
+BRANCHES="`git $GITPATH branch -a | grep linux` origin/master"
+for B in $BRANCHES ; do
+    COMMITS="`git $GITPATH rev-list --pretty=oneline "$B" |
+             grep "$TEXT" | cut -d " " -f 1`"
+    for C in $COMMITS ; do
+        if [ "${KNOWNCOMMITS/$C/}" == "$KNOWNCOMMITS" ] ; then
+            TAG="`git $GITPATH describe --contains --all "$C"`"
+            DESC="`git $GITPATH --no-pager log -1 --pretty=short "$C"`"
+            DESC="`echo "$DESC" | tr -s "\n" `"
+            DESC="${TAG/tags\//\n${GOOD}label${NORMAL}  }\n${DESC}"
+            DESC="${DESC/commit/${GOOD}commit${NORMAL}}"
+            DESC="${DESC/Author:/${GOOD}author${NORMAL}}"
+            DESC="${DESC/    /${GOOD}title${NORMAL}  ${BRACKET}}${NORMAL}\n"
+            echo -e "$DESC"
+
+            VERSION="${TAG/tags\/v/}"
+            VERSION="${VERSION/~*/}"
+            VERSION="${VERSION/^*/}"
+
+            if [ "$B" != "origin/master" ] ; then
+                THISBRANCH="${B/remotes\/origin\/linux-/}"
+                THISBRANCH="${THISBRANCH/.y/}"
+                REVISION="${THISBRANCH/*./}"
+                MAJORMINOR="${THISBRANCH/.${REVISION}/}"
+                let REVISION=REVISION+1
+                NEXTBRANCH="${MAJORMINOR}.${REVISION}"
+            fi
+            if [ -z "$LASTFIXEDBRANCH" ] ; then
+                    WHITEBOARD="$WHITEBOARD [linux <${VERSION}]"
+            else
+                if [ "${LASTFIXEDBRANCH}" == "${VERSION/-rc*/}" ] ; then
+                    WHITEBOARD="${WHITEBOARD} ${BAD}[linux >=" \
+                               "${LASTFIXEDBRANCH} <${VERSION}]${NORMAL}"
+                else
+                    WHITEBOARD="${WHITEBOARD} [linux >=" \
+                               "${LASTFIXEDBRANCH} <${VERSION}]"
+                fi
+            fi
+            LASTFIXEDBRANCH=$NEXTBRANCH
+            KNOWNCOMMITS="$KNOWNCOMMITS $C"
+        fi
+    done
+done
+echo -e "${GOOD}Whiteboard${NORMAL}\n${WHITEBOARD:1}\n"
diff --git a/tools/testsuite.py b/tools/testsuite.py
new file mode 100755
index 0000000..1a9287c
--- /dev/null
+++ b/tools/testsuite.py
@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+
+import unittest
+import sys
+import kernellib as lib
+import guidexml
+
+"""
+import matplotlib.pyplot as pyplot
+
+
+def draw_graph():
+    'Draws a graph x=kernel releases, y=amount of vulnerabilities'
+
+LINUX26_RELEASES = [
+'0', '1', '2', '3', '4', '5', '6', '7', '8', '8.1', '9', '10', '11', '11.1',
+'11.2', '11.3', '11.4', '11.5', '11.6', '11.7', '11.8', '11.9', '11.10',
+'11.11', '11.12', '12', '12.1', '12.2', '12.3', '12.4', '12.5', '12.6',
+'13', '13.1', '13.2', '13.3', '13.4', '13.5', '14', '14.1', '14.2', '14.3',
+'14.4', '14.5', '14.6', '14.7', '15', '15.1', '15.2', '15.3', '15.4', '15.5',
+'15.6', '15.7', '16', '16.1', '16.2', '16.3', '16.4', '16.5', '16.6', '16.7',
+'16.8', '16.9', '16.10', '16.11', '16.12', '16.13', '16.14', '16.15', '16.16',
+'16.17', '16.18', '16.19', '16.20', '16.21', '16.22', '16.23', '16.24',
+'16.25', '16.26', '16.27', '16.28', '16.29', '16.30', '16.31', '16.32',
+'16.33', '16.34', '16.35', '16.36', '16.37', '16.38', '16.39', '16.40',
+'16.41', '16.42', '16.43', '16.44', '16.45', '16.46', '16.47', '16.48',
+'16.49', '16.50', '16.51', '16.52', '16.53', '16.54', '16.55', '16.56',
+'16.57', '16.58', '16.59', '16.60', '16.61', '16.62', '17', '17.1', '17.2',
+'17.3', '17.4', '17.5', '17.6', '17.7', '17.8', '17.9', '17.10', '17.11',
+'17.12', '17.13', '17.14', '18', '18.1', '18.2', '18.3', '18.4', '18.5',
+'18.6', '18.7', '18.8', '19', '19.1', '19.2', '19.3', '19.4', '19.5', '19.6',
+'19.7', '20', '20.1', '20.2', '20.3', '20.4', '20.5', '20.6', '20.7', '20.8',
+'20.9', '20.10', '20.11', '20.12', '20.13', '20.14', '20.15', '20.16', '20.17',
+'20.18', '20.19', '20.20', '20.21', '21', '21.1', '21.2', '21.3', '21.4',
+'21.5', '21.6', '21.7', '22', '22.1', '22.2', '22.3', '22.4', '22.5', '22.6',
+'22.7', '22.8', '22.9', '22.10', '22.11', '22.12', '22.13', '22.14', '22.15',
+'22.16', '22.17', '22.18', '22.19', '23', '23.1', '23.2', '23.3', '23.4',
+'23.5', '23.6', '23.7', '23.8', '23.9', '23.10', '23.11', '23.12', '23.13',
+'23.14', '23.15', '23.16', '23.17', '24', '24.1', '24.2', '24.3', '24.4',
+'24.5', '24.6', '24.7', '25', '25.1', '25.2', '25.3', '25.4', '25.5', '25.6',
+'25.7', '25.8', '25.9', '25.10', '25.11', '25.12', '25.13', '25.14', '25.15',
+'25.16', '25.17', '25.18', '25.19', '25.20', '26', '26.1', '26.2', '26.3',
+'26.4', '26.5', '26.6', '26.7', '26.8', '27', '27.1', '27.2', '27.3', '27.4',
+'27.5', '27.6', '27.7', '27.8', '27.9', '27.10', '27.11', '27.12', '27.13',
+'27.14', '27.15', '27.16', '27.17', '27.18', '27.19', '27.20', '27.21',
+'27.22', '27.23', '27.24', '27.25', '27.26', '27.27', '27.28', '28', '28.1',
+'28.2', '28.3', '28.4', '28.5', '28.6', '28.7', '28.8', '28.9', '28.10', '29',
+'29.1', '29.2', '29.3', '29.4', '29.5', '29.6', '30', '30.1', '30.2', '30.3',
+'30.4', '30.5'
+]
+
+vul, line, value = [], [], []
+
+for item in LINUX26_RELEASES:
+    kernel = lib.Kernel('gentoo-sources')
+    kernel.revision = ''
+    kernel.version = '2.6.' + item
+    best = lib.best_version(kernel.source)
+    schedule = lib.parse_cve_files('out', kernel, best, 'all') #FIXME
+    vul.append(len(schedule.canfix))
+    if '.' not in item:
+        line.append(len(schedule.canfix))
+        value.append(len(vul) - 1)
+
+pyplot.plot(vul, color='#7A5ADA', linewidth=1)
+for i in range(0, len(line)):
+    pyplot.bar(value[i], line[i], color='#7A5ADA', edgecolor='#FFFFFF')
+
+pyplot.axis('tight')
+pyplot.ylabel('Number of vulnerabilities')
+pyplot.xlabel('Kernel release')
+pyplot.show()
+
+
+def write_guidexml():
+    'Testcases for guidexml'
+
+doc = guidexml.Document('mydocument', '2.0', 'Sehr nuetzliches infos', 'en',
+                       '/doc/en/bla')
+
+doc.addAuthor('creator', 'asyme', 'asymmail@googlemail.de')
+doc.addAuthor('creator', 'rbu')
+
+chap = guidexml.Chapter('1 chapter')
+sect = guidexml.Section('1 section')
+
+taguri = guidexml.uri("http://google.de")
+tagpara = guidexml.paragraph("hello world, Visit: %s.das ist ein sehr langer' \
+                            'text mit mehr als MAXLENGTH Zeichen." %
+                            repr(taguri))
+
+doc.append(chap)
+chap.append(sect)
+sect.append(tagpara)
+
+doc.create('/', 'file')
+
+
+def test_kernel_version():
+    'Test if kernel releases are parsed correctly'
+
+    kernelcases = {
+    '2.6.29.1' :
+            {'source'   : 'vanilla',
+             'revision' : 'r0',
+             'version'  : '2.6.29.1'},
+    '2.6.18-xen-r11' :
+            {'source'   : 'xen',
+             'revision' : 'r11',
+             'version'  : '2.6.18'},
+    '2.6.27-gentoo-r2' :
+            {'source'   : 'gentoo',
+             'revision' : 'r2',
+             'version'  : '2.6.27'},
+    '2.6.28-hardened-r77' :
+            {'source'   : 'hardened',
+             'revision' : 'r77',
+             'version'  : '2.6.28'},
+    '2.6.28-r7' :
+            {'source'   : 'vanilla',
+             'revision' : 'r7',
+             'version'  : '2.6.28'},
+    '2.6.28-rc8' :
+            {'source'   : 'vanilla',
+             'revision' : 'r0',
+             'version'  : '2.6.28_rc8'},
+    '2.6.28-rc8-git6' :
+            {'source'   : 'git',
+             'revision' : 'r6',
+             'version'  : '2.6.28_rc8'},
+    '2' :
+            {'source'   : 'vanilla',
+             'revision' : 'r0',
+             'version'  : '2'},
+    '2.6.28-hardened-r7777' :
+            {'source'   : 'hardened',
+             'revision' : 'r0',
+             'version'  : '2.6.28'},
+    '2.6.2833-hardened-r7' :
+            None,
+
+}
+
+    suite = unittest.TestSuite()
+
+    for name, dict in kernelcases.iteritems():
+        newtest = KernelVersionTestCase(name, dict)
+        suite.addTest(newtest)
+
+    unittest.TextTestRunner(verbosity=2).run(suite)
+
+
+class KernelVersionTestCase(unittest.TestCase):
+
+    def __init__(self, name, dict):
+        unittest.TestCase.__init__(self, 'testFunction')
+        self.name = name
+        self.dict = dict
+
+    def testFunction(self):
+        rvalue = lib.extract_version_from(self.name)
+        self.assertEqual(self.dict, rvalue)
+"""
+
+
+def test_interval():
+    'Test if a kernel release is affected'
+
+    kernelcase = ['gentoo', '2.6.30', 'r4', '2.6.30-5']
+    intervalcases = {
+        '[linux >=2.6.27 <=2.6.30] [gp <2.6.30.3]' : False,
+        '[linux >=2.6.27 <=2.6.30] [gp <2.6.30.6]' : True,
+        '[linux >=2.6.28 <=2.6.31] [gp <=2.6.30.5]' : True,
+        '[linux >=2.6.26 <2.6.28] [linux >2.6.29 <=2.6.30]' : True,
+
+    }
+
+    suite = unittest.TestSuite()
+
+    kernel = lib.Kernel(kernelcase[0])
+    kernel.version = kernelcase[1]
+    kernel.revision = kernelcase[2]
+    kernel_gp = lib.Genpatch(kernelcase[3])
+
+    for wb, affected in intervalcases.iteritems():
+        interval_list = lib.interval_from_whiteboard(wb)
+
+        newtest = IntervalTestCase(interval_list, kernel, kernel_gp, affected)
+        suite.addTest(newtest)
+
+    unittest.TextTestRunner(verbosity=2).run(suite)
+
+
+class IntervalTestCase(unittest.TestCase):
+
+    def __init__(self, interval_list, kernel, kernel_gp, affected):
+        unittest.TestCase.__init__(self, 'testFunction')
+        self.interval_list = interval_list
+        self.kernel = kernel
+        self.kernel_gp = kernel_gp
+        self.item = lib.Vulnerability('000000')
+        self.affected = affected
+
+    def testFunction(self):
+        rvalue = lib.is_affected(self.interval_list, self.kernel,
+                                self.kernel_gp, self.item)
+        self.assertEqual(self.affected, rvalue)
+
+
+if __name__ == '__main__':
+    test_interval()
+
-- 
cgit v1.2.3-65-gdbad