diff options
| -rwxr-xr-x | bin/euscan | 3 | ||||
| -rw-r--r-- | euscanwww/djeuscan/admin.py | 11 | ||||
| -rw-r--r-- | euscanwww/djeuscan/models.py | 10 | ||||
| -rw-r--r-- | euscanwww/djeuscan/templates/euscan/_package_details.html | 122 | ||||
| -rw-r--r-- | euscanwww/djeuscan/templates/euscan/package.html | 169 | ||||
| -rw-r--r-- | euscanwww/djeuscan/templates/euscan/problem.html | 57 | ||||
| -rw-r--r-- | euscanwww/djeuscan/urls.py | 2 | ||||
| -rw-r--r-- | euscanwww/djeuscan/views.py | 62 | ||||
| -rw-r--r-- | euscanwww/htdocs/img/bug.png | bin | 0 -> 925 bytes | |||
| -rw-r--r-- | pym/euscan/handlers/watch.py | 143 | ||||
| -rwxr-xr-x | setup.py | 13 |
11 files changed, 399 insertions, 193 deletions
@@ -5,7 +5,7 @@ Distributed under the terms of the GNU General Public License v2 """ from __future__ import print_function - +import os # Meta @@ -19,7 +19,6 @@ __description__ = "A tool to detect new upstream releases." # Imports import sys -import os import getopt from errno import EINTR, EINVAL from httplib import HTTPConnection diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py index a64007d..0322e18 100644 --- a/euscanwww/djeuscan/admin.py +++ b/euscanwww/djeuscan/admin.py @@ -7,7 +7,7 @@ from django.contrib import admin class EuscanResultAdmin(admin.ModelAdmin): search_fields = ('package__name', 'package__category') - list_filter = ('datetime', ) + list_filter = ('datetime', 'package__category') ordering = ["-datetime"] @@ -31,6 +31,12 @@ class VersionAdmin(admin.ModelAdmin): list_filter = ('overlay', 'packaged', 'alive') +class ProblemReportAdmin(admin.ModelAdmin): + list_display = ('package', 'subject', 'datetime') + search_fields = ('package__name', 'package__category') + list_filter = ('datetime', 'package__category') + ordering = ["-datetime"] + admin.site.register(Package, PackageAdmin) admin.site.register(Herd, HerdAdmin) @@ -52,4 +58,5 @@ admin.site.register(HerdAssociation) admin.site.register(CategoryAssociation) admin.site.register(MaintainerAssociation) admin.site.register(PackageAssociation) -admin.site.register(ProblemReport) + +admin.site.register(ProblemReport, ProblemReportAdmin) diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py index 0c38909..6626a41 100644 --- a/euscanwww/djeuscan/models.py +++ b/euscanwww/djeuscan/models.py @@ -1,3 +1,5 @@ +import json + from django.db import models from django.core.validators import RegexValidator, validate_email, URLValidator from django.core.exceptions import ValidationError @@ -215,6 +217,14 @@ class EuscanResult(models.Model): self.full_clean() super(EuscanResult, self).save(*args, **kwargs) + def messages(self): + result = json.loads(self.result) + + if result and self.package.cp() in result: + return result[self.package.cp()]['messages'] + else: + return "" + def __unicode__(self): return '[%s] %s/%s' % ( self.datetime, self.package.category, self.package.name diff --git a/euscanwww/djeuscan/templates/euscan/_package_details.html b/euscanwww/djeuscan/templates/euscan/_package_details.html new file mode 100644 index 0000000..1e04911 --- /dev/null +++ b/euscanwww/djeuscan/templates/euscan/_package_details.html @@ -0,0 +1,122 @@ +{% load djeuscan_helpers %} +{% load url from future %} + +<dl> + {% if package.description %} + <dt>Description</dt> + <dd>{{ package.description }}</dd> + {% endif %} + {% if package.homepage %} + <dt>Homepage</dt> + <dd> + {% for homepage in package.homepages %} + <a href="{{ homepage }}">{{ homepage }}</a><br /> + {% endfor %} + </dd> + {% endif %} + {% if package.herds.all %} + <dt>Herds</dt> + <dd> + {% for herd in package.herds.all %} + <a href="{% url "herd" herd.herd %}"> + {{ herd.herd }} + </a> + <{{ herd.email }}> + {% endfor %} + </dd> + {% endif %} + {% if package.maintainers.all %} + <dt>Maintainers</dt> + <dd> + {% for maintainer in package.maintainers.all %} + {% if maintainer.name != maintainer.email %} + <a href="{% url "maintainer" maintainer.id %}"> + {{ maintainer.name }} + </a> + <{{ maintainer.email }}> + {% else %} + <a href="{% url "maintainer" maintainer.id %}"> + {{ maintainer.email }} + </a> + {% endif %} + {% endfor %} + </dd> + {% endif %} + {% if packaged %} + <dt>Packaged Versions</dt> + <dd> + <ul> + {% for version in packaged %} + <li id="{{ version.version }}-{{version.revision }}:{{ version.slot }}-[{{ version.overlay }}]"> + {% if version.overlay == "gentoo" %} + <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" /> + {% else %} + <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" /> + {% endif %} + + {% if version.vtype and not version.vtype|is_stable %} + <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" /> + {% endif %} + + {{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}] + </li> + {% endfor %} + </ul> + </dd> + {% endif %} + {% if upstream %} + <dt>Upstream versions</dt> + <dd> + <ul> + {% for version in upstream %} + <li> + <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" /> + + {% if version.vtype and not version.vtype|is_stable %} + <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" /> + {% endif %} + + {{ version.version }} - {{ version.urls }} + {% if confidence < 100 %}({{ version.confidence }}%){% endif %} + </li> + {% endfor %} + </ul> + </dd> + {% endif %} + {% if vlog %} + <dt>Version history</dt> + <dd> + <ul class="log"> + {% for version in vlog %} + {% if version.action == version.VERSION_ADDED %} + <li class="added"> + {% else %} + <li class="removed"> + {% endif %} + {% if version.overlay == "gentoo" %} + <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" /> + {% elif version.overlay %} + <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" /> + {% else %} + <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" /> + {% endif %} + + {% if version.vtype and not version.vtype|is_stable %} + <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" /> + {% endif %} + + {{ version }} - {{ version.datetime }} + + </li> + {% endfor %} + </ul> + </dd> + {% endif %} + {% if log %} + <dt>euscan log</dt> + <dd> + <p>Date: {{ log.datetime }} + <pre class="log">{{ msg|ansi_to_html|safe }}</pre> + </dd> + {% endif %} +</dl> diff --git a/euscanwww/djeuscan/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html index 2357318..8c90efe 100644 --- a/euscanwww/djeuscan/templates/euscan/package.html +++ b/euscanwww/djeuscan/templates/euscan/package.html @@ -1,7 +1,6 @@ {% extends "euscan/_datatable.html" %} {% load sub %} -{% load djeuscan_helpers %} {% load url from future %} {% block meta %} @@ -26,12 +25,6 @@ {% block content %} -{% if thanks_for_reporting %} - <div class="alert alert-success"> - <strong>Thanks!</strong> Your report has been sent to admins - </div> -{% endif %} - <div class="refresh-alert alert {% if not refreshed %}hide{% endif %}"> A refresh request is in progress, please wait... </div> @@ -56,158 +49,18 @@ </span> {% endif %} </h2> -<dl> - {% if package.description %} - <dt>Description</dt> - <dd>{{ package.description }}</dd> - {% endif %} - {% if package.homepage %} - <dt>Homepage</dt> - <dd> - {% for homepage in package.homepages %} - <a href="{{ homepage }}">{{ homepage }}</a><br /> - {% endfor %} - </dd> - {% endif %} - {% if package.herds.all %} - <dt>Herds</dt> - <dd> - {% for herd in package.herds.all %} - <a href="{% url "herd" herd.herd %}"> - {{ herd.herd }} - </a> - <{{ herd.email }}> - {% endfor %} - </dd> - {% endif %} - {% if package.maintainers.all %} - <dt>Maintainers</dt> - <dd> - {% for maintainer in package.maintainers.all %} - {% if maintainer.name != maintainer.email %} - <a href="{% url "maintainer" maintainer.id %}"> - {{ maintainer.name }} - </a> - <{{ maintainer.email }}> - {% else %} - <a href="{% url "maintainer" maintainer.id %}"> - {{ maintainer.email }} - </a> - {% endif %} - {% endfor %} - </dd> - {% endif %} - {% if packaged %} - <dt>Packaged Versions</dt> - <dd> - <ul> - {% for version in packaged %} - <li id="{{ version.version }}-{{version.revision }}:{{ version.slot }}-[{{ version.overlay }}]"> - {% if version.overlay == "gentoo" %} - <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" /> - {% else %} - <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" /> - {% endif %} - - {% if version.vtype and not version.vtype|is_stable %} - <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" /> - {% endif %} - - {{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}] - </li> - {% endfor %} - </ul> - </dd> - {% endif %} - {% if upstream %} - <dt>Upstream versions</dt> - <dd> - <ul> - {% for version in upstream %} - <li> - <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" /> - - {% if version.vtype and not version.vtype|is_stable %} - <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" /> - {% endif %} - - {{ version.version }} - {{ version.urls }} - {% if confidence < 100 %}({{ version.confidence }}%){% endif %} - </li> - {% endfor %} - </ul> - </dd> - {% endif %} - <dt>Version history</dt> - <dd> - <ul class="log"> - {% for version in vlog %} - {% if version.action == version.VERSION_ADDED %} - <li class="added"> - {% else %} - <li class="removed"> - {% endif %} - {% if version.overlay == "gentoo" %} - <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" /> - {% elif version.overlay %} - <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" /> - {% else %} - <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" /> - {% endif %} - - {% if version.vtype and not version.vtype|is_stable %} - <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" /> - {% endif %} - - {{ version }} - {{ version.datetime }} - - </li> - {% endfor %} - </ul> - </dt> - {% if log %} - <dt>euscan log</dt> - <dd> - <p>Date: {{ log.datetime }} - <pre class="log">{{ msg|ansi_to_html|safe }}</pre> - </dd> - {% endif %} -</dl> + +{% include "euscan/_package_details.html" %} <hr/> -<h3 class="report-problems-toggle cursor-pointer">Report problems</h3> -<div class="report-problems hide"> - <form action="." method="post" class="form-horizontal"> - <div class="control-group {% if problem_form.version.errors %}error{% endif %}"> - <label class="control-label" for="version">{{ problem_form.version.label }}</label> - <div class="controls"> - {{ problem_form.version }} - <span class="help-inline">{{ problem_form.version.errors.as_text }}</span> - </div> - </div> - <div class="control-group {% if problem_form.subject.errors %}error{% endif %}"> - <label class="control-label" for="subject">{{ problem_form.subject.label }}</label> - <div class="controls"> - {{ problem_form.subject }} - <span class="help-inline">{{ problem_form.subject.errors.as_text }}</span> - </div> - </div> - <div class="control-group {% if problem_form.message.errors %}error{% endif %}"> - <label class="control-label" for="message">{{ problem_form.message.label }}</label> - <div class="controls"> - {{ problem_form.message }} - <span class="help-inline">{{ problem_form.message.errors.as_text }}</span> - </div> - </div> - - <div class="control-group"> - <div class="controls"> - <input class="btn" type="submit" value="Submit" /> - </div> - </div> - </form> -</div> +{% if user.is_authenticated %} + <form class="pull-right" action="{% url "problem" package.category package.name %}" method="get"> + <button class="btn bug-button"> + <img src="{{ STATIC_URL}}/img/bug.png" alt="Report problems" > + </button> + </form> +{% endif %} <script type="text/javascript"> $(document).ready(function () { @@ -235,10 +88,6 @@ }); }); - $(".report-problems-toggle").click(function () { - $(".report-problems").toggle("slow"); - }); - }); </script> diff --git a/euscanwww/djeuscan/templates/euscan/problem.html b/euscanwww/djeuscan/templates/euscan/problem.html new file mode 100644 index 0000000..828b7ab --- /dev/null +++ b/euscanwww/djeuscan/templates/euscan/problem.html @@ -0,0 +1,57 @@ +{% extends "euscan/_datatable.html" %} + +{% load sub %} +{% load djeuscan_helpers %} +{% load url from future %} + +{% block title %} +{{ block.super }} - Report Problem: {{ package.category }}/{{ package.name }} +{% endblock %} + +{% block content %} + +{% if thanks_for_reporting %} + <div class="alert alert-success"> + <strong>Thanks!</strong> Your report has been sent to admins + </div> +{% endif %} + +<h2> + Report Problem: {{ package.category }}/{{ package.name }} +</h2> + +{% include "euscan/_package_details.html" %} +<hr/> +<div class="report-problems"> + <form action="{% url "problem" package.category package.name %}" method="post" class="form-horizontal"> + <div class="control-group {% if form.version.errors %}error{% endif %}"> + <label class="control-label" for="version">{{ form.version.label }}</label> + <div class="controls"> + {{ form.version }} + <span class="help-inline">{{ form.version.errors.as_text }}</span> + </div> + </div> + <div class="control-group {% if form.subject.errors %}error{% endif %}"> + <label class="control-label" for="subject">{{ form.subject.label }}</label> + <div class="controls"> + {{ form.subject }} + <span class="help-inline">{{ form.subject.errors.as_text }}</span> + </div> + </div> + <div class="control-group {% if form.message.errors %}error{% endif %}"> + <label class="control-label" for="message">{{ form.message.label }}</label> + <div class="controls"> + {{ form.message }} + <span class="help-inline">{{ form.message.errors.as_text }}</span> + </div> + </div> + + <div class="control-group"> + <div class="controls"> + <input class="btn" type="submit" value="Submit" /> + </div> + </div> + </form> +</div> + +{% endblock %} diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py index e4611d7..b1b1f88 100644 --- a/euscanwww/djeuscan/urls.py +++ b/euscanwww/djeuscan/urls.py @@ -22,6 +22,8 @@ package_patterns = patterns('djeuscan.views', 'unfavourite/$'), 'unfavourite_package', name="unfavourite_package"), url((r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/' 'refresh$'), "refresh_package", name="refresh_package"), + url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/problem$', + 'problem', name="problem"), ) categories_patterns = patterns('djeuscan.views', diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py index 3d96cb0..68b3d07 100644 --- a/euscanwww/djeuscan/views.py +++ b/euscanwww/djeuscan/views.py @@ -1,7 +1,6 @@ """ Views """ import inspect -import json from annoying.decorators import render_to, ajax_request from django.http import Http404 @@ -203,13 +202,6 @@ def package(request, category, package): log = log[0] if log else None vlog = VersionLog.objects.for_package(package, order=True) - result = json.loads(log.result) if log else None - - if result and package.cp() in result: - msg = result[package.cp()]['messages'] - else: - msg = "" - try: last_scan = EuscanResult.objects.for_package(package).latest().datetime except EuscanResult.DoesNotExist: @@ -230,32 +222,52 @@ def package(request, category, package): except RefreshPackageQuery.DoesNotExist: refreshed = False + return { + 'package': package, + 'packaged': packaged, + 'upstream': upstream, + 'log': log.messages(), + 'vlog': vlog, + 'msg': log.messages() if log else "", + 'last_scan': last_scan, + 'favourited': favourited, + 'refreshed': refreshed, + } + + +@login_required +@render_to('euscan/problem.html') +def problem(request, category, package): + package = get_object_or_404(Package, category=category, name=package) + packaged = Version.objects.filter(package=package, packaged=True) + upstream = Version.objects.filter(package=package, packaged=False) + + log = EuscanResult.objects.filter(package=package).\ + order_by('-datetime')[:1] + log = log[0] if log else None + thanks_for_reporting = False + if request.method == "POST": - problem_form = ProblemReportForm(package, request.POST) - if problem_form.is_valid(): + form = ProblemReportForm(package, request.POST) + if form.is_valid(): ProblemReport( package=package, - version=problem_form.cleaned_data["version"], - subject=problem_form.cleaned_data["subject"], - message=problem_form.cleaned_data["message"], + version=form.cleaned_data["version"], + subject=form.cleaned_data["subject"], + message=form.cleaned_data["message"], ).save() thanks_for_reporting = True else: - problem_form = ProblemReportForm(package) + form = ProblemReportForm(package) return { + 'form': form, + 'thanks_for_reporting': thanks_for_reporting, 'package': package, 'packaged': packaged, 'upstream': upstream, - 'log': log, - 'vlog': vlog, - 'msg': msg, - 'last_scan': last_scan, - 'favourited': favourited, - 'refreshed': refreshed, - 'problem_form': problem_form, - 'thanks_for_reporting': thanks_for_reporting + 'msg': log.messages() if log else "", } @@ -264,8 +276,10 @@ def world(request): world_form = WorldForm() packages_form = PackagesForm() - return {'world_form': world_form, - 'packages_form': packages_form} + return { + 'world_form': world_form, + 'packages_form': packages_form + } @render_to('euscan/world_scan.html') diff --git a/euscanwww/htdocs/img/bug.png b/euscanwww/htdocs/img/bug.png Binary files differnew file mode 100644 index 0000000..c33bfb9 --- /dev/null +++ b/euscanwww/htdocs/img/bug.png diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/watch.py new file mode 100644 index 0000000..a129281 --- /dev/null +++ b/pym/euscan/handlers/watch.py @@ -0,0 +1,143 @@ +import re +import urllib2 + +import portage + +from euscan.handlers import generic +from euscan import output, helpers + +PRIORITY = 100 + +HANDLER_NAME = "watch" +CONFIDENCE = 100.0 + + +is_pattern = r"\([^\/]+\)" + + +def can_handle(pkg, url): + try: + return pkg.metadata._xml_tree.find("upstream").find("watch") \ + is not None + except AttributeError: + return False + + +def parse_mangles(mangles, string): + for mangle in mangles: + # convert regex from perl format to python format + # there are some regex in this format: s/pattern/replacement/ + m = re.match(r"s/(.*[^\\])/(.*)/", mangle) + if not m: + # or in this format s|pattern|replacement| + m = re.match(r"s\|(.*[^\\])\|(.*)\|", mangle) + pattern, repl = m.groups() + repl = re.sub(r"\$(\d+)", r"\\\1", repl) + string = re.sub(pattern, repl, string) + return string + + +def clean_results(results, versionmangle, urlmangle): + ret = [] + + for path, version, _, _ in results: + version = parse_mangles(versionmangle, version) + path = parse_mangles(urlmangle, path) + ret.append((path, version, HANDLER_NAME, CONFIDENCE)) + + return ret + + +def parse_watch(pkg): + for watch_tag in pkg.metadata._xml_tree.find("upstream").findall("watch"): + try: + base, file_pattern = watch_tag.text.split(" ")[:2] + except ValueError: + base, file_pattern = watch_tag.text, None + + # the file pattern can be in the base url + pattern_regex = r"/([^/]*\([^/]*\)[^/]*)$" + match = re.search(pattern_regex, base) + if match: + file_pattern = match.group(1) + base = base.replace(file_pattern, "") + + # handle sf.net specially + base = base.replace( + "http://sf.net/", "http://qa.debian.org/watch/sf.php/" + ) + + vmangle = watch_tag.attrib.get("uversionmangle", None) or \ + watch_tag.attrib.get("versionmangle", None) + versionmangle = vmangle.split(";") if vmangle else [] + + umangle = watch_tag.attrib.get("downloadurlmangle", None) + urlmangle = umangle.split(";") if umangle else [] + + yield (base, file_pattern, versionmangle, urlmangle) + + +def handle_directory_patterns(base, file_pattern): + """ + Directory pattern matching + e.g.: base: ftp://ftp.nessus.org/pub/nessus/nessus-([\d\.]+)/src/ + file_pattern: nessus-core-([\d\.]+)\.tar\.gz + """ + splitted = base.split("/") + i = 0 + basedir = [] + for elem in splitted: + if re.search(is_pattern, elem): + break + basedir.append(elem) + i += 1 + basedir = "/".join(basedir) + directory_pattern = splitted[i] + final = "/".join(splitted[i + 1:]) + + try: + fp = helpers.urlopen(basedir) + except urllib2.URLError: + return [] + except IOError: + return [] + + if not fp: + return [] + + data = fp.read() + + if basedir.startswith("ftp://"): + scan_data = generic.scan_ftp(data, basedir, directory_pattern) + else: + scan_data = generic.scan_html(data, basedir, directory_pattern) + + return [("/".join((basedir, path, final)), file_pattern) + for _, path in scan_data] + + +def scan(pkg, url): + output.einfo("Using watch data") + + cp, ver, rev = portage.pkgsplit(pkg.cpv) + + results = [] + for base, file_pattern, versionmangle, urlmangle in parse_watch(pkg): + if not re.search(is_pattern, base): + steps = [(base, file_pattern)] + res = generic.scan_directory_recursive( + cp, ver, rev, "", steps, url + ) + else: + res = [] + for step in handle_directory_patterns(base, file_pattern): + res += generic.scan_directory_recursive( + cp, ver, rev, "", [step], url + ) + + results += clean_results(res, versionmangle, urlmangle) + return results + + +def brute_force(pkg, url): + return [] @@ -70,7 +70,7 @@ packages = [ ] tests_require = [ - 'factory-boy==1.1.3', + 'factory-boy>=1.1.3', ] setup( @@ -88,10 +88,13 @@ setup( ('master' if __version__ == '9999' else ('euscan-%s' % __version__)) ), install_requires=[ - 'Django==1.4', 'django-annoying==0.7.6', 'South==0.7.4', - 'django-piston==0.2.3', 'BeautifulSoup==3.2.1', 'matplotlib==1.1.0', - 'django-celery==3.0.1', 'django-registration==0.8', - 'python-ldap==2.4.10', 'django-auth-ldap==1.1', + # Command line utility + 'BeautifulSoup>=3.2.1', + # Web interface + 'Django>=1.4', 'django-annoying>=0.7.6', 'South>=0.7', + 'django-piston>=0.2.3', 'matplotlib>=1.1.0', + 'django-celery>=3.0.1', 'django-registration>=0.8', + 'python-ldap>=2.4.10', 'django-auth-ldap>=1.1', ], package_dir={'': 'pym'}, packages=packages, |
