Merge branch 'master' of https://github.com/iksaif/euscan

Conflicts:
	bin/euscan_patch_metadata
	euscanwww/djeuscan/views.py
	pym/euscan/handlers/__init__.py
	pym/euscan/handlers/kde.py
	pym/euscan/handlers/url/cpan.py
	pym/euscan/handlers/url/generic.py
	pym/euscan/handlers/url/github.py
	pym/euscan/handlers/url/php.py
	pym/euscan/handlers/url/pypi.py
	pym/euscan/handlers/url/rubygem.py
	pym/euscan/scan.py

Signed-off-by: volpino <fox91@anche.no>

11 files changed, 399 insertions, 193 deletions

diff --git a/bin/euscan b/bin/euscan
index e0371b4..441d479 100755
--- a/bin/euscan
+++ b/bin/euscan
@@ -5,7 +5,7 @@ Distributed under the terms of the GNU General Public License v2
 """
 
 from __future__ import print_function
-
+import os
 
 # Meta
 
@@ -19,7 +19,6 @@ __description__ = "A tool to detect new upstream releases."
 # Imports
 
 import sys
-import os
 import getopt
 from errno import EINTR, EINVAL
 from httplib import HTTPConnection
diff --git a/euscanwww/djeuscan/admin.py b/euscanwww/djeuscan/admin.py
index a64007d..0322e18 100644
--- a/euscanwww/djeuscan/admin.py
+++ b/euscanwww/djeuscan/admin.py
@@ -7,7 +7,7 @@ from django.contrib import admin
 
 class EuscanResultAdmin(admin.ModelAdmin):
     search_fields = ('package__name', 'package__category')
-    list_filter = ('datetime', )
+    list_filter = ('datetime', 'package__category')
     ordering = ["-datetime"]
 
 
@@ -31,6 +31,12 @@ class VersionAdmin(admin.ModelAdmin):
     list_filter = ('overlay', 'packaged', 'alive')
 
 
+class ProblemReportAdmin(admin.ModelAdmin):
+    list_display = ('package', 'subject', 'datetime')
+    search_fields = ('package__name', 'package__category')
+    list_filter = ('datetime', 'package__category')
+    ordering = ["-datetime"]
+
 admin.site.register(Package, PackageAdmin)
 
 admin.site.register(Herd, HerdAdmin)
@@ -52,4 +58,5 @@ admin.site.register(HerdAssociation)
 admin.site.register(CategoryAssociation)
 admin.site.register(MaintainerAssociation)
 admin.site.register(PackageAssociation)
-admin.site.register(ProblemReport)
+
+admin.site.register(ProblemReport, ProblemReportAdmin)
diff --git a/euscanwww/djeuscan/models.py b/euscanwww/djeuscan/models.py
index 0c38909..6626a41 100644
--- a/euscanwww/djeuscan/models.py
+++ b/euscanwww/djeuscan/models.py
@@ -1,3 +1,5 @@
+import json
+
 from django.db import models
 from django.core.validators import RegexValidator, validate_email, URLValidator
 from django.core.exceptions import ValidationError
@@ -215,6 +217,14 @@ class EuscanResult(models.Model):
         self.full_clean()
         super(EuscanResult, self).save(*args, **kwargs)
 
+    def messages(self):
+        result = json.loads(self.result)
+
+        if result and self.package.cp() in result:
+            return result[self.package.cp()]['messages']
+        else:
+            return ""
+
     def __unicode__(self):
         return '[%s] %s/%s' % (
             self.datetime, self.package.category, self.package.name
diff --git a/euscanwww/djeuscan/templates/euscan/_package_details.html b/euscanwww/djeuscan/templates/euscan/_package_details.html
new file mode 100644
index 0000000..1e04911
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/_package_details.html
@@ -0,0 +1,122 @@
+{% load djeuscan_helpers %}
+{% load url from future %}
+
+<dl>
+  {% if package.description %}
+  <dt>Description</dt>
+  <dd>{{ package.description }}</dd>
+  {% endif %}
+  {% if package.homepage %}
+  <dt>Homepage</dt>
+  <dd>
+    {% for homepage in package.homepages %}
+    <a href="{{ homepage }}">{{ homepage }}</a><br />
+    {% endfor %}
+  </dd>
+  {% endif %}
+  {% if package.herds.all %}
+  <dt>Herds</dt>
+  <dd>
+    {% for herd in package.herds.all %}
+    <a href="{% url "herd" herd.herd %}">
+      {{ herd.herd }}
+    </a>
+    &lt;{{ herd.email }}&gt;
+    {% endfor %}
+  </dd>
+  {% endif %}
+  {% if package.maintainers.all %}
+  <dt>Maintainers</dt>
+  <dd>
+    {% for maintainer in package.maintainers.all %}
+      {% if maintainer.name != maintainer.email %}
+       <a href="{% url "maintainer" maintainer.id %}">
+         {{ maintainer.name }}
+       </a>
+       &lt;{{ maintainer.email }}&gt;
+      {% else %}
+       <a href="{% url "maintainer" maintainer.id %}">
+       {{ maintainer.email }}
+       </a>
+      {% endif %}
+    {% endfor %}
+  </dd>
+  {% endif %}
+  {% if packaged %}
+  <dt>Packaged Versions</dt>
+  <dd>
+    <ul>
+      {% for version in packaged %}
+      <li id="{{ version.version }}-{{version.revision }}:{{ version.slot }}-[{{ version.overlay }}]">
+        {% if version.overlay == "gentoo" %}
+          <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
+        {% else %}
+          <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
+        {% endif %}
+
+        {% if version.vtype and not version.vtype|is_stable %}
+          <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
+        {% endif %}
+
+        {{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}]
+      </li>
+      {% endfor %}
+    </ul>
+  </dd>
+  {% endif %}
+  {% if upstream %}
+  <dt>Upstream versions</dt>
+  <dd>
+    <ul>
+      {% for version in upstream %}
+      <li>
+        <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
+
+        {% if version.vtype and not version.vtype|is_stable %}
+          <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
+        {% endif %}
+
+        {{ version.version }} - {{ version.urls }}
+	  {% if confidence < 100 %}({{ version.confidence }}%){% endif %}
+      </li>
+      {% endfor %}
+     </ul>
+  </dd>
+  {% endif %}
+  {% if vlog %}
+  <dt>Version history</dt>
+  <dd>
+    <ul class="log">
+      {% for version in vlog %}
+      {% if version.action == version.VERSION_ADDED %}
+      <li class="added">
+      {% else %}
+      <li class="removed">
+      {% endif %}
+        {% if version.overlay == "gentoo" %}
+          <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
+        {% elif version.overlay %}
+          <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
+        {% else %}
+          <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
+        {% endif %}
+
+        {% if version.vtype and not version.vtype|is_stable %}
+          <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
+        {% endif %}
+
+        {{ version }} - {{ version.datetime }}
+
+      </li>
+      {% endfor %}
+    </ul>
+  </dd>
+  {% endif %}
+  {% if log %}
+  <dt>euscan log</dt>
+  <dd>
+    <p>Date: {{ log.datetime }}
+    <pre class="log">{{ msg|ansi_to_html|safe }}</pre>
+  </dd>
+  {% endif %}
+</dl>
diff --git a/euscanwww/djeuscan/templates/euscan/package.html b/euscanwww/djeuscan/templates/euscan/package.html
index 2357318..8c90efe 100644
--- a/euscanwww/djeuscan/templates/euscan/package.html
+++ b/euscanwww/djeuscan/templates/euscan/package.html
@@ -1,7 +1,6 @@
 {% extends "euscan/_datatable.html" %}
 
 {% load sub %}
-{% load djeuscan_helpers %}
 {% load url from future %}
 
 {% block meta %}
@@ -26,12 +25,6 @@
 
 {% block content %}
 
-{% if thanks_for_reporting %}
-  <div class="alert alert-success">
-    <strong>Thanks!</strong> Your report has been sent to admins
-  </div>
-{% endif %}
-
 <div class="refresh-alert alert {% if not refreshed %}hide{% endif %}">
   A refresh request is in progress, please wait...
 </div>
@@ -56,158 +49,18 @@
     </span>
   {% endif %}
 </h2>
-<dl>
-  {% if package.description %}
-  <dt>Description</dt>
-  <dd>{{ package.description }}</dd>
-  {% endif %}
-  {% if package.homepage %}
-  <dt>Homepage</dt>
-  <dd>
-    {% for homepage in package.homepages %}
-    <a href="{{ homepage }}">{{ homepage }}</a><br />
-    {% endfor %}
-  </dd>
-  {% endif %}
-  {% if package.herds.all %}
-  <dt>Herds</dt>
-  <dd>
-    {% for herd in package.herds.all %}
-    <a href="{% url "herd" herd.herd %}">
-      {{ herd.herd }}
-    </a>
-    &lt;{{ herd.email }}&gt;
-    {% endfor %}
-  </dd>
-  {% endif %}
-  {% if package.maintainers.all %}
-  <dt>Maintainers</dt>
-  <dd>
-    {% for maintainer in package.maintainers.all %}
-      {% if maintainer.name != maintainer.email %}
-       <a href="{% url "maintainer" maintainer.id %}">
-         {{ maintainer.name }}
-       </a>
-       &lt;{{ maintainer.email }}&gt;
-      {% else %}
-       <a href="{% url "maintainer" maintainer.id %}">
-       {{ maintainer.email }}
-       </a>
-      {% endif %}
-    {% endfor %}
-  </dd>
-  {% endif %}
-  {% if packaged %}
-  <dt>Packaged Versions</dt>
-  <dd>
-    <ul>
-      {% for version in packaged %}
-      <li id="{{ version.version }}-{{version.revision }}:{{ version.slot }}-[{{ version.overlay }}]">
-        {% if version.overlay == "gentoo" %}
-          <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
-        {% else %}
-          <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
-        {% endif %}
-
-        {% if version.vtype and not version.vtype|is_stable %}
-          <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
-        {% endif %}
-
-        {{ version.version }}-{{ version.revision }} :{{ version.slot }} [{{ version.overlay }}]
-      </li>
-      {% endfor %}
-    </ul>
-  </dd>
-  {% endif %}
-  {% if upstream %}
-  <dt>Upstream versions</dt>
-  <dd>
-    <ul>
-      {% for version in upstream %}
-      <li>
-        <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
-
-        {% if version.vtype and not version.vtype|is_stable %}
-          <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
-        {% endif %}
-
-        {{ version.version }} - {{ version.urls }}
-	  {% if confidence < 100 %}({{ version.confidence }}%){% endif %}
-      </li>
-      {% endfor %}
-     </ul>
-  </dd>
-  {% endif %}
-  <dt>Version history</dt>
-  <dd>
-    <ul class="log">
-      {% for version in vlog %}
-      {% if version.action == version.VERSION_ADDED %}
-      <li class="added">
-      {% else %}
-      <li class="removed">
-      {% endif %}
-        {% if version.overlay == "gentoo" %}
-          <img src="{{ STATIC_URL }}img/gentoo-icon.png" alt="gentoo" title="In Gentoo" />
-        {% elif version.overlay %}
-          <img src="{{ STATIC_URL }}img/overlay-icon.png" alt="overlays" title="In Overlays" />
-        {% else %}
-          <img src="{{ STATIC_URL }}img/upstream-icon.png" alt="upstream" title="Upstream" />
-        {% endif %}
-
-        {% if version.vtype and not version.vtype|is_stable %}
-          <img src="{{ STATIC_URL }}img/unstable-icon.png" alt="unstable" title="{{ version.vtype }}" />
-        {% endif %}
-
-        {{ version }} - {{ version.datetime }}
-
-      </li>
-      {% endfor %}
-    </ul>
-  </dt>
-  {% if log %}
-  <dt>euscan log</dt>
-  <dd>
-    <p>Date: {{ log.datetime }}
-    <pre class="log">{{ msg|ansi_to_html|safe }}</pre>
-  </dd>
-  {% endif %}
-</dl>
+
+{% include "euscan/_package_details.html" %}
 
 <hr/>
 
-<h3 class="report-problems-toggle cursor-pointer">Report problems</h3>
-<div class="report-problems hide">
-  <form action="." method="post" class="form-horizontal">
-    <div class="control-group {% if problem_form.version.errors %}error{% endif %}">
-      <label class="control-label" for="version">{{ problem_form.version.label }}</label>
-      <div class="controls">
-        {{ problem_form.version }}
-        <span class="help-inline">{{ problem_form.version.errors.as_text }}</span>
-      </div>
-    </div>
-    <div class="control-group {% if problem_form.subject.errors %}error{% endif %}">
-      <label class="control-label" for="subject">{{ problem_form.subject.label }}</label>
-      <div class="controls">
-        {{ problem_form.subject }}
-        <span class="help-inline">{{ problem_form.subject.errors.as_text }}</span>
-      </div>
-    </div>
-    <div class="control-group {% if problem_form.message.errors %}error{% endif %}">
-      <label class="control-label" for="message">{{ problem_form.message.label }}</label>
-      <div class="controls">
-        {{ problem_form.message }}
-        <span class="help-inline">{{ problem_form.message.errors.as_text }}</span>
-      </div>
-    </div>
-
-    <div class="control-group">
-      <div class="controls">
-        <input class="btn" type="submit" value="Submit" />
-      </div>
-    </div>
-  </form>
-</div>
+{% if user.is_authenticated %}
+    <form class="pull-right" action="{% url "problem" package.category package.name %}" method="get">
+      <button class="btn bug-button">
+        <img src="{{ STATIC_URL}}/img/bug.png" alt="Report problems" >
+      </button>
+    </form>
+{% endif %}
 
 <script type="text/javascript">
   $(document).ready(function () {
@@ -235,10 +88,6 @@
       });
     });
 
-    $(".report-problems-toggle").click(function () {
-      $(".report-problems").toggle("slow");
-    });
-
   });
 </script>
 
diff --git a/euscanwww/djeuscan/templates/euscan/problem.html b/euscanwww/djeuscan/templates/euscan/problem.html
new file mode 100644
index 0000000..828b7ab
--- /dev/null
+++ b/euscanwww/djeuscan/templates/euscan/problem.html
@@ -0,0 +1,57 @@
+{% extends "euscan/_datatable.html" %}
+
+{% load sub %}
+{% load djeuscan_helpers %}
+{% load url from future %}
+
+{% block title %}
+{{ block.super }} - Report Problem: {{ package.category }}/{{ package.name }}
+{% endblock %}
+
+{% block content %}
+
+{% if thanks_for_reporting %}
+  <div class="alert alert-success">
+    <strong>Thanks!</strong> Your report has been sent to admins
+  </div>
+{% endif %}
+
+<h2>
+  Report Problem: {{ package.category }}/{{ package.name }}
+</h2>
+
+{% include "euscan/_package_details.html" %}
+<hr/>
+<div class="report-problems">
+  <form action="{% url "problem" package.category package.name %}" method="post" class="form-horizontal">
+    <div class="control-group {% if form.version.errors %}error{% endif %}">
+      <label class="control-label" for="version">{{ form.version.label }}</label>
+      <div class="controls">
+        {{ form.version }}
+        <span class="help-inline">{{ form.version.errors.as_text }}</span>
+      </div>
+    </div>
+    <div class="control-group {% if form.subject.errors %}error{% endif %}">
+      <label class="control-label" for="subject">{{ form.subject.label }}</label>
+      <div class="controls">
+        {{ form.subject }}
+        <span class="help-inline">{{ form.subject.errors.as_text }}</span>
+      </div>
+    </div>
+    <div class="control-group {% if form.message.errors %}error{% endif %}">
+      <label class="control-label" for="message">{{ form.message.label }}</label>
+      <div class="controls">
+        {{ form.message }}
+        <span class="help-inline">{{ form.message.errors.as_text }}</span>
+      </div>
+    </div>
+
+    <div class="control-group">
+      <div class="controls">
+        <input class="btn" type="submit" value="Submit" />
+      </div>
+    </div>
+  </form>
+</div>
+
+{% endblock %}
diff --git a/euscanwww/djeuscan/urls.py b/euscanwww/djeuscan/urls.py
index e4611d7..b1b1f88 100644
--- a/euscanwww/djeuscan/urls.py
+++ b/euscanwww/djeuscan/urls.py
@@ -22,6 +22,8 @@ package_patterns = patterns('djeuscan.views',
          'unfavourite/$'), 'unfavourite_package', name="unfavourite_package"),
     url((r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/'
          'refresh$'), "refresh_package", name="refresh_package"),
+    url(r'^(?P<category>[\w+][\w+.-]*)/(?P<package>[\w+][\w+.-]*)/problem$',
+        'problem', name="problem"),
 )
 
 categories_patterns = patterns('djeuscan.views',
diff --git a/euscanwww/djeuscan/views.py b/euscanwww/djeuscan/views.py
index 3d96cb0..68b3d07 100644
--- a/euscanwww/djeuscan/views.py
+++ b/euscanwww/djeuscan/views.py
@@ -1,7 +1,6 @@
 """ Views """
 
 import inspect
-import json
 from annoying.decorators import render_to, ajax_request
 
 from django.http import Http404
@@ -203,13 +202,6 @@ def package(request, category, package):
     log = log[0] if log else None
     vlog = VersionLog.objects.for_package(package, order=True)
 
-    result = json.loads(log.result) if log else None
-
-    if result and package.cp() in result:
-        msg = result[package.cp()]['messages']
-    else:
-        msg = ""
-
     try:
         last_scan = EuscanResult.objects.for_package(package).latest().datetime
     except EuscanResult.DoesNotExist:
@@ -230,32 +222,52 @@ def package(request, category, package):
     except RefreshPackageQuery.DoesNotExist:
         refreshed = False
 
+    return {
+        'package': package,
+        'packaged': packaged,
+        'upstream': upstream,
+        'log': log.messages(),
+        'vlog': vlog,
+        'msg': log.messages() if log else "",
+        'last_scan': last_scan,
+        'favourited': favourited,
+        'refreshed': refreshed,
+    }
+
+
+@login_required
+@render_to('euscan/problem.html')
+def problem(request, category, package):
+    package = get_object_or_404(Package, category=category, name=package)
+    packaged = Version.objects.filter(package=package, packaged=True)
+    upstream = Version.objects.filter(package=package, packaged=False)
+
+    log = EuscanResult.objects.filter(package=package).\
+                               order_by('-datetime')[:1]
+    log = log[0] if log else None
+
     thanks_for_reporting = False
+
     if request.method == "POST":
-        problem_form = ProblemReportForm(package, request.POST)
-        if problem_form.is_valid():
+        form = ProblemReportForm(package, request.POST)
+        if form.is_valid():
             ProblemReport(
                 package=package,
-                version=problem_form.cleaned_data["version"],
-                subject=problem_form.cleaned_data["subject"],
-                message=problem_form.cleaned_data["message"],
+                version=form.cleaned_data["version"],
+                subject=form.cleaned_data["subject"],
+                message=form.cleaned_data["message"],
             ).save()
             thanks_for_reporting = True
     else:
-        problem_form = ProblemReportForm(package)
+        form = ProblemReportForm(package)
 
     return {
+        'form': form,
+        'thanks_for_reporting': thanks_for_reporting,
         'package': package,
         'packaged': packaged,
         'upstream': upstream,
-        'log': log,
-        'vlog': vlog,
-        'msg': msg,
-        'last_scan': last_scan,
-        'favourited': favourited,
-        'refreshed': refreshed,
-        'problem_form': problem_form,
-        'thanks_for_reporting': thanks_for_reporting
+        'msg': log.messages() if log else "",
     }
 
 
@@ -264,8 +276,10 @@ def world(request):
     world_form = WorldForm()
     packages_form = PackagesForm()
 
-    return {'world_form': world_form,
-            'packages_form': packages_form}
+    return {
+        'world_form': world_form,
+        'packages_form': packages_form
+    }
 
 
 @render_to('euscan/world_scan.html')
diff --git a/euscanwww/htdocs/img/bug.png b/euscanwww/htdocs/img/bug.png
new file mode 100644
index 0000000..c33bfb9
--- /dev/null
+++ b/euscanwww/htdocs/img/bug.png
diff --git a/pym/euscan/handlers/watch.py b/pym/euscan/handlers/watch.py
new file mode 100644
index 0000000..a129281
--- /dev/null
+++ b/pym/euscan/handlers/watch.py
@@ -0,0 +1,143 @@
+import re
+import urllib2
+
+import portage
+
+from euscan.handlers import generic
+from euscan import output, helpers
+
+PRIORITY = 100
+
+HANDLER_NAME = "watch"
+CONFIDENCE = 100.0
+
+
+is_pattern = r"\([^\/]+\)"
+
+
+def can_handle(pkg, url):
+    try:
+        return pkg.metadata._xml_tree.find("upstream").find("watch") \
+               is not None
+    except AttributeError:
+        return False
+
+
+def parse_mangles(mangles, string):
+    for mangle in mangles:
+        # convert regex from perl format to python format
+        # there are some regex in this format: s/pattern/replacement/
+        m = re.match(r"s/(.*[^\\])/(.*)/", mangle)
+        if not m:
+            # or in this format s|pattern|replacement|
+            m = re.match(r"s\|(.*[^\\])\|(.*)\|", mangle)
+        pattern, repl = m.groups()
+        repl = re.sub(r"\$(\d+)", r"\\\1", repl)
+        string = re.sub(pattern, repl, string)
+    return string
+
+
+def clean_results(results, versionmangle, urlmangle):
+    ret = []
+
+    for path, version, _, _ in results:
+        version = parse_mangles(versionmangle, version)
+        path = parse_mangles(urlmangle, path)
+        ret.append((path, version, HANDLER_NAME, CONFIDENCE))
+
+    return ret
+
+
+def parse_watch(pkg):
+    for watch_tag in pkg.metadata._xml_tree.find("upstream").findall("watch"):
+        try:
+            base, file_pattern = watch_tag.text.split(" ")[:2]
+        except ValueError:
+            base, file_pattern = watch_tag.text, None
+
+        # the file pattern can be in the base url
+        pattern_regex = r"/([^/]*\([^/]*\)[^/]*)$"
+        match = re.search(pattern_regex, base)
+        if match:
+            file_pattern = match.group(1)
+            base = base.replace(file_pattern, "")
+
+        # handle sf.net specially
+        base = base.replace(
+            "http://sf.net/", "http://qa.debian.org/watch/sf.php/"
+        )
+
+        vmangle = watch_tag.attrib.get("uversionmangle", None) or \
+                  watch_tag.attrib.get("versionmangle", None)
+        versionmangle = vmangle.split(";") if vmangle else []
+
+        umangle = watch_tag.attrib.get("downloadurlmangle", None)
+        urlmangle = umangle.split(";") if umangle else []
+
+        yield (base, file_pattern, versionmangle, urlmangle)
+
+
+def handle_directory_patterns(base, file_pattern):
+    """
+    Directory pattern matching
+    e.g.: base: ftp://ftp.nessus.org/pub/nessus/nessus-([\d\.]+)/src/
+          file_pattern: nessus-core-([\d\.]+)\.tar\.gz
+    """
+    splitted = base.split("/")
+    i = 0
+    basedir = []
+    for elem in splitted:
+        if re.search(is_pattern, elem):
+            break
+        basedir.append(elem)
+        i += 1
+    basedir = "/".join(basedir)
+    directory_pattern = splitted[i]
+    final = "/".join(splitted[i + 1:])
+
+    try:
+        fp = helpers.urlopen(basedir)
+    except urllib2.URLError:
+        return []
+    except IOError:
+        return []
+
+    if not fp:
+        return []
+
+    data = fp.read()
+
+    if basedir.startswith("ftp://"):
+        scan_data = generic.scan_ftp(data, basedir, directory_pattern)
+    else:
+        scan_data = generic.scan_html(data, basedir, directory_pattern)
+
+    return [("/".join((basedir, path, final)), file_pattern)
+            for _, path in scan_data]
+
+
+def scan(pkg, url):
+    output.einfo("Using watch data")
+
+    cp, ver, rev = portage.pkgsplit(pkg.cpv)
+
+    results = []
+    for base, file_pattern, versionmangle, urlmangle in parse_watch(pkg):
+        if not re.search(is_pattern, base):
+            steps = [(base, file_pattern)]
+            res = generic.scan_directory_recursive(
+                cp, ver, rev, "", steps, url
+            )
+        else:
+            res = []
+            for step in handle_directory_patterns(base, file_pattern):
+                res += generic.scan_directory_recursive(
+                    cp, ver, rev, "", [step], url
+                )
+
+        results += clean_results(res, versionmangle, urlmangle)
+    return results
+
+
+def brute_force(pkg, url):
+    return []
diff --git a/setup.py b/setup.py
index 85856aa..9b7179f 100755
--- a/setup.py
+++ b/setup.py
@@ -70,7 +70,7 @@ packages = [
 ]
 
 tests_require = [
-  'factory-boy==1.1.3',
+  'factory-boy>=1.1.3',
 ]
 
 setup(
@@ -88,10 +88,13 @@ setup(
         ('master' if __version__ == '9999' else ('euscan-%s' % __version__))
     ),
     install_requires=[
-        'Django==1.4', 'django-annoying==0.7.6', 'South==0.7.4',
-        'django-piston==0.2.3', 'BeautifulSoup==3.2.1', 'matplotlib==1.1.0',
-        'django-celery==3.0.1', 'django-registration==0.8',
-        'python-ldap==2.4.10', 'django-auth-ldap==1.1',
+        # Command line utility
+        'BeautifulSoup>=3.2.1',
+        # Web interface
+        'Django>=1.4', 'django-annoying>=0.7.6', 'South>=0.7',
+        'django-piston>=0.2.3',  'matplotlib>=1.1.0',
+        'django-celery>=3.0.1', 'django-registration>=0.8',
+        'python-ldap>=2.4.10', 'django-auth-ldap>=1.1',
     ],
     package_dir={'': 'pym'},
     packages=packages,