dev-python/twisted: Work on the test patches, separate some for upstream submission

Separate several test fixes and improve them for possible inclusion upstream.
Determined that there were some python anomlies occuring causing some utf8 test failures.
Isolate those with an UTF8_OVERRIDES environment variable.  Then run them separately
after the main twisted test run.
Adjust the ebuild to suit the patch and test changes.

Package-Manager: Portage-2.3.5, Repoman-2.3.2_p30

4 files changed, 157 insertions, 106 deletions

diff --git a/dev-python/twisted/files/test_main.patch b/dev-python/twisted/files/test_main.patch
new file mode 100644
index 000000000000..bfef40450d52
--- /dev/null
+++ b/dev-python/twisted/files/test_main.patch
@@ -0,0 +1,73 @@
+From 2c3c28f5dbbd61bcfa5c548d1d423fffbaf2132d Mon Sep 17 00:00:00 2001
+From: Brian Dolbec <dolsen@gentoo.org>
+Date: Fri, 31 Mar 2017 09:32:18 -0700
+Subject: [PATCH] tests/test_main.py: Fix test_twisted to handle differntly
+ sorted options
+
+Some systems retuned the usage with '__main__.py' instead of the command 'trial'
+So, substitute that out if it exists.
+The options returned via python can be a different sort order than is output via the
+command --help.  So break up the lines into a list and check equality, lines are neither
+missing or extra.
+---
+ src/twisted/test/test_main.py | 34 ++++++++++++++++++++++++++++++++--
+ 1 file changed, 32 insertions(+), 2 deletions(-)
+
+diff --git a/src/twisted/test/test_main.py b/src/twisted/test/test_main.py
+index 572769018..b010a389e 100644
+--- a/src/twisted/test/test_main.py
++++ b/src/twisted/test/test_main.py
+@@ -18,6 +18,10 @@ from twisted.trial.unittest import TestCase
+ 
+ class MainTests(TestCase):
+     """Test that twisted scripts can be invoked as modules."""
++    # this test just does not work correctly on Gentoo
++    # the output has '__main__.py' instead of 'trial'
++    # I have only been able to get 2.7 working correctly
++    # with replacing the value with what is expected.
+     def test_twisted(self):
+         """Invoking python -m twisted should execute twist."""
+         cmd = sys.executable
+@@ -28,11 +32,37 @@ class MainTests(TestCase):
+ 
+         def processEnded(ign):
+             f = p.outF
+-            output = f.getvalue().replace(b'\r\n', b'\n')
++            # Some systems may return __main__.py instead of the command name expected
++            output = f.getvalue().replace(b'\r\n', b'\n').replace(b"__main__.py", b"trial")
+ 
+             options = TwistOptions()
+             message = '{}\n'.format(options).encode('utf-8')
+-            self.assertEqual(output, message)
++            # NOTE: python may return the  options in a different order
++            # than is output via the command --help option
++            # so we must break up the text and compare that lines
++            # are not missing or extra from what is expected
++            a = output.split(b'\n')
++            b = message.split(b'\n')
++            extras = []
++            missing = []
++            equal_len = (len(a) == len(b))
++            for i in a:
++                if i not in b:
++                    extras.append(i)
++            for i in b:
++                if i not in a:
++                    missing.append(i)
++
++            self.assertTrue(equal_len,
++                msg="Usage reported a different number of lines than expected")
++            self.assertTrue(extras == [],
++                msg="Usage returned these extra lines not expected: %s"
++                    % '\n'.join(extras)
++            )
++            self.assertTrue(missing == [],
++                msg="Usage was missing these expected lines: %s"
++                    % '\n'.join(missing)
++            )
+         return d.addCallback(processEnded)
+ 
+     def test_twisted_import(self):
+-- 
+2.12.1
+
diff --git a/dev-python/twisted/files/twisted-16.6.0-test-fixes.patch b/dev-python/twisted/files/twisted-16.6.0-test-fixes.patch
index a04cafd1ccb5..3ce04aa3f684 100644
--- a/dev-python/twisted/files/twisted-16.6.0-test-fixes.patch
+++ b/dev-python/twisted/files/twisted-16.6.0-test-fixes.patch
@@ -1,7 +1,7 @@
-From 7fddbadde3f1f65c1ef78223e6af98a066a2315b Mon Sep 17 00:00:00 2001
+From 91b6d8b5b9d602152fb7148c6e2921463b93a8a5 Mon Sep 17 00:00:00 2001
 From: Brian Dolbec <dolsen@gentoo.org>
-Date: Wed, 29 Mar 2017 18:28:45 -0700
-Subject: [PATCH] Twisted-16.6.0 test fixes
+Date: Fri, 31 Mar 2017 10:55:32 -0700
+Subject: [PATCH] twisted test overrides
 
 ---
  src/twisted/internet/test/test_gireactor.py |  3 ++-
@@ -10,14 +10,11 @@ Subject: [PATCH] Twisted-16.6.0 test fixes
  src/twisted/pair/test/test_rawudp.py        | 10 +++++++++-
  src/twisted/pair/test/test_tuntap.py        | 16 ++++++++++++++++
  src/twisted/python/test/test_dist3.py       |  2 ++
- src/twisted/test/test_ident.py              |  7 ++++++-
- src/twisted/test/test_main.py               |  8 +++++++-
+ src/twisted/test/test_ident.py              |  5 ++++-
  src/twisted/test/test_plugin.py             |  6 ++++++
  src/twisted/test/test_policies.py           |  5 +++++
- src/twisted/test/test_reflect.py            |  3 ++-
- src/twisted/test/test_twistd.py             | 21 ++++++++++++++-------
  src/twisted/test/test_udp.py                |  6 ++++++
- 13 files changed, 92 insertions(+), 12 deletions(-)
+ 10 files changed, 67 insertions(+), 3 deletions(-)
 
 diff --git a/src/twisted/internet/test/test_gireactor.py b/src/twisted/internet/test/test_gireactor.py
 index 43147fdce..6333218e7 100644
@@ -169,17 +166,10 @@ index 3ce2bdd60..494674bc7 100644
 +
 +ModulesToInstallTests.skip = "This is an upstream distribution test only"
 diff --git a/src/twisted/test/test_ident.py b/src/twisted/test/test_ident.py
-index d86b840e5..028778a2d 100644
+index d86b840e5..3cc40261f 100644
 --- a/src/twisted/test/test_ident.py
 +++ b/src/twisted/test/test_ident.py
-@@ -6,13 +6,14 @@
- Test cases for twisted.protocols.ident module.
- """
- 
-+import os
- import struct
- 
- from twisted.protocols import ident
+@@ -12,7 +12,7 @@ from twisted.protocols import ident
  from twisted.python import failure
  from twisted.internet import error
  from twisted.internet import defer
@@ -188,15 +178,7 @@ index d86b840e5..028778a2d 100644
  
  from twisted.trial import unittest
  from twisted.test.proto_helpers import StringTransport
-@@ -23,6 +24,7 @@ try:
- except ImportError:
-     import __builtin__ as builtins
- 
-+EMERGE_TEST_OVERRIDE = os.environ.get("EMERGE_TEST_OVERRIDE", False)
- 
- 
- class ClassParserTests(unittest.TestCase):
-@@ -216,6 +218,9 @@ class ProcMixinTests(unittest.TestCase):
+@@ -216,6 +216,9 @@ class ProcMixinTests(unittest.TestCase):
          """
          L{ident.ProcServerMixin.lookup} uses the Linux TCP process table.
          """
@@ -206,26 +188,6 @@ index d86b840e5..028778a2d 100644
          open_calls = []
  
          def mocked_open(*args, **kwargs):
-diff --git a/src/twisted/test/test_main.py b/src/twisted/test/test_main.py
-index 572769018..60b795f96 100644
---- a/src/twisted/test/test_main.py
-+++ b/src/twisted/test/test_main.py
-@@ -18,8 +18,14 @@ from twisted.trial.unittest import TestCase
- 
- class MainTests(TestCase):
-     """Test that twisted scripts can be invoked as modules."""
--    def test_twisted(self):
-+    # this test just does not work correctly on Gentoo
-+    # the output has '__main__.py' instead of 'trial'
-+    # I have only been able to get 2.7 working correctly
-+    # with replacing the value with what is expected.
-+    def _test_twisted(self):
-         """Invoking python -m twisted should execute twist."""
-+        if EMERGE_TEST_OVERRIDE:
-+            return
-         cmd = sys.executable
-         p = Accumulator()
-         d = p.endedDeferred = defer.Deferred()
 diff --git a/src/twisted/test/test_plugin.py b/src/twisted/test/test_plugin.py
 index a23caa72b..a6d61858c 100644
 --- a/src/twisted/test/test_plugin.py
@@ -287,66 +249,6 @@ index 3d92633d6..c08809a66 100644
          open_calls = []
          open_rvalues = []
  
-diff --git a/src/twisted/test/test_reflect.py b/src/twisted/test/test_reflect.py
-index 5348fc65e..2f431cef7 100644
---- a/src/twisted/test/test_reflect.py
-+++ b/src/twisted/test/test_reflect.py
-@@ -551,7 +551,8 @@ class SafeStrTests(TestCase):
-         value unchanged.
-         """
-         x = b't\xc3\xbcst'
--        self.assertEqual(reflect.safe_str(x), x)
-+        y = [b't\xc3\xbcst', b't\\xfcst']
-+        self.assertEqual(reflect.safe_str(x) in y, True)
- 
- 
-     def test_workingUtf8_3(self):
-diff --git a/src/twisted/test/test_twistd.py b/src/twisted/test/test_twistd.py
-index 04dc83600..654f8cba3 100644
---- a/src/twisted/test/test_twistd.py
-+++ b/src/twisted/test/test_twistd.py
-@@ -78,6 +78,7 @@ if getattr(os, 'setuid', None) is None:
- else:
-     setuidSkip = None
- 
-+EMERGE_TEST_OVERRIDE = os.environ.get("EMERGE_TEST_OVERRIDE", False)
- 
- 
- def patchUserDatabase(patch, user, uid, group, gid):
-@@ -1778,9 +1779,12 @@ class DaemonizeTests(unittest.TestCase):
-         message is Unicode, the child encodes the message as ascii
-         with backslash Unicode code points.
-         """
--        self.assertErrorWritten(raised=u"\u2022",
--                                reported=b'1 RuntimeError: \\u2022')
--
-+        if _PY3:
-+            self.assertErrorWritten(raised=u"\u2022",
-+                                    reported=b'1 RuntimeError: \\u2022')
-+        else:
-+            self.assertErrorWritten(raised=u"\u2022",
-+                                    reported=b'1 RuntimeError: \xe2\x80\xa2')
- 
- 
-     def assertErrorInParentBehavior(self, readData, errorMessage,
-@@ -1879,10 +1883,13 @@ class DaemonizeTests(unittest.TestCase):
-         unicode and too long, it's truncated by the child, even if
-         this splits a unicode escape sequence.
-         """
--        self.assertErrorWritten(
--            raised=u"\u2022" * 30,
--            reported=b'1 RuntimeError: ' + b'\\u2022' * 14,
--        )
-+        if not EMERGE_TEST_OVERRIDE or _PY3:
-+            self.assertErrorWritten(
-+                raised=u"\u2022" * 30,
-+                reported=b'1 RuntimeError: ' + b'\\u2022' * 14,
-+            )
-+        else:
-+            pass
- 
- 
-     def test_hooksCalled(self):
 diff --git a/src/twisted/test/test_udp.py b/src/twisted/test/test_udp.py
 index 6cf4583b2..86b513704 100644
 --- a/src/twisted/test/test_udp.py
diff --git a/dev-python/twisted/files/utf8_overrides.patch b/dev-python/twisted/files/utf8_overrides.patch
new file mode 100644
index 000000000000..41f48cebfd84
--- /dev/null
+++ b/dev-python/twisted/files/utf8_overrides.patch
@@ -0,0 +1,64 @@
+From f8b2e95cc9bd1cbae565e1b4d576950961edc9a7 Mon Sep 17 00:00:00 2001
+From: Brian Dolbec <dolsen@gentoo.org>
+Date: Fri, 31 Mar 2017 09:40:16 -0700
+Subject: [PATCH] UTF8 test overrides:  The DaemonizeTests SafeStrTests tests
+ may need to be run independantly
+
+Some other tests may leave python in a state that returns a different form of the b'\\u2022'
+bytestring (b'\xe2\x80\xa2') which causes the tests to fail.
+In StafeStrTests, the returned 't\\xfcst' != 't\xc3\xbcst' originally sent, but is just
+the unicode equivalent.
+
+This adds an environment override which can be used to skip these test during a full
+"trial twisted" run.  The DaemonizeTests, SafeStrTests can then be run independantly
+with a clean python interpreter.
+---
+ src/twisted/test/test_reflect.py | 6 ++++++
+ src/twisted/test/test_twistd.py  | 3 +++
+ 2 files changed, 9 insertions(+)
+
+diff --git a/src/twisted/test/test_reflect.py b/src/twisted/test/test_reflect.py
+index ff0c7fc9e..0c13b949b 100644
+--- a/src/twisted/test/test_reflect.py
++++ b/src/twisted/test/test_reflect.py
+@@ -19,6 +19,9 @@ from twisted.python.reflect import (
+     accumulateMethods, prefixedMethods, prefixedMethodNames,
+     addMethodNamesToDict, fullyQualifiedName)
+ 
++UTF8_OVERRIDES = os.environ.get("UTF8_OVERRIDES", False)
++
++
+ 
+ class Base(object):
+     """
+@@ -553,6 +556,9 @@ class SafeStrTests(TestCase):
+         x = b't\xc3\xbcst'
+         self.assertEqual(reflect.safe_str(x), x)
+ 
++    if UTF8_OVERRIDES:
++        test_workingUtf8_2.skip = "test_workingUtf8_2 requires to be run independantly of other tests"
++
+ 
+     def test_workingUtf8_3(self):
+         """
+diff --git a/src/twisted/test/test_twistd.py b/src/twisted/test/test_twistd.py
+index b74fe4a08..d55be16b9 100644
+--- a/src/twisted/test/test_twistd.py
++++ b/src/twisted/test/test_twistd.py
+@@ -78,6 +78,7 @@ if getattr(os, 'setuid', None) is None:
+ else:
+     setuidSkip = None
+ 
++UTF8_OVERRIDES = os.environ.get("UTF8_OVERRIDES", False)
+ 
+ 
+ def patchUserDatabase(patch, user, uid, group, gid):
+@@ -1913,3 +1914,5 @@ class DaemonizeTests(unittest.TestCase):
+ 
+ if _twistd_unix is None:
+     DaemonizeTests.skip = "twistd unix support not available"
++elif UTF8_OVERRIDES:
++    DaemonizeTests.skip = "twistd.DaemonizeTests testing needs to be run separately"
+-- 
+2.12.1
+
diff --git a/dev-python/twisted/twisted-16.6.0.ebuild b/dev-python/twisted/twisted-16.6.0-r1.ebuild
index e8d10cecd299..74f7447e1fb9 100644
--- a/dev-python/twisted/twisted-16.6.0.ebuild
+++ b/dev-python/twisted/twisted-16.6.0-r1.ebuild
@@ -67,6 +67,8 @@ DEPEND="
 PATCHES=(
 	# Respect TWISTED_DISABLE_WRITING_OF_PLUGIN_CACHE variable.
 	"${FILESDIR}/${PN}-16.5.0-respect_TWISTED_DISABLE_WRITING_OF_PLUGIN_CACHE.patch"
+	"${FILESDIR}/test_main.patch"
+	"${FILESDIR}/utf8_overrides.patch"
 	"${FILESDIR}/${PN}-16.6.0-test-fixes.patch"
 )
 
@@ -95,6 +97,7 @@ python_test() {
 	distutils_install_for_testing
 
 	export EMERGE_TEST_OVERRIDE=1
+	export UTF8_OVERRIDES=1
 	# workaround for the eclass not installing the entry points
 	# in the test environment.  copy the old 16.3.2 start script
 	# to run the tests with
@@ -105,6 +108,15 @@ python_test() {
 	if ! "${TEST_DIR}"/trial twisted; then
 		die "Tests failed with ${EPYTHON}"
 	fi
+	# due to an anomoly in the tests, python doesn't return the correct form
+	# of the escape sequence. So run those test separately with a clean python interpreter
+	export UTF8_OVERRIDES=0
+	if ! "${TEST_DIR}"/trial twisted.test.test_twistd.DaemonizeTests; then
+		die "DaemonizeTests failed with ${EPYTHON}"
+	fi
+	if ! "${TEST_DIR}"/trial twisted.test.test_reflect.SafeStrTests; then
+		die "SafeStrTests failed with ${EPYTHON}"
+	fi
 
 	popd > /dev/null || die
 }