From 10f5d967455f48368d137411cdb36f110632a55f Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 19 Oct 2016 10:56:14 -0700 Subject: [PATCH 001/113] Upgrade asis. --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 08961bd..c84894f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,4 +1,4 @@ -asis==0.2.3 +asis==0.3.0 bottle==0.12.9 colorama==0.3.7 coverage==4.0.3 From 37d7e3f9c8c54e75eea98ae4e4bc92ea97cb9517 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 19 Oct 2016 10:56:23 -0700 Subject: [PATCH 002/113] Update url-py. --- .travis.yml | 1 + dev-requirements-py3.txt | 2 +- dev-requirements.txt | 2 +- requirements.txt | 2 +- setup.py | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index afac707..26ff55e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ language: python +dist: trusty python: - "3.4" diff --git a/dev-requirements-py3.txt b/dev-requirements-py3.txt index d416d75..270993c 100644 --- a/dev-requirements-py3.txt +++ b/dev-requirements-py3.txt @@ -9,4 +9,4 @@ python-termstyle==0.1.10 rednose==1.1.1 requests==2.10.0 six==1.10.0 -url==0.1.7 +url==0.3.0 diff --git a/dev-requirements.txt b/dev-requirements.txt index c84894f..57bbe03 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -14,5 +14,5 @@ python-termstyle==0.1.10 rednose==1.1.1 requests==2.10.0 six==1.10.0 -url==0.1.7 +url==0.3.0 wheel==0.24.0 diff --git a/requirements.txt b/requirements.txt index 06ff91c..53b0dbd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -url==0.1.7 +url==0.3.0 requests==2.10.0 python-dateutil==2.5.3 diff --git a/setup.py b/setup.py index b747da4..ca858a0 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ ], install_requires = [ 'python-dateutil>=1.5, !=2.0', - 'url<0.2.0', + 'url>=0.2.0', 'requests' ], classifiers = [ From 493eef49d1141d5910a395069c185c0a5cf0ebde Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Mon, 10 Oct 2016 08:28:48 -0700 Subject: [PATCH 003/113] Drop the use of requests.Session. --- reppy/cache.py | 3 +-- tests/test_cache.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/reppy/cache.py b/reppy/cache.py index 3172e10..307b06b 100644 --- a/reppy/cache.py +++ b/reppy/cache.py @@ -43,7 +43,6 @@ class RobotsCache(object): def __init__(self, *args, **kwargs): # The provided args and kwargs are used when fetching robots.txt with # a `requests.get` - self.session = kwargs.pop('session', requests.Session()) self.args = args self.disallow_forbidden = kwargs.pop('disallow_forbidden', True) self.kwargs = kwargs @@ -78,7 +77,7 @@ def fetch(self, url, *args, **kwargs): # First things first, fetch the thing robots_url = Utility.roboturl(url) logger.debug('Fetching %s' % robots_url) - req = self.session.get(robots_url, *args, **kwargs) + req = requests.get(robots_url, *args, **kwargs) ttl = max(self.min_ttl, Utility.get_ttl(req.headers, self.default_ttl)) # And now parse the thing and return it return parser.Rules(robots_url, req.status_code, req.content, diff --git a/tests/test_cache.py b/tests/test_cache.py index 995f9ac..180b87a 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -114,7 +114,7 @@ def test_add(self): def test_server_error(self): '''Make sure we can catch server errors''' - with mock.patch.object(self.robots.session, 'get', side_effect=TypeError): + with mock.patch('reppy.cache.requests.get', side_effect=TypeError): self.assertRaises(ServerError, self.robots.allowed, 'http://localhost:8080/foo', 'rogerbot') From 040526752da5499f390adcb2e3666b630300a080 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Mon, 10 Oct 2016 12:56:46 -0700 Subject: [PATCH 004/113] Add a directive class. --- reppy/directive.py | 87 ++++++++++++++++++++++++++++++++++++++ tests/test_directive.py | 94 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+) create mode 100644 reppy/directive.py create mode 100644 tests/test_directive.py diff --git a/reppy/directive.py b/reppy/directive.py new file mode 100644 index 0000000..5c95a91 --- /dev/null +++ b/reppy/directive.py @@ -0,0 +1,87 @@ +'''Single allow / disallow directives.''' + +import re + +try: + # Python 3 + from urllib.parse import unquote +except ImportError: + # Python 2 + from urllib import unquote + + +class BaseDirective(object): + '''A allow/disallow directive.''' + + @classmethod + def sanitize(cls, query): + '''Canonicalize query for comparison.''' + if (not query) or (query[0] != '/'): + query = '/' + query + # TODO(dan): this is the legacy implementation, but should be replaced + return unquote(query.replace('%2f', '%252f')) + + @classmethod + def parse(cls, line): + '''Return a directive.''' + key, rule = line.split(':', 1) + key = key.lower().strip() + if key == 'allowed': + allowed = True + elif key == 'disallowed': + allowed = False + else: + raise ValueError('Unknown directive in line %s' % line) + + rule = rule.strip() + priority = len(rule) + if (not rule) or (rule == '/') or (rule == '*'): + return AllDirective(priority, rule, allowed) + elif ('*' not in rule) and ('$' not in rule): + return StringDirective(priority, rule, allowed) + else: + return RegexDirective(priority, rule, allowed) + + def __init__(self, priority, pattern, allowed): + self.priority = priority + self.pattern = self.sanitize(pattern) + self.allowed = allowed + + def match(self, query): + '''Return true if the query matches this pattern.''' + raise NotImplementedError('BaseDirective does not implement match') + + def __lt__(self, other): + return self.priority < other.priority + + +class StringDirective(BaseDirective): + '''A directive that uses a string comparison.''' + + def match(self, query): + '''Return true if the query matches this pattern.''' + return query.startswith(self.pattern) + + +class AllDirective(BaseDirective): + '''A directive that applies to all URLs.''' + + def match(self, query): + '''Return true if the query matches this pattern.''' + return True + + +class RegexDirective(BaseDirective): + '''A directive that uses a regex.''' + + # For collapsing multiple asterisks into one + RE_ASTERISK = re.compile(r'\*+') + + def __init__(self, priority, pattern, allowed): + BaseDirective.__init__(self, priority, pattern, allowed) + pattern = re.escape(self.RE_ASTERISK.sub('*', self.pattern)) + self._regex = re.compile(pattern.replace('\*', '.*').replace('\$', '$')) + + def match(self, query): + '''Return true if the query matches this pattern.''' + return self._regex.match(query) diff --git a/tests/test_directive.py b/tests/test_directive.py new file mode 100644 index 0000000..d16f555 --- /dev/null +++ b/tests/test_directive.py @@ -0,0 +1,94 @@ +import unittest + +from reppy import directive + + +class BaseDirectiveTest(unittest.TestCase): + '''Tests about the BaseDirectiveTest.''' + + def test_recognizes_allowed(self): + '''Recognizes the allowed directive.''' + self.assertTrue(directive.BaseDirective.parse('Allowed:').allowed) + + def test_recognizes_disallowed(self): + '''Recognizes the disallowed directive.''' + self.assertFalse(directive.BaseDirective.parse('Disallowed:').allowed) + + def test_exception_for_unrecognized(self): + '''Raises an exception for unrecognized directives.''' + with self.assertRaises(ValueError): + directive.BaseDirective.parse('Foo: bar') + + def test_case_insensitive(self): + '''Ignores the case of the directive.''' + self.assertTrue(directive.BaseDirective.parse('aLLowed:').allowed) + + def test_ignores_passing(self): + '''Ignores the padding around the directive.''' + self.assertTrue(directive.BaseDirective.parse(' Allowed :').allowed) + + def test_returns_all_directive(self): + '''Returns an AllDirective when possible.''' + self.assertIsInstance( + directive.BaseDirective.parse('Allowed:'), + directive.AllDirective) + + def test_returns_string_directive(self): + '''Returns a StringDirective when possible.''' + self.assertIsInstance( + directive.BaseDirective.parse('Allowed: /some/path'), + directive.StringDirective) + + def test_returns_regex_directive(self): + '''Returns a RegexDirective when necessary.''' + self.assertIsInstance( + directive.BaseDirective.parse('Allowed: /some/*/wildcard'), + directive.RegexDirective) + + def test_raises_on_match(self): + '''Does not implement match.''' + with self.assertRaises(NotImplementedError): + directive.BaseDirective(0, '', True).match('path') + + def test_supports_less_than(self): + '''Can be compared.''' + self.assertLess( + directive.BaseDirective(0, '', True), + directive.BaseDirective(1, 'a', True)) + + +class StringDirectiveTest(unittest.TestCase): + '''Tests about StringDirective.''' + + def test_matches_prefix(self): + '''The query must start with the provided pattern to not match.''' + self.assertTrue( + directive.StringDirective(0, '/prefix', True).match('/prefix/url')) + + def test_does_not_match_prefix(self): + '''The query must not start with the provided pattern to not match.''' + self.assertFalse( + directive.StringDirective(0, '/prefix', True).match('/different')) + + +class AllDirectiveTest(unittest.TestCase): + '''Tests about the AllDirective.''' + + def test_matches_everything(self): + '''Matches everything.''' + self.assertTrue( + directive.AllDirective(0, '', True).match('/anything')) + + +class RegexDirectiveTest(unittest.TestCase): + '''Tests about the RegexDirective.''' + + def test_matches_beginning(self): + '''Must match at the beginning to match.''' + self.assertTrue( + directive.RegexDirective(0, '/foo/*', True).match('/foo/bar')) + + def test_does_not_match_middle(self): + '''Does not match on the middle of the query.''' + self.assertFalse( + directive.RegexDirective(0, '/foo/*', True).match('/whiz/foo/bar')) From 2a89ed5d95cc019b0ac6ccee7ef17c95a87b5bb5 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 12 Oct 2016 17:04:13 -0700 Subject: [PATCH 005/113] Rebase me into failing directive test. --- tests/test_directive.py | 142 ++++++++++++++++++++++++++++++++++------ 1 file changed, 123 insertions(+), 19 deletions(-) diff --git a/tests/test_directive.py b/tests/test_directive.py index d16f555..0082561 100644 --- a/tests/test_directive.py +++ b/tests/test_directive.py @@ -4,45 +4,32 @@ class BaseDirectiveTest(unittest.TestCase): - '''Tests about the BaseDirectiveTest.''' + '''Tests about the BaseDirective.''' def test_recognizes_allowed(self): '''Recognizes the allowed directive.''' - self.assertTrue(directive.BaseDirective.parse('Allowed:').allowed) + self.assertTrue(directive.BaseDirective.allow('').allowed) def test_recognizes_disallowed(self): '''Recognizes the disallowed directive.''' - self.assertFalse(directive.BaseDirective.parse('Disallowed:').allowed) - - def test_exception_for_unrecognized(self): - '''Raises an exception for unrecognized directives.''' - with self.assertRaises(ValueError): - directive.BaseDirective.parse('Foo: bar') - - def test_case_insensitive(self): - '''Ignores the case of the directive.''' - self.assertTrue(directive.BaseDirective.parse('aLLowed:').allowed) - - def test_ignores_passing(self): - '''Ignores the padding around the directive.''' - self.assertTrue(directive.BaseDirective.parse(' Allowed :').allowed) + self.assertFalse(directive.BaseDirective.disallow('').allowed) def test_returns_all_directive(self): '''Returns an AllDirective when possible.''' self.assertIsInstance( - directive.BaseDirective.parse('Allowed:'), + directive.BaseDirective.allow(''), directive.AllDirective) def test_returns_string_directive(self): '''Returns a StringDirective when possible.''' self.assertIsInstance( - directive.BaseDirective.parse('Allowed: /some/path'), + directive.BaseDirective.allow('/some/path'), directive.StringDirective) def test_returns_regex_directive(self): '''Returns a RegexDirective when necessary.''' self.assertIsInstance( - directive.BaseDirective.parse('Allowed: /some/*/wildcard'), + directive.BaseDirective.allow('/some/*/wildcard'), directive.RegexDirective) def test_raises_on_match(self): @@ -60,6 +47,11 @@ def test_supports_less_than(self): class StringDirectiveTest(unittest.TestCase): '''Tests about StringDirective.''' + def test_exact_match(self): + '''The query may be an exact match.''' + self.assertTrue( + directive.StringDirective(0, '/prefix', True).match('/prefix')) + def test_matches_prefix(self): '''The query must start with the provided pattern to not match.''' self.assertTrue( @@ -70,6 +62,46 @@ def test_does_not_match_prefix(self): self.assertFalse( directive.StringDirective(0, '/prefix', True).match('/different')) + def test_escaped_rule(self): + '''Handles the case where the rule is escaped.''' + rule = directive.StringDirective(0, '/a%3cd.html', True) + self.assertTrue(rule.match('/a') + class AllDirectiveTest(unittest.TestCase): '''Tests about the AllDirective.''' @@ -79,6 +111,12 @@ def test_matches_everything(self): self.assertTrue( directive.AllDirective(0, '', True).match('/anything')) + def test_str(self): + '''Has a resonable string representation.''' + rule = directive.AllDirective(0, '', True) + self.assertEqual( + str(rule), '') + class RegexDirectiveTest(unittest.TestCase): '''Tests about the RegexDirective.''' @@ -92,3 +130,69 @@ def test_does_not_match_middle(self): '''Does not match on the middle of the query.''' self.assertFalse( directive.RegexDirective(0, '/foo/*', True).match('/whiz/foo/bar')) + + def test_any_with_query(self): + '''Matches any path with a query.''' + self.assertTrue( + directive.RegexDirective(0, '/*?', True).match('/path?query')) + + def test_middle_wildcard(self): + '''Matches wildcards in the middle.''' + rule = directive.RegexDirective(0, '/folder*name/', True) + self.assertTrue(rule.match('/folder-middle-name/')) + + def test_matches_end(self): + '''Can match the end of a path.''' + rule = directive.RegexDirective(0, '/*.gif$', True) + self.assertTrue(rule.match('/funny.gif')) + self.assertFalse(rule.match('/funny.gifs/path')) + + def test_many_wildcards(self): + '''Multiple consecutive wildcards are compressed into one.''' + rule = directive.RegexDirective(0, '/********************************.css', True) + self.assertTrue(rule.match('/style.css')) + + def test_multiple_wildcards(self): + '''Multiple wildcards are supported.''' + rule = directive.RegexDirective(0, '/one/*/three/*/five', True) + self.assertTrue(rule.match('/one/two/three/four/five')) + + def test_escaped_rule(self): + '''Handles the case where the rule is escaped.''' + rule = directive.RegexDirective(0, '/a%3cd*', True) + self.assertTrue(rule.match('/a') From e398e135cdb3979b8158edd1e70a5a610a040a7c Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 12 Oct 2016 17:04:23 -0700 Subject: [PATCH 006/113] Rebase me into passing directive test. --- reppy/directive.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/reppy/directive.py b/reppy/directive.py index 5c95a91..168838a 100644 --- a/reppy/directive.py +++ b/reppy/directive.py @@ -22,17 +22,18 @@ def sanitize(cls, query): return unquote(query.replace('%2f', '%252f')) @classmethod - def parse(cls, line): - '''Return a directive.''' - key, rule = line.split(':', 1) - key = key.lower().strip() - if key == 'allowed': - allowed = True - elif key == 'disallowed': - allowed = False - else: - raise ValueError('Unknown directive in line %s' % line) + def allow(cls, rule): + '''Return an allowed directive.''' + return cls.parse(rule, True) + @classmethod + def disallow(cls, rule): + '''Return a disallowed directive.''' + return cls.parse(rule, False) + + @classmethod + def parse(cls, rule, allowed): + '''Return a directive.''' rule = rule.strip() priority = len(rule) if (not rule) or (rule == '/') or (rule == '*'): @@ -54,13 +55,17 @@ def match(self, query): def __lt__(self, other): return self.priority < other.priority + def __str__(self): + return '<%s priority=%s, pattern=%s, allowed=%s>' % ( + type(self).__name__, self.priority, self.pattern, self.allowed) + class StringDirective(BaseDirective): '''A directive that uses a string comparison.''' - def match(self, query): + def match(self, sanitized): '''Return true if the query matches this pattern.''' - return query.startswith(self.pattern) + return sanitized.startswith(self.pattern) class AllDirective(BaseDirective): @@ -82,6 +87,6 @@ def __init__(self, priority, pattern, allowed): pattern = re.escape(self.RE_ASTERISK.sub('*', self.pattern)) self._regex = re.compile(pattern.replace('\*', '.*').replace('\$', '$')) - def match(self, query): + def match(self, sanitized): '''Return true if the query matches this pattern.''' - return self._regex.match(query) + return bool(self._regex.match(sanitized)) From d55b97c2a18f187dd5cf5742f3867e8d6702fb05 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 12 Oct 2016 17:04:34 -0700 Subject: [PATCH 007/113] Failing test about util.pairs. --- tests/test_util.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tests/test_util.py diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..b1802ea --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,68 @@ +import unittest + +from reppy import util + + +class PairsTest(unittest.TestCase): + '''Tests about how we make pairs.''' + + def test_skips_blanks(self): + '''Skips blank lines.''' + content = ''' + hello + + world + ''' + lines = [ + ('hello', ''), + ('world', '') + ] + self.assertEqual(lines, list(util.pairs(content))) + + def test_strips_keys_and_values(self): + '''Strips leading and trailing whitespace on keys and values.''' + content = ''' + hello: world\t + ''' + lines = [ + ('hello', 'world') + ] + self.assertEqual(lines, list(util.pairs(content))) + + def test_lowercases_keys(self): + '''Makes the key lowercase.''' + content = ''' + HeLLo: World + ''' + lines = [ + ('hello', 'World') + ] + self.assertEqual(lines, list(util.pairs(content))) + + def test_does_not_trip_on_multiple_colons(self): + '''Does not fail when multiple colons are present.''' + content = ''' + hello: world: 2 + ''' + lines = [ + ('hello', 'world: 2') + ] + self.assertEqual(lines, list(util.pairs(content))) + + def test_skips_comments(self): + '''Skips comment lines.''' + content = ''' + # This is a comment. + ''' + lines = [] + self.assertEqual(lines, list(util.pairs(content))) + + def test_omits_comments(self): + '''Removes the comment portion of a line.''' + content = ''' + hello: world # this is a comment + ''' + lines = [ + ('hello', 'world') + ] + self.assertEqual(lines, list(util.pairs(content))) From 055ddea36457ef885300c919b46b1117cadcba0c Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 12 Oct 2016 17:04:51 -0700 Subject: [PATCH 008/113] Passing test about util.pairs. --- reppy/util.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 reppy/util.py diff --git a/reppy/util.py b/reppy/util.py new file mode 100644 index 0000000..dc0c722 --- /dev/null +++ b/reppy/util.py @@ -0,0 +1,18 @@ +'''Utility functions.''' + +import string + + +def pairs(content): + '''A generator of lowercase, stripped key-value pairs in contents' lines.''' + for line in content.strip().split('\n'): + # Remove any comments + if '#' in line: + line, _ = line.split('#', 1) + + key, _, value = [l.strip() for l in line.strip().partition(':')] + + if not key: + continue + + yield (key.lower(), value) From f7cc895934bdde4616cf93d8ab6aed8e0c42529e Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 12 Oct 2016 17:23:26 -0700 Subject: [PATCH 009/113] Failing tests about an Agent class. --- tests/test_agent.py | 63 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/test_agent.py diff --git a/tests/test_agent.py b/tests/test_agent.py new file mode 100644 index 0000000..c66bea3 --- /dev/null +++ b/tests/test_agent.py @@ -0,0 +1,63 @@ +import unittest + +from reppy.agent import Agent + + +class AgentTest(unittest.TestCase): + '''Tests about the Agent.''' + + def test_basic(self): + '''Parses a basic example.''' + agent = Agent().allow('/').disallow('/foo') + self.assertEqual(len(agent.directives), 2) + + def test_checks_allowed(self): + '''Answers the allowed question.''' + agent = Agent().allow('/path') + self.assertTrue(agent.allowed('/path')) + self.assertTrue(agent.allowed('/elsewhere')) + + def test_honors_longest_first_priority(self): + '''The longest matching rule takes priority.''' + agent = Agent().disallow('/path').allow('/path/exception') + self.assertTrue(agent.allowed('/path/exception')) + self.assertFalse(agent.allowed('/path')) + + def test_robots_txt_allowed(self): + '''Robots.txt is always allowed.''' + agent = Agent().disallow('/robots.txt') + self.assertTrue(agent.allowed('/robots.txt')) + + def test_disallow_none(self): + '''Recognizes the "Disallow:" form of "Allow: /"''' + agent = Agent().disallow('') + self.assertTrue(agent.allowed('/anything')) + + def test_escaped_rule(self): + '''Handles the case where the rule is escaped.''' + agent = Agent().disallow('/a%3cd.html') + self.assertFalse(agent.allowed('/a Date: Wed, 12 Oct 2016 17:23:38 -0700 Subject: [PATCH 010/113] Passing tests about an Agent class. --- reppy/agent.py | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 reppy/agent.py diff --git a/reppy/agent.py b/reppy/agent.py new file mode 100644 index 0000000..e29cbe5 --- /dev/null +++ b/reppy/agent.py @@ -0,0 +1,56 @@ +'''Holds the directives for a single user agent.''' + +import url as URL + +from . import logger +from .directive import BaseDirective + + +class Agent(object): + '''The directives for a single user agent.''' + + def __init__(self, directives=None, delay=None): + self._directives = directives or list() + self.delay = None + self._sorted = True + + @property + def directives(self): + '''Keep directives in descending order of precedence.''' + if not self._sorted: + self._directives.sort(reverse=True) + self._sorted = True + return self._directives + + def allow(self, query): + '''Add an allow directive.''' + self._directives.append(BaseDirective.allow(query)) + self._sorted = False + return self + + def disallow(self, query): + '''Add a disallow directive.''' + if not query: + # Special case: "Disallow:" means "Allow: /" + self._directives.append(BaseDirective.allow(query)) + else: + self._directives.append(BaseDirective.disallow(query)) + self._sorted = False + return self + + def url_allowed(self, url): + '''Return true if the path in url is allowed. Otherwise, false''' + parsed = URL.parse(url).defrag().deuserinfo() + parsed.scheme = '' + parsed.host = '' + return self.allowed(str(parsed)) + + def allowed(self, query): + '''Return true if the query is allowed. Otherwise, false.''' + if query == '/robots.txt': + return True + sanitized = BaseDirective.sanitize(query) + for directive in self.directives: + if directive.match(sanitized): + return directive.allowed + return True From dc224cf169a90feb02dd61db90f6d861af217375 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 13 Oct 2016 08:38:24 -0700 Subject: [PATCH 011/113] Failing tests about util.parse_date. --- tests/test_util.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_util.py b/tests/test_util.py index b1802ea..324dea9 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -66,3 +66,27 @@ def test_omits_comments(self): ('hello', 'world') ] self.assertEqual(lines, list(util.pairs(content))) + + +class ParseDateTest(unittest.TestCase): + '''Tests about parsing dates provided in headers.''' + + def test_invalid(self): + '''Raises an error on an unparseable date.''' + with self.assertRaises(ValueError): + util.parse_date('Not a real date') + + def test_imf_fixdate(self): + '''Successfully parses a IMF fixdate.''' + self.assertEqual( + util.parse_date('Sun, 06 Nov 1994 08:49:37 GMT'), 784111777) + + def test_rfc_850(self): + '''An obsolete date format that is also supported.''' + self.assertEqual( + util.parse_date('Sunday, 06-Nov-94 08:49:37 GMT'), 784111777) + + def test_asctime(self): + '''An obsolete date format that is also supported.''' + self.assertEqual( + util.parse_date('Sun Nov 6 08:49:37 1994'), 784111777) From 6caea62dfd9cac3cced449f6ba8ce6b3722e19c7 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 13 Oct 2016 08:38:33 -0700 Subject: [PATCH 012/113] Passing tests about util.parse_date. --- reppy/util.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/reppy/util.py b/reppy/util.py index dc0c722..8f16a4c 100644 --- a/reppy/util.py +++ b/reppy/util.py @@ -1,5 +1,6 @@ '''Utility functions.''' +import email import string @@ -16,3 +17,15 @@ def pairs(content): continue yield (key.lower(), value) + + +def parse_date(string): + '''Return a timestamp for the provided datestring, described by RFC 7231.''' + parsed = email.utils.parsedate_tz(string) + if parsed is None: + raise ValueError("Invalid time.") + if parsed[9] is None: + # Default time zone is GMT/UTC + parsed = list(parsed) + parsed[9] = 0 + return email.utils.mktime_tz(parsed) From e46adf52d488699ea142cfa06250344326735685 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 13 Oct 2016 08:59:52 -0700 Subject: [PATCH 013/113] Failing test about ttl policies. --- tests/test_ttl.py | 133 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 tests/test_ttl.py diff --git a/tests/test_ttl.py b/tests/test_ttl.py new file mode 100644 index 0000000..ec88898 --- /dev/null +++ b/tests/test_ttl.py @@ -0,0 +1,133 @@ +import unittest + +import mock + +from reppy import ttl + + +class TTLPolicyBaseTest(unittest.TestCase): + '''Tests about TTLPolicyBase.''' + + def test_does_not_implement_ttl(self): + '''Does not implement the ttl method.''' + with self.assertRaises(NotImplementedError): + ttl.TTLPolicyBase().ttl(object()) + + def test_implements_expires(self): + '''Expires is based off of ttl.''' + policy = ttl.TTLPolicyBase() + with mock.patch.object(policy, 'ttl', return_value=10): + with mock.patch.object(ttl.time, 'time', return_value=100): + self.assertEqual(policy.expires(object()), 110) + + +class HeaderWithDefaultPolicyTest(unittest.TestCase): + '''Tests about HeaderWithDefaultPolicy.''' + + def test_no_store(self): + '''Returns the minimum when no-store present.''' + response = mock.Mock(headers={ + 'cache_control': 'no-store' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 10) + + def test_must_revalidate(self): + '''Returns the minimum when must-revalidate present.''' + response = mock.Mock(headers={ + 'cache_control': 'must-revalidate' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 10) + + def test_no_cache(self): + '''Returns the minimum when no-cache present.''' + response = mock.Mock(headers={ + 'cache_control': 'no-cache' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 10) + + def test_s_maxage(self): + '''Returns the parsed s-maxage.''' + response = mock.Mock(headers={ + 'cache_control': 's-maxage=15' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 15) + + def test_max_age(self): + '''Returns the parsed max-age.''' + response = mock.Mock(headers={ + 'cache_control': 'max-age=15' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 15) + + def test_default_for_malformed_maxage(self): + '''Returns the default when maxage cannot be parsed.''' + response = mock.Mock(headers={ + 'cache_control': 'max-age=not-a-number' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 20) + + def test_multiple_cache_control(self): + '''Can walk through multiple cache control configs.''' + response = mock.Mock(headers={ + 'cache_control': 'foo, max-age=15' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 15) + + def test_expires_with_no_date(self): + '''Uses the host computer's date when the Date header is absent.''' + expires = 'Thu, 13 Oct 2016 15:50:54 GMT' + response = mock.Mock(headers={ + 'expires': expires + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + timestamp = ttl.parse_date(expires) + expected = 60 + with mock.patch.object(ttl.time, 'time', return_value=timestamp - expected): + self.assertEqual(policy.ttl(response), expected) + + def test_expires_with_malformed_date(self): + '''Uses the host computer's date when the Date header is unparseable.''' + expires = 'Thu, 13 Oct 2016 15:50:54 GMT' + response = mock.Mock(headers={ + 'expires': expires, + 'date': 'not parseable as a date' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + timestamp = ttl.parse_date(expires) + expected = 60 + with mock.patch.object(ttl.time, 'time', return_value=timestamp - expected): + self.assertEqual(policy.ttl(response), expected) + + def test_expires_with_date(self): + '''Uses the Date header when present.''' + response = mock.Mock(headers={ + 'expires': 'Thu, 13 Oct 2016 15:50:54 GMT', + 'date': 'Thu, 13 Oct 2016 15:49:54 GMT' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 60) + + def test_malformed_expires(self): + '''Returns the default when the Expires header is malformed.''' + response = mock.Mock(headers={ + 'expires': 'not parseable as a date' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 20) + + def test_cache_control_precedence(self): + '''Cache control is used before expires.''' + response = mock.Mock(headers={ + 'cache_control': 'max-age=30', + 'expires': 'Thu, 13 Oct 2016 15:50:54 GMT', + 'date': 'Thu, 13 Oct 2016 15:49:54 GMT' + }) + policy = ttl.HeaderWithDefaultPolicy(20, 10) + self.assertEqual(policy.ttl(response), 30) From e1ffc5284d1ad3ec8a3e2554bee68afc70948964 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 13 Oct 2016 09:26:59 -0700 Subject: [PATCH 014/113] Passing test about ttl policies. --- reppy/ttl.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 reppy/ttl.py diff --git a/reppy/ttl.py b/reppy/ttl.py new file mode 100644 index 0000000..0f6ae8f --- /dev/null +++ b/reppy/ttl.py @@ -0,0 +1,64 @@ +'''Policies for setting the TTL on Robots objects.''' + +import time + +from . import logger +from .util import parse_date + + +class TTLPolicyBase(object): + '''Policy for setting the TTL on Robots objects.''' + + def ttl(self, response): + '''Get the caching TTL for a response.''' + raise NotImplementedError('TTLPolicyBase does not implement ttl.') + + def expires(self, response): + '''Determine when a response should expire.''' + return time.time() + self.ttl(response) + + +class HeaderWithDefaultPolicy(TTLPolicyBase): + '''TTL is based on headers, but falls back to a default, clamps to a minimum.''' + + def __init__(self, default, minimum): + self.default = default + self.minimum = minimum + + def ttl(self, response): + '''Get the ttl from headers.''' + # If max-age is specified in Cache-Control, use it and ignore any + # Expires header, as per RFC2616 Sec. 13.2.4. + cache_control = response.headers.get('cache_control') + if cache_control is not None: + for directive in cache_control.split(','): + name, _, value = directive.lower().partition('=') + name = name.strip() + if name in ('no-store', 'must-revalidate', 'no-cache'): + return max(self.minimum, 0) + elif name in ('s-maxage', 'max-age'): + try: + return max(self.minimum, int(value.strip())) + except ValueError: + logger.exception('Could not parse %s=%s', name, value) + + # Check the Expires header + expires = response.headers.get('expires') + if expires is not None: + # Evaluate the expiration relative to the server-provided date + date = response.headers.get('date') + if date is not None: + try: + date = parse_date(date) + except ValueError: + logger.exception('Could not parse date string %s', date) + date = time.time() + else: + date = time.time() + + try: + return max(self.minimum, parse_date(expires) - date) + except ValueError: + logger.exception('Could not parse date string %s', expires) + + return self.default From 6e3023e1399def6a505c0cad7552220f100e966a Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 13 Oct 2016 20:34:11 -0700 Subject: [PATCH 015/113] Failing tests about robots.txt. --- tests/asis/test_content_too_big/robots.txt | 5 + tests/asis/test_fetch_status_200/robots.txt | 5 + tests/asis/test_fetch_status_401/robots.txt | 2 + tests/asis/test_fetch_status_403/robots.txt | 2 + tests/asis/test_fetch_status_4XX/robots.txt | 2 + tests/asis/test_fetch_status_5XX/robots.txt | 2 + tests/test_robots.py | 377 ++++++++++++++++++++ 7 files changed, 395 insertions(+) create mode 100644 tests/asis/test_content_too_big/robots.txt create mode 100644 tests/asis/test_fetch_status_200/robots.txt create mode 100644 tests/asis/test_fetch_status_401/robots.txt create mode 100644 tests/asis/test_fetch_status_403/robots.txt create mode 100644 tests/asis/test_fetch_status_4XX/robots.txt create mode 100644 tests/asis/test_fetch_status_5XX/robots.txt create mode 100644 tests/test_robots.py diff --git a/tests/asis/test_content_too_big/robots.txt b/tests/asis/test_content_too_big/robots.txt new file mode 100644 index 0000000..1d914e6 --- /dev/null +++ b/tests/asis/test_content_too_big/robots.txt @@ -0,0 +1,5 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: / diff --git a/tests/asis/test_fetch_status_200/robots.txt b/tests/asis/test_fetch_status_200/robots.txt new file mode 100644 index 0000000..1d914e6 --- /dev/null +++ b/tests/asis/test_fetch_status_200/robots.txt @@ -0,0 +1,5 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: / diff --git a/tests/asis/test_fetch_status_401/robots.txt b/tests/asis/test_fetch_status_401/robots.txt new file mode 100644 index 0000000..666123e --- /dev/null +++ b/tests/asis/test_fetch_status_401/robots.txt @@ -0,0 +1,2 @@ +HTTP/1.0 401 Unauthorized +Content-Type: text/plain diff --git a/tests/asis/test_fetch_status_403/robots.txt b/tests/asis/test_fetch_status_403/robots.txt new file mode 100644 index 0000000..87e5f45 --- /dev/null +++ b/tests/asis/test_fetch_status_403/robots.txt @@ -0,0 +1,2 @@ +HTTP/1.0 403 Forbidden +Content-Type: text/plain diff --git a/tests/asis/test_fetch_status_4XX/robots.txt b/tests/asis/test_fetch_status_4XX/robots.txt new file mode 100644 index 0000000..1187c9d --- /dev/null +++ b/tests/asis/test_fetch_status_4XX/robots.txt @@ -0,0 +1,2 @@ +HTTP/1.0 404 Not Found +Content-Type: text/plain diff --git a/tests/asis/test_fetch_status_5XX/robots.txt b/tests/asis/test_fetch_status_5XX/robots.txt new file mode 100644 index 0000000..dcedffd --- /dev/null +++ b/tests/asis/test_fetch_status_5XX/robots.txt @@ -0,0 +1,2 @@ +HTTP/1.0 500 Internal Server Error +Content-Type: text/plain diff --git a/tests/test_robots.py b/tests/test_robots.py new file mode 100644 index 0000000..578b6a5 --- /dev/null +++ b/tests/test_robots.py @@ -0,0 +1,377 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- + +'''These are unit tests that are derived from the rfc at +http://www.robotstxt.org/norobots-rfc.txt''' + +import codecs +import contextlib +import os +import unittest + +import asis +import mock + +from reppy import robots + + +class RobotsTest(unittest.TestCase): + '''Tests about our Robots class.''' + + @contextlib.contextmanager + def server(self, *segments): + '''Run an asis server with the provided path.''' + path = os.path.join('tests', 'asis', *segments) + with asis.Server(path, server='gevent', port=8080).greenlet(): + yield + + def test_expired(self): + '''Returns true if expired.''' + with mock.patch.object(robots.time, 'time', return_value=10): + robot = robots.Robots.parse('http://example.com/robots.txt', '', expires=5) + self.assertTrue(robot.expired) + + def test_not_expired(self): + '''Returns false if not expired.''' + with mock.patch.object(robots.time, 'time', return_value=10): + robot = robots.Robots.parse('http://example.com/robots.txt', '', expires=15) + self.assertFalse(robot.expired) + + def test_ttl(self): + '''Returns the time remaining until expiration.''' + with mock.patch.object(robots.time, 'time', return_value=10): + robot = robots.Robots.parse('http://example.com/robots.txt', '', expires=15) + self.assertEqual(robot.ttl, 5) + + def test_disallow_first(self): + '''Handles a missing User-Agent line.''' + with self.assertRaises(ValueError): + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + Disallow: /path + ''') + + def test_malformed_crawl_delay(self): + '''Handles a malformed delay.''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-agent: * + Crawl-delay: word + ''') + self.assertEqual(robot.agent('agent').delay, None) + + def test_honors_default_agents(self): + '''Honors the default user agent when that's all that's available.''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-agent: * + Disallow: /tmp + + User-agent: other-agent + Allow: /tmp + ''') + self.assertFalse(robot.allowed('/tmp', 'agent')) + self.assertTrue(robot.allowed('/path', 'agent')) + + def test_honors_specific_agent(self): + '''Honors the specific user agent if a match is found.''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-agent: * + Disallow: /tmp + + User-agent: agent + Allow: /tmp + ''') + self.assertTrue(robot.allowed('/tmp', 'agent')) + self.assertTrue(robot.allowed('/path', 'agent')) + + def test_grouping(self): + '''Multiple consecutive User-Agent lines are allowed.''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-agent: one + User-agent: two + Disallow: /tmp + ''') + self.assertFalse(robot.allowed('/tmp', 'one')) + self.assertFalse(robot.allowed('/tmp', 'two')) + + def test_grouping_unknown_keys(self): + ''' + When we encounter unknown keys, we should disregard any grouping that may have + happened between user agent rules. + + This is an example from the wild. Despite `Noindex` not being a valid directive, + we'll not consider the '*' and 'ia_archiver' rules together. + ''' + rules = robots.Robots.parse('http://example.com/robots.txt', ''' + User-agent: * + Disallow: /content/2/ + User-agent: * + Noindex: /gb.html + Noindex: /content/2/ + User-agent: ia_archiver + Disallow: / + ''') + self.assertTrue(rules.allowed('/foo', 'agent')) + self.assertTrue(not rules.allowed('/bar', 'ia_archiver')) + + def test_separates_agents(self): + '''Hands back an appropriate agent.''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-agent: one + Crawl-delay: 1 + + User-agent: two + Crawl-delay: 2 + ''') + self.assertNotEqual( + robot.agent('one').delay, + robot.agent('two').delay) + + def test_exposes_sitemaps(self): + '''Finds and exposes sitemaps.''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + Sitemap: http://a.com/sitemap.xml + Sitemap: http://b.com/sitemap.xml + ''') + self.assertEqual(robot.sitemaps, [ + 'http://a.com/sitemap.xml', 'http://b.com/sitemap.xml' + ]) + + def test_case_insensitivity(self): + '''Make sure user agent matches are case insensitive''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-agent: agent + ''') + self.assertEqual(robot.agent('agent'), robot.agent('aGeNt')) + + def test_empty(self): + '''Makes sure we can parse an empty robots.txt''' + robot = robots.Robots.parse('http://example.com/robots.txt', '') + self.assertEqual(robot.agent('agent'), None) + self.assertEqual(robot.sitemaps, []) + self.assertTrue(robot.allowed('/', 'agent')) + + def test_comments(self): + '''Robust against comments.''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-Agent: * # comment saying it's the default agent + Allow: / + ''') + self.assertNotEqual(robot.agent('agent'), None) + + def test_url_allowed(self): + '''Can accept a url string.''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-Agent: agent + Disallow: / + ''') + self.assertFalse(robot.url_allowed('http://example.com/path', 'agent')) + + def test_url_allowed_none_matching(self): + '''Can accept a url string when there are no matching agents.''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-Agent: other + Disallow: / + ''') + self.assertTrue(robot.url_allowed('http://example.com/path', 'agent')) + + def test_skip_malformed_line(self): + '''If there is no colon in a line, then we must skip it''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + User-Agent: agent + Disallow /no/colon/in/this/line + ''') + self.assertTrue(robot.allowed('/no/colon/in/this/line', 'agent')) + + def test_fetch_status_200(self): + '''A 200 parses things normally.''' + with self.server('test_fetch_status_200'): + robot = robots.Robots.fetch('http://localhost:8080/robots.txt') + self.assertFalse(robot.allowed('/path', 'agent')) + + def test_fetch_status_401(self): + '''A 401 gives us an AllowNone Robots.''' + with self.server('test_fetch_status_401'): + robot = robots.Robots.fetch('http://localhost:8080/robots.txt') + self.assertIsInstance(robot, robots.AllowNone) + + def test_fetch_status_403(self): + '''A 403 gives us an AllowNone Robots.''' + with self.server('test_fetch_status_403'): + robot = robots.Robots.fetch('http://localhost:8080/robots.txt') + self.assertIsInstance(robot, robots.AllowNone) + + def test_fetch_status_4XX(self): + '''A 4XX gives us an AllowAll Robots.''' + with self.server('test_fetch_status_4XX'): + robot = robots.Robots.fetch('http://localhost:8080/robots.txt') + self.assertIsInstance(robot, robots.AllowAll) + + def test_fetch_status_5XX(self): + '''A server error raises an exception.''' + with self.server('test_fetch_status_5XX'): + with self.assertRaises(robots.exceptions.BadStatusCode): + robots.Robots.fetch('http://localhost:8080/robots.txt') + + def test_content_too_big(self): + '''Raises an exception if the content is too big.''' + with self.server('test_content_too_big'): + with self.assertRaises(robots.exceptions.ReppyException): + robots.Robots.fetch('http://localhost:8080/robots.txt', max_size=5) + + def test_ssl_exception(self): + '''Raises a ReppyException on SSL errors.''' + with self.server('test_ssl_exception'): + with self.assertRaises(robots.exceptions.SSLException): + robots.Robots.fetch('https://localhost:8080/robots.txt') + + def test_connection_exception(self): + '''Raises a ReppyException on connection errors.''' + with self.assertRaises(robots.exceptions.ConnectionException): + robots.Robots.fetch('http://localhost:8080/robots.txt') + + def test_malformed_url(self): + '''Raises a ReppyException on malformed URLs.''' + with self.assertRaises(robots.exceptions.MalformedUrl): + robots.Robots.fetch('gobbledygook') + + def test_excessive_redirects(self): + '''Raises a ReppyException on too many redirects.''' + with self.server('test_excessive_redirects'): + with self.assertRaises(robots.exceptions.ExcessiveRedirects): + robots.Robots.fetch('http://localhost:8080/robots.txt') + + def test_utf8_bom(self): + '''If there's a utf-8 BOM, we should parse it as such''' + robot = robots.Robots.parse('http://example.com/robots.txt', + codecs.BOM_UTF8 + b''' + User-Agent: agent + Allow: /path + + User-Agent: other + Disallow: /path + ''') + self.assertTrue(robot.url_allowed('http://example.com/path', 'agent')) + self.assertFalse(robot.url_allowed('http://example.com/path', 'other')) + + def test_utf16_bom(self): + '''If there's a utf-16 BOM, we should parse it as such''' + robot = robots.Robots.parse('http://example.com/robots.txt', + codecs.BOM_UTF16 + b''' + User-Agent: agent + Allow: /path + + User-Agent: other + Disallow: /path + ''') + self.assertTrue(robot.url_allowed('http://example.com/path', 'agent')) + self.assertFalse(robot.url_allowed('http://example.com/path', 'other')) + + def test_rfc_example(self): + '''Tests the example provided by the RFC.''' + robot = robots.Robots.parse('http://www.fict.org', ''' + # /robots.txt for http://www.fict.org/ + # comments to webmaster@fict.org + + User-agent: unhipbot + Disallow: / + + User-agent: webcrawler + User-agent: excite + Disallow: + + User-agent: * + Disallow: /org/plans.html + Allow: /org/ + Allow: /serv + Allow: /~mak + Disallow: / + ''') + + # The unhip bot + self.assertFalse(robot.allowed('/', 'unhipbot')) + self.assertFalse(robot.allowed('/index.html', 'unhipbot')) + self.assertTrue(robot.allowed('/robots.txt', 'unhipbot')) + self.assertFalse(robot.allowed('/server.html', 'unhipbot')) + self.assertFalse(robot.allowed('/services/fast.html', 'unhipbot')) + self.assertFalse(robot.allowed('/services/slow.html', 'unhipbot')) + self.assertFalse(robot.allowed('/orgo.gif', 'unhipbot')) + self.assertFalse(robot.allowed('/org/about.html', 'unhipbot')) + self.assertFalse(robot.allowed('/org/plans.html', 'unhipbot')) + self.assertFalse(robot.allowed('/%7Ejim/jim.html', 'unhipbot')) + self.assertFalse(robot.allowed('/%7Emak/mak.html', 'unhipbot')) + + # The webcrawler agent + self.assertTrue(robot.allowed('/', 'webcrawler')) + self.assertTrue(robot.allowed('/index.html', 'webcrawler')) + self.assertTrue(robot.allowed('/robots.txt', 'webcrawler')) + self.assertTrue(robot.allowed('/server.html', 'webcrawler')) + self.assertTrue(robot.allowed('/services/fast.html', 'webcrawler')) + self.assertTrue(robot.allowed('/services/slow.html', 'webcrawler')) + self.assertTrue(robot.allowed('/orgo.gif', 'webcrawler')) + self.assertTrue(robot.allowed('/org/about.html', 'webcrawler')) + self.assertTrue(robot.allowed('/org/plans.html', 'webcrawler')) + self.assertTrue(robot.allowed('/%7Ejim/jim.html', 'webcrawler')) + self.assertTrue(robot.allowed('/%7Emak/mak.html', 'webcrawler')) + + # The excite agent + self.assertTrue(robot.allowed('/', 'excite')) + self.assertTrue(robot.allowed('/index.html', 'excite')) + self.assertTrue(robot.allowed('/robots.txt', 'excite')) + self.assertTrue(robot.allowed('/server.html', 'excite')) + self.assertTrue(robot.allowed('/services/fast.html', 'excite')) + self.assertTrue(robot.allowed('/services/slow.html', 'excite')) + self.assertTrue(robot.allowed('/orgo.gif', 'excite')) + self.assertTrue(robot.allowed('/org/about.html', 'excite')) + self.assertTrue(robot.allowed('/org/plans.html', 'excite')) + self.assertTrue(robot.allowed('/%7Ejim/jim.html', 'excite')) + self.assertTrue(robot.allowed('/%7Emak/mak.html', 'excite')) + + # All others + self.assertFalse(robot.allowed('/', 'anything')) + self.assertFalse(robot.allowed('/index.html', 'anything')) + self.assertTrue(robot.allowed('/robots.txt', 'anything')) + self.assertTrue(robot.allowed('/server.html', 'anything')) + self.assertTrue(robot.allowed('/services/fast.html', 'anything')) + self.assertTrue(robot.allowed('/services/slow.html', 'anything')) + self.assertFalse(robot.allowed('/orgo.gif', 'anything')) + self.assertTrue(robot.allowed('/org/about.html', 'anything')) + self.assertFalse(robot.allowed('/org/plans.html', 'anything')) + self.assertFalse(robot.allowed('/%7Ejim/jim.html', 'anything')) + self.assertTrue(robot.allowed('/%7Emak/mak.html', 'anything')) + + +class AllowNoneTest(unittest.TestCase): + '''Tests about the AllowNone Robots class.''' + + def test_allow(self): + '''Allows nothing.''' + robot = robots.AllowNone('http://example.com/robots.txt') + self.assertFalse(robot.allowed('/', 'agent')) + + def test_allow_robots_txt(self): + '''Allows robots.txt.''' + robot = robots.AllowNone('http://example.com/robots.txt') + self.assertTrue(robot.allowed('/robots.txt', 'agent')) + + def test_allow_url(self): + '''Allows nothing.''' + robot = robots.AllowNone('http://example.com/robots.txt') + self.assertFalse(robot.url_allowed('http://example.com/', 'agent')) + + def test_allow_robots_txt_url(self): + '''Allows robots.txt url.''' + robot = robots.AllowNone('http://example.com/robots.txt') + self.assertTrue(robot.url_allowed('http://example.com/robots.txt', 'agent')) + + +class AllowAllTest(unittest.TestCase): + '''Tests about the AllowAll Robots class.''' + + def test_allow(self): + '''Allows nothing.''' + robot = robots.AllowAll('http://example.com/robots.txt') + self.assertTrue(robot.allowed('/', 'agent')) + + def test_allow_url(self): + '''Allows nothing.''' + robot = robots.AllowAll('http://example.com/robots.txt') + self.assertTrue(robot.url_allowed('http://example.com/', 'agent')) From 510ffd8d2b274cbc1a0f16887c6f258df49e436d Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 13 Oct 2016 20:34:24 -0700 Subject: [PATCH 016/113] Passing tests about robots.txt. --- reppy/robots.py | 150 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 reppy/robots.py diff --git a/reppy/robots.py b/reppy/robots.py new file mode 100644 index 0000000..c79b6da --- /dev/null +++ b/reppy/robots.py @@ -0,0 +1,150 @@ +'''A class holding a parsed robots.txt.''' + +from contextlib import closing +import time + +import requests +from requests.exceptions import ( + SSLError, + ConnectionError, + URLRequired, + MissingSchema, + InvalidSchema, + InvalidURL, + TooManyRedirects) + +from .agent import Agent +from .ttl import HeaderWithDefaultPolicy +from . import util, logger, exceptions + + +class Robots(object): + '''A class holding a parsed robots.txt.''' + + # The default TTL policy is to cache for 3600 seconds or what's provided in the + # headers, and a minimum of 600 seconds + DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) + + @classmethod + def fetch(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): + '''Get the robots.txt at the provided URL.''' + try: + # Limit the size of the request + kwargs['stream'] = True + with closing(requests.get(url, *args, **kwargs)) as res: + content = res.raw.read(amt=max_size, decode_content=True) + # Try to read an additional byte, to see if the response is too big + if res.raw.read(amt=1, decode_content=True): + raise exceptions.ContentTooLong( + 'Content larger than %s bytes' % max_size) + + # Get the TTL policy's ruling on the ttl + expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) + + if res.status_code == 200: + return cls.parse(url, content, expires) + elif res.status_code in (401, 403): + return AllowNone(url, expires) + elif res.status_code >= 400 and res.status_code < 500: + return AllowAll(url, expires) + else: + raise exceptions.BadStatusCode( + 'Got %i for %s' % (res.status_code, url), res.status_code) + except SSLError as exc: + raise exceptions.SSLException(exc) + except ConnectionError as exc: + raise exceptions.ConnectionException(exc) + except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: + raise exceptions.MalformedUrl(exc) + except TooManyRedirects as exc: + raise exceptions.ExcessiveRedirects(exc) + + @classmethod + def parse(cls, url, content, expires=None): + '''Parse the provided lines as a robots.txt.''' + sitemaps = [] + agents = {} + agent = None + last_agent = False + for key, value in util.pairs(content): + # Consecutive User-Agent lines mean they all have the same rules + if key == 'user-agent': + if not last_agent: + agent = Agent() + agents[value] = agent + last_agent = True + continue + else: + last_agent = False + + if key == 'sitemap': + sitemaps.append(value) + elif key in ('disallow', 'allow', 'crawl-delay'): + if agent is None: + raise ValueError( + 'Directive "%s" must be preceeded by User-Agent' % key) + + if key == 'disallow': + agent.disallow(value) + elif key == 'allow': + agent.allow(value) + else: + try: + agent.delay = float(value) + except ValueError: + logger.warn('Could not parse crawl delay: %s', value) + else: + logger.warn('Unknown directive "%s"' % key) + + return Robots(url, agents, sitemaps, expires) + + def __init__(self, url, agents, sitemaps, expires=None): + self.url = url + self.agents = agents + self.sitemaps = sitemaps + self.expires = expires + self.default = self.agents.get('*', None) + + @property + def expired(self): + '''True if the current time is past its expiration.''' + return time.time() > self.expires + + @property + def ttl(self): + '''Remaining time for this response to be considered valid.''' + return max(self.expires - time.time(), 0) + + def agent(self, name): + '''Get the rules for the agent with the provided name.''' + return self.agents.get(name.lower(), self.default) + + def url_allowed(self, url, agent): + '''Return true if agent may fetch the path in url.''' + found = self.agent(agent) + if found is None: + return True + return found.url_allowed(url) + + def allowed(self, path, agent): + '''Return true if agent may fetch path.''' + found = self.agent(agent) + if found is None: + return True + return found.allowed(path) + + +class AllowNone(Robots): + '''No requests are allowed.''' + + def __init__(self, url, expires=None): + Robots.__init__(self, url, sitemaps=[], expires=expires, agents={ + '*': Agent().disallow('/') + }) + + +class AllowAll(Robots): + '''All requests are allowed.''' + + def __init__(self, url, expires=None): + Robots.__init__(self, url, {}, [], expires=expires) From d1ac48de8f9cbdcee40bc897ff2a20ea4b2f8f7c Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 13 Oct 2016 09:30:52 -0700 Subject: [PATCH 017/113] Slash and burn. --- reppy/__init__.py | 155 +-- reppy/cache.py | 140 --- reppy/exceptions.py | 10 +- reppy/parser.py | 261 ----- tests/asis/test_bad_status_codes/robots.txt | 2 - tests/asis/test_delay/robots.txt | 7 - tests/asis/test_expires/robots.txt | 6 - tests/asis/test_sitemaps/robots.txt | 8 - tests/test_cache.py | 175 ---- tests/test_mozscape.py | 110 --- tests/test_old_mozscape.py | 698 -------------- tests/test_parse.py | 994 -------------------- tests/test_utility.py | 80 -- 13 files changed, 12 insertions(+), 2634 deletions(-) delete mode 100644 reppy/cache.py delete mode 100644 reppy/parser.py delete mode 100644 tests/asis/test_bad_status_codes/robots.txt delete mode 100644 tests/asis/test_delay/robots.txt delete mode 100644 tests/asis/test_expires/robots.txt delete mode 100644 tests/asis/test_sitemaps/robots.txt delete mode 100644 tests/test_cache.py delete mode 100644 tests/test_mozscape.py delete mode 100644 tests/test_old_mozscape.py delete mode 100755 tests/test_parse.py delete mode 100644 tests/test_utility.py diff --git a/reppy/__init__.py b/reppy/__init__.py index 8a5fd65..703e41a 100755 --- a/reppy/__init__.py +++ b/reppy/__init__.py @@ -1,156 +1,15 @@ -#! /usr/bin/env python -# -# Copyright (c) 2011 SEOmoz -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +'''Robots.txt parsing.''' -'''A robot exclusion protocol parser. Because I could not find a good one.''' - -__maintainer__ = 'Dan Lecocq' -__copyright__ = '2011-2014 SEOmoz' -__license__ = 'SEOmoz' -__version__ = '0.3.0' -__author__ = 'Dan Lecocq' -__status__ = 'Development' -__email__ = 'dan@moz.com' - -##################################################### -# All things logging -##################################################### import logging logger = logging.getLogger('reppy') handler = logging.StreamHandler() -handler.setFormatter(logging.Formatter('%(message)s')) +formatter = logging.Formatter( + ' | '.join([ + '[%(asctime)s]', + 'PID %(process)d', + '%(levelname)s in %(module)s:%(funcName)s@%(lineno)s => %(message)s' + ])) handler.setLevel(logging.DEBUG) logger.addHandler(handler) logger.setLevel(logging.ERROR) - -##################################################### -# A couple utilities -##################################################### -import sys -import re -import time -import email.utils -try: - from urllib import parse as urlparse -except ImportError: - # Python 2 - import urlparse - -if sys.version_info[0] == 3: - long = int - -##################################################### -# Import our exceptions at the global level -##################################################### -from .exceptions import ServerError, ReppyException - -class Utility(object): - '''Utility methods''' - @staticmethod - def hostname(url): - '''Return a normalized, canonicalized version of the url's hostname''' - return urlparse.urlparse(url).netloc - - @staticmethod - def roboturl(url): - '''Return a normalized uri to the robots.txt''' - parsed = urlparse.urlparse(url) - return '%s://%s/robots.txt' % (parsed.scheme, parsed.netloc) - - @staticmethod - def short_user_agent(strng): - '''Return a default user agent string to match, based on strng. For - example, for 'MyUserAgent/1.0', it will generate 'MyUserAgent' ''' - index = strng.find('/') - if index == -1: - return strng - return strng[0:index] - - @staticmethod - def parse_time(strng): - '''Parse an HTTP-style (i.e. email-style) time into a timestamp''' - v = email.utils.parsedate_tz(strng) - if v is None: - # Reject bad data - raise ValueError("Invalid time.") - if v[9] is None: - # Default time zone is GMT/UTC - v = list(v) # @$%?? Dutch - v[9] = 0 - v = tuple(v) - return email.utils.mktime_tz(v) - - @staticmethod - def get_ttl(headers, default): - '''Extract the correct ttl from the provided headers, or default''' - # Now, we'll determine the expiration - ttl = None - # If max-age is specified in Cache-Control, use it and ignore any - # Expires header, as per RFC2616 Sec. 13.2.4. - if headers.get('cache-control') is not None: - for directive in headers['cache-control'].split(','): - tokens = directive.lower().partition('=') - t_name = tokens[0].strip() - t_value = tokens[2].strip() - # If we're not allowed to cache, then expires is now - if t_name in ('no-store', 'must-revalidate'): - return 0 - elif t_name == 'no-cache' and t_value == '': - # Only honor no-cache if there is no =value after it - return 0 - elif t_name == 's-maxage': - try: - # Since s-maxage should override max-age, return - return long(t_value) - except ValueError: - # Couldn't parse s-maxage as an integer - continue - elif t_name == 'max-age': - try: - ttl = long(t_value) - except ValueError: - # Couldn't parse max-age as an integer - continue - - # We should honor cache-control first, so if we found anything at - # all, we should return that - if ttl is not None: - return ttl - - # Otherwise, we should use the expires tag - expires = headers.get('expires') - date = headers.get('date') - if expires: - if date is None: - base = time.time() - else: - try: - base = Utility.parse_time(date) - except ValueError: - base = time.time() - try: - return Utility.parse_time(expires) - base - except ValueError: - pass - - return ttl or default diff --git a/reppy/cache.py b/reppy/cache.py deleted file mode 100644 index 307b06b..0000000 --- a/reppy/cache.py +++ /dev/null @@ -1,140 +0,0 @@ -#! /usr/bin/env python -# -# Copyright (c) 2011 SEOmoz -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -'''Caching fetch robots.txt files''' - - -import time -import requests -from requests.exceptions import (SSLError, ConnectionError, URLRequired, MissingSchema, -InvalidSchema, InvalidURL, TooManyRedirects) - -from . import parser, logger, exceptions, Utility -from .parser import string_types - - -class RobotsCache(object): - '''This object represents a cache of robots.txt rules for various sites. - A policy for eviction can be set (or not set) by inheriting from this class - ''' - default_ttl = 3600 - min_ttl = 60 - - def __init__(self, *args, **kwargs): - # The provided args and kwargs are used when fetching robots.txt with - # a `requests.get` - self.args = args - self.disallow_forbidden = kwargs.pop('disallow_forbidden', True) - self.kwargs = kwargs - # A mapping of hostnames to their robots.txt rules - self._cache = {} - - def find(self, url, fetch_if_missing=False, honor_ttl=True): - '''Finds the rules associated with the particular url. Optionally, it - can fetch the rules if they are missing.''' - canonical = Utility.hostname(url) - cached = self._cache.get(canonical) - # If it's expired, we should get rid of it - if honor_ttl and cached and cached.expired: - del self._cache[canonical] - cached = None - # Should we fetch it if it's missing? - if not cached and fetch_if_missing: - return self.cache(url, *self.args, **self.kwargs) - return cached - - def cache(self, url, *args, **kwargs): - '''Like `fetch`, but caches the results, and does eviction''' - fetched = self.fetch(url, *args, **kwargs) - self._cache[Utility.hostname(url)] = fetched - return fetched - - def fetch(self, url, *args, **kwargs): - '''Fetch the robots.txt rules associated with the url. This does /not/ - cache the results. Any additional args are passed into `requests.get` - ''' - try: - # First things first, fetch the thing - robots_url = Utility.roboturl(url) - logger.debug('Fetching %s' % robots_url) - req = requests.get(robots_url, *args, **kwargs) - ttl = max(self.min_ttl, Utility.get_ttl(req.headers, self.default_ttl)) - # And now parse the thing and return it - return parser.Rules(robots_url, req.status_code, req.content, - time.time() + ttl, - disallow_forbidden=self.disallow_forbidden) - except SSLError as exc: - raise exceptions.SSLException(exc) - except ConnectionError as exc: - raise exceptions.ConnectionException(exc) - except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: - raise exceptions.MalformedUrl(exc) - except TooManyRedirects as exc: - raise exceptions.ExcessiveRedirects(exc) - except exceptions.BadStatusCode as exc: - raise exceptions.BadStatusCode(exc) - except Exception as exc: - raise exceptions.ServerError(exc) - - def add(self, rules): - '''Add a rules object to the cache. This is in case you ever want to - ''' - self._cache[Utility.hostname(rules.url)] = rules - - def allowed(self, url, agent): - '''Check whether the provided url is allowed for the provided user - agent. The agent may be a short or long version''' - if hasattr(url, '__iter__') and not isinstance(url, string_types): - results = [self.allowed(u, agent) for u in url] - return [u for u, allowed in zip(url, results) if allowed] - return self.find(url, fetch_if_missing=True).allowed( - url, Utility.short_user_agent(agent)) - - def disallowed(self, url, agent): - '''Check whether the provided url is disallowed. Equivalent to: - not obj.allowed(url, agent)''' - if hasattr(url, '__iter__') and not isinstance(url, string_types): - results = [self.allowed(u, agent) for u in url] - return [u for u, allowed in zip(url, results) if not allowed] - return not self.allowed(url, agent) - - def delay(self, url, agent): - '''Return the crawl delay rule''' - return self.find(url, fetch_if_missing=True).delay( - Utility.short_user_agent(agent)) - - def sitemaps(self, url): - '''Get the sitemaps of the provided url''' - return self.find(url, fetch_if_missing=True).sitemaps - - def clear(self): - '''Clear the cache''' - self._cache = {} - - # Context Manager -- As a context manager, this always remembers to clear - # the cache - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.clear() diff --git a/reppy/exceptions.py b/reppy/exceptions.py index 8e6d6ad..d3913b2 100644 --- a/reppy/exceptions.py +++ b/reppy/exceptions.py @@ -28,13 +28,13 @@ class ReppyException(Exception): '''Any internal exception''' pass +class ContentTooLong(ReppyException): + '''Robots.txt content is too long.''' + pass + class ServerError(ReppyException): '''When the remote server returns an error''' - def __init__(self, *args, **kwargs): - self.status = kwargs.get('status') - if self.status is None and len(args) >= 2: - self.status = args[1] - ReppyException.__init__(self, *args, **kwargs) + pass class SSLException(ReppyException): '''An SSL error.''' diff --git a/reppy/parser.py b/reppy/parser.py deleted file mode 100644 index 4a429d7..0000000 --- a/reppy/parser.py +++ /dev/null @@ -1,261 +0,0 @@ -#! /usr/bin/env python -# -# Copyright (c) 2011 SEOmoz -# -# Permission is hereby granted, free of charge, to any person obtaining -# a copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be -# included in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -'''Classes for parsing robots.txt files''' - -import sys -import re -import time -import codecs -try: - from urllib import parse as urlparse - from urllib.parse import unquote -except ImportError: - # Python 2 - import urlparse - from urllib import unquote - -from . import logger, exceptions, Utility - -if sys.version_info[0] == 3: - string_types = str - text_type = str - binary_type = bytes -else: - string_types = basestring, - text_type = unicode - binary_type = str - - -class Agent(object): - '''Represents attributes for a given robot''' - pathRE = re.compile(r'^([^\/]+\/\/)?([^\/]+)?(/?.+?)$', re.M) - - def __init__(self): - self.allowances = [] - self.delay = None - - @staticmethod - def extract_path(url): - '''Extracts path (with parameters) from given url, if url is already a - path (starts with /) it's rerurned without modifications. In case url - is empty or only contains domain without trailing slash, returns a - single slash.''' - # This method was actually contributed by Wiktor Bachnik (wbachnik) - # but because I forgot to rebase before my branch, this is going to - # appear in a different commit :-/ - if len(url) == 0: - # empty, assume / - path = '/' - elif url[0] == '/': - # url is already a path - path = url - else: - # url is a proper url scheme://host/... - parts = urlparse.urlsplit(url) - needed_parts = urlparse.SplitResult(scheme='', netloc='', - path=parts.path, query=parts.query, fragment='') - path = needed_parts.geturl() - if len(path) == 0: - # case for http://example.com - path = '/' - # If there was a question mark in the url, but no query string - # then we must still preserve the question mark. - if (not parts.query) and ('?' in url): - path = path + '?' - return path - - def allowed(self, url): - '''Can I fetch a given URL?''' - path = unquote(self.extract_path(url).replace('%2f', '%252f')) - if path == '/robots.txt': - return True - allowed = [a for a in self.allowances if a[1].match(path)] - if allowed: - return max(allowed)[2] - else: - return True - -RE_ASTERISK = re.compile(r'\*+') - -class Rules(object): - '''A class that represents a set of agents, and can select them - appropriately. Associated with one robots.txt file.''' - def __init__(self, url, status, content, expires, disallow_forbidden=True): - self.agents = {} - self.sitemaps = [] - # Information about this - self.url = url - self.status = status - self.expires = expires - if status == 200: - self.parse(content) - elif status in (401, 403): - logger.warn('Access forbidden to site %s (%i)' % ( - url, status)) - if disallow_forbidden: - self.parse('''User-agent: *\nDisallow: /''') - else: - self.parse('') - elif status >= 400 and status < 500: - logger.info('Assuming unrestricted access %s (%i)' % ( - url, status)) - self.parse('') - elif status >= 500: - raise exceptions.BadStatusCode( - 'Remote server returned ServerError %i' % status, status) - else: - raise exceptions.ReppyException( - exceptions.ServerError( - 'Remote server returned status %i' % status, status)) - - @property - def ttl(self): - '''Get the time left before expiration''' - return self.expires - time.time() - - @property - def expired(self): - '''Has this rules object expired?''' - return self.ttl <= 0 - - @staticmethod - def _regex_rule(rule): - '''Make a regex that matches the patterns expressable in robots.txt''' - # If the string doesn't start with a forward slash, we'll insert it - # anyways: - # http://code.google.com/web/controlcrawlindex/docs/robots_txt.html - # As such, the only permissible start characters for a rule like this - # are - # '*' and '/' - if rule and rule[0] != '/' and rule[0] != '*': - rule = '/' + rule - # collapse many **** into a single one - rule = RE_ASTERISK.sub('*', rule) - tmp = re.escape(unquote(rule.replace('%2f', '%252f'))) - return re.compile(tmp.replace('\*', '.*').replace('\$', '$')) - - def __getitem__(self, agent): - '''Find the agent given a string for it''' - agent = Utility.short_user_agent(agent) - return self.agents.get(agent.lower(), self.agents.get('*')) - - def parse(self, content): - '''Parse the given string and store the resultant rules''' - # The agent we're currently working with - cur = Agent() - - # If we didn't get a header indicating unicode, we have an 8-bit - # string here. Suspect undeclared UTF-8 or UTF-16 and look for a - # leading BOM. If there is one, attempt to decode. If the decoding - # fails, proclaim the robots.txt file to be garbage and ignore it. - if isinstance(content, binary_type): - try: - if content.startswith(codecs.BOM_UTF8): - content = content.decode('utf-8').lstrip( - text_type(codecs.BOM_UTF8, 'utf-8')) - elif content.startswith(codecs.BOM_UTF16): - content = content.decode('utf-16') - else: - content = content.decode('utf-8', 'ignore') - except UnicodeDecodeError: # pragma: no cover - # This is a very rare and difficult-to-reproduce exception - logger.error('Too much garbage! Ignoring %s' % self.url) - self.agents['*'] = Agent() - return - - # The name of the current agent. There are a couple schools of thought - # here. For example, by including a default agent, the robots.txt's - # author's intent is clearly accommodated if a Disallow line appears - # before the a User-Agent line. However, how hard is it to follow the - # standard? If you're writing a robots.txt, you should be able to - # write it correctly. - curname = '*' - last = '' - for rawline in content.splitlines(): - # Throw away any leading or trailing whitespace - line = rawline.strip() - - # Throw away comments, and ignore blank lines - octothorpe = line.find('#') - if octothorpe >= 0: - line = line[:octothorpe] - if line == '': - continue - - # Non-silently ignore lines with no ':' delimiter - if ':' not in line: - logger.warn('Skipping garbled robots.txt line %s' % - repr(rawline)) - continue - - # Looks valid. Split and interpret it - key, val = [x.strip() for x in line.split(':', 1)] - key = key.lower() - if key == 'user-agent' or key == 'useragent': - # Store the current working agent - if cur: - self.agents[curname] = cur - curname = val.lower() - if last != 'user-agent' and last != 'useragent': - # If the last line was a user agent, then all lines - # below also apply to the last user agent. So, we'll - # have this user agent point to the one we declared - # for the previously-listed agent - cur = self.agents.get(curname, None) or Agent() - elif cur and key == 'disallow': - if len(val): - cur.allowances.append( - (len(val), self._regex_rule(val), False)) - elif cur and key == 'allow': - cur.allowances.append( - (len(val), self._regex_rule(val), True)) - elif cur and key == 'crawl-delay': - try: - cur.delay = float(val) - except: - logger.warn('Unable to parse crawl delay "%s" as float.', val) - elif cur and key == 'sitemap': - self.sitemaps.append(val) - else: - logger.warn('Unknown key in robots.txt line %s' % - repr(rawline)) - last = key - - # Now store the user agent that we've been working on - self.agents[curname] = cur or Agent() - - def allowed(self, url, agent): - '''We try to perform a good match, then a * match''' - if hasattr(url, '__iter__') and not isinstance(url, string_types): - results = [self[agent].allowed(u) for u in url] - return [u for u, allowed in zip(url, results) if allowed] - return self[agent].allowed(url) - - def disallowed(self, url, agent): - '''For completeness''' - return not self.allowed(url, agent) - - def delay(self, agent): - '''How fast can the specified agent legally crawl this site?''' - return self[agent].delay diff --git a/tests/asis/test_bad_status_codes/robots.txt b/tests/asis/test_bad_status_codes/robots.txt deleted file mode 100644 index c6ca0c4..0000000 --- a/tests/asis/test_bad_status_codes/robots.txt +++ /dev/null @@ -1,2 +0,0 @@ -HTTP/1.0 500 ServerError -Content-Type: text/plain diff --git a/tests/asis/test_delay/robots.txt b/tests/asis/test_delay/robots.txt deleted file mode 100644 index 32264b2..0000000 --- a/tests/asis/test_delay/robots.txt +++ /dev/null @@ -1,7 +0,0 @@ -HTTP/1.0 200 OK -Content-Length: 29 -Content-Type: text/plain -Cache-Control: no-cache - -User-Agent: * -Crawl-Delay: 5 diff --git a/tests/asis/test_expires/robots.txt b/tests/asis/test_expires/robots.txt deleted file mode 100644 index 8e571ae..0000000 --- a/tests/asis/test_expires/robots.txt +++ /dev/null @@ -1,6 +0,0 @@ -HTTP/1.0 200 OK -Content-Length: 28 -Content-Type: text/plain -Cache-Control: no-cache - -# This should not be cached diff --git a/tests/asis/test_sitemaps/robots.txt b/tests/asis/test_sitemaps/robots.txt deleted file mode 100644 index b6b26d2..0000000 --- a/tests/asis/test_sitemaps/robots.txt +++ /dev/null @@ -1,8 +0,0 @@ -HTTP/1.0 200 OK -Content-Length: 99 -Content-Type: text/plain -Cache-Control: no-cache - -Sitemap: http://localhost:8080/a -Sitemap: http://localhost:8080/b -Sitemap: http://localhost:8080/c diff --git a/tests/test_cache.py b/tests/test_cache.py deleted file mode 100644 index 180b87a..0000000 --- a/tests/test_cache.py +++ /dev/null @@ -1,175 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -'''These are unit tests that are derived from the rfc at -http://www.robotstxt.org/norobots-rfc.txt''' - -import unittest -import sys -import mock - -if sys.version_info[0] == 3 or '__pypy__' in sys.builtin_module_names: - # We cannot run these tests on Python 3 yet, because they rely - # on the asis module and gevent, both of which are not available. - raise unittest.SkipTest() - -# We need to monkey-patch socket -from gevent import monkey; monkey.patch_all() -import asis -import reppy -import logging -from reppy.cache import RobotsCache -from reppy.exceptions import (ServerError, ConnectionException, MalformedUrl, SSLException, -ExcessiveRedirects, BadStatusCode) -reppy.logger.setLevel(logging.FATAL) - - -class TestCache(unittest.TestCase): - def setUp(self): - self.robots = RobotsCache() - - def test_404(self): - '''When we get a 404, assume free range''' - with asis.Server('tests/asis/test_404', port=8080): - self.assertEqual(self.robots.allowed( - 'http://localhost:8080/foo', 'rogerbot'), True) - - def test_caching(self): - '''We should be able to cache results''' - with asis.Server('tests/asis/test_caching', port=8080): - self.assertEqual( - self.robots.find('http://localhost:8080/foo'), None) - self.robots.allowed('http://localhost:8080/foo', 'rogerbot') - self.assertNotEqual( - self.robots.find('http://localhost:8080/foo'), None) - - def test_context_manager(self): - '''When using as a context manager, it should clear afterwards''' - with asis.Server('tests/asis/test_context_manager', port=8080): - with self.robots: - self.assertEqual( - self.robots.find('http://localhost:8080/foo'), None) - self.robots.allowed('http://localhost:8080/foo', 'rogerbot') - self.assertNotEqual( - self.robots.find('http://localhost:8080/foo'), None) - # And now, we should have it no longer cached - self.assertEqual( - self.robots.find('http://localhost:8080/foo'), None) - - def test_expires(self): - '''Should be able to recognize expired rules''' - with asis.Server('tests/asis/test_expires', port=8080): - old_ttl = self.robots.min_ttl - self.robots.min_ttl = 0 - self.assertNotEqual( - self.robots.find('http://localhost:8080/foo', fetch_if_missing=True), None) - # If we ignore the TTL, it should still be there. - self.assertNotEqual( - self.robots.find('http://localhost:8080/foo', fetch_if_missing=False, honor_ttl=False), None) - # However, if we honor the TTL, it should be missing in the cache. - self.assertEqual( - self.robots.find('http://localhost:8080/foo', fetch_if_missing=False), None) - self.robots.min_ttl = old_ttl - - def test_clear(self): - '''Should be able to explicitly clear rules''' - with asis.Server('tests/asis/test_clear', port=8080): - self.assertEqual( - self.robots.find('http://localhost:8080/foo'), None) - self.robots.allowed('http://localhost:8080/foo', 'rogerbot') - self.assertNotEqual( - self.robots.find('http://localhost:8080/foo'), None) - # Now if we clear the rules, we should not find it - self.robots.clear() - self.assertEqual( - self.robots.find('http://localhost:8080/foo'), None) - - def test_fetch(self): - '''Ensure that 'fetch' doesn't cache''' - with asis.Server('tests/asis/test_fetch', port=8080): - self.assertNotEqual( - self.robots.fetch('http://localhost:8080/foo'), None) - self.assertEqual( - self.robots.find('http://localhost:8080/foo'), None) - - def test_cache(self): - '''Ensure we can ask it to cache a result''' - with asis.Server('tests/asis/test_cache', port=8080): - self.assertEqual( - self.robots.find('http://localhost:8080/foo'), None) - self.assertNotEqual( - self.robots.cache('http://localhost:8080/foo'), None) - self.assertNotEqual( - self.robots.find('http://localhost:8080/foo'), None) - - def test_add(self): - '''We should be able to add rules that we get''' - with asis.Server('tests/asis/test_add', port=8080): - self.assertEqual( - self.robots.find('http://localhost:8080/foo'), None) - self.robots.add(self.robots.fetch( - 'http://localhost:8080/foo')) - self.assertNotEqual( - self.robots.find('http://localhost:8080/foo'), None) - - def test_server_error(self): - '''Make sure we can catch server errors''' - with mock.patch('reppy.cache.requests.get', side_effect=TypeError): - self.assertRaises(ServerError, self.robots.allowed, - 'http://localhost:8080/foo', 'rogerbot') - - def test_disallowed(self): - '''Check the disallowed interface''' - with asis.Server('tests/asis/test_disallowed', port=8080): - self.assertFalse(self.robots.disallowed( - 'http://localhost:8080/foo', 'rogerbot')) - urls = [ - 'http://localhost:8080/foo', - 'http://localhost:8080/bar' - ] - self.assertEqual(self.robots.allowed(urls, 'rogerbot'), urls) - self.assertEqual(self.robots.disallowed(urls, 'rogerbot'), []) - - def test_delay(self): - '''Check the delay interface''' - with asis.Server('tests/asis/test_delay', port=8080): - self.assertEqual(self.robots.delay( - 'http://localhost:8080/foo', 'rogerbot'), 5) - - def test_sitemaps(self): - '''Check the sitemaps interface''' - with asis.Server('tests/asis/test_sitemaps', port=8080): - self.assertEqual( - self.robots.sitemaps('http://localhost:8080/foo'), [ - 'http://localhost:8080/a', - 'http://localhost:8080/b', - 'http://localhost:8080/c' - ]) - - def test_dns_exception(self): - '''Raises an exception if url does not resolve.''' - self.assertRaises(ConnectionException, self.robots.allowed, - 'http://does-not-resolve', 'rogerbot') - - def test_malformed_url(self): - '''Raises an exception if the url is malformed.''' - self.assertRaises(MalformedUrl, self.robots.allowed, - 'hhttp://moz.com', 'rogerbot') - - def test_ssl_exception(self): - '''Raises an exception if there is an ssl error.''' - with asis.Server('tests/asis/test_ssl_exception', port=8080): - self.assertRaises(SSLException, self.robots.allowed, - 'https://localhost:8080', 'rogerbot') - - def test_excessive_redirects(self): - '''Raises an exception if there are too many redirects.''' - with asis.Server('tests/asis/test_excessive_redirects', port=8080): - self.assertRaises(ExcessiveRedirects, self.robots.allowed, - 'http://localhost:8080/one', 'rogerbot') - - def test_bad_status_codes(self): - '''Raises an exception if there is a 5xx status code.''' - with asis.Server('tests/asis/test_bad_status_codes', port=8080): - self.assertRaises(BadStatusCode, self.robots.allowed, - 'http://localhost:8080', 'rogerbot') diff --git a/tests/test_mozscape.py b/tests/test_mozscape.py deleted file mode 100644 index 6931c29..0000000 --- a/tests/test_mozscape.py +++ /dev/null @@ -1,110 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -'''Tests cribbed from linkscape/processing/test/robotstxt.test.cc''' - -import unittest - -import reppy -import logging -from reppy import Utility -reppy.logger.setLevel(logging.FATAL) - -MYNAME = 'reppy' - -class TestMozscape(unittest.TestCase): - @staticmethod - def parse(strng): - '''Helper to parse a string as a Rules object''' - return reppy.parser.Rules('http://example.com/robots.txt', 200, strng, 0) - - def test_disallow_all(self): - rules = self.parse("User-agent: *\nDisallow: /\n") - self.assertFalse(rules.allowed("/", MYNAME)) - self.assertFalse(rules.allowed("/cgi-bin//", MYNAME)) - self.assertFalse(rules.allowed("/tmp/", MYNAME)) - self.assertFalse(rules.allowed("/~joe/", MYNAME)) - self.assertFalse(rules.allowed("/~bob/", MYNAME)) - self.assertFalse(rules.allowed("/stuff/to/read/", MYNAME)) - - def test_disallow_specific(self): - robots_txt = ( "User-agent: *\n" - "Disallow: /cgi-bin/\n" - "Disallow: /tmp/\n" - "Disallow: /~joe/\n" ) - rules = self.parse(robots_txt) - self.assertTrue(rules.allowed("/", MYNAME)) - self.assertFalse(rules.allowed("/cgi-bin//", MYNAME)) - self.assertFalse(rules.allowed("/tmp/", MYNAME)) - self.assertFalse(rules.allowed("/~joe/", MYNAME)) - self.assertTrue(rules.allowed("/~bob/", MYNAME)) - self.assertTrue(rules.allowed("/stuff/to/read/", MYNAME)) - - def test_complete_access(self): - rules = self.parse("User-agent: *\nDisallow:\n") - self.assertTrue(rules.allowed("/", MYNAME)) - self.assertTrue(rules.allowed("/cgi-bin//", MYNAME)) - self.assertTrue(rules.allowed("/tmp/", MYNAME)) - self.assertTrue(rules.allowed("/~joe/", MYNAME)) - self.assertTrue(rules.allowed("/~bob/", MYNAME)) - self.assertTrue(rules.allowed("/stuff/to/read/", MYNAME)) - - def test_disable_another_robot(self): - rules = self.parse("User-agent: BadBot\nDisallow: /\n") - self.assertTrue(rules.allowed("/", MYNAME)) - self.assertTrue(rules.allowed("/cgi-bin//", MYNAME)) - self.assertTrue(rules.allowed("/tmp/", MYNAME)) - self.assertTrue(rules.allowed("/~joe/", MYNAME)) - self.assertTrue(rules.allowed("/~bob/", MYNAME)) - self.assertTrue(rules.allowed("/stuff/to/read/", MYNAME)) - - def test_disable_our_robot(self): - rules = self.parse("User-agent: " + MYNAME + "\nDisallow: /\n") - self.assertFalse(rules.allowed("/", MYNAME)) - self.assertFalse(rules.allowed("/cgi-bin//", MYNAME)) - self.assertFalse(rules.allowed("/tmp/", MYNAME)) - self.assertFalse(rules.allowed("/~joe/", MYNAME)) - self.assertFalse(rules.allowed("/~bob/", MYNAME)) - self.assertFalse(rules.allowed("/stuff/to/read/", MYNAME)) - - def test_allow_another_robot(self): - robots_txt = ( "User-agent: GoodBot\n" - "Disallow:\n" - "\n" - "User-agent: *\n" - "Disallow: /\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/", MYNAME)) - self.assertFalse(rules.allowed("/cgi-bin//", MYNAME)) - self.assertFalse(rules.allowed("/tmp/", MYNAME)) - self.assertFalse(rules.allowed("/~joe/", MYNAME)) - self.assertFalse(rules.allowed("/~bob/", MYNAME)) - self.assertFalse(rules.allowed("/stuff/to/read/", MYNAME)) - - def test_allow_our_robot(self): - robots_txt = ( "User-agent: " + MYNAME + "\n" - "Disallow:\n" - "\n" - "User-agent: *\n" - "Disallow: /\n" ) - rules = self.parse(robots_txt) - self.assertTrue(rules.allowed("/", MYNAME)) - self.assertTrue(rules.allowed("/cgi-bin//", MYNAME)) - self.assertTrue(rules.allowed("/tmp/", MYNAME)) - self.assertTrue(rules.allowed("/~joe/", MYNAME)) - self.assertTrue(rules.allowed("/~bob/", MYNAME)) - self.assertTrue(rules.allowed("/stuff/to/read/", MYNAME)) - - def test_allow_variation(self): - robots_txt = ( "User-agent: " + MYNAME + "\n" - "Disallow: /\n" - "Allow: /tmp/\n" - "Allow: /stuff/\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/", MYNAME)) - self.assertFalse(rules.allowed("/cgi-bin//", MYNAME)) - self.assertTrue(rules.allowed("/tmp/", MYNAME)) - self.assertFalse(rules.allowed("/~joe/", MYNAME)) - self.assertFalse(rules.allowed("/~bob/", MYNAME)) - self.assertTrue(rules.allowed("/stuff/to/read/", MYNAME)) - diff --git a/tests/test_old_mozscape.py b/tests/test_old_mozscape.py deleted file mode 100644 index ad30366..0000000 --- a/tests/test_old_mozscape.py +++ /dev/null @@ -1,698 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -'''Tests cribbed from linkscape/processing/test/robotstxt.test.old.cc''' - -import unittest - -import reppy -import logging -from reppy import Utility -reppy.logger.setLevel(logging.FATAL) - -MYNAME = 'rogerbot' - -class TestOldMozscape(unittest.TestCase): - @staticmethod - def parse(strng): - '''Helper to parse a string as a Rules object''' - return reppy.parser.Rules('http://example.com/robots.txt', 200, strng, 0) - - def test_wwwseomozorg(self): - robots_txt = ( "../resources.test/rep/www.seomoz.org\n" - "User-agent: *\n" - "Disallow: /blogdetail.php?ID=537\n" - "Disallow: /tracker\n" - "\n" - "Sitemap: http://www.seomoz.org/sitemap.xml.gz\n" - "Sitemap: http://files.wistia.com/sitemaps/seomoz_video_sitemap.xml\n" ) - rules = self.parse(robots_txt) - # Basic functionality, and lack of case sensitivity. - for agent in [ 'reppy', 'rEpPy' ]: - self.assertTrue(rules.allowed("/blog", agent)) - self.assertFalse(rules.allowed("/blogdetail.php?ID=537", agent)) - self.assertFalse(rules.allowed("/tracker", agent)) - - def test_allowall(self): - rules = self.parse("User-agent: *\nDisallow:") - for agent in [ "reppy", "oijsdofijsdofijsodifj" ]: - self.assertTrue(rules.allowed("/", agent)) - self.assertTrue(rules.allowed("/foo", agent)) - self.assertTrue(rules.allowed("/foo.html", agent)) - self.assertTrue(rules.allowed("/foo/bar", agent)) - self.assertTrue(rules.allowed("/foo/bar.html", agent)) - - def test_disallowall(self): - rules = self.parse("User-agent: *\nDisallow: /\n") - for agent in [ "reppy", "oijsdofijsdofijsodifj" ]: - self.assertFalse(rules.allowed("/", agent)) - self.assertFalse(rules.allowed("/foo", agent)) - self.assertFalse(rules.allowed("/foo.html", agent)) - self.assertFalse(rules.allowed("/foo/bar", agent)) - self.assertFalse(rules.allowed("/foo/bar.html", agent)) - - def test_no_googlebot_folder(self): - robots_txt = ( "User-agent: Googlebot\n" - "Disallow: /no-google/\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/no-google/", "googlebot")) - self.assertFalse(rules.allowed("/no-google/something", "googlebot")) - self.assertFalse(rules.allowed("/no-google/something.html", "googlebot")) - self.assertTrue(rules.allowed("/", "googlebot")) - self.assertTrue(rules.allowed("/somethingelse", "googlebot")) - - def test_no_googlebot_file(self): - robots_txt = ( "User-agent: Googlebot\n" - "Disallow: /no-google/blocked-page.html\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/no-google/blocked-page.html", "googlebot")) - self.assertTrue(rules.allowed("/", "googlebot")) - self.assertTrue(rules.allowed("/no-google", "googlebot")) - self.assertTrue(rules.allowed("/no-google/someotherfolder", "googlebot")) - self.assertTrue(rules.allowed("/no-google/someotherfolder/somefile", "googlebot")) - - def test_rogerbot_only(self): - robots_txt = ( "User-agent: *\n" - "Disallow: /no-bots/block-all-bots-except-rogerbot-page.html \t\t\t\t\n" - "\n" - "User-agent: rogerbot\n" - "Allow: /no-bots/block-all-bots-except-rogerbot-page.html\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/no-bots/block-all-bots-except-rogerbot-page.html", "notroger")) - self.assertTrue(rules.allowed("/", "notroger")) - self.assertTrue(rules.allowed("/no-bots/block-all-bots-except-rogerbot-page.html", "rogerbot")) - self.assertTrue(rules.allowed("/", "rogerbot")) - - def test_allow_certain_pages_only(self): - robots_txt = ( "User-agent: *\n" - "Allow: /onepage.html\n" - "Allow: /oneotherpage.php\n" - "Disallow: /\n" - "Allow: /subfolder/page1.html\n" - "Allow: /subfolder/page2.php\n" - "Disallow: /subfolder/\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/", "reppy")) - self.assertFalse(rules.allowed("/foo", "reppy")) - self.assertFalse(rules.allowed("/bar.html", "reppy")) - self.assertTrue(rules.allowed("/onepage.html", "reppy")) - self.assertTrue(rules.allowed("/oneotherpage.php", "reppy")) - self.assertFalse(rules.allowed("/subfolder", "reppy")) - self.assertFalse(rules.allowed("/subfolder/", "reppy")) - self.assertFalse(rules.allowed("/subfolder/aaaaa", "reppy")) - self.assertTrue(rules.allowed("/subfolder/page1.html", "reppy")) - self.assertTrue(rules.allowed("/subfolder/page2.php", "reppy")) - - def test_no_gifs_or_jpgs(self): - robots_txt = ( "User-agent: *\n" - "Disallow: /*.gif$\n" - "Disallow: /*.jpg$\n" ) - rules = self.parse(robots_txt) - - self.assertTrue(rules.allowed("/", "reppy")) - self.assertTrue(rules.allowed("/foo", "reppy")) - self.assertTrue(rules.allowed("/foo.html", "reppy")) - self.assertTrue(rules.allowed("/foo/bar", "reppy")) - self.assertTrue(rules.allowed("/foo/bar.html", "reppy")) - self.assertFalse(rules.allowed("/test.jpg", "reppy")) - self.assertFalse(rules.allowed("/foo/test.jpg", "reppy")) - self.assertFalse(rules.allowed("/foo/bar/test.jpg", "reppy")) - self.assertTrue(rules.allowed("/the-jpg-extension-is-awesome.html", "reppy")) - - # Edge cases where the wildcard could match in multiple places - self.assertFalse(rules.allowed("/jpg.jpg", "reppy")) - self.assertFalse(rules.allowed("/foojpg.jpg", "reppy")) - self.assertFalse(rules.allowed("/bar/foojpg.jpg", "reppy")) - self.assertFalse(rules.allowed("/.jpg.jpg", "reppy")) - self.assertFalse(rules.allowed("/.jpg/.jpg", "reppy")) - self.assertFalse(rules.allowed("/test.gif", "reppy")) - self.assertFalse(rules.allowed("/foo/test.gif", "reppy")) - self.assertFalse(rules.allowed("/foo/bar/test.gif", "reppy")) - self.assertTrue(rules.allowed("/the-gif-extension-is-awesome.html", "reppy")) - - def test_block_subdirectory_wildcard(self): - robots_txt = ( "User-agent: *\n" - "Disallow: /private*/\n" ) - rules = self.parse(robots_txt) - - self.assertTrue(rules.allowed("/", "reppy")) - self.assertTrue(rules.allowed("/foo", "reppy")) - self.assertTrue(rules.allowed("/foo.html", "reppy")) - self.assertTrue(rules.allowed("/foo/bar", "reppy")) - self.assertTrue(rules.allowed("/foo/bar.html", "reppy")) - - # Disallow clause ends with a slash, so these shouldn't match - self.assertTrue(rules.allowed("/private", "reppy")) - self.assertTrue(rules.allowed("/privates", "reppy")) - self.assertTrue(rules.allowed("/privatedir", "reppy")) - self.assertFalse(rules.allowed("/private/", "reppy")) - self.assertFalse(rules.allowed("/private/foo", "reppy")) - self.assertFalse(rules.allowed("/private/foo/bar.html", "reppy")) - self.assertFalse(rules.allowed("/privates/", "reppy")) - self.assertFalse(rules.allowed("/privates/foo", "reppy")) - self.assertFalse(rules.allowed("/privates/foo/bar.html", "reppy")) - self.assertFalse(rules.allowed("/privatedir/", "reppy")) - self.assertFalse(rules.allowed("/privatedir/foo", "reppy")) - self.assertFalse(rules.allowed("/privatedir/foo/bar.html", "reppy")) - - def test_block_urls_with_question_marks(self): - robots_txt = ( "User-agent: *\n" - "Disallow: /*?\n" ) - rules = self.parse(robots_txt) - self.assertTrue(rules.allowed("/", "reppy")) - self.assertTrue(rules.allowed("/foo", "reppy")) - self.assertTrue(rules.allowed("/foo.html", "reppy")) - self.assertTrue(rules.allowed("/foo/bar", "reppy")) - self.assertTrue(rules.allowed("/foo/bar.html", "reppy")) - self.assertFalse(rules.allowed("/?", "reppy")) - self.assertFalse(rules.allowed("/foo?q=param", "reppy")) - self.assertFalse(rules.allowed("/foo.html?q=param", "reppy")) - self.assertFalse(rules.allowed("/foo/bar?q=param", "reppy")) - self.assertFalse(rules.allowed("/foo/bar.html?q=param&bar=baz", "reppy")) - - def test_no_question_marks_except_at_end(self): - robots_txt = ( "User-agent: *\n" - "Allow: /*?$\n" - "Disallow: /*?\n" ) - rules = self.parse(robots_txt) - self.assertTrue(rules.allowed("/", "reppy")) - self.assertTrue(rules.allowed("/foo", "reppy")) - self.assertTrue(rules.allowed("/foo.html", "reppy")) - self.assertTrue(rules.allowed("/foo/bar", "reppy")) - self.assertTrue(rules.allowed("/foo/bar.html", "reppy")) - self.assertTrue(rules.allowed("/?", "reppy")) - self.assertTrue(rules.allowed("/foo/bar.html?", "reppy")) - self.assertFalse(rules.allowed("/foo?q=param", "reppy")) - self.assertFalse(rules.allowed("/foo.html?q=param", "reppy")) - self.assertFalse(rules.allowed("/foo/bar?q=param", "reppy")) - self.assertFalse(rules.allowed("/foo/bar.html?q=param&bar=baz", "reppy")) - - def test_wildcard_edge_cases(self): - robots_txt = ( "User-agent: *\n" - "Disallow: /*one\n" - "Disallow: /two*three\n" - "Disallow: /irrelevant/four*five\n" - "Disallow: /six*\n" - "Disallow: /foo/*/seven*/eight*nine\n" - "Disallow: /foo/*/*ten$\n" - "\n" - "Disallow: /*products/default.aspx\n" - "Disallow: /*/feed/$\n" ) - rules = self.parse(robots_txt) - self.assertTrue(rules.allowed("/", "reppy")) - self.assertTrue(rules.allowed("/foo", "reppy")) - self.assertTrue(rules.allowed("/foo.html", "reppy")) - self.assertTrue(rules.allowed("/foo/bar", "reppy")) - self.assertTrue(rules.allowed("/foo/bar.html", "reppy")) - self.assertFalse(rules.allowed("/one", "reppy")) - self.assertFalse(rules.allowed("/aaaone", "reppy")) - self.assertFalse(rules.allowed("/aaaaoneaaa", "reppy")) - self.assertFalse(rules.allowed("/oneaaaa", "reppy")) - self.assertFalse(rules.allowed("/twothree", "reppy")) - self.assertFalse(rules.allowed("/twoaaathree", "reppy")) - self.assertFalse(rules.allowed("/twoaaaathreeaaa", "reppy")) - self.assertFalse(rules.allowed("/twothreeaaa", "reppy")) - self.assertFalse(rules.allowed("/irrelevant/fourfive", "reppy")) - self.assertFalse(rules.allowed("/irrelevant/fouraaaafive", "reppy")) - self.assertFalse(rules.allowed("/irrelevant/fouraaafiveaaaa", "reppy")) - self.assertFalse(rules.allowed("/irrelevant/fourfiveaaa", "reppy")) - self.assertFalse(rules.allowed("/six", "reppy")) - self.assertFalse(rules.allowed("/sixaaaa", "reppy")) - self.assertFalse(rules.allowed("/products/default.aspx", "reppy")) - self.assertFalse(rules.allowed("/author/admin/feed/", "reppy")) - - def test_allow_edge_cases(self): - robots_txt = ( "User-agent: *\n" - "Disallow:\t/somereallylongfolder/\n" - "Allow:\t\t/*.jpg\n" - "\n" - "Disallow:\t/sales-secrets.php\n" - "Allow: \t\t/sales-secrets.php\n" - "\n" - "Disallow:\t/folder\n" - "Allow:\t\t/folder/\n" - "\n" - "Allow:\t\t/folder2\n" - "Disallow:\t/folder2/\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/somereallylongfolder/", "reppy")) - self.assertFalse(rules.allowed("/somereallylongfolder/aaaa", "reppy")) - self.assertFalse(rules.allowed("/somereallylongfolder/test.jpg", "reppy")) - self.assertTrue(rules.allowed("/sales-secrets.php", "reppy")) - self.assertTrue(rules.allowed("/folder/page", "reppy")) - self.assertTrue(rules.allowed("/folder/page2", "reppy")) - - def test_redundant_allow(self): - robots_txt = ( "User-agent: *\n" - "Disallow: /en/\n" - "Disallow: /files/documentation/\n" - "Disallow: /files/\n" - "Disallow: /de/careers/\n" - "Disallow: /images/\n" - "\n" - "Disallow: /print_mode.yes/\n" - "Disallow: /?product=lutensit&print_mode=yes&googlebot=nocrawl\n" - "Allow: /\n" - "Disallow: /search/\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/print_mode.yes/", "reppy")) - self.assertFalse(rules.allowed("/print_mode.yes/foo", "reppy")) - self.assertFalse(rules.allowed("/search/", "reppy")) - self.assertFalse(rules.allowed("/search/foo", "reppy")) - - # Some comments, wildcards, and anchor tests -- this was a legacy test - # ported from urlexclude - def test_legacy_test_1(self): - robots_txt = ( "user-agent: * #a comment!\n" - "disallow: /Blerf\n" - "disallow: /Blerg$\n" - "disallow: /blerf/*/print.html$#a comment\n" - "disallow: /blerf/*/blim/blerf$\n" - "disallow: /plerf/*/blim/blim$\n" - "\tuser-agent: BLERF\n" - " DisALLOW: \tblerfPage\n" - "blerf:blah\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/Blerf/blah", "reppy")) - self.assertTrue(rules.allowed("/Blerg/blah", "reppy")) - self.assertTrue(rules.allowed("/blerf/blah", "reppy")) - self.assertFalse(rules.allowed("/Blerg", "reppy")) - self.assertFalse(rules.allowed("/blerf/some/subdirs/print.html", "reppy")) - self.assertTrue(rules.allowed("/blerf/some/subdirs/print.html?extra=stuff", "reppy")) - self.assertFalse(rules.allowed("/blerf/some/sub/dirs/blim/blim/blerf", "reppy")) - self.assertFalse(rules.allowed("/plerf/some/sub/dirs/blim/blim", "reppy")) - - def test_legacy_test_2(self): - robots_txt = ( "User-agent: *\n" - "Allow: /searchhistory/\n" - "Disallow: /news?output=xhtml&\n" - "Allow: /news?output=xhtml\n" - "Disallow: /search\n" - "Disallow: /groups\n" - "Disallow: /images\n" - "Disallow: /catalogs\n" - "Disallow: /catalogues\n" - "Disallow: /news\n" - "Disallow: /nwshp\n" - "Allow: /news?btcid=\n" - "Disallow: /news?btcid=*&\n" - "Allow: /news?btaid=\n" - "Disallow: /news?btaid=*&\n" - "Disallow: /?\n" - "Disallow: /addurl/image?\n" - "Disallow: /pagead/\n" - "Disallow: /relpage/\n" - "Disallow: /relcontent\n" - "Disallow: /sorry/\n" - "Disallow: /imgres\n" - "Disallow: /keyword/\n" - "Disallow: /u/\n" - "Disallow: /univ/\n" - "Disallow: /cobrand\n" - "Disallow: /custom\n" - "Disallow: /advanced_group_search\n" - "Disallow: /advanced_search\n" - "Disallow: /googlesite\n" - "Disallow: /preferences\n" - "Disallow: /setprefs\n" - "Disallow: /swr\n" - "Disallow: /url\n" - "Disallow: /default\n" - "Disallow: /m?\n" - "Disallow: /m/?\n" - "Disallow: /m/lcb\n" - "Disallow: /m/search?\n" - "Disallow: /wml?\n" - "Disallow: /wml/?\n" - "Disallow: /wml/search?\n" - "Disallow: /xhtml?\n" - "Disallow: /xhtml/?\n" - "Disallow: /xhtml/search?\n" - "Disallow: /xml?\n" - "Disallow: /imode?\n" - "Disallow: /imode/?\n" - "Disallow: /imode/search?\n" - "Disallow: /jsky?\n" - "Disallow: /jsky/?\n" - "Disallow: /jsky/search?\n" - "Disallow: /pda?\n" - "Disallow: /pda/?\n" - "Disallow: /pda/search?\n" - "Disallow: /sprint_xhtml\n" - "Disallow: /sprint_wml\n" - "Disallow: /pqa\n" - "Disallow: /palm\n" - "Disallow: /gwt/\n" - "Disallow: /purchases\n" - "Disallow: /hws\n" - "Disallow: /bsd?\n" - "Disallow: /linux?\n" - "Disallow: /mac?\n" - "Disallow: /microsoft?\n" - "Disallow: /unclesam?\n" - "Disallow: /answers/search?q=\n" - "Disallow: /local?\n" - "Disallow: /local_url\n" - "Disallow: /froogle?\n" - "Disallow: /products?\n" - "Disallow: /froogle_\n" - "Disallow: /product_\n" - "Disallow: /products_\n" - "Disallow: /print\n" - "Disallow: /books\n" - "Disallow: /patents?\n" - "Disallow: /scholar?\n" - "Disallow: /complete\n" - "Disallow: /sponsoredlinks\n" - "Disallow: /videosearch?\n" - "Disallow: /videopreview?\n" - "Disallow: /videoprograminfo?\n" - "Disallow: /maps?\n" - "Disallow: /mapstt?\n" - "Disallow: /mapslt?\n" - "Disallow: /maps/stk/\n" - "Disallow: /mapabcpoi?\n" - "Disallow: /translate?\n" - "Disallow: /ie?\n" - "Disallow: /sms/demo?\n" - "Disallow: /katrina?\n" - "Disallow: /blogsearch?\n" - "Disallow: /blogsearch/\n" - "Disallow: /blogsearch_feeds\n" - "Disallow: /advanced_blog_search\n" - "Disallow: /reader/\n" - "Disallow: /uds/\n" - "Disallow: /chart?\n" - "Disallow: /transit?\n" - "Disallow: /mbd?\n" - "Disallow: /extern_js/\n" - "Disallow: /calendar/feeds/\n" - "Disallow: /calendar/ical/\n" - "Disallow: /cl2/feeds/\n" - "Disallow: /cl2/ical/\n" - "Disallow: /coop/directory\n" - "Disallow: /coop/manage\n" - "Disallow: /trends?\n" - "Disallow: /trends/music?\n" - "Disallow: /notebook/search?\n" - "Disallow: /music\n" - "Disallow: /browsersync\n" - "Disallow: /call\n" - "Disallow: /archivesearch?\n" - "Disallow: /archivesearch/url\n" - "Disallow: /archivesearch/advanced_search\n" - "Disallow: /base/search?\n" - "Disallow: /base/reportbadoffer\n" - "Disallow: /base/s2\n" - "Disallow: /urchin_test/\n" - "Disallow: /movies?\n" - "Disallow: /codesearch?\n" - "Disallow: /codesearch/feeds/search?\n" - "Disallow: /wapsearch?\n" - "Disallow: /safebrowsing\n" - "Disallow: /reviews/search?\n" - "Disallow: /orkut/albums\n" - "Disallow: /jsapi\n" - "Disallow: /views?\n" - "Disallow: /c/\n" - "Disallow: /cbk\n" - "Disallow: /recharge/dashboard/car\n" - "Disallow: /recharge/dashboard/static/\n" - "Disallow: /translate_c?\n" - "Disallow: /s2/profiles/me\n" - "Allow: /s2/profiles\n" - "Disallow: /s2\n" - "Disallow: /transconsole/portal/\n" - "Disallow: /gcc/\n" - "Disallow: /aclk\n" - "Disallow: /cse?\n" - "Disallow: /tbproxy/\n" - "Disallow: /MerchantSearchBeta/\n" - "Disallow: /ime/\n" - "Disallow: /websites?\n" - "Disallow: /shenghuo/search?\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/?as_q=ethics&ie=UTF-8&ui=blg&bl_url=centrerion.blogspot.com&x=0&y=0&ui=blg", "reppy")) - - # Real world example with several similar disallow rules - def test_legacy_test_3(self): - robots_txt = ( "User-agent: *\n" - "Allow: /searchhistory/\n" - "Disallow: /news?output=xhtml&\n" - "Allow: /news?output=xhtml\n" - "Disallow: /search\n" - "Disallow: /groups\n" - "Disallow: /images\n" - "Disallow: /catalogs\n" - "Disallow: /catalogues\n" - "Disallow: /news\n" - "Disallow: /nwshp\n" - "Allow: /news?btcid=\n" - "Disallow: /news?btcid=*&\n" - "Allow: /news?btaid=\n" - "Disallow: /news?btaid=*&\n" - "Disallow: /?\n" - "Disallow: /addurl/image?\n" - "Disallow: /pagead/\n" - "Disallow: /relpage/\n" - "Disallow: /relcontent\n" - "Disallow: /sorry/\n" - "Disallow: /imgres\n" - "Disallow: /keyword/\n" - "Disallow: /u/\n" - "Disallow: /univ/\n" - "Disallow: /cobrand\n" - "Disallow: /custom\n" - "Disallow: /advanced_group_search\n" - "Disallow: /advanced_search\n" - "Disallow: /googlesite\n" - "Disallow: /preferences\n" - "Disallow: /setprefs\n" - "Disallow: /swr\n" - "Disallow: /url\n" - "Disallow: /default\n" - "Disallow: /m?\n" - "Disallow: /m/?\n" - "Disallow: /m/lcb\n" - "Disallow: /m/search?\n" - "Disallow: /wml?\n" - "Disallow: /wml/?\n" - "Disallow: /wml/search?\n" - "Disallow: /xhtml?\n" - "Disallow: /xhtml/?\n" - "Disallow: /xhtml/search?\n" - "Disallow: /xml?\n" - "Disallow: /imode?\n" - "Disallow: /imode/?\n" - "Disallow: /imode/search?\n" - "Disallow: /jsky?\n" - "Disallow: /jsky/?\n" - "Disallow: /jsky/search?\n" - "Disallow: /pda?\n" - "Disallow: /pda/?\n" - "Disallow: /pda/search?\n" - "Disallow: /sprint_xhtml\n" - "Disallow: /sprint_wml\n" - "Disallow: /pqa\n" - "Disallow: /palm\n" - "Disallow: /gwt/\n" - "Disallow: /purchases\n" - "Disallow: /hws\n" - "Disallow: /bsd?\n" - "Disallow: /linux?\n" - "Disallow: /mac?\n" - "Disallow: /microsoft?\n" - "Disallow: /unclesam?\n" - "Disallow: /answers/search?q=\n" - "Disallow: /local?\n" - "Disallow: /local_url\n" - "Disallow: /froogle?\n" - "Disallow: /products?\n" - "Disallow: /froogle_\n" - "Disallow: /product_\n" - "Disallow: /products_\n" - "Disallow: /print\n" - "Disallow: /books\n" - "Disallow: /patents?\n" - "Disallow: /scholar?\n" - "Disallow: /complete\n" - "Disallow: /sponsoredlinks\n" - "Disallow: /videosearch?\n" - "Disallow: /videopreview?\n" - "Disallow: /videoprograminfo?\n" - "Disallow: /maps?\n" - "Disallow: /mapstt?\n" - "Disallow: /mapslt?\n" - "Disallow: /maps/stk/\n" - "Disallow: /mapabcpoi?\n" - "Disallow: /translate?\n" - "Disallow: /ie?\n" - "Disallow: /sms/demo?\n" - "Disallow: /katrina?\n" - "Disallow: /blogsearch?\n" - "Disallow: /blogsearch/\n" - "Disallow: /blogsearch_feeds\n" - "Disallow: /advanced_blog_search\n" - "Disallow: /reader/\n" - "Disallow: /uds/\n" - "Disallow: /chart?\n" - "Disallow: /transit?\n" - "Disallow: /mbd?\n" - "Disallow: /extern_js/\n" - "Disallow: /calendar/feeds/\n" - "Disallow: /calendar/ical/\n" - "Disallow: /cl2/feeds/\n" - "Disallow: /cl2/ical/\n" - "Disallow: /coop/directory\n" - "Disallow: /coop/manage\n" - "Disallow: /trends?\n" - "Disallow: /trends/music?\n" - "Disallow: /notebook/search?\n" - "Disallow: /music\n" - "Disallow: /browsersync\n" - "Disallow: /call\n" - "Disallow: /archivesearch?\n" - "Disallow: /archivesearch/url\n" - "Disallow: /archivesearch/advanced_search\n" - "Disallow: /base/search?\n" - "Disallow: /base/reportbadoffer\n" - "Disallow: /base/s2\n" - "Disallow: /urchin_test/\n" - "Disallow: /movies?\n" - "Disallow: /codesearch?\n" - "Disallow: /codesearch/feeds/search?\n" - "Disallow: /wapsearch?\n" - "Disallow: /safebrowsing\n" - "Disallow: /reviews/search?\n" - "Disallow: /orkut/albums\n" - "Disallow: /jsapi\n" - "Disallow: /views?\n" - "Disallow: /c/\n" - "Disallow: /cbk\n" - "Disallow: /recharge/dashboard/car\n" - "Disallow: /recharge/dashboard/static/\n" - "Disallow: /translate_c?\n" - "Disallow: /s2/profiles/me\n" - "Allow: /s2/profiles\n" - "Disallow: /s2\n" - "Disallow: /transconsole/portal/\n" - "Disallow: /gcc/\n" - "Disallow: /aclk\n" - "Disallow: /cse?\n" - "Disallow: /tbproxy/\n" - "Disallow: /MerchantSearchBeta/\n" - "Disallow: /ime/\n" - "Disallow: /websites?\n" - "Disallow: /shenghuo/search?\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/archivesearch?q=stalin&scoring=t&hl=en&sa=N&sugg=d&as_ldate=1900&as_hdate=1919&lnav=hist2", "reppy")) - - # Real world example - def test_legacy_test_4(self): - robots_txt = ( "User-agent: scooter\n" - "Disallow: /\n" - "\n" - "User-agent: wget\n" - "User-agent: webzip\n" - "Disallow: /\n" - "\n" - "User-agent: *\n" - "Disallow:\n" ) - rules = self.parse(robots_txt) - self.assertTrue(rules.allowed("/index.html", "reppy")) - - # Real world example - def test_legacy_test_5(self): - robots_txt = ( "# Alexa\n" - "User-agent: ia_archiver\n" - "Disallow: /utils/date_picker/\n" - "# Ask Jeeves\n" - "User-agent: Teoma\n" - "Disallow: /utils/date_picker/\n" - "# Google\n" - "User-agent: googlebot\n" - "Disallow: /utils/date_picker/\n" - "# MSN\n" - "User-agent: MSNBot\n" - "Disallow: /utils/date_picker/\n" - "# Yahoo!\n" - "User-agent: Slurp\n" - "Disallow: /utils/date_picker/\n" - "# Baidu\n" - "User-agent: baiduspider\n" - "Disallow: /utils/date_picker/\n" - "# All the rest go away\n" - "User-agent: *\n" - "Disallow: /\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/", "reppy")) - - # Real world example with multiple consecutive user agent directives - def test_legacy_test_6(self): - robots_txt = ( "User-agent: reppy\n" - "User-agent:snowball\n" - "Disallow:/\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/", "reppy")) - - # Real world example, r.e. phpBB - def test_legacy_test_7(self): - robots_txt = ( "User-agent: Googlebot-Image\n" - "Disallow: /\n" - "\n" - "User-agent: *\n" - "Crawl-delay: 7\n" - "\n" - "Disallow: /faq.php\n" - "Disallow: /groupcp.php\n" - "Disallow: /login.php\n" - "Disallow: /memberlist.php\n" - "Disallow: /merge.php\n" - "Disallow: /modcp.php\n" - "Disallow: /posting.php\n" - "Disallow: /phpBB2/posting.php\n" - "Disallow: /privmsg.php\n" - "Disallow: /profile.php\n" - "Disallow: /search.php\n" - "Disallow: /phpBB2/faq.php\n" - "Disallow: /phpBB2/groupcp.php\n" - "Disallow: /phpBB2/login.php\n" - "Disallow: /phpBB2/memberlist.php\n" - "Disallow: /phpBB2/merge.php\n" - "Disallow: /phpBB2/modcp.php\n" - "Disallow: /phpBB2/posting.php\n" - "Disallow: /phpBB2/posting.php\n" - "Disallow: /phpBB2/privmsg.php\n" - "Disallow: /phpBB2/profile.php\n" - "Disallow: /phpBB2/search.php\n" - "\n" - "Disallow: /admin/\n" - "Disallow: /images/\n" - "Disallow: /includes/\n" - "Disallow: /install/\n" - "Disallow: /modcp/\n" - "Disallow: /templates/\n" - "Disallow: /phpBB2/admin/\n" - "Disallow: /phpBB2/images/\n" - "Disallow: /phpBB2/includes/\n" - "Disallow: /phpBB2/install/\n" - "Disallow: /phpBB2/modcp/\n" - "Disallow: /phpBB2/templates/\n" - "\n" - "Disallow: /trac/\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/phpBB2/posting.php?mode=reply&t=895", "reppy")) - - # Is this pertinent to reppy, or would this have been sanitized by - # the time it reaches the parsing stage? - def test_utf8bom(self): - robots_txt = ( "\357\273\277User-agent: *\n" - "Disallow: /Product/List\n" - "Disallow: /Product/Search\n" - "Disallow: /Product/TopSellers\n" - "Disallow: /Product/UploadImage\n" - "Disallow: /WheelPit\n" - "Disallow: /iwwida.pvx\n" ) - rules = self.parse(robots_txt) - self.assertFalse(rules.allowed("/WheelPit", "reppy")) - diff --git a/tests/test_parse.py b/tests/test_parse.py deleted file mode 100755 index 9cbfa02..0000000 --- a/tests/test_parse.py +++ /dev/null @@ -1,994 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -'''These are unit tests that are derived from the rfc at -http://www.robotstxt.org/norobots-rfc.txt''' - -import random -import unittest - -import reppy -import logging -from reppy.parser import Rules -from reppy.exceptions import ReppyException, BadStatusCode -reppy.logger.setLevel(logging.FATAL) - - -class TestParse(unittest.TestCase): - @staticmethod - def parse(strng): - '''Helper to parse a string as a Rules object''' - return Rules('http://example.com/robots.txt', 200, strng, 0) - - def test_basic(self): - '''Basic robots.txt parsing''' - # Test beginning matching - rules = self.parse(''' - User-agent: * - Disallow: /tmp''') - self.assertTrue(not rules.allowed('/tmp', 't')) - self.assertTrue(not rules.allowed('/tmp.html', 't')) - self.assertTrue(not rules.allowed('/tmp/a.html', 't')) - - rules = self.parse(''' - User-agent: * - Disallow: /tmp/''') - self.assertTrue(rules.allowed('/tmp', 't')) - self.assertTrue(not rules.allowed('/tmp/', 't')) - self.assertTrue(not rules.allowed('/tmp/a.html', 't')) - # Make sure we can also provide a full url - self.assertTrue(rules.allowed('http://example.com/tmp', 't')) - self.assertTrue(not rules.allowed('http://example.com/tmp/', 't')) - - def test_unquoting(self): - '''Ensures we correctly interpret url-escaped entities''' - # Now test escaping entities - rules = self.parse(''' - User-agent: * - Disallow: /a%3cd.html''') - self.assertTrue(not rules.allowed('/a%3cd.html', 't')) - self.assertTrue(not rules.allowed('/a%3Cd.html', 't')) - # And case indpendent - rules = self.parse(''' - User-agent: * - Disallow: /a%3Cd.html''') - self.assertTrue(not rules.allowed('/a%3cd.html', 't')) - self.assertTrue(not rules.allowed('/a%3Cd.html', 't')) - # And escape the urls themselves - rules = self.parse(''' - User-agent: * - Disallow: /%7ejoe/index.html''') - self.assertTrue(not rules.allowed('/~joe/index.html', 't')) - self.assertTrue(not rules.allowed('/%7ejoe/index.html', 't')) - - def test_unquoting_forward_slash(self): - '''Ensures escaped forward slashes are not matched with unescaped''' - rules = self.parse(''' - User-agent: * - Disallow: /a%2fb.html''') - self.assertTrue(not rules.allowed('/a%2fb.html', 't')) - self.assertTrue(rules.allowed('/a/b.html', 't')) - rules = self.parse(''' - User-agent: * - Disallow: /a/b.html''') - self.assertTrue(rules.allowed('/a%2fb.html', 't')) - self.assertTrue(not rules.allowed('/a/b.html', 't')) - - def test_rfc_example(self): - '''Tests the example provided in the RFC''' - rules = self.parse('''# /robots.txt for http://www.fict.org/ - # comments to webmaster@fict.org - - User-agent: unhipbot - Disallow: / - - User-agent: webcrawler - User-agent: excite - Disallow: - - User-agent: * - Disallow: /org/plans.html - Allow: /org/ - Allow: /serv - Allow: /~mak - Disallow: /''') - # The unhip bot - self.assertTrue(not rules.allowed('/', 'unhipbot')) - self.assertTrue(not rules.allowed('/index.html', 'unhipbot')) - self.assertTrue(rules.allowed('/robots.txt', 'unhipbot')) - self.assertTrue(not rules.allowed('/server.html', 'unhipbot')) - self.assertTrue(not rules.allowed('/services/fast.html', 'unhipbot')) - self.assertTrue(not rules.allowed('/services/slow.html', 'unhipbot')) - self.assertTrue(not rules.allowed('/orgo.gif', 'unhipbot')) - self.assertTrue(not rules.allowed('/org/about.html', 'unhipbot')) - self.assertTrue(not rules.allowed('/org/plans.html', 'unhipbot')) - self.assertTrue(not rules.allowed('/%7Ejim/jim.html', 'unhipbot')) - self.assertTrue(not rules.allowed('/%7Emak/mak.html', 'unhipbot')) - # The webcrawler agent - self.assertTrue(rules.allowed('/', 'webcrawler')) - self.assertTrue(rules.allowed('/index.html', 'webcrawler')) - self.assertTrue(rules.allowed('/robots.txt', 'webcrawler')) - self.assertTrue(rules.allowed('/server.html', 'webcrawler')) - self.assertTrue(rules.allowed('/services/fast.html', 'webcrawler')) - self.assertTrue(rules.allowed('/services/slow.html', 'webcrawler')) - self.assertTrue(rules.allowed('/orgo.gif', 'webcrawler')) - self.assertTrue(rules.allowed('/org/about.html', 'webcrawler')) - self.assertTrue(rules.allowed('/org/plans.html', 'webcrawler')) - self.assertTrue(rules.allowed('/%7Ejim/jim.html', 'webcrawler')) - self.assertTrue(rules.allowed('/%7Emak/mak.html', 'webcrawler')) - # The excite agent - self.assertTrue(rules.allowed('/', 'excite')) - self.assertTrue(rules.allowed('/index.html', 'excite')) - self.assertTrue(rules.allowed('/robots.txt', 'excite')) - self.assertTrue(rules.allowed('/server.html', 'excite')) - self.assertTrue(rules.allowed('/services/fast.html', 'excite')) - self.assertTrue(rules.allowed('/services/slow.html', 'excite')) - self.assertTrue(rules.allowed('/orgo.gif', 'excite')) - self.assertTrue(rules.allowed('/org/about.html', 'excite')) - self.assertTrue(rules.allowed('/org/plans.html', 'excite')) - self.assertTrue(rules.allowed('/%7Ejim/jim.html', 'excite')) - self.assertTrue(rules.allowed('/%7Emak/mak.html', 'excite')) - # All others - self.assertTrue(not rules.allowed('/', 'anything')) - self.assertTrue(not rules.allowed('/index.html', 'anything')) - self.assertTrue(rules.allowed('/robots.txt', 'anything')) - self.assertTrue(rules.allowed('/server.html', 'anything')) - self.assertTrue(rules.allowed('/services/fast.html', 'anything')) - self.assertTrue(rules.allowed('/services/slow.html', 'anything')) - self.assertTrue(not rules.allowed('/orgo.gif', 'anything')) - self.assertTrue(rules.allowed('/org/about.html', 'anything')) - self.assertTrue(not rules.allowed('/org/plans.html', 'anything')) - self.assertTrue(not rules.allowed('/%7Ejim/jim.html', 'anything')) - self.assertTrue(rules.allowed('/%7Emak/mak.html', 'anything')) - - def test_crawl_delay(self): - '''Ensures we can parse crawl delays per agent''' - rules = self.parse(''' - User-agent: agent - Crawl-delay: 5 - User-agent: * - Crawl-delay: 1''') - self.assertEqual(rules.delay('agent'), 5) - self.assertEqual(rules.delay('testing'), 1) - - def test_sitemaps(self): - '''Ensures we can read all the sitemaps''' - rules = self.parse(''' - User-agent: agent - Sitemap: http://a.com/sitemap.xml - Sitemap: http://b.com/sitemap.xml''') - self.assertEqual(rules.sitemaps, [ - 'http://a.com/sitemap.xml', 'http://b.com/sitemap.xml']) - - def test_batch_queries(self): - '''Ensures batch queries run as expected''' - rules = self.parse(''' - User-agent: * - Disallow: /a - Disallow: /b - Allow: /b/''') - test_urls = ['/a/hello', '/a/howdy', '/b', '/b/hello'] - allowed = ['/b/hello'] - self.assertEqual(rules.allowed(test_urls, 't'), allowed) - - def test_wildcard(self): - '''Ensures wildcard matching works as expected''' - rules = self.parse(''' - User-agent: * - Disallow: /hello/*/are/you''') - testUrls = ['/hello/', '/hello/how/are/you', '/hi/how/are/you/'] - allowed = ['/hello/', '/hi/how/are/you/'] - self.assertEqual(rules.allowed(testUrls, 't'), allowed) - - def test_disallowed(self): - '''Make sure disallowed is the opposite of allowed''' - rules = self.parse(''' - User-agent: * - Disallow: /0xa - Disallow: /0xb - Disallow: /0xc - Disallow: /0xd - Disallow: /0xe - Disallow: /0xf''') - for i in range(1000): - path = hex(random.randint(0, 16)) - self.assertNotEqual( - rules.allowed(path, 't'), rules.disallowed(path, 't')) - - def test_case_insensitivity(self): - '''Make sure user agent matches are case insensitive''' - rules = self.parse(''' - User-agent: agent - Disallow: /a''') - self.assertTrue(rules.disallowed('/a', 'Agent')) - self.assertTrue(rules.disallowed('/a', 'aGent')) - self.assertTrue(rules.disallowed('/a', 'AGeNt')) - self.assertTrue(rules.disallowed('/a', 'AGENT')) - - def test_query(self): - '''Make sure user agent matches are case insensitive''' - rules = self.parse(''' - User-agent: agent - Disallow: /a?howdy''') - self.assertTrue(rules.allowed('/a', 'agent')) - self.assertTrue(not rules.allowed('/a?howdy', 'agent')) - self.assertTrue(not rules.allowed('/a?howdy#fragment', 'agent')) - self.assertTrue(rules.allowed('/a?heyall', 'agent')) - - def test_allow_all(self): - '''Ensures an empty disallow directive allows all''' - rules = self.parse(''' - User-agent: * - Disallow: ''') - agent = 'dotbot' - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/foo', agent)) - self.assertTrue(rules.allowed('/foo.html', agent)) - self.assertTrue(rules.allowed('/foo/bar', agent)) - self.assertTrue(rules.allowed('/foo/bar.html', agent)) - - def test_disallow_all(self): - '''Ensures we can disallow all paths''' - rules = self.parse(''' - User-agent: * - Disallow: /''') - agent = 'dotbot' - self.assertTrue(not rules.allowed('/', agent)) - self.assertTrue(not rules.allowed('/foo', agent)) - self.assertTrue(not rules.allowed('/foo.html', agent)) - self.assertTrue(not rules.allowed('/foo/bar', agent)) - self.assertTrue(not rules.allowed('/foo/bar.html', agent)) - - def test_allow_certain_pages_only(self): - '''Tests explicit path allowances, even when they'd otherwise be - disallowed''' - rules = self.parse(''' - User-agent: * - Allow: /onepage.html - Allow: /oneotherpage.php - Disallow: / - Allow: /subfolder/page1.html - Allow: /subfolder/page2.php - Disallow: /subfolder/''') - agent = 'dotbot' - self.assertTrue(not rules.allowed('/', agent)) - self.assertTrue(not rules.allowed('/foo', agent)) - self.assertTrue(not rules.allowed('/bar.html', agent)) - self.assertTrue(rules.allowed('/onepage.html', agent)) - self.assertTrue(rules.allowed('/oneotherpage.php', agent)) - self.assertTrue(not rules.allowed('/subfolder', agent)) - self.assertTrue(not rules.allowed('/subfolder/', agent)) - self.assertTrue(not rules.allowed('/subfolder/aaaaa', agent)) - self.assertTrue(rules.allowed('/subfolder/page1.html', agent)) - self.assertTrue(rules.allowed('/subfolder/page2.php', agent)) - - def test_empty(self): - '''Makes sure we can part an empty robots.txt''' - rules = self.parse('') - self.assertTrue(rules.allowed('/', 't')) - self.assertEqual(rules.delay('t'), None) - self.assertEqual(rules.sitemaps, []) - - def test_relative(self): - '''Basically, we should treat 'hello' like '/hello' ''' - rules = self.parse(''' - User-agent: * - Disallow: hello - Disallow: *goodbye''') - agent = 'dotbot' - self.assertTrue(not rules.allowed('/hello', agent)) - self.assertTrue(not rules.allowed('/hello/everyone', agent)) - - def test_grouping_unknown_keys(self): - '''When we encounter unknown keys, we should disregard any grouping - that may have happened between user agent rules.''' - # For example, in the wild, we encountered: - # - # User-agent: * - # Disallow: /content/2/ - # User-agent: * - # Noindex: /gb.html - # Noindex: /content/2/ - # User-agent: ia_archiver - # Disallow: / - # - # Since 'Noindex' is an unknown directive, the rules for ia_archiver - # were getting glommed into the rules for *, when that was - # clearly not the intention. - rules = self.parse(''' - User-agent: * - Disallow: /content/2/ - User-agent: * - Noindex: /gb.html - Noindex: /content/2/ - User-agent: ia_archiver - Disallow: / - ''') - self.assertTrue(rules.allowed('/foo', 'dotbot')) - self.assertTrue(not rules.allowed('/bar', 'ia_archiver')) - - def test_utf8_bom(self): - '''If there's a utf-8 BOM, we should parse it as such''' - import codecs - rules = self.parse(codecs.BOM_UTF8 + b'''User-agent: foo - Allow: /foo - - User-agent: * - Disallow: /foo - ''') - self.assertTrue(rules.allowed('/foo', 'foo')) - self.assertTrue(not rules.allowed('/foo', 'other')) - - def test_utf16_bom(self): - '''If there's a utf-16 BOM, we should parse it as such''' - rules = self.parse(b'''User-agent: foo - Allow: /foo - - User-agent: * - Disallow: /foo - '''.decode('utf-8').encode('utf-16')) - self.assertTrue(rules.allowed('/foo', 'foo')) - self.assertTrue(not rules.allowed('/foo', 'other')) - - def test_ascii(self): - '''If we get bytes without encoding, we should parse it as such''' - rules = self.parse(b'''User-agent: foo - Allow: /foo - - User-agent: * - Disallow: /foo - ''') - self.assertTrue(rules.allowed('/foo', 'foo')) - self.assertTrue(not rules.allowed('/foo', 'other')) - - def test_skip_line(self): - '''If there is no colon in a line, then we must skip it''' - rules = self.parse('''User-agent: foo - Allow: /foo - Disallow /bar''') - # This should not throw an exception, and it should not have a disallow - self.assertTrue(rules.allowed('/bar', 'foo')) - - ########################################################################### - # Status code issues - ########################################################################### - def test_status_forbidden(self): - '''Make sure that when we get a forbidden status, that we believe - we're not allowed to crawl a site''' - rules = Rules('http://example.com/robots.txt', 401, '', 0) - self.assertTrue(not rules.allowed('/foo', 't')) - self.assertTrue(not rules.allowed('http://example.com/foo', 't')) - - def test_status_forbidden_allow(self): - '''Test that if the flag is given, we allow all sites when robots.txt - is forbidden''' - rules = Rules('http://example.com/robots.txt', 401, '', 0, disallow_forbidden=False) - self.assertTrue(rules.allowed('/foo', 't')) - self.assertTrue(rules.allowed('http://example.com/foo', 't')) - - def test_status_allowed(self): - '''If no robots.txt exists, we're given free range''' - rules = Rules('http://example.com/robots.txt', 404, '', 0) - self.assertTrue(rules.allowed('/foo', 't')) - self.assertTrue(rules.allowed('http://example.com/foo', 't')) - - def test_server_error(self): - '''Make sure that if there's a server error, it raises an exception''' - self.assertRaises(BadStatusCode, Rules, - 'http://example.com/robots.txt', 500, '', 0) - - ########################################################################### - # Here are some tests from other internal robots.txt parsers - ########################################################################### - def test_www_seomoz_org(self): - rules = self.parse(''' - User-agent: * - Disallow: /blogdetail.php?ID=537 - Disallow: /tracker - - Sitemap: http://www.seomoz.org/sitemap.xml.gz - Sitemap: http://files.wistia.com/sitemaps/seomoz_video_sitemap.xml - ''') - agent = 'dotbot' - self.assertTrue(rules.allowed('/blog', agent)) - self.assertTrue(not rules.allowed('/blogdetail.php?ID=537', agent)) - self.assertTrue(not rules.allowed('/tracker', agent)) - agent = 'DoTbOt' - self.assertTrue(rules.allowed('/blog', agent)) - self.assertTrue(not rules.allowed('/blogdetail.php?ID=537', agent)) - self.assertTrue(not rules.allowed('/tracker', agent)) - - def test_allow_all(self): - # Now test escaping entities - rules = self.parse(''' - User-agent: * - Disallow: ''') - agent = 'dotbot' - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/foo', agent)) - self.assertTrue(rules.allowed('/foo.html', agent)) - self.assertTrue(rules.allowed('/foo/bar', agent)) - self.assertTrue(rules.allowed('/foo/bar.html', agent)) - agent = 'oijsdofijsdofijsodifj' - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/foo', agent)) - self.assertTrue(rules.allowed('/foo.html', agent)) - self.assertTrue(rules.allowed('/foo/bar', agent)) - self.assertTrue(rules.allowed('/foo/bar.html', agent)) - - def test_disallow_all(self): - # But not with foward slash - rules = self.parse(''' - User-agent: * - Disallow: /''') - agent = 'dotbot' - self.assertTrue(not rules.allowed('/', agent)) - self.assertTrue(not rules.allowed('/foo', agent)) - self.assertTrue(not rules.allowed('/foo.html', agent)) - self.assertTrue(not rules.allowed('/foo/bar', agent)) - self.assertTrue(not rules.allowed('/foo/bar.html', agent)) - agent = 'oijsdofijsdofijsodifj' - self.assertTrue(not rules.allowed('/', agent)) - self.assertTrue(not rules.allowed('/foo', agent)) - self.assertTrue(not rules.allowed('/foo.html', agent)) - self.assertTrue(not rules.allowed('/foo/bar', agent)) - self.assertTrue(not rules.allowed('/foo/bar.html', agent)) - - def test_no_googlebot_folder(self): - rules = self.parse(''' - User-agent: Googlebot - Disallow: /no-google/''') - agent = 'googlebot' - self.assertTrue(not rules.allowed('/no-google/', agent)) - self.assertTrue(not rules.allowed('/no-google/something', agent)) - self.assertTrue(not rules.allowed('/no-google/something.html', agent)) - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/somethingelse', agent)) - - def test_no_googlebot_file(self): - rules = self.parse(''' - User-agent: Googlebot - Disallow: /no-google/blocked-page.html''') - agent = 'googlebot' - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/no-google/someotherfolder', agent)) - self.assertTrue(rules.allowed('/no-google/someotherfolder/somefile', agent)) - self.assertTrue(not rules.allowed('/no-google/blocked-page.html', agent)) - - def test_rogerbot_only(self): - rules = self.parse(''' - User-agent: * - Disallow: /no-bots/block-all-bots-except-rogerbot-page.html - - User-agent: rogerbot - Allow: /no-bots/block-all-bots-except-rogerbot-page.html''') - agent = 'notroger' - self.assertTrue(not rules.allowed('/no-bots/block-all-bots-except-rogerbot-page.html', agent)) - self.assertTrue(rules.allowed('/', agent)) - agent = 'rogerbot' - self.assertTrue(rules.allowed('/no-bots/block-all-bots-except-rogerbot-page.html', agent)) - self.assertTrue(rules.allowed('/', agent)) - - def test_allow_certain_pages_only(self): - rules = self.parse(''' - User-agent: * - Allow: /onepage.html - Allow: /oneotherpage.php - Disallow: / - Allow: /subfolder/page1.html - Allow: /subfolder/page2.php - Disallow: /subfolder/''') - agent = 'dotbot' - self.assertTrue(not rules.allowed('/', agent)) - self.assertTrue(not rules.allowed('/foo', agent)) - self.assertTrue(not rules.allowed('/bar.html', agent)) - self.assertTrue(rules.allowed('/onepage.html', agent)) - self.assertTrue(rules.allowed('/oneotherpage.php', agent)) - self.assertTrue(not rules.allowed('/subfolder', agent)) - self.assertTrue(not rules.allowed('/subfolder/', agent)) - self.assertTrue(not rules.allowed('/subfolder/aaaaa', agent)) - self.assertTrue(rules.allowed('/subfolder/page1.html', agent)) - self.assertTrue(rules.allowed('/subfolder/page2.php', agent)) - - def test_no_gifs_or_jpgs(self): - rules = self.parse(''' - User-agent: * - Disallow: /*.gif$ - Disallow: /*.jpg$''') - agent = 'dotbot' - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/foo', agent)) - self.assertTrue(rules.allowed('/foo.html', agent)) - self.assertTrue(rules.allowed('/foo/bar', agent)) - self.assertTrue(rules.allowed('/foo/bar.html', agent)) - self.assertTrue(not rules.allowed('/test.jpg', agent)) - self.assertTrue(not rules.allowed('/foo/test.jpg', agent)) - self.assertTrue(not rules.allowed('/foo/bar/test.jpg', agent)) - self.assertTrue(rules.allowed('/the-jpg-extension-is-awesome.html', agent)) - self.assertTrue(not rules.allowed('/jpg.jpg', agent)) - self.assertTrue(not rules.allowed('/foojpg.jpg', agent)) - self.assertTrue(not rules.allowed('/bar/foojpg.jpg', agent)) - self.assertTrue(not rules.allowed('/.jpg.jpg', agent)) - self.assertTrue(not rules.allowed('/.jpg/.jpg', agent)) - self.assertTrue(not rules.allowed('/test.gif', agent)) - self.assertTrue(not rules.allowed('/foo/test.gif', agent)) - self.assertTrue(not rules.allowed('/foo/bar/test.gif', agent)) - self.assertTrue(rules.allowed('/the-gif-extension-is-awesome.html', agent)) - - def test_block_subdirectory_wildcard(self): - rules = self.parse(''' - User-agent: * - Disallow: /private*/''') - agent = 'dotbot' - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/foo', agent)) - self.assertTrue(rules.allowed('/foo.html', agent)) - self.assertTrue(rules.allowed('/foo/bar', agent)) - self.assertTrue(rules.allowed('/foo/bar.html', agent)) - self.assertTrue(rules.allowed('/private', agent)) - self.assertTrue(rules.allowed('/privates', agent)) - self.assertTrue(rules.allowed('/privatedir', agent)) - self.assertTrue(not rules.allowed('/private/', agent)) - self.assertTrue(not rules.allowed('/private/foo', agent)) - self.assertTrue(not rules.allowed('/private/foo/bar.html', agent)) - self.assertTrue(not rules.allowed('/privates/', agent)) - self.assertTrue(not rules.allowed('/privates/foo', agent)) - self.assertTrue(not rules.allowed('/privates/foo/bar.html', agent)) - self.assertTrue(not rules.allowed('/privatedir/', agent)) - self.assertTrue(not rules.allowed('/privatedir/foo', agent)) - self.assertTrue(not rules.allowed('/privatedir/foo/bar.html', agent)) - - def test_block_urls_with_question_marks(self): - rules = self.parse(''' - User-agent: * - Disallow: /*?''') - agent = 'dotbot' - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/foo', agent)) - self.assertTrue(rules.allowed('/foo.html', agent)) - self.assertTrue(rules.allowed('/foo/bar', agent)) - self.assertTrue(rules.allowed('/foo/bar.html', agent)) - self.assertTrue(not rules.allowed('/?', agent)) - self.assertTrue(not rules.allowed('/foo?q=param', agent)) - self.assertTrue(not rules.allowed('/foo.html?q=param', agent)) - self.assertTrue(not rules.allowed('/foo/bar?q=param', agent)) - self.assertTrue(not rules.allowed('/foo/bar.html?q=param&bar=baz', agent)) - - def test_no_question_except_at_end(self): - rules = self.parse(''' - User-agent: * - Allow: /*?$ - Disallow: /*?''') - agent = 'dotbot' - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/foo', agent)) - self.assertTrue(rules.allowed('/foo.html', agent)) - self.assertTrue(rules.allowed('/foo/bar', agent)) - self.assertTrue(rules.allowed('/foo/bar.html', agent)) - self.assertTrue(rules.allowed('/?', agent)) - self.assertTrue(rules.allowed('/foo/bar.html?', agent)) - self.assertTrue(not rules.allowed('/foo?q=param', agent)) - self.assertTrue(not rules.allowed('/foo.html?q=param', agent)) - self.assertTrue(not rules.allowed('/foo/bar?q=param', agent)) - self.assertTrue(not rules.allowed('/foo/bar.html?q=param&bar=baz', agent)) - - def test_wildcard_edge_cases(self): - rules = self.parse(''' - User-agent: * - Disallow: /*one - Disallow: /two*three - Disallow: /irrelevant/four*five - Disallow: /six* - Disallow: /foo/*/seven*/eight*nine - Disallow: /foo/*/*ten$ - - Disallow: /*products/default.aspx - Disallow: /*/feed/$''') - agent = 'dotbot' - self.assertTrue(rules.allowed('/', agent)) - self.assertTrue(rules.allowed('/foo', agent)) - self.assertTrue(rules.allowed('/foo.html', agent)) - self.assertTrue(rules.allowed('/foo/bar', agent)) - self.assertTrue(rules.allowed('/foo/bar.html', agent)) - self.assertTrue(not rules.allowed('/one', agent)) - self.assertTrue(not rules.allowed('/aaaone', agent)) - self.assertTrue(not rules.allowed('/aaaaoneaaa', agent)) - self.assertTrue(not rules.allowed('/oneaaaa', agent)) - self.assertTrue(not rules.allowed('/twothree', agent)) - self.assertTrue(not rules.allowed('/twoaaathree', agent)) - self.assertTrue(not rules.allowed('/twoaaaathreeaaa', agent)) - self.assertTrue(not rules.allowed('/twothreeaaa', agent)) - self.assertTrue(not rules.allowed('/irrelevant/fourfive', agent)) - self.assertTrue(not rules.allowed('/irrelevant/fouraaaafive', agent)) - self.assertTrue(not rules.allowed('/irrelevant/fouraaafiveaaaa', agent)) - self.assertTrue(not rules.allowed('/irrelevant/fourfiveaaa', agent)) - self.assertTrue(not rules.allowed('/six', agent)) - self.assertTrue(not rules.allowed('/sixaaaa', agent)) - self.assertTrue(not rules.allowed('/foo/aaa/seven/eightnine', agent)) - self.assertTrue(not rules.allowed('/foo/aaa/seventeen/eightteennineteen', agent)) - self.assertTrue(not rules.allowed('/foo/aaa/ten', agent)) - self.assertTrue(not rules.allowed('/foo/bbb/often', agent)) - self.assertTrue(rules.allowed('/foo/aaa/tenaciousd', agent)) - self.assertTrue(not rules.allowed('/products/default.aspx', agent)) - self.assertTrue(not rules.allowed('/author/admin/feed/', agent)) - - def test_many_wildcards(self): - rules = self.parse(''' - User-agent: * - Allow: /***************************************.css''') - agent = 'dotbot' - self.assertTrue(rules.allowed('/blog/a-really-long-url-string-that-takes-a-very-long-time-to-process-without-combining-many-asterisks', agent)) - - def test_allow_edge_cases(self): - rules = self.parse(''' - User-agent: * - Disallow: /somereallylongfolder/ - Allow: /*.jpg - - Disallow: /sales-secrets.php - Allow: /sales-secrets.php - - Disallow: /folder - Allow: /folder/ - - Allow: /folder2 - Disallow: /folder2/''') - agent = 'dotbot' - self.assertTrue(not rules.allowed('/somereallylongfolder/', agent)) - self.assertTrue(not rules.allowed('/somereallylongfolder/aaaa', agent)) - self.assertTrue(not rules.allowed('/somereallylongfolder/test.jpg', agent)) - self.assertTrue(rules.allowed('/sales-secrets.php', agent)) - self.assertTrue(rules.allowed('/folder/page', agent)) - self.assertTrue(rules.allowed('/folder/page2', agent)) - - def test_redundant_allow(self): - rules = self.parse(''' - User-agent: * - Disallow: /en/ - Disallow: /files/documentation/ - Disallow: /files/ - Disallow: /de/careers/ - Disallow: /images/ - - Disallow: /print_mode.yes/ - Disallow: /?product=lutensit&print_mode=yes&googlebot=nocrawl - Allow: / - Disallow: /search/''') - agent = 'dotbot' - self.assertTrue(not rules.allowed('/print_mode.yes/', agent)) - self.assertTrue(not rules.allowed('/print_mode.yes/foo', agent)) - self.assertTrue(not rules.allowed('/search/', agent)) - self.assertTrue(not rules.allowed('/search/foo', agent)) - - def test_legacy(self): - rules = self.parse(''' - user-agent: * #a comment! - disallow: /Blerf - disallow: /Blerg$ - disallow: /blerf/*/print.html$#a comment - disallow: /blerf/*/blim/blerf$ - disallow: /plerf/*/blim/blim$ - user-agent: BLERF - DisALLOW: blerfPage - blerf:blah''') - agent = 'dotbot' - self.assertTrue(not rules.allowed('/Blerf/blah', agent)) - self.assertTrue(rules.allowed('/Blerg/blah', agent)) - self.assertTrue(rules.allowed('/blerf/blah', agent)) - self.assertTrue(not rules.allowed('/Blerg', agent)) - self.assertTrue(not rules.allowed( - '/blerf/some/subdirs/print.html', agent)) - self.assertTrue(rules.allowed( - '/blerf/some/subdirs/print.html?extra=stuff', agent)) - self.assertTrue(not rules.allowed( - '/blerf/some/sub/dirs/blim/blim/blerf', agent)) - self.assertTrue(not rules.allowed( - '/plerf/some/sub/dirs/blim/blim', agent)) - rules = self.parse(''' - User-agent: * - Allow: /searchhistory/ - Disallow: /news?output=xhtml& - Allow: /news?output=xhtml - Disallow: /search - Disallow: /groups - Disallow: /images - Disallow: /catalogs - Disallow: /catalogues - Disallow: /news - Disallow: /nwshp - Allow: /news?btcid= - Disallow: /news?btcid=*& - Allow: /news?btaid= - Disallow: /news?btaid=*& - Disallow: /? - Disallow: /addurl/image? - Disallow: /pagead/ - Disallow: /relpage/ - Disallow: /relcontent - Disallow: /sorry/ - Disallow: /imgres - Disallow: /keyword/ - Disallow: /u/ - Disallow: /univ/ - Disallow: /cobrand - Disallow: /custom - Disallow: /advanced_group_search - Disallow: /advanced_search - Disallow: /googlesite - Disallow: /preferences - Disallow: /setprefs - Disallow: /swr - Disallow: /url - Disallow: /default - Disallow: /m? - Disallow: /m/? - Disallow: /m/lcb - Disallow: /m/search? - Disallow: /wml? - Disallow: /wml/? - Disallow: /wml/search? - Disallow: /xhtml? - Disallow: /xhtml/? - Disallow: /xhtml/search? - Disallow: /xml? - Disallow: /imode? - Disallow: /imode/? - Disallow: /imode/search? - Disallow: /jsky? - Disallow: /jsky/? - Disallow: /jsky/search? - Disallow: /pda? - Disallow: /pda/? - Disallow: /pda/search? - Disallow: /sprint_xhtml - Disallow: /sprint_wml - Disallow: /pqa - Disallow: /palm - Disallow: /gwt/ - Disallow: /purchases - Disallow: /hws - Disallow: /bsd? - Disallow: /linux? - Disallow: /mac? - Disallow: /microsoft? - Disallow: /unclesam? - Disallow: /answers/search?q= - Disallow: /local? - Disallow: /local_url - Disallow: /froogle? - Disallow: /products? - Disallow: /froogle_ - Disallow: /product_ - Disallow: /products_ - Disallow: /print - Disallow: /books - Disallow: /patents? - Disallow: /scholar? - Disallow: /complete - Disallow: /sponsoredlinks - Disallow: /videosearch? - Disallow: /videopreview? - Disallow: /videoprograminfo? - Disallow: /maps? - Disallow: /mapstt? - Disallow: /mapslt? - Disallow: /maps/stk/ - Disallow: /mapabcpoi? - Disallow: /translate? - Disallow: /ie? - Disallow: /sms/demo? - Disallow: /katrina? - Disallow: /blogsearch? - Disallow: /blogsearch/ - Disallow: /blogsearch_feeds - Disallow: /advanced_blog_search - Disallow: /reader/ - Disallow: /uds/ - Disallow: /chart? - Disallow: /transit? - Disallow: /mbd? - Disallow: /extern_js/ - Disallow: /calendar/feeds/ - Disallow: /calendar/ical/ - Disallow: /cl2/feeds/ - Disallow: /cl2/ical/ - Disallow: /coop/directory - Disallow: /coop/manage - Disallow: /trends? - Disallow: /trends/music? - Disallow: /notebook/search? - Disallow: /music - Disallow: /browsersync - Disallow: /call - Disallow: /archivesearch? - Disallow: /archivesearch/url - Disallow: /archivesearch/advanced_search - Disallow: /base/search? - Disallow: /base/reportbadoffer - Disallow: /base/s2 - Disallow: /urchin_test/ - Disallow: /movies? - Disallow: /codesearch? - Disallow: /codesearch/feeds/search? - Disallow: /wapsearch? - Disallow: /safebrowsing - Disallow: /reviews/search? - Disallow: /orkut/albums - Disallow: /jsapi - Disallow: /views? - Disallow: /c/ - Disallow: /cbk - Disallow: /recharge/dashboard/car - Disallow: /recharge/dashboard/static/ - Disallow: /translate_c? - Disallow: /s2/profiles/me - Allow: /s2/profiles - Disallow: /s2 - Disallow: /transconsole/portal/ - Disallow: /gcc/ - Disallow: /aclk - Disallow: /cse? - Disallow: /tbproxy/ - Disallow: /MerchantSearchBeta/ - Disallow: /ime/ - Disallow: /websites? - Disallow: /shenghuo/search?''') - self.assertTrue(not rules.allowed('/?as_q=ethics&ie=UTF-8&ui=blg&bl_url=centrerion.blogspot.com&x=0&y=0&ui=blg', agent)) - self.assertTrue(not rules.allowed('/archivesearch?q=stalin&scoring=t&hl=en&sa=N&sugg=d&as_ldate=1900&as_hdate=1919&lnav=hist2', agent)) - rules = self.parse(''' - User-agent: scooter - Disallow: / - - User-agent: wget - User-agent: webzip - Disallow: / - - User-agent: * - Disallow:''') - self.assertTrue(rules.allowed('/index.html', agent)) - - rules = self.parse(''' - # Alexa - User-agent: ia_archiver - Disallow: /utils/date_picker/ - # Ask Jeeves - User-agent: Teoma - Disallow: /utils/date_picker/ - # Google - User-agent: googlebot - Disallow: /utils/date_picker/ - # MSN - User-agent: MSNBot - Disallow: /utils/date_picker/ - # Yahoo! - User-agent: Slurp - Disallow: /utils/date_picker/ - # Baidu - User-agent: baiduspider - Disallow: /utils/date_picker/ - # All the rest go away - User-agent: * - Disallow: /''') - self.assertTrue(not rules.allowed('/', agent)) - - rules = self.parse(''' - User-agent: dotbot - User-agent:snowball - Disallow:/''') - self.assertTrue(not rules.allowed('/', agent)) - - rules = self.parse(''' - User-agent: Googlebot-Image - Disallow: / - User-agent: * - Crawl-delay: 7 - Disallow: /faq.php - Disallow: /groupcp.php - Disallow: /login.php - Disallow: /memberlist.php - Disallow: /merge.php - Disallow: /modcp.php - Disallow: /posting.php - Disallow: /phpBB2/posting.php - Disallow: /privmsg.php - Disallow: /profile.php - Disallow: /search.php - Disallow: /phpBB2/faq.php - Disallow: /phpBB2/groupcp.php - Disallow: /phpBB2/login.php - Disallow: /phpBB2/memberlist.php - Disallow: /phpBB2/merge.php - Disallow: /phpBB2/modcp.php - Disallow: /phpBB2/posting.php - Disallow: /phpBB2/posting.php - Disallow: /phpBB2/privmsg.php - Disallow: /phpBB2/profile.php - Disallow: /phpBB2/search.php - Disallow: /admin/ - Disallow: /images/ - Disallow: /includes/ - Disallow: /install/ - Disallow: /modcp/ - Disallow: /templates/ - Disallow: /phpBB2/admin/ - Disallow: /phpBB2/images/ - Disallow: /phpBB2/includes/ - Disallow: /phpBB2/install/ - Disallow: /phpBB2/modcp/ - Disallow: /phpBB2/templates/ - Disallow: /trac/''') - self.assertTrue(not rules.allowed( - '/phpBB2/posting.php?mode=reply&t=895', agent)) - - rules = self.parse(''' - User-agent: * - Disallow: /Product/List - Disallow: /Product/Search - Disallow: /Product/TopSellers - Disallow: /Product/UploadImage - Disallow: /WheelPit - Disallow: /iwwida.pvx''') - self.assertTrue(not rules.allowed('/WheelPit', agent)) - - def test_trailing_question(self): - '''Make sure a trailing question mark doesn't muss anything up''' - rules = self.parse(''' - User-agent: * - Disallow: /search - Disallow: /sdch - Disallow: /groups - Disallow: /images - Disallow: /catalogs - Allow: /catalogs/about - Allow: /catalogs/p?''') - agent = 'dotbot' - base_url = 'http://example.com' - self.assertTrue(not rules.allowed('/catalogs/foo', agent)) - self.assertTrue(not rules.allowed('/catalogs/p', agent)) - self.assertTrue(rules.allowed('/catalogs/p?', agent)) - # Make sure it works for full urls as well - self.assertTrue(not rules.allowed(base_url + '/catalogs/foo', agent)) - self.assertTrue(not rules.allowed(base_url + '/catalogs/p', agent)) - self.assertTrue(rules.allowed(base_url + '/catalogs/p?', agent)) - - def test_disallow_all_url(self): - '''Make sure base url without trailing slash is disallowed - in case Disallow: / rule is used.''' - base_url = 'http://example.com' - rules = self.parse(''' - User-agent: * - Disallow: /''') - agent = 'dotbot' - # all urls should be blocked according to - # http://www.robotstxt.org/orig.html#code - self.assertTrue(not rules.allowed(base_url, agent)) - self.assertTrue(not rules.allowed(base_url + '/', agent)) - self.assertTrue(not rules.allowed(base_url + '/foo.html', agent)) - - def test_extract_path(self): - '''Make sure we can correctly extract example paths''' - from reppy.parser import Agent - self.assertEqual('/', Agent.extract_path('')) - self.assertEqual('/', Agent.extract_path('http://example.com')) - self.assertEqual('/foo', Agent.extract_path('/foo')) - self.assertEqual('/foo', Agent.extract_path('http://example.com/foo')) - self.assertEqual('/foo/', Agent.extract_path('/foo/')) - self.assertEqual('/foo/', - Agent.extract_path('http://example.com/foo/')) - self.assertEqual('/foo/bar', Agent.extract_path('/foo/bar')) - self.assertEqual('/foo/bar', - Agent.extract_path('http://example.com/foo/bar')) - self.assertEqual('/foo/bar?a=b', Agent.extract_path('/foo/bar?a=b')) - self.assertEqual('/foo/bar?a=b', - Agent.extract_path('http://example.com/foo/bar?a=b')) - - def test_malformed_crawl_delay(self): - '''Ignore crawl delays that are too malformed to parse.''' - rules = self.parse(''' - User-agent: * - Crawl-delay: - ''') - self.assertEqual(rules.delay('rogerbot'), None) - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_utility.py b/tests/test_utility.py deleted file mode 100644 index f154c18..0000000 --- a/tests/test_utility.py +++ /dev/null @@ -1,80 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- - -'''These are unit tests that are derived from the rfc at -http://www.robotstxt.org/norobots-rfc.txt''' - -import unittest - -import reppy -import logging -from reppy import Utility -reppy.logger.setLevel(logging.FATAL) - - -class TestUtility(unittest.TestCase): - '''Make sure our utility functions work''' - def test_cache_control(self): - '''Make sure parsing of the ttl with cache control works''' - for directive in ('no-cache', 'no-store', 'must-revalidate'): - self.assertEqual(Utility.get_ttl({ - 'cache-control': directive - }, 5), 0) - - # Make sure that we can honor s-maxage - for directive in ('s-maxage=10,foo', 's-maxage = 10'): - self.assertEqual(Utility.get_ttl({ - 'cache-control': directive - }, 5), 10) - # If we can't parse it as an integer, then we'll skip it - self.assertEqual(Utility.get_ttl({ - 'cache-control': 's-maxage = not int' - }, 5), 5) - - # Make sure we can honor max-age - for directive in ('max-age=10,foo', 'max-age = 10'): - self.assertEqual(Utility.get_ttl({ - 'cache-control': directive - }, 5), 10) - # If we can't parse it as an integer, then we'll skip it - self.assertEqual(Utility.get_ttl({ - 'cache-control': 'max-age = not int' - }, 5), 5) - - def test_expires(self): - '''Make sure we can honor Expires''' - # Test a plain-and-simple expires, using now as a default time - import datetime - ttl = Utility.get_ttl({ - 'expires': ( - datetime.datetime.utcnow() + datetime.timedelta(seconds=10) - ).strftime('%a, %d %b %Y %H:%M:%S %z') - }, 5) - self.assertLess(ttl, 11) - self.assertGreater(ttl, 9) - - # Make sure this works when a date is provided - now = datetime.datetime.utcnow() - ttl = self.assertEqual(Utility.get_ttl({ - 'expires': ( - now + datetime.timedelta(seconds=10) - ).strftime('%a, %d %b %Y %H:%M:%S %z'), - 'date': ( - now - ).strftime('%a, %d %b %Y %H:%M:%S %z') - }, 5), 10) - - # If the date is unparseable, use 'now' - ttl = Utility.get_ttl({ - 'expires': ( - datetime.datetime.utcnow() + datetime.timedelta(seconds=10) - ).strftime('%a, %d %b %Y %H:%M:%S %z'), - 'date': 'not a valid time' - }, 5) - self.assertLess(ttl, 11) - self.assertGreater(ttl, 9) - - # Lastly, if the 'expires' header is unparseable, then pass - ttl = self.assertEqual(Utility.get_ttl({ - 'expires': 'not a valid time' - }, 5), 5) From be2e1e34960863d9af97e58f7ffff87bd302ddc9 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 19 Oct 2016 11:00:50 -0700 Subject: [PATCH 018/113] Update bench.py. --- bench.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/bench.py b/bench.py index 3a6ce47..3a4486b 100755 --- a/bench.py +++ b/bench.py @@ -6,29 +6,48 @@ import sys import time -from reppy.cache import RobotsCache -from reppy.parser import Rules - +from reppy.robots import Robots content = ''' -User-agent: '*' -Allow: / -''' +# /robots.txt for http://www.fict.org/ +# comments to webmaster@fict.org + +User-agent: unhipbot +Disallow: / -cache = RobotsCache() -cache.add(Rules('http://example.com/', 200, content, sys.maxint)) +User-agent: webcrawler +User-agent: excite +Disallow: + +User-agent: * +Disallow: /org/plans.html +Allow: /org/ +Allow: /serv +Allow: /~mak +Disallow: / +''' @contextmanager -def timer(count): +def timer(name, count): '''Time this block.''' start = time.time() try: yield count finally: duration = time.time() - start + print(name) + print('=' * 10) print('Total: %s' % duration) print(' Avg: %s' % (duration / count)) print(' Rate: %s' % (count / duration)) + print('') + + +with timer('Parse', 100000) as count: + for _ in xrange(count): + Robots.parse('http://example.com/robots.txt', content) + -with timer(100000) as count: +parsed = Robots.parse('http://example.com/robots.txt', content) +with timer('Evaluate', 100000) as count: for _ in xrange(count): - cache.allowed('http://example.com/page', 'agent') + parsed.allowed('/org/example.html', 'other-bot') From 22935118e878dd27355ee095dadca383b62f1be4 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 19 Oct 2016 11:01:27 -0700 Subject: [PATCH 019/113] Require 100% coverage. --- .coveragerc | 2 ++ setup.cfg | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..64de7b2 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +concurrency = gevent diff --git a/setup.cfg b/setup.cfg index a3f27d1..00703b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ rednose=1 exe=1 cover-package=reppy cover-branches=1 -#cover-min-percentage=100 +cover-min-percentage=100 cover-inclusive=1 cover-erase=1 logging-clear-handlers=1 From 56903f5ed8ff7ecca62b8facd8f70404bfe8f29c Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Fri, 28 Oct 2016 11:50:31 -0700 Subject: [PATCH 020/113] Drop pypi and python 3 in Travis, temporarily. --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26ff55e..85985d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,10 @@ language: python dist: trusty python: - - "3.4" - - "3.3" + # - "3.4" + # - "3.3" - "2.7" - - "pypy" + # - "pypy" install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install -r dev-requirements.txt; fi From 9ffd69efa44e60f05794444fe4d2200541dc491e Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Tue, 1 Nov 2016 08:37:17 -0700 Subject: [PATCH 021/113] Add dependency on rep-cpp. Rebase me once this revision lands on master. --- .gitmodules | 3 +++ reppy/rep-cpp | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 reppy/rep-cpp diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..6ce8f04 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "reppy/rep-cpp"] + path = reppy/rep-cpp + url = https://github.com/seomoz/rep-cpp diff --git a/reppy/rep-cpp b/reppy/rep-cpp new file mode 160000 index 0000000..a12fe2d --- /dev/null +++ b/reppy/rep-cpp @@ -0,0 +1 @@ +Subproject commit a12fe2d53e64191e9554ac0603d93443625577b7 From 55709d0410d4c0df9adf02e4ab074a3eacb086b3 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Mon, 31 Oct 2016 09:03:59 -0700 Subject: [PATCH 022/113] Add vagrant provisioning. --- Vagrantfile | 20 ++++++++++++++++ scripts/vagrant/provision.sh | 45 ++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 Vagrantfile create mode 100755 scripts/vagrant/provision.sh diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..01e9517 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,20 @@ +# Encoding: utf-8 +# -*- mode: ruby -*- +# vi: set ft=ruby : + +ENV['VAGRANT_DEFAULT_PROVIDER'] = 'virtualbox' + +# http://docs.vagrantup.com/v2/ +Vagrant.configure('2') do |config| + config.vm.box = 'ubuntu/trusty64' + config.vm.hostname = 'reppy' + config.ssh.forward_agent = true + + config.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", "1024"] + vb.customize ["modifyvm", :id, "--cpus", "2"] + vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] + end + + config.vm.provision :shell, path: 'scripts/vagrant/provision.sh', privileged: false +end diff --git a/scripts/vagrant/provision.sh b/scripts/vagrant/provision.sh new file mode 100755 index 0000000..a8c3db9 --- /dev/null +++ b/scripts/vagrant/provision.sh @@ -0,0 +1,45 @@ +#! /usr/bin/env bash + +set -e + +sudo apt-get update +sudo apt-get install -y tar curl git + +# Libraries required to build a complete python with pyenv: +# https://github.com/yyuu/pyenv/wiki +sudo apt-get install -y make build-essential libssl-dev zlib1g-dev libbz2-dev \ + libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev + +# Install pyenv +if [ ! -d ~/.pyenv ]; then + git clone https://github.com/yyuu/pyenv.git ~/.pyenv + echo ' + # Pyenv + export PYENV_ROOT="$HOME/.pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + eval "$(pyenv init -)" + ' >> ~/.bash_profile + source ~/.bash_profile + hash +fi + +pushd /vagrant + + # Submodules + git submodule update --init --recursive + + # Install our python version + pyenv install --skip-existing + pyenv rehash + + # Install a virtualenv + pip install virtualenv + if [ ! -d venv ]; then + virtualenv venv + fi + source venv/bin/activate + + # Lastly, our dependencies + pip install -r requirements.txt + +popd From 12a655a5b07834074c9b31fa6cd5715b056d5ab8 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Tue, 1 Nov 2016 08:28:12 -0700 Subject: [PATCH 023/113] Add dev dependency on Cython. --- dev-requirements-py3.txt | 1 + dev-requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/dev-requirements-py3.txt b/dev-requirements-py3.txt index 270993c..0a2c807 100644 --- a/dev-requirements-py3.txt +++ b/dev-requirements-py3.txt @@ -1,5 +1,6 @@ colorama==0.3.7 coverage==4.0.3 +Cython==0.25.1 mock==2.0.0 nose==1.3.7 pbr==1.9.1 diff --git a/dev-requirements.txt b/dev-requirements.txt index 57bbe03..ef26086 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,6 +2,7 @@ asis==0.3.0 bottle==0.12.9 colorama==0.3.7 coverage==4.0.3 +cython==0.25.1 funcsigs==1.0.2 gevent==1.1.1 greenlet==0.4.9 From dffff12c691dce167935f660833f8f9763952373 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 3 Nov 2016 15:48:55 -0700 Subject: [PATCH 024/113] Add dependency on six. --- requirements.txt | 1 + setup.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53b0dbd..b771e7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ url==0.3.0 requests==2.10.0 +six==1.10.0 python-dateutil==2.5.3 diff --git a/setup.py b/setup.py index ca858a0..ef42a1a 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,8 @@ install_requires = [ 'python-dateutil>=1.5, !=2.0', 'url>=0.2.0', - 'requests' + 'requests', + 'six' ], classifiers = [ 'License :: OSI Approved :: MIT License', From 82684d464b3f7d5b580a83f44507961a8ff60c92 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Tue, 1 Nov 2016 08:27:49 -0700 Subject: [PATCH 025/113] Use rep-cpp. --- .coveragerc | 2 +- .gitignore | 21 +- Makefile | 5 +- reppy/__init__.py | 2 + reppy/agent.py | 56 - reppy/directive.py | 92 - reppy/rep-cpp | 2 +- reppy/robots.cpp | 9346 +++++++++++++++++++++++++++++++++++++++ reppy/robots.pxd | 38 + reppy/robots.py | 150 - reppy/robots.pyx | 170 + reppy/util.py | 22 +- setup.py | 36 +- tests/test_agent.py | 72 +- tests/test_directive.py | 198 - tests/test_robots.py | 80 +- tests/test_util.py | 65 - 17 files changed, 9714 insertions(+), 643 deletions(-) delete mode 100644 reppy/agent.py delete mode 100644 reppy/directive.py create mode 100644 reppy/robots.cpp create mode 100644 reppy/robots.pxd delete mode 100644 reppy/robots.py create mode 100644 reppy/robots.pyx delete mode 100644 tests/test_directive.py diff --git a/.coveragerc b/.coveragerc index 64de7b2..d9a48b4 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,2 @@ [run] -concurrency = gevent +plugins = Cython.Coverage diff --git a/.gitignore b/.gitignore index de4d44b..5471fd0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,16 @@ +# Binaries and build artifacts *.pyc -build/* -dist/* -reppy.egg-info/* +*.o +*.so +build/ +dist/ +reppy.egg-info/ + +# Dev artifacts +venv* + +# Coverage .coverage -htmlcov/ -.tox -*.egg -*.egg-info + +# Vagrant +.vagrant/ diff --git a/Makefile b/Makefile index e803e74..5f816b1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,10 @@ .PHONY: test -test: +test: reppy/robots.so nosetests --with-coverage tests +reppy/%.so: reppy/%.* reppy/rep-cpp/src/* reppy/rep-cpp/include/* reppy/rep-cpp/deps/url-cpp/include/* reppy/rep-cpp/deps/url-cpp/src/* + python setup.py build_ext --inplace + install: python setup.py install diff --git a/reppy/__init__.py b/reppy/__init__.py index 703e41a..a513c79 100755 --- a/reppy/__init__.py +++ b/reppy/__init__.py @@ -13,3 +13,5 @@ handler.setLevel(logging.DEBUG) logger.addHandler(handler) logger.setLevel(logging.ERROR) + +from .robots import Robots, Agent diff --git a/reppy/agent.py b/reppy/agent.py deleted file mode 100644 index e29cbe5..0000000 --- a/reppy/agent.py +++ /dev/null @@ -1,56 +0,0 @@ -'''Holds the directives for a single user agent.''' - -import url as URL - -from . import logger -from .directive import BaseDirective - - -class Agent(object): - '''The directives for a single user agent.''' - - def __init__(self, directives=None, delay=None): - self._directives = directives or list() - self.delay = None - self._sorted = True - - @property - def directives(self): - '''Keep directives in descending order of precedence.''' - if not self._sorted: - self._directives.sort(reverse=True) - self._sorted = True - return self._directives - - def allow(self, query): - '''Add an allow directive.''' - self._directives.append(BaseDirective.allow(query)) - self._sorted = False - return self - - def disallow(self, query): - '''Add a disallow directive.''' - if not query: - # Special case: "Disallow:" means "Allow: /" - self._directives.append(BaseDirective.allow(query)) - else: - self._directives.append(BaseDirective.disallow(query)) - self._sorted = False - return self - - def url_allowed(self, url): - '''Return true if the path in url is allowed. Otherwise, false''' - parsed = URL.parse(url).defrag().deuserinfo() - parsed.scheme = '' - parsed.host = '' - return self.allowed(str(parsed)) - - def allowed(self, query): - '''Return true if the query is allowed. Otherwise, false.''' - if query == '/robots.txt': - return True - sanitized = BaseDirective.sanitize(query) - for directive in self.directives: - if directive.match(sanitized): - return directive.allowed - return True diff --git a/reppy/directive.py b/reppy/directive.py deleted file mode 100644 index 168838a..0000000 --- a/reppy/directive.py +++ /dev/null @@ -1,92 +0,0 @@ -'''Single allow / disallow directives.''' - -import re - -try: - # Python 3 - from urllib.parse import unquote -except ImportError: - # Python 2 - from urllib import unquote - - -class BaseDirective(object): - '''A allow/disallow directive.''' - - @classmethod - def sanitize(cls, query): - '''Canonicalize query for comparison.''' - if (not query) or (query[0] != '/'): - query = '/' + query - # TODO(dan): this is the legacy implementation, but should be replaced - return unquote(query.replace('%2f', '%252f')) - - @classmethod - def allow(cls, rule): - '''Return an allowed directive.''' - return cls.parse(rule, True) - - @classmethod - def disallow(cls, rule): - '''Return a disallowed directive.''' - return cls.parse(rule, False) - - @classmethod - def parse(cls, rule, allowed): - '''Return a directive.''' - rule = rule.strip() - priority = len(rule) - if (not rule) or (rule == '/') or (rule == '*'): - return AllDirective(priority, rule, allowed) - elif ('*' not in rule) and ('$' not in rule): - return StringDirective(priority, rule, allowed) - else: - return RegexDirective(priority, rule, allowed) - - def __init__(self, priority, pattern, allowed): - self.priority = priority - self.pattern = self.sanitize(pattern) - self.allowed = allowed - - def match(self, query): - '''Return true if the query matches this pattern.''' - raise NotImplementedError('BaseDirective does not implement match') - - def __lt__(self, other): - return self.priority < other.priority - - def __str__(self): - return '<%s priority=%s, pattern=%s, allowed=%s>' % ( - type(self).__name__, self.priority, self.pattern, self.allowed) - - -class StringDirective(BaseDirective): - '''A directive that uses a string comparison.''' - - def match(self, sanitized): - '''Return true if the query matches this pattern.''' - return sanitized.startswith(self.pattern) - - -class AllDirective(BaseDirective): - '''A directive that applies to all URLs.''' - - def match(self, query): - '''Return true if the query matches this pattern.''' - return True - - -class RegexDirective(BaseDirective): - '''A directive that uses a regex.''' - - # For collapsing multiple asterisks into one - RE_ASTERISK = re.compile(r'\*+') - - def __init__(self, priority, pattern, allowed): - BaseDirective.__init__(self, priority, pattern, allowed) - pattern = re.escape(self.RE_ASTERISK.sub('*', self.pattern)) - self._regex = re.compile(pattern.replace('\*', '.*').replace('\$', '$')) - - def match(self, sanitized): - '''Return true if the query matches this pattern.''' - return bool(self._regex.match(sanitized)) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index a12fe2d..d59ee14 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit a12fe2d53e64191e9554ac0603d93443625577b7 +Subproject commit d59ee1486e78e8dea53dca38acd0d874117b96da diff --git a/reppy/robots.cpp b/reppy/robots.cpp new file mode 100644 index 0000000..1fae419 --- /dev/null +++ b/reppy/robots.cpp @@ -0,0 +1,9346 @@ +/* Generated by Cython 0.25.1 */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) + #error Cython requires Python 2.6+ or Python 3.2+. +#else +#define CYTHON_ABI "0_25_1" +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#ifndef HAVE_LONG_LONG + #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) + #define HAVE_LONG_LONG + #endif +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#ifdef PYPY_VERSION + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#elif defined(PYSTON_VERSION) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #if PY_MAJOR_VERSION < 3 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #ifndef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if PY_VERSION_HEX < 0x030300F0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif +#endif +#if !defined(CYTHON_FAST_PYCCALL) +#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) +#endif +#if CYTHON_USE_PYLONG_INTERNALS + #include "longintrepr.h" + #undef SHIFT + #undef BASE + #undef MASK +#endif +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) + #define Py_OptimizeFlag 0 +#endif +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#if PY_MAJOR_VERSION < 3 + #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyClass_Type +#else + #define __Pyx_BUILTIN_MODULE_NAME "builtins" + #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyType_Type +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #define __Pyx_PyCFunctionFast _PyCFunctionFast +#endif +#if CYTHON_FAST_PYCCALL +#define __Pyx_PyFastCFunction_Check(func)\ + ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) +#else +#define __Pyx_PyFastCFunction_Check(func) 0 +#endif +#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) +#else + #define CYTHON_PEP393_ENABLED 0 + #define PyUnicode_1BYTE_KIND 1 + #define PyUnicode_2BYTE_KIND 2 + #define PyUnicode_4BYTE_KIND 4 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) + #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_PYSTON + #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) +#else + #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) +#endif +#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) + #define PyObject_ASCII(o) PyObject_Repr(o) +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBaseString_Type PyUnicode_Type + #define PyStringObject PyUnicodeObject + #define PyString_Type PyUnicode_Type + #define PyString_Check PyUnicode_Check + #define PyString_CheckExact PyUnicode_CheckExact +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) + #define __Pyx_PyBaseString_CheckExact(obj) PyUnicode_CheckExact(obj) +#else + #define __Pyx_PyBaseString_Check(obj) (PyString_Check(obj) || PyUnicode_Check(obj)) + #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#endif +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#if PY_MAJOR_VERSION >= 3 + #define PyIntObject PyLongObject + #define PyInt_Type PyLong_Type + #define PyInt_Check(op) PyLong_Check(op) + #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define PyInt_FromString PyLong_FromString + #define PyInt_FromUnicode PyLong_FromUnicode + #define PyInt_FromLong PyLong_FromLong + #define PyInt_FromSize_t PyLong_FromSize_t + #define PyInt_FromSsize_t PyLong_FromSsize_t + #define PyInt_AsLong PyLong_AsLong + #define PyInt_AS_LONG PyLong_AS_LONG + #define PyInt_AsSsize_t PyLong_AsSsize_t + #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask + #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask + #define PyNumber_Int PyNumber_Long +#endif +#if PY_MAJOR_VERSION >= 3 + #define PyBoolObject PyLongObject +#endif +#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_PYPY + #ifndef PyUnicode_InternFromString + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) + #endif +#endif +#if PY_VERSION_HEX < 0x030200A4 + typedef long Py_hash_t; + #define __Pyx_PyInt_FromHash_t PyInt_FromLong + #define __Pyx_PyInt_AsHash_t PyInt_AsLong +#else + #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t + #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t +#endif +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) +#else + #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) +#endif +#if CYTHON_USE_ASYNC_SLOTS + #if PY_VERSION_HEX >= 0x030500B1 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods + #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) + #else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif +#else + #define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) + +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #define CYTHON_INLINE inline +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(U other) { return *ptr == other; }; + template bool operator !=(U other) { return *ptr != other; }; + private: + T *ptr; +}; + +#if defined(WIN32) || defined(MS_WINDOWS) + #define _USE_MATH_DEFINES +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + + +#define __PYX_ERR(f_index, lineno, Ln_error) \ +{ \ + __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ +} + +#if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif + +#ifndef __PYX_EXTERN_C + #ifdef __cplusplus + #define __PYX_EXTERN_C extern "C" + #else + #define __PYX_EXTERN_C extern + #endif +#endif + +#define __PYX_HAVE__reppy__robots +#define __PYX_HAVE_API__reppy__robots +#include +#include +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include +#include "rep-cpp/include/directive.h" +#include "rep-cpp/include/agent.h" +#include "rep-cpp/include/robots.h" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#ifdef PYREX_WITHOUT_ASSERTIONS +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; + const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) && defined (_M_X64) + #define __Pyx_sst_abs(value) _abs64(value) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +#define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if PY_MAJOR_VERSION < 3 + #define __Pyx_PyStr_FromString __Pyx_PyBytes_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#else + #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString + #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize +#endif +#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#if PY_MAJOR_VERSION < 3 +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ + const Py_UNICODE *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen +#endif +#define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) +#define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +#define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) +#define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) +#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +#if CYTHON_ASSUME_SAFE_MACROS +#define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#else +#define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#endif +#define __pyx_PyFloat_AsFloat(x) ((float) __pyx_PyFloat_AsDouble(x)) +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#else +#define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) +#endif +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII +static int __Pyx_sys_getdefaultencoding_not_ascii; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + PyObject* ascii_chars_u = NULL; + PyObject* ascii_chars_b = NULL; + const char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + if (strcmp(default_encoding_c, "ascii") == 0) { + __Pyx_sys_getdefaultencoding_not_ascii = 0; + } else { + char ascii_chars[128]; + int c; + for (c = 0; c < 128; c++) { + ascii_chars[c] = c; + } + __Pyx_sys_getdefaultencoding_not_ascii = 1; + ascii_chars_u = PyUnicode_DecodeASCII(ascii_chars, 128, NULL); + if (!ascii_chars_u) goto bad; + ascii_chars_b = PyUnicode_AsEncodedString(ascii_chars_u, default_encoding_c, NULL); + if (!ascii_chars_b || !PyBytes_Check(ascii_chars_b) || memcmp(ascii_chars, PyBytes_AS_STRING(ascii_chars_b), 128) != 0) { + PyErr_Format( + PyExc_ValueError, + "This module compiled with c_string_encoding=ascii, but default encoding '%.200s' is not a superset of ascii.", + default_encoding_c); + goto bad; + } + Py_DECREF(ascii_chars_u); + Py_DECREF(ascii_chars_b); + } + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + Py_XDECREF(ascii_chars_u); + Py_XDECREF(ascii_chars_b); + return -1; +} +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT && PY_MAJOR_VERSION >= 3 +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#else +#define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#if __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +static char* __PYX_DEFAULT_STRING_ENCODING; +static int __Pyx_init_sys_getdefaultencoding_params(void) { + PyObject* sys; + PyObject* default_encoding = NULL; + char* default_encoding_c; + sys = PyImport_ImportModule("sys"); + if (!sys) goto bad; + default_encoding = PyObject_CallMethod(sys, (char*) (const char*) "getdefaultencoding", NULL); + Py_DECREF(sys); + if (!default_encoding) goto bad; + default_encoding_c = PyBytes_AsString(default_encoding); + if (!default_encoding_c) goto bad; + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; + strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); + Py_DECREF(default_encoding); + return 0; +bad: + Py_XDECREF(default_encoding); + return -1; +} +#endif +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ + +static PyObject *__pyx_m; +static PyObject *__pyx_d; +static PyObject *__pyx_b; +static PyObject *__pyx_empty_tuple; +static PyObject *__pyx_empty_bytes; +static PyObject *__pyx_empty_unicode; +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * __pyx_cfilenm= __FILE__; +static const char *__pyx_filename; + + +static const char *__pyx_f[] = { + "reppy/robots.pxd", + "reppy/robots.pyx", + "stringsource", +}; + +/*--- Type declarations ---*/ +struct __pyx_obj_5reppy_6robots_Agent; +struct __pyx_obj_5reppy_6robots_Robots; +struct __pyx_obj_5reppy_6robots_AllowNone; +struct __pyx_obj_5reppy_6robots_AllowAll; +struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; + +/* "reppy/robots.pyx":39 + * return agent + * + * cdef class Agent: # <<<<<<<<<<<<<< + * '''Wrapper around rep-cpp's Rep::Agent class.''' + * + */ +struct __pyx_obj_5reppy_6robots_Agent { + PyObject_HEAD + Rep::Agent agent; +}; + + +/* "reppy/robots.pyx":110 + * return as_string(CppRobots.robotsUrl(as_bytes(url))) + * + * cdef class Robots: # <<<<<<<<<<<<<< + * '''Wrapper around rep-cpp's Rep::Robots class.''' + * + */ +struct __pyx_obj_5reppy_6robots_Robots { + PyObject_HEAD + Rep::Robots *robots; + PyObject *url; + PyObject *expires; +}; + + +/* "reppy/robots.pyx":159 + * + * + * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< + * '''No requests are allowed.''' + * + */ +struct __pyx_obj_5reppy_6robots_AllowNone { + struct __pyx_obj_5reppy_6robots_Robots __pyx_base; +}; + + +/* "reppy/robots.pyx":166 + * + * + * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< + * '''All requests are allowed.''' + * + */ +struct __pyx_obj_5reppy_6robots_AllowAll { + struct __pyx_obj_5reppy_6robots_Robots __pyx_base; +}; + + +/* "cfunc.to_py":64 + * + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< + * def wrap(object value): + * """wrap(value)""" + */ +struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py { + PyObject_HEAD + PyObject *(*__pyx_v_f)(PyObject *); +}; + + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, int); + void (*DECREF)(void*, PyObject*, int); + void (*GOTREF)(void*, PyObject*, int); + void (*GIVEREF)(void*, PyObject*, int); + void* (*SetupContext)(const char*, int, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; +#ifdef WITH_THREAD + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__);\ + } +#else + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) +#endif + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) + #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_getattr)) + return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); +#endif + return PyObject_GetAttr(obj, attr_name); +} +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* Profile.proto */ +#ifndef CYTHON_PROFILE +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON + #define CYTHON_PROFILE 0 +#else + #define CYTHON_PROFILE 1 +#endif +#endif +#ifndef CYTHON_TRACE_NOGIL + #define CYTHON_TRACE_NOGIL 0 +#else + #if CYTHON_TRACE_NOGIL && !defined(CYTHON_TRACE) + #define CYTHON_TRACE 1 + #endif +#endif +#ifndef CYTHON_TRACE + #define CYTHON_TRACE 0 +#endif +#if CYTHON_TRACE + #undef CYTHON_PROFILE_REUSE_FRAME +#endif +#ifndef CYTHON_PROFILE_REUSE_FRAME + #define CYTHON_PROFILE_REUSE_FRAME 0 +#endif +#if CYTHON_PROFILE || CYTHON_TRACE + #include "compile.h" + #include "frameobject.h" + #include "traceback.h" + #if CYTHON_PROFILE_REUSE_FRAME + #define CYTHON_FRAME_MODIFIER static + #define CYTHON_FRAME_DEL(frame) + #else + #define CYTHON_FRAME_MODIFIER + #define CYTHON_FRAME_DEL(frame) Py_CLEAR(frame) + #endif + #define __Pyx_TraceDeclarations\ + static PyCodeObject *__pyx_frame_code = NULL;\ + CYTHON_FRAME_MODIFIER PyFrameObject *__pyx_frame = NULL;\ + int __Pyx_use_tracing = 0; + #define __Pyx_TraceFrameInit(codeobj)\ + if (codeobj) __pyx_frame_code = (PyCodeObject*) codeobj; + #ifdef WITH_THREAD + #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error)\ + if (nogil) {\ + if (CYTHON_TRACE_NOGIL) {\ + PyThreadState *tstate;\ + PyGILState_STATE state = PyGILState_Ensure();\ + tstate = PyThreadState_GET();\ + if (unlikely(tstate->use_tracing) && !tstate->tracing &&\ + (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {\ + __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&__pyx_frame_code, &__pyx_frame, funcname, srcfile, firstlineno);\ + }\ + PyGILState_Release(state);\ + if (unlikely(__Pyx_use_tracing < 0)) goto_error;\ + }\ + } else {\ + PyThreadState* tstate = PyThreadState_GET();\ + if (unlikely(tstate->use_tracing) && !tstate->tracing &&\ + (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {\ + __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&__pyx_frame_code, &__pyx_frame, funcname, srcfile, firstlineno);\ + if (unlikely(__Pyx_use_tracing < 0)) goto_error;\ + }\ + } + #else + #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error)\ + { PyThreadState* tstate = PyThreadState_GET();\ + if (unlikely(tstate->use_tracing) && !tstate->tracing &&\ + (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {\ + __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&__pyx_frame_code, &__pyx_frame, funcname, srcfile, firstlineno);\ + if (unlikely(__Pyx_use_tracing < 0)) goto_error;\ + }\ + } + #endif + #define __Pyx_TraceException()\ + if (likely(!__Pyx_use_tracing)); else {\ + PyThreadState* tstate = PyThreadState_GET();\ + if (tstate->use_tracing &&\ + (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {\ + tstate->tracing++;\ + tstate->use_tracing = 0;\ + PyObject *exc_info = __Pyx_GetExceptionTuple(tstate);\ + if (exc_info) {\ + if (CYTHON_TRACE && tstate->c_tracefunc)\ + tstate->c_tracefunc(\ + tstate->c_traceobj, __pyx_frame, PyTrace_EXCEPTION, exc_info);\ + tstate->c_profilefunc(\ + tstate->c_profileobj, __pyx_frame, PyTrace_EXCEPTION, exc_info);\ + Py_DECREF(exc_info);\ + }\ + tstate->use_tracing = 1;\ + tstate->tracing--;\ + }\ + } + static void __Pyx_call_return_trace_func(PyThreadState *tstate, PyFrameObject *frame, PyObject *result) { + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + tstate->tracing++; + tstate->use_tracing = 0; + if (CYTHON_TRACE && tstate->c_tracefunc) + tstate->c_tracefunc(tstate->c_traceobj, frame, PyTrace_RETURN, result); + if (tstate->c_profilefunc) + tstate->c_profilefunc(tstate->c_profileobj, frame, PyTrace_RETURN, result); + CYTHON_FRAME_DEL(frame); + tstate->use_tracing = 1; + tstate->tracing--; + PyErr_Restore(type, value, traceback); + } + #ifdef WITH_THREAD + #define __Pyx_TraceReturn(result, nogil)\ + if (likely(!__Pyx_use_tracing)); else {\ + if (nogil) {\ + if (CYTHON_TRACE_NOGIL) {\ + PyThreadState *tstate;\ + PyGILState_STATE state = PyGILState_Ensure();\ + tstate = PyThreadState_GET();\ + if (tstate->use_tracing) {\ + __Pyx_call_return_trace_func(tstate, __pyx_frame, (PyObject*)result);\ + }\ + PyGILState_Release(state);\ + }\ + } else {\ + PyThreadState* tstate = PyThreadState_GET();\ + if (tstate->use_tracing) {\ + __Pyx_call_return_trace_func(tstate, __pyx_frame, (PyObject*)result);\ + }\ + }\ + } + #else + #define __Pyx_TraceReturn(result, nogil)\ + if (likely(!__Pyx_use_tracing)); else {\ + PyThreadState* tstate = PyThreadState_GET();\ + if (tstate->use_tracing) {\ + __Pyx_call_return_trace_func(tstate, __pyx_frame, (PyObject*)result);\ + }\ + } + #endif + static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const char *srcfile, int firstlineno); + static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, const char *funcname, const char *srcfile, int firstlineno); +#else + #define __Pyx_TraceDeclarations + #define __Pyx_TraceFrameInit(codeobj) + #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error) if (1); else goto_error; + #define __Pyx_TraceException() + #define __Pyx_TraceReturn(result, nogil) +#endif +#if CYTHON_TRACE + static int __Pyx_call_line_trace_func(PyThreadState *tstate, PyFrameObject *frame, int lineno) { + int ret; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + __Pyx_PyFrame_SetLineNumber(frame, lineno); + tstate->tracing++; + tstate->use_tracing = 0; + ret = tstate->c_tracefunc(tstate->c_traceobj, frame, PyTrace_LINE, NULL); + tstate->use_tracing = 1; + tstate->tracing--; + if (likely(!ret)) { + PyErr_Restore(type, value, traceback); + } else { + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + } + return ret; + } + #ifdef WITH_THREAD + #define __Pyx_TraceLine(lineno, nogil, goto_error)\ + if (likely(!__Pyx_use_tracing)); else {\ + if (nogil) {\ + if (CYTHON_TRACE_NOGIL) {\ + int ret = 0;\ + PyThreadState *tstate;\ + PyGILState_STATE state = PyGILState_Ensure();\ + tstate = PyThreadState_GET();\ + if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) {\ + ret = __Pyx_call_line_trace_func(tstate, __pyx_frame, lineno);\ + }\ + PyGILState_Release(state);\ + if (unlikely(ret)) goto_error;\ + }\ + } else {\ + PyThreadState* tstate = PyThreadState_GET();\ + if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) {\ + int ret = __Pyx_call_line_trace_func(tstate, __pyx_frame, lineno);\ + if (unlikely(ret)) goto_error;\ + }\ + }\ + } + #else + #define __Pyx_TraceLine(lineno, nogil, goto_error)\ + if (likely(!__Pyx_use_tracing)); else {\ + PyThreadState* tstate = PyThreadState_GET();\ + if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) {\ + int ret = __Pyx_call_line_trace_func(tstate, __pyx_frame, lineno);\ + if (unlikely(ret)) goto_error;\ + }\ + } + #endif +#else + #define __Pyx_TraceLine(lineno, nogil, goto_error) if (1); else goto_error; +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* GetModuleGlobalName.proto */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ + const char* function_name); + +/* ArgTypeTest.proto */ +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact); + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#endif + +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyObjectLookupSpecial.proto */ +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 +static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); +#if PY_MAJOR_VERSION < 3 + if (unlikely(PyInstance_Check(obj))) + return __Pyx_PyObject_GetAttrStr(obj, attr_name); +#endif + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#else +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* PyObjectCallNoArg.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); +#else +#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* PyIntBinop.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); +#else +#define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ + PyObject_RichCompare(op1, op2, Py_EQ) + #endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* WriteUnraisableException.proto */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); + +/* CythonFunction.proto */ +#define __Pyx_CyFunction_USED 1 +#include +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { + PyCFunctionObject func; +#if PY_VERSION_HEX < 0x030500A0 + PyObject *func_weakreflist; +#endif + PyObject *func_dict; + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; + PyObject *func_classobj; + void *defaults; + int defaults_pyobjects; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; +} __pyx_CyFunctionObject; +static PyTypeObject *__pyx_CyFunctionType = 0; +#define __Pyx_CyFunction_NewEx(ml, flags, qualname, self, module, globals, code)\ + __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, qualname, self, module, globals, code) +static PyObject *__Pyx_CyFunction_New(PyTypeObject *, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *self, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, + size_t size, + int pyobjects); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(void); + +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS +static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { + PyListObject* L = (PyListObject*) list; + Py_ssize_t len = Py_SIZE(list); + if (likely(L->allocated > len)) { + Py_INCREF(x); + PyList_SET_ITEM(list, len, x); + Py_SIZE(list) = len+1; + return 0; + } + return PyList_Append(list, x); +} +#else +#define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) +#endif + +/* IncludeStringH.proto */ +#include + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* ClassMethod.proto */ +#include "descrobject.h" +static PyObject* __Pyx_Method_ClassMethod(PyObject *method); + +/* CodeObjectCache.proto */ +typedef struct { + PyCodeObject* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; +}; +static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static PyCodeObject *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* None.proto */ +#include + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(void); + +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); + + +/* Module declarations from 'libc.string' */ + +/* Module declarations from 'libcpp.string' */ + +/* Module declarations from 'libcpp.vector' */ + +/* Module declarations from 'libcpp' */ + +/* Module declarations from 'reppy.robots' */ +static PyTypeObject *__pyx_ptype_5reppy_6robots_Agent = 0; +static PyTypeObject *__pyx_ptype_5reppy_6robots_Robots = 0; +static PyTypeObject *__pyx_ptype_5reppy_6robots_AllowNone = 0; +static PyTypeObject *__pyx_ptype_5reppy_6robots_AllowAll = 0; +static PyTypeObject *__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; +static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *); /*proto*/ +static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *); /*proto*/ +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyObject_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyUnicode_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyStr_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyBytes_string_to_py_std__in_string(std::string const &); /*proto*/ +static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_string(std::string const &); /*proto*/ +static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*)(PyObject *)); /*proto*/ +static PyObject *__pyx_convert_vector_to_py_std_3a__3a_string(const std::vector &); /*proto*/ +#define __Pyx_MODULE_NAME "reppy.robots" +int __pyx_module_is_main_reppy__robots = 0; + +/* Implementation of 'reppy.robots' */ +static PyObject *__pyx_builtin_ValueError; +static PyObject *__pyx_builtin_map; +static PyObject *__pyx_builtin_range; +static const char __pyx_k__9[] = ""; +static const char __pyx_k_PY3[] = "PY3"; +static const char __pyx_k_amt[] = "amt"; +static const char __pyx_k_cls[] = "cls"; +static const char __pyx_k_exc[] = "exc"; +static const char __pyx_k_get[] = "get"; +static const char __pyx_k_map[] = "map"; +static const char __pyx_k_raw[] = "raw"; +static const char __pyx_k_res[] = "res"; +static const char __pyx_k_six[] = "six"; +static const char __pyx_k_ttl[] = "ttl"; +static const char __pyx_k_url[] = "url"; +static const char __pyx_k_args[] = "args"; +static const char __pyx_k_exit[] = "__exit__"; +static const char __pyx_k_init[] = "__init__"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "name"; +static const char __pyx_k_path[] = "path"; +static const char __pyx_k_read[] = "read"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_time[] = "time"; +static const char __pyx_k_util[] = "util"; +static const char __pyx_k_wrap[] = "wrap"; +static const char __pyx_k_agent[] = "agent"; +static const char __pyx_k_enter[] = "__enter__"; +static const char __pyx_k_fetch[] = "fetch"; +static const char __pyx_k_parse[] = "parse"; +static const char __pyx_k_range[] = "range"; +static const char __pyx_k_utf_8[] = "utf-8"; +static const char __pyx_k_value[] = "value"; +static const char __pyx_k_decode[] = "decode"; +static const char __pyx_k_encode[] = "encode"; +static const char __pyx_k_import[] = "__import__"; +static const char __pyx_k_kwargs[] = "kwargs"; +static const char __pyx_k_logger[] = "logger"; +static const char __pyx_k_robots[] = "robots"; +static const char __pyx_k_stream[] = "stream"; +static const char __pyx_k_closing[] = "closing"; +static const char __pyx_k_content[] = "content"; +static const char __pyx_k_default[] = "default"; +static const char __pyx_k_expires[] = "expires"; +static const char __pyx_k_minimum[] = "minimum"; +static const char __pyx_k_SSLError[] = "SSLError"; +static const char __pyx_k_max_size[] = "max_size"; +static const char __pyx_k_requests[] = "requests"; +static const char __pyx_k_InvalidURL[] = "InvalidURL"; +static const char __pyx_k_ValueError[] = "ValueError"; +static const char __pyx_k_contextlib[] = "contextlib"; +static const char __pyx_k_exceptions[] = "exceptions"; +static const char __pyx_k_robots_url[] = "robots_url"; +static const char __pyx_k_ttl_policy[] = "ttl_policy"; +static const char __pyx_k_FetchMethod[] = "FetchMethod"; +static const char __pyx_k_Got_i_for_s[] = "Got %i for %s"; +static const char __pyx_k_ParseMethod[] = "ParseMethod"; +static const char __pyx_k_URLRequired[] = "URLRequired"; +static const char __pyx_k_cfunc_to_py[] = "cfunc.to_py"; +static const char __pyx_k_from_robots[] = "from_robots"; +static const char __pyx_k_status_code[] = "status_code"; +static const char __pyx_k_MalformedUrl[] = "MalformedUrl"; +static const char __pyx_k_SSLException[] = "SSLException"; +static const char __pyx_k_reppy_robots[] = "reppy.robots"; +static const char __pyx_k_stringsource[] = "stringsource"; +static const char __pyx_k_BadStatusCode[] = "BadStatusCode"; +static const char __pyx_k_InvalidSchema[] = "InvalidSchema"; +static const char __pyx_k_MissingSchema[] = "MissingSchema"; +static const char __pyx_k_ContentTooLong[] = "ContentTooLong"; +static const char __pyx_k_decode_content[] = "decode_content"; +static const char __pyx_k_ConnectionError[] = "ConnectionError"; +static const char __pyx_k_RobotsUrlMethod[] = "RobotsUrlMethod"; +static const char __pyx_k_FromRobotsMethod[] = "FromRobotsMethod"; +static const char __pyx_k_TooManyRedirects[] = "TooManyRedirects"; +static const char __pyx_k_DEFAULT_TTL_POLICY[] = "DEFAULT_TTL_POLICY"; +static const char __pyx_k_ExcessiveRedirects[] = "ExcessiveRedirects"; +static const char __pyx_k_ConnectionException[] = "ConnectionException"; +static const char __pyx_k_User_agent_Disallow[] = "User-agent: *\nDisallow: /"; +static const char __pyx_k_requests_exceptions[] = "requests.exceptions"; +static const char __pyx_k_HeaderWithDefaultPolicy[] = "HeaderWithDefaultPolicy"; +static const char __pyx_k_vagrant_reppy_robots_pyx[] = "/vagrant/reppy/robots.pyx"; +static const char __pyx_k_Content_larger_than_s_bytes[] = "Content larger than %s bytes"; +static const char __pyx_k_Pyx_CFunc_object____object___t[] = "__Pyx_CFunc_object____object___to_py..wrap"; +static PyObject *__pyx_n_s_BadStatusCode; +static PyObject *__pyx_n_s_ConnectionError; +static PyObject *__pyx_n_s_ConnectionException; +static PyObject *__pyx_n_s_ContentTooLong; +static PyObject *__pyx_kp_s_Content_larger_than_s_bytes; +static PyObject *__pyx_n_s_DEFAULT_TTL_POLICY; +static PyObject *__pyx_n_s_ExcessiveRedirects; +static PyObject *__pyx_n_s_FetchMethod; +static PyObject *__pyx_n_s_FromRobotsMethod; +static PyObject *__pyx_kp_s_Got_i_for_s; +static PyObject *__pyx_n_s_HeaderWithDefaultPolicy; +static PyObject *__pyx_n_s_InvalidSchema; +static PyObject *__pyx_n_s_InvalidURL; +static PyObject *__pyx_n_s_MalformedUrl; +static PyObject *__pyx_n_s_MissingSchema; +static PyObject *__pyx_n_s_PY3; +static PyObject *__pyx_n_s_ParseMethod; +static PyObject *__pyx_n_s_Pyx_CFunc_object____object___t; +static PyObject *__pyx_n_s_RobotsUrlMethod; +static PyObject *__pyx_n_s_SSLError; +static PyObject *__pyx_n_s_SSLException; +static PyObject *__pyx_n_s_TooManyRedirects; +static PyObject *__pyx_n_s_URLRequired; +static PyObject *__pyx_kp_b_User_agent_Disallow; +static PyObject *__pyx_n_s_ValueError; +static PyObject *__pyx_n_s__9; +static PyObject *__pyx_kp_b__9; +static PyObject *__pyx_n_s_agent; +static PyObject *__pyx_n_s_amt; +static PyObject *__pyx_n_s_args; +static PyObject *__pyx_n_s_cfunc_to_py; +static PyObject *__pyx_n_s_closing; +static PyObject *__pyx_n_s_cls; +static PyObject *__pyx_n_s_content; +static PyObject *__pyx_n_s_contextlib; +static PyObject *__pyx_n_s_decode; +static PyObject *__pyx_n_s_decode_content; +static PyObject *__pyx_n_s_default; +static PyObject *__pyx_n_s_encode; +static PyObject *__pyx_n_s_enter; +static PyObject *__pyx_n_s_exc; +static PyObject *__pyx_n_s_exceptions; +static PyObject *__pyx_n_s_exit; +static PyObject *__pyx_n_s_expires; +static PyObject *__pyx_n_s_fetch; +static PyObject *__pyx_n_s_from_robots; +static PyObject *__pyx_n_s_get; +static PyObject *__pyx_n_s_import; +static PyObject *__pyx_n_s_init; +static PyObject *__pyx_n_s_kwargs; +static PyObject *__pyx_n_s_logger; +static PyObject *__pyx_n_s_main; +static PyObject *__pyx_n_s_map; +static PyObject *__pyx_n_s_max_size; +static PyObject *__pyx_n_s_minimum; +static PyObject *__pyx_n_s_name; +static PyObject *__pyx_n_s_parse; +static PyObject *__pyx_n_s_path; +static PyObject *__pyx_n_s_range; +static PyObject *__pyx_n_s_raw; +static PyObject *__pyx_n_s_read; +static PyObject *__pyx_n_s_reppy_robots; +static PyObject *__pyx_n_s_requests; +static PyObject *__pyx_n_s_requests_exceptions; +static PyObject *__pyx_n_s_res; +static PyObject *__pyx_n_s_robots; +static PyObject *__pyx_n_s_robots_url; +static PyObject *__pyx_n_s_six; +static PyObject *__pyx_n_s_status_code; +static PyObject *__pyx_n_s_stream; +static PyObject *__pyx_kp_s_stringsource; +static PyObject *__pyx_n_s_test; +static PyObject *__pyx_n_s_time; +static PyObject *__pyx_n_s_ttl; +static PyObject *__pyx_n_s_ttl_policy; +static PyObject *__pyx_n_s_url; +static PyObject *__pyx_kp_s_utf_8; +static PyObject *__pyx_n_s_util; +static PyObject *__pyx_kp_s_vagrant_reppy_robots_pyx; +static PyObject *__pyx_n_s_value; +static PyObject *__pyx_n_s_wrap; +static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_robots, std::string __pyx_v_name); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_2disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_4allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_content, PyObject *__pyx_v_expires); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, PyObject *__pyx_v_url); /* proto */ +static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_url, std::string __pyx_v_content, PyObject *__pyx_v_expires); /* proto */ +static void __pyx_pf_5reppy_6robots_6Robots_2__dealloc__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_4allowed(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_name); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_name); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ +static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6robots_AllowNone *__pyx_v_self, PyObject *__pyx_v_url, PyObject *__pyx_v_expires); /* proto */ +static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6robots_AllowAll *__pyx_v_self, PyObject *__pyx_v_url, PyObject *__pyx_v_expires); /* proto */ +static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_value); /* proto */ +static PyObject *__pyx_tp_new_5reppy_6robots_Agent(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5reppy_6robots_Robots(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5reppy_6robots_AllowNone(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5reppy_6robots_AllowAll(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_int_1; +static PyObject *__pyx_int_200; +static PyObject *__pyx_int_400; +static PyObject *__pyx_int_401; +static PyObject *__pyx_int_403; +static PyObject *__pyx_int_500; +static PyObject *__pyx_int_600; +static PyObject *__pyx_int_3600; +static PyObject *__pyx_int_1048576; +static PyObject *__pyx_tuple_; +static PyObject *__pyx_tuple__2; +static PyObject *__pyx_tuple__6; +static PyObject *__pyx_tuple__7; +static PyObject *__pyx_tuple__10; +static PyObject *__pyx_tuple__12; +static PyObject *__pyx_tuple__13; +static PyObject *__pyx_tuple__14; +static PyObject *__pyx_tuple__15; +static PyObject *__pyx_codeobj__3; +static PyObject *__pyx_codeobj__4; +static PyObject *__pyx_codeobj__5; +static PyObject *__pyx_codeobj__8; +static PyObject *__pyx_codeobj__11; + +/* "reppy/robots.pyx":21 + * from . import util, logger, exceptions + * + * cdef as_bytes(value): # <<<<<<<<<<<<<< + * if isinstance(value, bytes): + * return value + */ + +static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + __Pyx_RefNannySetupContext("as_bytes", 0); + __Pyx_TraceCall("as_bytes", __pyx_f[1], 21, 0, __PYX_ERR(1, 21, __pyx_L1_error)); + + /* "reppy/robots.pyx":22 + * + * cdef as_bytes(value): + * if isinstance(value, bytes): # <<<<<<<<<<<<<< + * return value + * return value.encode('utf-8') + */ + __Pyx_TraceLine(22,0,__PYX_ERR(1, 22, __pyx_L1_error)) + __pyx_t_1 = PyBytes_Check(__pyx_v_value); + __pyx_t_2 = (__pyx_t_1 != 0); + if (__pyx_t_2) { + + /* "reppy/robots.pyx":23 + * cdef as_bytes(value): + * if isinstance(value, bytes): + * return value # <<<<<<<<<<<<<< + * return value.encode('utf-8') + * + */ + __Pyx_TraceLine(23,0,__PYX_ERR(1, 23, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_value); + __pyx_r = __pyx_v_value; + goto __pyx_L0; + + /* "reppy/robots.pyx":22 + * + * cdef as_bytes(value): + * if isinstance(value, bytes): # <<<<<<<<<<<<<< + * return value + * return value.encode('utf-8') + */ + } + + /* "reppy/robots.pyx":24 + * if isinstance(value, bytes): + * return value + * return value.encode('utf-8') # <<<<<<<<<<<<<< + * + * cdef as_string(value): + */ + __Pyx_TraceLine(24,0,__PYX_ERR(1, 24, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 24, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 24, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":21 + * from . import util, logger, exceptions + * + * cdef as_bytes(value): # <<<<<<<<<<<<<< + * if isinstance(value, bytes): + * return value + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("reppy.robots.as_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":26 + * return value.encode('utf-8') + * + * cdef as_string(value): # <<<<<<<<<<<<<< + * if six.PY3: + * if isinstance(value, bytes): + */ + +static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_t_3; + int __pyx_t_4; + __Pyx_RefNannySetupContext("as_string", 0); + __Pyx_TraceCall("as_string", __pyx_f[1], 26, 0, __PYX_ERR(1, 26, __pyx_L1_error)); + + /* "reppy/robots.pyx":27 + * + * cdef as_string(value): + * if six.PY3: # <<<<<<<<<<<<<< + * if isinstance(value, bytes): + * return value.decode('utf-8') + */ + __Pyx_TraceLine(27,0,__PYX_ERR(1, 27, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_six); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 27, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_PY3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 27, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(1, 27, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_3) { + + /* "reppy/robots.pyx":28 + * cdef as_string(value): + * if six.PY3: + * if isinstance(value, bytes): # <<<<<<<<<<<<<< + * return value.decode('utf-8') + * return value + */ + __Pyx_TraceLine(28,0,__PYX_ERR(1, 28, __pyx_L1_error)) + __pyx_t_3 = PyBytes_Check(__pyx_v_value); + __pyx_t_4 = (__pyx_t_3 != 0); + if (__pyx_t_4) { + + /* "reppy/robots.pyx":29 + * if six.PY3: + * if isinstance(value, bytes): + * return value.decode('utf-8') # <<<<<<<<<<<<<< + * return value + * + */ + __Pyx_TraceLine(29,0,__PYX_ERR(1, 29, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_decode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 29, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 29, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":28 + * cdef as_string(value): + * if six.PY3: + * if isinstance(value, bytes): # <<<<<<<<<<<<<< + * return value.decode('utf-8') + * return value + */ + } + + /* "reppy/robots.pyx":27 + * + * cdef as_string(value): + * if six.PY3: # <<<<<<<<<<<<<< + * if isinstance(value, bytes): + * return value.decode('utf-8') + */ + } + + /* "reppy/robots.pyx":30 + * if isinstance(value, bytes): + * return value.decode('utf-8') + * return value # <<<<<<<<<<<<<< + * + * + */ + __Pyx_TraceLine(30,0,__PYX_ERR(1, 30, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_value); + __pyx_r = __pyx_v_value; + goto __pyx_L0; + + /* "reppy/robots.pyx":26 + * return value.encode('utf-8') + * + * cdef as_string(value): # <<<<<<<<<<<<<< + * if six.PY3: + * if isinstance(value, bytes): + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("reppy.robots.as_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":33 + * + * + * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< + * '''Construct an Agent from a CppAgent.''' + * agent = Agent() + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5reppy_6robots_FromRobotsMethod[] = "Construct an Agent from a CppAgent."; +static PyMethodDef __pyx_mdef_5reppy_6robots_1FromRobotsMethod = {"FromRobotsMethod", (PyCFunction)__pyx_pw_5reppy_6robots_1FromRobotsMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_FromRobotsMethod}; +static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_cls = 0; + struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_robots = 0; + std::string __pyx_v_name; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("FromRobotsMethod (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_cls,&__pyx_n_s_robots,&__pyx_n_s_name,0}; + PyObject* values[3] = {0,0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_robots)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 1); __PYX_ERR(1, 33, __pyx_L3_error) + } + case 2: + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 2); __PYX_ERR(1, 33, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "FromRobotsMethod") < 0)) __PYX_ERR(1, 33, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + } + __pyx_v_cls = values[0]; + __pyx_v_robots = ((struct __pyx_obj_5reppy_6robots_Robots *)values[1]); + __pyx_v_name = __pyx_convert_string_from_py_std__in_string(values[2]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 33, __pyx_L3_error) + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 33, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("reppy.robots.FromRobotsMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_robots), __pyx_ptype_5reppy_6robots_Robots, 1, "robots", 0))) __PYX_ERR(1, 33, __pyx_L1_error) + __pyx_r = __pyx_pf_5reppy_6robots_FromRobotsMethod(__pyx_self, __pyx_v_cls, __pyx_v_robots, __pyx_v_name); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_robots, std::string __pyx_v_name) { + struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_agent = NULL; + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_TraceFrameInit(__pyx_codeobj__3) + __Pyx_RefNannySetupContext("FromRobotsMethod", 0); + __Pyx_TraceCall("FromRobotsMethod", __pyx_f[1], 33, 0, __PYX_ERR(1, 33, __pyx_L1_error)); + + /* "reppy/robots.pyx":35 + * def FromRobotsMethod(cls, Robots robots, const string& name): + * '''Construct an Agent from a CppAgent.''' + * agent = Agent() # <<<<<<<<<<<<<< + * agent.agent = robots.robots.agent(name) + * return agent + */ + __Pyx_TraceLine(35,0,__PYX_ERR(1, 35, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 35, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_agent = ((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_t_1); + __pyx_t_1 = 0; + + /* "reppy/robots.pyx":36 + * '''Construct an Agent from a CppAgent.''' + * agent = Agent() + * agent.agent = robots.robots.agent(name) # <<<<<<<<<<<<<< + * return agent + * + */ + __Pyx_TraceLine(36,0,__PYX_ERR(1, 36, __pyx_L1_error)) + __pyx_v_agent->agent = __pyx_v_robots->robots->agent(__pyx_v_name); + + /* "reppy/robots.pyx":37 + * agent = Agent() + * agent.agent = robots.robots.agent(name) + * return agent # <<<<<<<<<<<<<< + * + * cdef class Agent: + */ + __Pyx_TraceLine(37,0,__PYX_ERR(1, 37, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_agent)); + __pyx_r = ((PyObject *)__pyx_v_agent); + goto __pyx_L0; + + /* "reppy/robots.pyx":33 + * + * + * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< + * '''Construct an Agent from a CppAgent.''' + * agent = Agent() + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.FromRobotsMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF((PyObject *)__pyx_v_agent); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":47 + * + * @property + * def delay(self): # <<<<<<<<<<<<<< + * '''The delay associated with this agent.''' + * cdef float value = self.agent.delay() + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_5delay_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_5delay_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_5delay___get__(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self) { + float __pyx_v_value; + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_TraceCall("__get__", __pyx_f[1], 47, 0, __PYX_ERR(1, 47, __pyx_L1_error)); + + /* "reppy/robots.pyx":49 + * def delay(self): + * '''The delay associated with this agent.''' + * cdef float value = self.agent.delay() # <<<<<<<<<<<<<< + * if value > 0: + * return value + */ + __Pyx_TraceLine(49,0,__PYX_ERR(1, 49, __pyx_L1_error)) + __pyx_v_value = __pyx_v_self->agent.delay(); + + /* "reppy/robots.pyx":50 + * '''The delay associated with this agent.''' + * cdef float value = self.agent.delay() + * if value > 0: # <<<<<<<<<<<<<< + * return value + * return None + */ + __Pyx_TraceLine(50,0,__PYX_ERR(1, 50, __pyx_L1_error)) + __pyx_t_1 = ((__pyx_v_value > 0.0) != 0); + if (__pyx_t_1) { + + /* "reppy/robots.pyx":51 + * cdef float value = self.agent.delay() + * if value > 0: + * return value # <<<<<<<<<<<<<< + * return None + * + */ + __Pyx_TraceLine(51,0,__PYX_ERR(1, 51, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 51, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":50 + * '''The delay associated with this agent.''' + * cdef float value = self.agent.delay() + * if value > 0: # <<<<<<<<<<<<<< + * return value + * return None + */ + } + + /* "reppy/robots.pyx":52 + * if value > 0: + * return value + * return None # <<<<<<<<<<<<<< + * + * def allow(self, path): + */ + __Pyx_TraceLine(52,0,__PYX_ERR(1, 52, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(Py_None); + __pyx_r = Py_None; + goto __pyx_L0; + + /* "reppy/robots.pyx":47 + * + * @property + * def delay(self): # <<<<<<<<<<<<<< + * '''The delay associated with this agent.''' + * cdef float value = self.agent.delay() + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("reppy.robots.Agent.delay.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":54 + * return None + * + * def allow(self, path): # <<<<<<<<<<<<<< + * '''Allow the provided path.''' + * self.agent.allow(as_bytes(path)) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_1allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ +static char __pyx_doc_5reppy_6robots_5Agent_allow[] = "Allow the provided path."; +static PyObject *__pyx_pw_5reppy_6robots_5Agent_1allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("allow (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_allow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_5Agent_allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + std::string __pyx_t_2; + __Pyx_RefNannySetupContext("allow", 0); + __Pyx_TraceCall("allow", __pyx_f[1], 54, 0, __PYX_ERR(1, 54, __pyx_L1_error)); + + /* "reppy/robots.pyx":56 + * def allow(self, path): + * '''Allow the provided path.''' + * self.agent.allow(as_bytes(path)) # <<<<<<<<<<<<<< + * return self + * + */ + __Pyx_TraceLine(56,0,__PYX_ERR(1, 56, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 56, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_self->agent.allow(__pyx_t_2); + + /* "reppy/robots.pyx":57 + * '''Allow the provided path.''' + * self.agent.allow(as_bytes(path)) + * return self # <<<<<<<<<<<<<< + * + * def disallow(self, path): + */ + __Pyx_TraceLine(57,0,__PYX_ERR(1, 57, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_self)); + __pyx_r = ((PyObject *)__pyx_v_self); + goto __pyx_L0; + + /* "reppy/robots.pyx":54 + * return None + * + * def allow(self, path): # <<<<<<<<<<<<<< + * '''Allow the provided path.''' + * self.agent.allow(as_bytes(path)) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Agent.allow", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":59 + * return self + * + * def disallow(self, path): # <<<<<<<<<<<<<< + * '''Disallow the provided path.''' + * self.agent.disallow(as_bytes(path)) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_3disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ +static char __pyx_doc_5reppy_6robots_5Agent_2disallow[] = "Disallow the provided path."; +static PyObject *__pyx_pw_5reppy_6robots_5Agent_3disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("disallow (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_2disallow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_5Agent_2disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + std::string __pyx_t_2; + __Pyx_RefNannySetupContext("disallow", 0); + __Pyx_TraceCall("disallow", __pyx_f[1], 59, 0, __PYX_ERR(1, 59, __pyx_L1_error)); + + /* "reppy/robots.pyx":61 + * def disallow(self, path): + * '''Disallow the provided path.''' + * self.agent.disallow(as_bytes(path)) # <<<<<<<<<<<<<< + * return self + * + */ + __Pyx_TraceLine(61,0,__PYX_ERR(1, 61, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_self->agent.disallow(__pyx_t_2); + + /* "reppy/robots.pyx":62 + * '''Disallow the provided path.''' + * self.agent.disallow(as_bytes(path)) + * return self # <<<<<<<<<<<<<< + * + * def allowed(self, path): + */ + __Pyx_TraceLine(62,0,__PYX_ERR(1, 62, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(((PyObject *)__pyx_v_self)); + __pyx_r = ((PyObject *)__pyx_v_self); + goto __pyx_L0; + + /* "reppy/robots.pyx":59 + * return self + * + * def disallow(self, path): # <<<<<<<<<<<<<< + * '''Disallow the provided path.''' + * self.agent.disallow(as_bytes(path)) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Agent.disallow", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":64 + * return self + * + * def allowed(self, path): # <<<<<<<<<<<<<< + * '''Is the provided URL allowed?''' + * return self.agent.allowed(as_bytes(path)) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_5allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ +static char __pyx_doc_5reppy_6robots_5Agent_4allowed[] = "Is the provided URL allowed?"; +static PyObject *__pyx_pw_5reppy_6robots_5Agent_5allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("allowed (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_4allowed(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_5Agent_4allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + std::string __pyx_t_2; + __Pyx_RefNannySetupContext("allowed", 0); + __Pyx_TraceCall("allowed", __pyx_f[1], 64, 0, __PYX_ERR(1, 64, __pyx_L1_error)); + + /* "reppy/robots.pyx":66 + * def allowed(self, path): + * '''Is the provided URL allowed?''' + * return self.agent.allowed(as_bytes(path)) # <<<<<<<<<<<<<< + * + * + */ + __Pyx_TraceLine(66,0,__PYX_ERR(1, 66, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":64 + * return self + * + * def allowed(self, path): # <<<<<<<<<<<<<< + * '''Is the provided URL allowed?''' + * return self.agent.allowed(as_bytes(path)) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Agent.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":69 + * + * + * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< + * '''Parse a robots.txt file.''' + * return cls(url, as_bytes(content), expires) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5reppy_6robots_2ParseMethod[] = "Parse a robots.txt file."; +static PyMethodDef __pyx_mdef_5reppy_6robots_3ParseMethod = {"ParseMethod", (PyCFunction)__pyx_pw_5reppy_6robots_3ParseMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_2ParseMethod}; +static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_cls = 0; + PyObject *__pyx_v_url = 0; + PyObject *__pyx_v_content = 0; + PyObject *__pyx_v_expires = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("ParseMethod (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_cls,&__pyx_n_s_url,&__pyx_n_s_content,&__pyx_n_s_expires,0}; + PyObject* values[4] = {0,0,0,0}; + values[3] = ((PyObject *)Py_None); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(1, 69, __pyx_L3_error) + } + case 2: + if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(1, 69, __pyx_L3_error) + } + case 3: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); + if (value) { values[3] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(1, 69, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_cls = values[0]; + __pyx_v_url = values[1]; + __pyx_v_content = values[2]; + __pyx_v_expires = values[3]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 69, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("reppy.robots.ParseMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5reppy_6robots_2ParseMethod(__pyx_self, __pyx_v_cls, __pyx_v_url, __pyx_v_content, __pyx_v_expires); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_content, PyObject *__pyx_v_expires) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + int __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + __Pyx_TraceFrameInit(__pyx_codeobj__4) + __Pyx_RefNannySetupContext("ParseMethod", 0); + __Pyx_TraceCall("ParseMethod", __pyx_f[1], 69, 0, __PYX_ERR(1, 69, __pyx_L1_error)); + + /* "reppy/robots.pyx":71 + * def ParseMethod(cls, url, content, expires=None): + * '''Parse a robots.txt file.''' + * return cls(url, as_bytes(content), expires) # <<<<<<<<<<<<<< + * + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): + */ + __Pyx_TraceLine(71,0,__PYX_ERR(1, 71, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_v_cls); + __pyx_t_3 = __pyx_v_cls; __pyx_t_4 = NULL; + __pyx_t_5 = 0; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + __pyx_t_5 = 1; + } + } + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 71, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 71, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__pyx_t_4) { + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; + } + __Pyx_INCREF(__pyx_v_url); + __Pyx_GIVEREF(__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_6, 0+__pyx_t_5, __pyx_v_url); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_2); + __Pyx_INCREF(__pyx_v_expires); + __Pyx_GIVEREF(__pyx_v_expires); + PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_5, __pyx_v_expires); + __pyx_t_2 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 71, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":69 + * + * + * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< + * '''Parse a robots.txt file.''' + * return cls(url, as_bytes(content), expires) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("reppy.robots.ParseMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":73 + * return cls(url, as_bytes(content), expires) + * + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< + * '''Get the robots.txt at the provided URL.''' + * try: + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5reppy_6robots_4FetchMethod[] = "Get the robots.txt at the provided URL."; +static PyMethodDef __pyx_mdef_5reppy_6robots_5FetchMethod = {"FetchMethod", (PyCFunction)__pyx_pw_5reppy_6robots_5FetchMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_4FetchMethod}; +static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_cls = 0; + PyObject *__pyx_v_url = 0; + PyObject *__pyx_v_ttl_policy = 0; + PyObject *__pyx_v_max_size = 0; + PyObject *__pyx_v_args = 0; + PyObject *__pyx_v_kwargs = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("FetchMethod (wrapper)", 0); + __pyx_v_kwargs = PyDict_New(); if (unlikely(!__pyx_v_kwargs)) return NULL; + __Pyx_GOTREF(__pyx_v_kwargs); + if (PyTuple_GET_SIZE(__pyx_args) > 4) { + __pyx_v_args = PyTuple_GetSlice(__pyx_args, 4, PyTuple_GET_SIZE(__pyx_args)); + if (unlikely(!__pyx_v_args)) { + __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; + __Pyx_RefNannyFinishContext(); + return NULL; + } + __Pyx_GOTREF(__pyx_v_args); + } else { + __pyx_v_args = __pyx_empty_tuple; __Pyx_INCREF(__pyx_empty_tuple); + } + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_cls,&__pyx_n_s_url,&__pyx_n_s_ttl_policy,&__pyx_n_s_max_size,0}; + PyObject* values[4] = {0,0,0,0}; + values[2] = ((PyObject *)Py_None); + values[3] = ((PyObject *)__pyx_int_1048576); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + default: + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(1, 73, __pyx_L3_error) + } + case 2: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_ttl_policy); + if (value) { values[2] = value; kw_args--; } + } + case 3: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_max_size); + if (value) { values[3] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + const Py_ssize_t used_pos_args = (pos_args < 4) ? pos_args : 4; + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(1, 73, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + default: + case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + case 1: + case 0: + goto __pyx_L5_argtuple_error; + } + } + __pyx_v_cls = values[0]; + __pyx_v_url = values[1]; + __pyx_v_ttl_policy = values[2]; + __pyx_v_max_size = values[3]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 73, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_DECREF(__pyx_v_args); __pyx_v_args = 0; + __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; + __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5reppy_6robots_4FetchMethod(__pyx_self, __pyx_v_cls, __pyx_v_url, __pyx_v_ttl_policy, __pyx_v_max_size, __pyx_v_args, __pyx_v_kwargs); + + /* function exit code */ + __Pyx_XDECREF(__pyx_v_args); + __Pyx_XDECREF(__pyx_v_kwargs); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs) { + PyObject *__pyx_v_res = NULL; + PyObject *__pyx_v_content = NULL; + PyObject *__pyx_v_expires = NULL; + PyObject *__pyx_v_exc = NULL; + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + int __pyx_t_13; + int __pyx_t_14; + int __pyx_t_15; + PyObject *__pyx_t_16 = NULL; + PyObject *__pyx_t_17 = NULL; + PyObject *__pyx_t_18 = NULL; + __Pyx_TraceFrameInit(__pyx_codeobj__5) + __Pyx_RefNannySetupContext("FetchMethod", 0); + __Pyx_TraceCall("FetchMethod", __pyx_f[1], 73, 0, __PYX_ERR(1, 73, __pyx_L1_error)); + + /* "reppy/robots.pyx":75 + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): + * '''Get the robots.txt at the provided URL.''' + * try: # <<<<<<<<<<<<<< + * # Limit the size of the request + * kwargs['stream'] = True + */ + __Pyx_TraceLine(75,0,__PYX_ERR(1, 75, __pyx_L3_error)) + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_1); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_3); + /*try:*/ { + + /* "reppy/robots.pyx":77 + * try: + * # Limit the size of the request + * kwargs['stream'] = True # <<<<<<<<<<<<<< + * with closing(requests.get(url, *args, **kwargs)) as res: + * content = res.raw.read(amt=max_size, decode_content=True) + */ + __Pyx_TraceLine(77,0,__PYX_ERR(1, 77, __pyx_L3_error)) + if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 77, __pyx_L3_error) + + /* "reppy/robots.pyx":78 + * # Limit the size of the request + * kwargs['stream'] = True + * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< + * content = res.raw.read(amt=max_size, decode_content=True) + * # Try to read an additional byte, to see if the response is too big + */ + __Pyx_TraceLine(78,0,__PYX_ERR(1, 78, __pyx_L3_error)) + /*with:*/ { + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_INCREF(__pyx_v_url); + __Pyx_GIVEREF(__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_url); + __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_8 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_5); + if (likely(__pyx_t_8)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_5, function); + } + } + if (!__pyx_t_8) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_5)) { + PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { + PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + { + __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_7, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + } + } + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 78, __pyx_L11_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_7); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_7, function); + } + } + if (__pyx_t_6) { + __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 78, __pyx_L11_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else { + __pyx_t_5 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 78, __pyx_L11_error) + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __pyx_t_5; + __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + /*try:*/ { + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_10, &__pyx_t_11, &__pyx_t_12); + __Pyx_XGOTREF(__pyx_t_10); + __Pyx_XGOTREF(__pyx_t_11); + __Pyx_XGOTREF(__pyx_t_12); + /*try:*/ { + __pyx_v_res = __pyx_t_7; + __pyx_t_7 = 0; + + /* "reppy/robots.pyx":79 + * kwargs['stream'] = True + * with closing(requests.get(url, *args, **kwargs)) as res: + * content = res.raw.read(amt=max_size, decode_content=True) # <<<<<<<<<<<<<< + * # Try to read an additional byte, to see if the response is too big + * if res.raw.read(amt=1, decode_content=True): + */ + __Pyx_TraceLine(79,0,__PYX_ERR(1, 79, __pyx_L17_error)) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 79, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 79, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 79, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_7); + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 79, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 79, __pyx_L17_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 79, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_v_content = __pyx_t_5; + __pyx_t_5 = 0; + + /* "reppy/robots.pyx":81 + * content = res.raw.read(amt=max_size, decode_content=True) + * # Try to read an additional byte, to see if the response is too big + * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< + * raise exceptions.ContentTooLong( + * 'Content larger than %s bytes' % max_size) + */ + __Pyx_TraceLine(81,0,__PYX_ERR(1, 81, __pyx_L17_error)) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 81, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = PyDict_New(); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 81, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 81, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 81, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (__pyx_t_13) { + + /* "reppy/robots.pyx":82 + * # Try to read an additional byte, to see if the response is too big + * if res.raw.read(amt=1, decode_content=True): + * raise exceptions.ContentTooLong( # <<<<<<<<<<<<<< + * 'Content larger than %s bytes' % max_size) + * + */ + __Pyx_TraceLine(82,0,__PYX_ERR(1, 82, __pyx_L17_error)) + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 82, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 82, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + + /* "reppy/robots.pyx":83 + * if res.raw.read(amt=1, decode_content=True): + * raise exceptions.ContentTooLong( + * 'Content larger than %s bytes' % max_size) # <<<<<<<<<<<<<< + * + * # Get the TTL policy's ruling on the ttl + */ + __Pyx_TraceLine(83,0,__PYX_ERR(1, 83, __pyx_L17_error)) + __pyx_t_5 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 83, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_7); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_7, function); + } + } + if (!__pyx_t_6) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_7)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_5}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_5}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else + #endif + { + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 82, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_5); + __pyx_t_5 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_Raise(__pyx_t_4, 0, 0, 0); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __PYX_ERR(1, 82, __pyx_L17_error) + + /* "reppy/robots.pyx":81 + * content = res.raw.read(amt=max_size, decode_content=True) + * # Try to read an additional byte, to see if the response is too big + * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< + * raise exceptions.ContentTooLong( + * 'Content larger than %s bytes' % max_size) + */ + } + + /* "reppy/robots.pyx":86 + * + * # Get the TTL policy's ruling on the ttl + * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) # <<<<<<<<<<<<<< + * + * if res.status_code == 200: + */ + __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L17_error)) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 86, __pyx_L17_error) + if (!__pyx_t_13) { + } else { + __Pyx_INCREF(__pyx_v_ttl_policy); + __pyx_t_7 = __pyx_v_ttl_policy; + goto __pyx_L26_bool_binop_done; + } + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_t_8); + __pyx_t_7 = __pyx_t_8; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_L26_bool_binop_done:; + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_8); + if (likely(__pyx_t_7)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); + __Pyx_INCREF(__pyx_t_7); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_8, function); + } + } + if (!__pyx_t_7) { + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_v_res); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_8)) { + PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { + PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_GOTREF(__pyx_t_4); + } else + #endif + { + __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_7); __pyx_t_7 = NULL; + __Pyx_INCREF(__pyx_v_res); + __Pyx_GIVEREF(__pyx_v_res); + PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_v_res); + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_expires = __pyx_t_4; + __pyx_t_4 = 0; + + /* "reppy/robots.pyx":88 + * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) + * + * if res.status_code == 200: # <<<<<<<<<<<<<< + * return cls.parse(url, content, expires) + * elif res.status_code in (401, 403): + */ + __Pyx_TraceLine(88,0,__PYX_ERR(1, 88, __pyx_L17_error)) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 88, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_8 = __Pyx_PyInt_EqObjC(__pyx_t_4, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 88, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 88, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (__pyx_t_13) { + + /* "reppy/robots.pyx":89 + * + * if res.status_code == 200: + * return cls.parse(url, content, expires) # <<<<<<<<<<<<<< + * elif res.status_code in (401, 403): + * return AllowNone(url, expires) + */ + __Pyx_TraceLine(89,0,__PYX_ERR(1, 89, __pyx_L17_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = NULL; + __pyx_t_14 = 0; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_5)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + __pyx_t_14 = 1; + } + } + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[4] = {__pyx_t_5, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; + __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_8); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[4] = {__pyx_t_5, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; + __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_8); + } else + #endif + { + __pyx_t_7 = PyTuple_New(3+__pyx_t_14); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 89, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_7); + if (__pyx_t_5) { + __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); __pyx_t_5 = NULL; + } + __Pyx_INCREF(__pyx_v_url); + __Pyx_GIVEREF(__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_7, 0+__pyx_t_14, __pyx_v_url); + __Pyx_INCREF(__pyx_v_content); + __Pyx_GIVEREF(__pyx_v_content); + PyTuple_SET_ITEM(__pyx_t_7, 1+__pyx_t_14, __pyx_v_content); + __Pyx_INCREF(__pyx_v_expires); + __Pyx_GIVEREF(__pyx_v_expires); + PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_14, __pyx_v_expires); + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_7, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_r = __pyx_t_8; + __pyx_t_8 = 0; + goto __pyx_L21_try_return; + + /* "reppy/robots.pyx":88 + * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) + * + * if res.status_code == 200: # <<<<<<<<<<<<<< + * return cls.parse(url, content, expires) + * elif res.status_code in (401, 403): + */ + } + + /* "reppy/robots.pyx":90 + * if res.status_code == 200: + * return cls.parse(url, content, expires) + * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< + * return AllowNone(url, expires) + * elif res.status_code >= 400 and res.status_code < 500: + */ + __Pyx_TraceLine(90,0,__PYX_ERR(1, 90, __pyx_L17_error)) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (!__pyx_t_15) { + } else { + __pyx_t_13 = __pyx_t_15; + goto __pyx_L29_bool_binop_done; + } + __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_13 = __pyx_t_15; + __pyx_L29_bool_binop_done:; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_15 = (__pyx_t_13 != 0); + if (__pyx_t_15) { + + /* "reppy/robots.pyx":91 + * return cls.parse(url, content, expires) + * elif res.status_code in (401, 403): + * return AllowNone(url, expires) # <<<<<<<<<<<<<< + * elif res.status_code >= 400 and res.status_code < 500: + * return AllowAll(url, expires) + */ + __Pyx_TraceLine(91,0,__PYX_ERR(1, 91, __pyx_L17_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 91, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_v_url); + __Pyx_GIVEREF(__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_url); + __Pyx_INCREF(__pyx_v_expires); + __Pyx_GIVEREF(__pyx_v_expires); + PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_v_expires); + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 91, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; + goto __pyx_L21_try_return; + + /* "reppy/robots.pyx":90 + * if res.status_code == 200: + * return cls.parse(url, content, expires) + * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< + * return AllowNone(url, expires) + * elif res.status_code >= 400 and res.status_code < 500: + */ + } + + /* "reppy/robots.pyx":92 + * elif res.status_code in (401, 403): + * return AllowNone(url, expires) + * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< + * return AllowAll(url, expires) + * else: + */ + __Pyx_TraceLine(92,0,__PYX_ERR(1, 92, __pyx_L17_error)) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 92, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_8 = PyObject_RichCompare(__pyx_t_4, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 92, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (__pyx_t_13) { + } else { + __pyx_t_15 = __pyx_t_13; + goto __pyx_L31_bool_binop_done; + } + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_4 = PyObject_RichCompare(__pyx_t_8, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 92, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 92, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_15 = __pyx_t_13; + __pyx_L31_bool_binop_done:; + if (__pyx_t_15) { + + /* "reppy/robots.pyx":93 + * return AllowNone(url, expires) + * elif res.status_code >= 400 and res.status_code < 500: + * return AllowAll(url, expires) # <<<<<<<<<<<<<< + * else: + * raise exceptions.BadStatusCode( + */ + __Pyx_TraceLine(93,0,__PYX_ERR(1, 93, __pyx_L17_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_INCREF(__pyx_v_url); + __Pyx_GIVEREF(__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_url); + __Pyx_INCREF(__pyx_v_expires); + __Pyx_GIVEREF(__pyx_v_expires); + PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_v_expires); + __pyx_t_8 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_4, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_r = __pyx_t_8; + __pyx_t_8 = 0; + goto __pyx_L21_try_return; + + /* "reppy/robots.pyx":92 + * elif res.status_code in (401, 403): + * return AllowNone(url, expires) + * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< + * return AllowAll(url, expires) + * else: + */ + } + + /* "reppy/robots.pyx":95 + * return AllowAll(url, expires) + * else: + * raise exceptions.BadStatusCode( # <<<<<<<<<<<<<< + * 'Got %i for %s' % (res.status_code, url), res.status_code) + * except SSLError as exc: + */ + __Pyx_TraceLine(95,0,__PYX_ERR(1, 95, __pyx_L17_error)) + /*else*/ { + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + + /* "reppy/robots.pyx":96 + * else: + * raise exceptions.BadStatusCode( + * 'Got %i for %s' % (res.status_code, url), res.status_code) # <<<<<<<<<<<<<< + * except SSLError as exc: + * raise exceptions.SSLException(exc) + */ + __Pyx_TraceLine(96,0,__PYX_ERR(1, 96, __pyx_L17_error)) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 96, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 96, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); + __Pyx_INCREF(__pyx_v_url); + __Pyx_GIVEREF(__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_url); + __pyx_t_4 = 0; + __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 96, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 96, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = NULL; + __pyx_t_14 = 0; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_7); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_7, function); + __pyx_t_14 = 1; + } + } + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_7)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_4, __pyx_t_5}; + __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_4, __pyx_t_5}; + __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } else + #endif + { + __pyx_t_16 = PyTuple_New(2+__pyx_t_14); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_16); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; + } + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_16, 0+__pyx_t_14, __pyx_t_4); + __Pyx_GIVEREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_14, __pyx_t_5); + __pyx_t_4 = 0; + __pyx_t_5 = 0; + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_Raise(__pyx_t_8, 0, 0, 0); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __PYX_ERR(1, 95, __pyx_L17_error) + } + + /* "reppy/robots.pyx":78 + * # Limit the size of the request + * kwargs['stream'] = True + * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< + * content = res.raw.read(amt=max_size, decode_content=True) + * # Try to read an additional byte, to see if the response is too big + */ + } + __pyx_L17_error:; + __Pyx_PyThreadState_assign + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + /*except:*/ { + __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 78, __pyx_L19_except_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_5 = PyTuple_Pack(3, __pyx_t_8, __pyx_t_7, __pyx_t_16); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 78, __pyx_L19_except_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_5, NULL); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 78, __pyx_L19_except_error) + __Pyx_GOTREF(__pyx_t_17); + __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_17); + __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; + if (__pyx_t_15 < 0) __PYX_ERR(1, 78, __pyx_L19_except_error) + __pyx_t_13 = ((!(__pyx_t_15 != 0)) != 0); + if (__pyx_t_13) { + __Pyx_GIVEREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_16); + __Pyx_ErrRestoreWithState(__pyx_t_8, __pyx_t_7, __pyx_t_16); + __pyx_t_8 = 0; __pyx_t_7 = 0; __pyx_t_16 = 0; + __PYX_ERR(1, 78, __pyx_L19_except_error) + } + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + goto __pyx_L18_exception_handled; + } + __pyx_L19_except_error:; + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_10); + __Pyx_XGIVEREF(__pyx_t_11); + __Pyx_XGIVEREF(__pyx_t_12); + __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); + goto __pyx_L3_error; + __pyx_L21_try_return:; + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_10); + __Pyx_XGIVEREF(__pyx_t_11); + __Pyx_XGIVEREF(__pyx_t_12); + __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); + goto __pyx_L14_return; + __pyx_L18_exception_handled:; + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_10); + __Pyx_XGIVEREF(__pyx_t_11); + __Pyx_XGIVEREF(__pyx_t_12); + __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); + } + } + /*finally:*/ { + /*normal exit:*/{ + if (__pyx_t_9) { + __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__6, NULL); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_12); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + } + goto __pyx_L16; + } + __pyx_L14_return: { + __pyx_t_12 = __pyx_r; + __pyx_r = 0; + if (__pyx_t_9) { + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__7, NULL); + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 78, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + } + __pyx_r = __pyx_t_12; + __pyx_t_12 = 0; + goto __pyx_L7_try_return; + } + __pyx_L16:; + } + goto __pyx_L36; + __pyx_L11_error:; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + goto __pyx_L3_error; + __pyx_L36:; + } + + /* "reppy/robots.pyx":75 + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): + * '''Get the robots.txt at the provided URL.''' + * try: # <<<<<<<<<<<<<< + * # Limit the size of the request + * kwargs['stream'] = True + */ + } + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L10_try_end; + __pyx_L3_error:; + __Pyx_PyThreadState_assign + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; + + /* "reppy/robots.pyx":97 + * raise exceptions.BadStatusCode( + * 'Got %i for %s' % (res.status_code, url), res.status_code) + * except SSLError as exc: # <<<<<<<<<<<<<< + * raise exceptions.SSLException(exc) + * except ConnectionError as exc: + */ + __Pyx_TraceLine(97,0,__PYX_ERR(1, 97, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 97, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + if (__pyx_t_14) { + __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_7, &__pyx_t_8) < 0) __PYX_ERR(1, 97, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_t_7); + __pyx_v_exc = __pyx_t_7; + + /* "reppy/robots.pyx":98 + * 'Got %i for %s' % (res.status_code, url), res.status_code) + * except SSLError as exc: + * raise exceptions.SSLException(exc) # <<<<<<<<<<<<<< + * except ConnectionError as exc: + * raise exceptions.ConnectionException(exc) + */ + __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L5_except_error)) + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_6, function); + } + } + if (!__pyx_t_4) { + __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; + __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_5); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; + __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_5); + } else + #endif + { + __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_v_exc); + __Pyx_GIVEREF(__pyx_v_exc); + PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + } + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_Raise(__pyx_t_5, 0, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 98, __pyx_L5_except_error) + } + + /* "reppy/robots.pyx":99 + * except SSLError as exc: + * raise exceptions.SSLException(exc) + * except ConnectionError as exc: # <<<<<<<<<<<<<< + * raise exceptions.ConnectionException(exc) + * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: + */ + __Pyx_TraceLine(99,0,__PYX_ERR(1, 99, __pyx_L5_except_error)) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 99, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (__pyx_t_14) { + __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 99, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GOTREF(__pyx_t_16); + __Pyx_INCREF(__pyx_t_7); + __pyx_v_exc = __pyx_t_7; + + /* "reppy/robots.pyx":100 + * raise exceptions.SSLException(exc) + * except ConnectionError as exc: + * raise exceptions.ConnectionException(exc) # <<<<<<<<<<<<<< + * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: + * raise exceptions.MalformedUrl(exc) + */ + __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L5_except_error)) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_18 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_18))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_18); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_18); + __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_18, function); + } + } + if (!__pyx_t_6) { + __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_18, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_18)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; + __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_5); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_18)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; + __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_5); + } else + #endif + { + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_INCREF(__pyx_v_exc); + __Pyx_GIVEREF(__pyx_v_exc); + PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_exc); + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_18, __pyx_t_4, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + } + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_Raise(__pyx_t_5, 0, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __PYX_ERR(1, 100, __pyx_L5_except_error) + } + + /* "reppy/robots.pyx":101 + * except ConnectionError as exc: + * raise exceptions.ConnectionException(exc) + * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: # <<<<<<<<<<<<<< + * raise exceptions.MalformedUrl(exc) + * except TooManyRedirects as exc: + */ + __Pyx_TraceLine(101,0,__PYX_ERR(1, 101, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_5); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (__pyx_t_14) { + __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_7) < 0) __PYX_ERR(1, 101, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GOTREF(__pyx_t_7); + __Pyx_INCREF(__pyx_t_8); + __pyx_v_exc = __pyx_t_8; + + /* "reppy/robots.pyx":102 + * raise exceptions.ConnectionException(exc) + * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: + * raise exceptions.MalformedUrl(exc) # <<<<<<<<<<<<<< + * except TooManyRedirects as exc: + * raise exceptions.ExcessiveRedirects(exc) + */ + __Pyx_TraceLine(102,0,__PYX_ERR(1, 102, __pyx_L5_except_error)) + __pyx_t_18 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __pyx_t_18 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_18 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_18)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_18); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + } + } + if (!__pyx_t_18) { + __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; + __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_GOTREF(__pyx_t_16); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; + __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_GOTREF(__pyx_t_16); + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_18); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_18); __pyx_t_18 = NULL; + __Pyx_INCREF(__pyx_v_exc); + __Pyx_GIVEREF(__pyx_v_exc); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_v_exc); + __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_Raise(__pyx_t_16, 0, 0, 0); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __PYX_ERR(1, 102, __pyx_L5_except_error) + } + + /* "reppy/robots.pyx":103 + * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: + * raise exceptions.MalformedUrl(exc) + * except TooManyRedirects as exc: # <<<<<<<<<<<<<< + * raise exceptions.ExcessiveRedirects(exc) + * + */ + __Pyx_TraceLine(103,0,__PYX_ERR(1, 103, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 103, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_7); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (__pyx_t_14) { + __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_5) < 0) __PYX_ERR(1, 103, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(__pyx_t_8); + __pyx_v_exc = __pyx_t_8; + + /* "reppy/robots.pyx":104 + * raise exceptions.MalformedUrl(exc) + * except TooManyRedirects as exc: + * raise exceptions.ExcessiveRedirects(exc) # <<<<<<<<<<<<<< + * + * def RobotsUrlMethod(cls, url): + */ + __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L5_except_error)) + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_6, function); + } + } + if (!__pyx_t_4) { + __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; + __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_16); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; + __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_16); + } else + #endif + { + __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_v_exc); + __Pyx_GIVEREF(__pyx_v_exc); + PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); + __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + } + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_Raise(__pyx_t_16, 0, 0, 0); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __PYX_ERR(1, 104, __pyx_L5_except_error) + } + goto __pyx_L5_except_error; + __pyx_L5_except_error:; + + /* "reppy/robots.pyx":75 + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): + * '''Get the robots.txt at the provided URL.''' + * try: # <<<<<<<<<<<<<< + * # Limit the size of the request + * kwargs['stream'] = True + */ + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L1_error; + __pyx_L7_try_return:; + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_1); + __Pyx_XGIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + goto __pyx_L0; + __pyx_L10_try_end:; + } + + /* "reppy/robots.pyx":73 + * return cls(url, as_bytes(content), expires) + * + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< + * '''Get the robots.txt at the provided URL.''' + * try: + */ + + /* function exit code */ + __pyx_r = Py_None; __Pyx_INCREF(Py_None); + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_7); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_XDECREF(__pyx_t_16); + __Pyx_XDECREF(__pyx_t_18); + __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_res); + __Pyx_XDECREF(__pyx_v_content); + __Pyx_XDECREF(__pyx_v_expires); + __Pyx_XDECREF(__pyx_v_exc); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":106 + * raise exceptions.ExcessiveRedirects(exc) + * + * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< + * '''Get the robots.txt URL that corresponds to the provided one.''' + * return as_string(CppRobots.robotsUrl(as_bytes(url))) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5reppy_6robots_6RobotsUrlMethod[] = "Get the robots.txt URL that corresponds to the provided one."; +static PyMethodDef __pyx_mdef_5reppy_6robots_7RobotsUrlMethod = {"RobotsUrlMethod", (PyCFunction)__pyx_pw_5reppy_6robots_7RobotsUrlMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_6RobotsUrlMethod}; +static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + CYTHON_UNUSED PyObject *__pyx_v_cls = 0; + PyObject *__pyx_v_url = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("RobotsUrlMethod (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_cls,&__pyx_n_s_url,0}; + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 106, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 106, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + } + __pyx_v_cls = values[0]; + __pyx_v_url = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 106, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5reppy_6robots_6RobotsUrlMethod(__pyx_self, __pyx_v_cls, __pyx_v_url); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, PyObject *__pyx_v_url) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + std::string __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + __Pyx_TraceFrameInit(__pyx_codeobj__8) + __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); + __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 106, 0, __PYX_ERR(1, 106, __pyx_L1_error)); + + /* "reppy/robots.pyx":108 + * def RobotsUrlMethod(cls, url): + * '''Get the robots.txt URL that corresponds to the provided one.''' + * return as_string(CppRobots.robotsUrl(as_bytes(url))) # <<<<<<<<<<<<<< + * + * cdef class Robots: + */ + __Pyx_TraceLine(108,0,__PYX_ERR(1, 108, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 108, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 108, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(Rep::Robots::robotsUrl(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 108, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 108, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":106 + * raise exceptions.ExcessiveRedirects(exc) + * + * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< + * '''Get the robots.txt URL that corresponds to the provided one.''' + * return as_string(CppRobots.robotsUrl(as_bytes(url))) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":127 + * cdef object expires + * + * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< + * self.url = url + * self.robots = new CppRobots(content) + */ + +/* Python wrapper */ +static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_url = 0; + std::string __pyx_v_content; + PyObject *__pyx_v_expires = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_url,&__pyx_n_s_content,&__pyx_n_s_expires,0}; + PyObject* values[3] = {0,0,0}; + values[2] = ((PyObject *)Py_None); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 127, __pyx_L3_error) + } + case 2: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); + if (value) { values[2] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 127, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_url = values[0]; + __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 127, __pyx_L3_error) + __pyx_v_expires = values[2]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 127, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5reppy_6robots_6Robots___init__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self), __pyx_v_url, __pyx_v_content, __pyx_v_expires); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_url, std::string __pyx_v_content, PyObject *__pyx_v_expires) { + int __pyx_r; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + Rep::Robots *__pyx_t_1; + __Pyx_RefNannySetupContext("__init__", 0); + __Pyx_TraceCall("__init__", __pyx_f[1], 127, 0, __PYX_ERR(1, 127, __pyx_L1_error)); + + /* "reppy/robots.pyx":128 + * + * def __init__(self, url, const string& content, expires=None): + * self.url = url # <<<<<<<<<<<<<< + * self.robots = new CppRobots(content) + * self.expires = expires + */ + __Pyx_TraceLine(128,0,__PYX_ERR(1, 128, __pyx_L1_error)) + __Pyx_INCREF(__pyx_v_url); + __Pyx_GIVEREF(__pyx_v_url); + __Pyx_GOTREF(__pyx_v_self->url); + __Pyx_DECREF(__pyx_v_self->url); + __pyx_v_self->url = __pyx_v_url; + + /* "reppy/robots.pyx":129 + * def __init__(self, url, const string& content, expires=None): + * self.url = url + * self.robots = new CppRobots(content) # <<<<<<<<<<<<<< + * self.expires = expires + * + */ + __Pyx_TraceLine(129,0,__PYX_ERR(1, 129, __pyx_L1_error)) + try { + __pyx_t_1 = new Rep::Robots(__pyx_v_content); + } catch(...) { + try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } + __PYX_ERR(1, 129, __pyx_L1_error) + } + __pyx_v_self->robots = __pyx_t_1; + + /* "reppy/robots.pyx":130 + * self.url = url + * self.robots = new CppRobots(content) + * self.expires = expires # <<<<<<<<<<<<<< + * + * def __dealloc__(self): + */ + __Pyx_TraceLine(130,0,__PYX_ERR(1, 130, __pyx_L1_error)) + __Pyx_INCREF(__pyx_v_expires); + __Pyx_GIVEREF(__pyx_v_expires); + __Pyx_GOTREF(__pyx_v_self->expires); + __Pyx_DECREF(__pyx_v_self->expires); + __pyx_v_self->expires = __pyx_v_expires; + + /* "reppy/robots.pyx":127 + * cdef object expires + * + * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< + * self.url = url + * self.robots = new CppRobots(content) + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_TraceReturn(Py_None, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":132 + * self.expires = expires + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * del self.robots + * + */ + +/* Python wrapper */ +static void __pyx_pw_5reppy_6robots_6Robots_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_pw_5reppy_6robots_6Robots_3__dealloc__(PyObject *__pyx_v_self) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); + __pyx_pf_5reppy_6robots_6Robots_2__dealloc__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); +} + +static void __pyx_pf_5reppy_6robots_6Robots_2__dealloc__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self) { + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__dealloc__", 0); + __Pyx_TraceCall("__dealloc__", __pyx_f[1], 132, 0, __PYX_ERR(1, 132, __pyx_L1_error)); + + /* "reppy/robots.pyx":133 + * + * def __dealloc__(self): + * del self.robots # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_TraceLine(133,0,__PYX_ERR(1, 133, __pyx_L1_error)) + delete __pyx_v_self->robots; + + /* "reppy/robots.pyx":132 + * self.expires = expires + * + * def __dealloc__(self): # <<<<<<<<<<<<<< + * del self.robots + * + */ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_WriteUnraisable("reppy.robots.Robots.__dealloc__", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); + __pyx_L0:; + __Pyx_TraceReturn(Py_None, 0); + __Pyx_RefNannyFinishContext(); +} + +/* "reppy/robots.pyx":136 + * + * @property + * def sitemaps(self): # <<<<<<<<<<<<<< + * '''Get all the sitemaps in this robots.txt.''' + * return map(as_string, self.robots.sitemaps()) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_8sitemaps_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_8sitemaps_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_TraceCall("__get__", __pyx_f[1], 136, 0, __PYX_ERR(1, 136, __pyx_L1_error)); + + /* "reppy/robots.pyx":138 + * def sitemaps(self): + * '''Get all the sitemaps in this robots.txt.''' + * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< + * + * def allowed(self, path, name): + */ + __Pyx_TraceLine(138,0,__PYX_ERR(1, 138, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 138, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 138, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 138, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); + __pyx_t_1 = 0; + __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 138, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":136 + * + * @property + * def sitemaps(self): # <<<<<<<<<<<<<< + * '''Get all the sitemaps in this robots.txt.''' + * return map(as_string, self.robots.sitemaps()) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("reppy.robots.Robots.sitemaps.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":140 + * return map(as_string, self.robots.sitemaps()) + * + * def allowed(self, path, name): # <<<<<<<<<<<<<< + * '''Is the provided path allowed for the provided agant?''' + * return self.robots.allowed(as_bytes(path), as_bytes(name)) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_5allowed(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5reppy_6robots_6Robots_4allowed[] = "Is the provided path allowed for the provided agant?"; +static PyObject *__pyx_pw_5reppy_6robots_6Robots_5allowed(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_path = 0; + PyObject *__pyx_v_name = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("allowed (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_path,&__pyx_n_s_name,0}; + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 140, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 140, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + } + __pyx_v_path = values[0]; + __pyx_v_name = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 140, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_4allowed(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self), __pyx_v_path, __pyx_v_name); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6Robots_4allowed(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_name) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + std::string __pyx_t_2; + std::string __pyx_t_3; + __Pyx_RefNannySetupContext("allowed", 0); + __Pyx_TraceCall("allowed", __pyx_f[1], 140, 0, __PYX_ERR(1, 140, __pyx_L1_error)); + + /* "reppy/robots.pyx":142 + * def allowed(self, path, name): + * '''Is the provided path allowed for the provided agant?''' + * return self.robots.allowed(as_bytes(path), as_bytes(name)) # <<<<<<<<<<<<<< + * + * def agent(self, name): + */ + __Pyx_TraceLine(142,0,__PYX_ERR(1, 142, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 142, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 142, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 142, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 142, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 142, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":140 + * return map(as_string, self.robots.sitemaps()) + * + * def allowed(self, path, name): # <<<<<<<<<<<<<< + * '''Is the provided path allowed for the provided agant?''' + * return self.robots.allowed(as_bytes(path), as_bytes(name)) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":144 + * return self.robots.allowed(as_bytes(path), as_bytes(name)) + * + * def agent(self, name): # <<<<<<<<<<<<<< + * '''Return the Agent that corresponds to name.''' + * return Agent.from_robots(self, as_bytes(name)) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_7agent(PyObject *__pyx_v_self, PyObject *__pyx_v_name); /*proto*/ +static char __pyx_doc_5reppy_6robots_6Robots_6agent[] = "Return the Agent that corresponds to name."; +static PyObject *__pyx_pw_5reppy_6robots_6Robots_7agent(PyObject *__pyx_v_self, PyObject *__pyx_v_name) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("agent (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_6agent(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self), ((PyObject *)__pyx_v_name)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_name) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + int __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + __Pyx_RefNannySetupContext("agent", 0); + __Pyx_TraceCall("agent", __pyx_f[1], 144, 0, __PYX_ERR(1, 144, __pyx_L1_error)); + + /* "reppy/robots.pyx":146 + * def agent(self, name): + * '''Return the Agent that corresponds to name.''' + * return Agent.from_robots(self, as_bytes(name)) # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_TraceLine(146,0,__PYX_ERR(1, 146, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = NULL; + __pyx_t_5 = 0; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + __pyx_t_5 = 1; + } + } + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + if (__pyx_t_4) { + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; + } + __Pyx_INCREF(((PyObject *)__pyx_v_self)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_self)); + PyTuple_SET_ITEM(__pyx_t_6, 0+__pyx_t_5, ((PyObject *)__pyx_v_self)); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":144 + * return self.robots.allowed(as_bytes(path), as_bytes(name)) + * + * def agent(self, name): # <<<<<<<<<<<<<< + * '''Return the Agent that corresponds to name.''' + * return Agent.from_robots(self, as_bytes(name)) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("reppy.robots.Robots.agent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":149 + * + * @property + * def expired(self): # <<<<<<<<<<<<<< + * '''True if the current time is past its expiration.''' + * return time.time() > self.expires + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_7expired_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_7expired_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_7expired___get__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_TraceCall("__get__", __pyx_f[1], 149, 0, __PYX_ERR(1, 149, __pyx_L1_error)); + + /* "reppy/robots.pyx":151 + * def expired(self): + * '''True if the current time is past its expiration.''' + * return time.time() > self.expires # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_TraceLine(151,0,__PYX_ERR(1, 151, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 151, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 151, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + if (likely(__pyx_t_2)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_3, function); + } + } + if (__pyx_t_2) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 151, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else { + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 151, __pyx_L1_error) + } + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 151, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":149 + * + * @property + * def expired(self): # <<<<<<<<<<<<<< + * '''True if the current time is past its expiration.''' + * return time.time() > self.expires + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("reppy.robots.Robots.expired.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":154 + * + * @property + * def ttl(self): # <<<<<<<<<<<<<< + * '''Remaining time for this response to be considered valid.''' + * return max(self.expires - time.time(), 0) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_3ttl_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_3ttl_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_3ttl___get__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + long __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + int __pyx_t_6; + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_TraceCall("__get__", __pyx_f[1], 154, 0, __PYX_ERR(1, 154, __pyx_L1_error)); + + /* "reppy/robots.pyx":156 + * def ttl(self): + * '''Remaining time for this response to be considered valid.''' + * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< + * + * + */ + __Pyx_TraceLine(156,0,__PYX_ERR(1, 156, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = 0; + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + } + } + if (__pyx_t_3) { + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 156, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else { + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 156, __pyx_L1_error) + } + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 156, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 156, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (__pyx_t_6) { + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 156, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_2 = __pyx_t_5; + __pyx_t_5 = 0; + } else { + __Pyx_INCREF(__pyx_t_4); + __pyx_t_2 = __pyx_t_4; + } + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_INCREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":154 + * + * @property + * def ttl(self): # <<<<<<<<<<<<<< + * '''Remaining time for this response to be considered valid.''' + * return max(self.expires - time.time(), 0) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("reppy.robots.Robots.ttl.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":162 + * '''No requests are allowed.''' + * + * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< + * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) + * + */ + +/* Python wrapper */ +static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_url = 0; + PyObject *__pyx_v_expires = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_url,&__pyx_n_s_expires,0}; + PyObject* values[2] = {0,0}; + values[1] = ((PyObject *)Py_None); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); + if (value) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 162, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_url = values[0]; + __pyx_v_expires = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 162, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5reppy_6robots_9AllowNone___init__(((struct __pyx_obj_5reppy_6robots_AllowNone *)__pyx_v_self), __pyx_v_url, __pyx_v_expires); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6robots_AllowNone *__pyx_v_self, PyObject *__pyx_v_url, PyObject *__pyx_v_expires) { + int __pyx_r; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannySetupContext("__init__", 0); + __Pyx_TraceCall("__init__", __pyx_f[1], 162, 0, __PYX_ERR(1, 162, __pyx_L1_error)); + + /* "reppy/robots.pyx":163 + * + * def __init__(self, url, expires=None): + * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< + * + * + */ + __Pyx_TraceLine(163,0,__PYX_ERR(1, 163, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = NULL; + __pyx_t_4 = 0; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + __pyx_t_4 = 1; + } + } + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 163, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 163, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + { + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (__pyx_t_3) { + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; + } + __Pyx_INCREF(((PyObject *)__pyx_v_self)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_self)); + PyTuple_SET_ITEM(__pyx_t_5, 0+__pyx_t_4, ((PyObject *)__pyx_v_self)); + __Pyx_INCREF(__pyx_v_url); + __Pyx_GIVEREF(__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_4, __pyx_v_url); + __Pyx_INCREF(__pyx_kp_b_User_agent_Disallow); + __Pyx_GIVEREF(__pyx_kp_b_User_agent_Disallow); + PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b_User_agent_Disallow); + __Pyx_INCREF(__pyx_v_expires); + __Pyx_GIVEREF(__pyx_v_expires); + PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "reppy/robots.pyx":162 + * '''No requests are allowed.''' + * + * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< + * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) + * + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_TraceReturn(Py_None, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":169 + * '''All requests are allowed.''' + * + * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< + * Robots.__init__(self, url, b'', expires) + */ + +/* Python wrapper */ +static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_url = 0; + PyObject *__pyx_v_expires = 0; + int __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__init__ (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_url,&__pyx_n_s_expires,0}; + PyObject* values[2] = {0,0}; + values[1] = ((PyObject *)Py_None); + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (kw_args > 0) { + PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); + if (value) { values[1] = value; kw_args--; } + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 169, __pyx_L3_error) + } + } else { + switch (PyTuple_GET_SIZE(__pyx_args)) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + break; + default: goto __pyx_L5_argtuple_error; + } + } + __pyx_v_url = values[0]; + __pyx_v_expires = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 169, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return -1; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5reppy_6robots_8AllowAll___init__(((struct __pyx_obj_5reppy_6robots_AllowAll *)__pyx_v_self), __pyx_v_url, __pyx_v_expires); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6robots_AllowAll *__pyx_v_self, PyObject *__pyx_v_url, PyObject *__pyx_v_expires) { + int __pyx_r; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; + __Pyx_RefNannySetupContext("__init__", 0); + __Pyx_TraceCall("__init__", __pyx_f[1], 169, 0, __PYX_ERR(1, 169, __pyx_L1_error)); + + /* "reppy/robots.pyx":170 + * + * def __init__(self, url, expires=None): + * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< + */ + __Pyx_TraceLine(170,0,__PYX_ERR(1, 170, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = NULL; + __pyx_t_4 = 0; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + __pyx_t_4 = 1; + } + } + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + { + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (__pyx_t_3) { + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; + } + __Pyx_INCREF(((PyObject *)__pyx_v_self)); + __Pyx_GIVEREF(((PyObject *)__pyx_v_self)); + PyTuple_SET_ITEM(__pyx_t_5, 0+__pyx_t_4, ((PyObject *)__pyx_v_self)); + __Pyx_INCREF(__pyx_v_url); + __Pyx_GIVEREF(__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_4, __pyx_v_url); + __Pyx_INCREF(__pyx_kp_b__9); + __Pyx_GIVEREF(__pyx_kp_b__9); + PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b__9); + __Pyx_INCREF(__pyx_v_expires); + __Pyx_GIVEREF(__pyx_v_expires); + PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "reppy/robots.pyx":169 + * '''All requests are allowed.''' + * + * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< + * Robots.__init__(self, url, b'', expires) + */ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_TraceReturn(Py_None, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + +static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { + Py_ssize_t __pyx_v_length; + char *__pyx_v_data; + std::string __pyx_r; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + char *__pyx_t_1; + __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); + __Pyx_TraceCall("__pyx_convert_string_from_py_std__in_string", __pyx_f[2], 13, 0, __PYX_ERR(2, 13, __pyx_L1_error)); + + /* "string.from_py":15 + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< + * return string(data, length) + * + */ + __Pyx_TraceLine(15,0,__PYX_ERR(2, 15, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(2, 15, __pyx_L1_error) + __pyx_v_data = __pyx_t_1; + + /* "string.from_py":16 + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * return string(data, length) # <<<<<<<<<<<<<< + * + * + */ + __Pyx_TraceLine(16,0,__PYX_ERR(2, 16, __pyx_L1_error)) + __pyx_r = std::string(__pyx_v_data, __pyx_v_length); + goto __pyx_L0; + + /* "string.from_py":13 + * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_L0:; + __Pyx_TraceReturn(Py_None, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":31 + * + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyObject_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyObject_string_to_py_std__in_string", 0); + __Pyx_TraceCall("__pyx_convert_PyObject_string_to_py_std__in_string", __pyx_f[2], 31, 0, __PYX_ERR(2, 31, __pyx_L1_error)); + + /* "string.to_py":32 + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyUnicode_FromStringAndSize(char*, size_t) + */ + __Pyx_TraceLine(32,0,__PYX_ERR(2, 32, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyObject_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 32, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":31 + * + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyObject_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":37 + * + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyUnicode_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyUnicode_string_to_py_std__in_string", 0); + __Pyx_TraceCall("__pyx_convert_PyUnicode_string_to_py_std__in_string", __pyx_f[2], 37, 0, __PYX_ERR(2, 37, __pyx_L1_error)); + + /* "string.to_py":38 + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyStr_FromStringAndSize(char*, size_t) + */ + __Pyx_TraceLine(38,0,__PYX_ERR(2, 38, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyUnicode_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":37 + * + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyUnicode_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":43 + * + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyStr_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyStr_string_to_py_std__in_string", 0); + __Pyx_TraceCall("__pyx_convert_PyStr_string_to_py_std__in_string", __pyx_f[2], 43, 0, __PYX_ERR(2, 43, __pyx_L1_error)); + + /* "string.to_py":44 + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyBytes_FromStringAndSize(char*, size_t) + */ + __Pyx_TraceLine(44,0,__PYX_ERR(2, 44, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyStr_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":43 + * + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyStr_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":49 + * + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyBytes_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyBytes_string_to_py_std__in_string", 0); + __Pyx_TraceCall("__pyx_convert_PyBytes_string_to_py_std__in_string", __pyx_f[2], 49, 0, __PYX_ERR(2, 49, __pyx_L1_error)); + + /* "string.to_py":50 + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * cdef extern from *: + * cdef object __Pyx_PyByteArray_FromStringAndSize(char*, size_t) + */ + __Pyx_TraceLine(50,0,__PYX_ERR(2, 50, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 50, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":49 + * + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyBytes_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "string.to_py":55 + * + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) + * + */ + +static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_string(std::string const &__pyx_v_s) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_PyByteArray_string_to_py_std__in_string", 0); + __Pyx_TraceCall("__pyx_convert_PyByteArray_string_to_py_std__in_string", __pyx_f[2], 55, 0, __PYX_ERR(2, 55, __pyx_L1_error)); + + /* "string.to_py":56 + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< + * + */ + __Pyx_TraceLine(56,0,__PYX_ERR(2, 56, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __Pyx_PyByteArray_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "string.to_py":55 + * + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("string.to_py.__pyx_convert_PyByteArray_string_to_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cfunc.to_py":65 + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): + * def wrap(object value): # <<<<<<<<<<<<<< + * """wrap(value)""" + * return f(value) + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_value); /*proto*/ +static char __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap[] = "wrap(value)"; +static PyMethodDef __pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap = {"wrap", (PyCFunction)__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, METH_O, __pyx_doc_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap}; +static PyObject *__pyx_pw_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap(PyObject *__pyx_self, PyObject *__pyx_v_value) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("wrap (wrapper)", 0); + __pyx_r = __pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(__pyx_self, ((PyObject *)__pyx_v_value)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_value) { + struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; + struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_outer_scope; + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("wrap", 0); + __pyx_outer_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_TraceCall("wrap", __pyx_f[2], 65, 0, __PYX_ERR(2, 65, __pyx_L1_error)); + + /* "cfunc.to_py":67 + * def wrap(object value): + * """wrap(value)""" + * return f(value) # <<<<<<<<<<<<<< + * return wrap + * + */ + __Pyx_TraceLine(67,0,__PYX_ERR(2, 67, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_cur_scope->__pyx_v_f(__pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 67, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "cfunc.to_py":65 + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): + * def wrap(object value): # <<<<<<<<<<<<<< + * """wrap(value)""" + * return f(value) + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py.wrap", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cfunc.to_py":64 + * + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< + * def wrap(object value): + * """wrap(value)""" + */ + +static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyObject *)) { + struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_cur_scope; + PyObject *__pyx_v_wrap = 0; + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__Pyx_CFunc_object____object___to_py", 0); + __pyx_cur_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(2, 64, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __Pyx_TraceCall("__Pyx_CFunc_object____object___to_py", __pyx_f[2], 64, 0, __PYX_ERR(2, 64, __pyx_L1_error)); + __pyx_cur_scope->__pyx_v_f = __pyx_v_f; + + /* "cfunc.to_py":65 + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): + * def wrap(object value): # <<<<<<<<<<<<<< + * """wrap(value)""" + * return f(value) + */ + __Pyx_TraceLine(65,0,__PYX_ERR(2, 65, __pyx_L1_error)) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_wrap = __pyx_t_1; + __pyx_t_1 = 0; + + /* "cfunc.to_py":68 + * """wrap(value)""" + * return f(value) + * return wrap # <<<<<<<<<<<<<< + * + * + */ + __Pyx_TraceLine(68,0,__PYX_ERR(2, 68, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_wrap); + __pyx_r = __pyx_v_wrap; + goto __pyx_L0; + + /* "cfunc.to_py":64 + * + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< + * def wrap(object value): + * """wrap(value)""" + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cfunc.to_py.__Pyx_CFunc_object____object___to_py", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_wrap); + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "vector.to_py":67 + * + * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") + * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): # <<<<<<<<<<<<<< + * return [X_to_py(v[i]) for i in range(v.size())] + * + */ + +static PyObject *__pyx_convert_vector_to_py_std_3a__3a_string(const std::vector &__pyx_v_v) { + size_t __pyx_v_i; + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + size_t __pyx_t_2; + size_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + __Pyx_RefNannySetupContext("__pyx_convert_vector_to_py_std_3a__3a_string", 0); + __Pyx_TraceCall("__pyx_convert_vector_to_py_std_3a__3a_string", __pyx_f[2], 67, 0, __PYX_ERR(2, 67, __pyx_L1_error)); + + /* "vector.to_py":68 + * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") + * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): + * return [X_to_py(v[i]) for i in range(v.size())] # <<<<<<<<<<<<<< + * + * + */ + __Pyx_TraceLine(68,0,__PYX_ERR(2, 68, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_v_v.size(); + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + __pyx_t_4 = __pyx_convert_PyObject_string_to_py_std__in_string((__pyx_v_v[__pyx_v_i])); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "vector.to_py":67 + * + * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") + * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): # <<<<<<<<<<<<<< + * return [X_to_py(v[i]) for i in range(v.size())] + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("vector.to_py.__pyx_convert_vector_to_py_std_3a__3a_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_tp_new_5reppy_6robots_Agent(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5reppy_6robots_Agent *p; + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5reppy_6robots_Agent *)o); + new((void*)&(p->agent)) Rep::Agent(); + return o; +} + +static void __pyx_tp_dealloc_5reppy_6robots_Agent(PyObject *o) { + struct __pyx_obj_5reppy_6robots_Agent *p = (struct __pyx_obj_5reppy_6robots_Agent *)o; + #if PY_VERSION_HEX >= 0x030400a1 + if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + __Pyx_call_destructor(p->agent); + (*Py_TYPE(o)->tp_free)(o); +} + +static PyObject *__pyx_getprop_5reppy_6robots_5Agent_delay(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5reppy_6robots_5Agent_5delay_1__get__(o); +} + +static PyMethodDef __pyx_methods_5reppy_6robots_Agent[] = { + {"allow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_1allow, METH_O, __pyx_doc_5reppy_6robots_5Agent_allow}, + {"disallow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_3disallow, METH_O, __pyx_doc_5reppy_6robots_5Agent_2disallow}, + {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_5allowed, METH_O, __pyx_doc_5reppy_6robots_5Agent_4allowed}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5reppy_6robots_Agent[] = { + {(char *)"delay", __pyx_getprop_5reppy_6robots_5Agent_delay, 0, (char *)"The delay associated with this agent.", 0}, + {0, 0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5reppy_6robots_Agent = { + PyVarObject_HEAD_INIT(0, 0) + "reppy.robots.Agent", /*tp_name*/ + sizeof(struct __pyx_obj_5reppy_6robots_Agent), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5reppy_6robots_Agent, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "Wrapper around rep-cpp's Rep::Agent class.", /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5reppy_6robots_Agent, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5reppy_6robots_Agent, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5reppy_6robots_Agent, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyObject *__pyx_tp_new_5reppy_6robots_Robots(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + struct __pyx_obj_5reppy_6robots_Robots *p; + PyObject *o; + if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) { + o = (*t->tp_alloc)(t, 0); + } else { + o = (PyObject *) PyBaseObject_Type.tp_new(t, __pyx_empty_tuple, 0); + } + if (unlikely(!o)) return 0; + p = ((struct __pyx_obj_5reppy_6robots_Robots *)o); + p->url = Py_None; Py_INCREF(Py_None); + p->expires = Py_None; Py_INCREF(Py_None); + return o; +} + +static void __pyx_tp_dealloc_5reppy_6robots_Robots(PyObject *o) { + struct __pyx_obj_5reppy_6robots_Robots *p = (struct __pyx_obj_5reppy_6robots_Robots *)o; + #if PY_VERSION_HEX >= 0x030400a1 + if (unlikely(Py_TYPE(o)->tp_finalize) && !_PyGC_FINALIZED(o)) { + if (PyObject_CallFinalizerFromDealloc(o)) return; + } + #endif + PyObject_GC_UnTrack(o); + { + PyObject *etype, *eval, *etb; + PyErr_Fetch(&etype, &eval, &etb); + ++Py_REFCNT(o); + __pyx_pw_5reppy_6robots_6Robots_3__dealloc__(o); + --Py_REFCNT(o); + PyErr_Restore(etype, eval, etb); + } + Py_CLEAR(p->url); + Py_CLEAR(p->expires); + (*Py_TYPE(o)->tp_free)(o); +} + +static int __pyx_tp_traverse_5reppy_6robots_Robots(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5reppy_6robots_Robots *p = (struct __pyx_obj_5reppy_6robots_Robots *)o; + if (p->url) { + e = (*v)(p->url, a); if (e) return e; + } + if (p->expires) { + e = (*v)(p->expires, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5reppy_6robots_Robots(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5reppy_6robots_Robots *p = (struct __pyx_obj_5reppy_6robots_Robots *)o; + tmp = ((PyObject*)p->url); + p->url = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->expires); + p->expires = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyObject *__pyx_getprop_5reppy_6robots_6Robots_sitemaps(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5reppy_6robots_6Robots_8sitemaps_1__get__(o); +} + +static PyObject *__pyx_getprop_5reppy_6robots_6Robots_expired(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5reppy_6robots_6Robots_7expired_1__get__(o); +} + +static PyObject *__pyx_getprop_5reppy_6robots_6Robots_ttl(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5reppy_6robots_6Robots_3ttl_1__get__(o); +} + +static PyMethodDef __pyx_methods_5reppy_6robots_Robots[] = { + {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_5allowed, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_6Robots_4allowed}, + {"agent", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_7agent, METH_O, __pyx_doc_5reppy_6robots_6Robots_6agent}, + {0, 0, 0, 0} +}; + +static struct PyGetSetDef __pyx_getsets_5reppy_6robots_Robots[] = { + {(char *)"sitemaps", __pyx_getprop_5reppy_6robots_6Robots_sitemaps, 0, (char *)"Get all the sitemaps in this robots.txt.", 0}, + {(char *)"expired", __pyx_getprop_5reppy_6robots_6Robots_expired, 0, (char *)"True if the current time is past its expiration.", 0}, + {(char *)"ttl", __pyx_getprop_5reppy_6robots_6Robots_ttl, 0, (char *)"Remaining time for this response to be considered valid.", 0}, + {0, 0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5reppy_6robots_Robots = { + PyVarObject_HEAD_INIT(0, 0) + "reppy.robots.Robots", /*tp_name*/ + sizeof(struct __pyx_obj_5reppy_6robots_Robots), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5reppy_6robots_Robots, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + "Wrapper around rep-cpp's Rep::Robots class.", /*tp_doc*/ + __pyx_tp_traverse_5reppy_6robots_Robots, /*tp_traverse*/ + __pyx_tp_clear_5reppy_6robots_Robots, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5reppy_6robots_Robots, /*tp_methods*/ + 0, /*tp_members*/ + __pyx_getsets_5reppy_6robots_Robots, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + __pyx_pw_5reppy_6robots_6Robots_1__init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5reppy_6robots_Robots, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyObject *__pyx_tp_new_5reppy_6robots_AllowNone(PyTypeObject *t, PyObject *a, PyObject *k) { + PyObject *o = __pyx_tp_new_5reppy_6robots_Robots(t, a, k); + if (unlikely(!o)) return 0; + return o; +} + +static PyMethodDef __pyx_methods_5reppy_6robots_AllowNone[] = { + {0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5reppy_6robots_AllowNone = { + PyVarObject_HEAD_INIT(0, 0) + "reppy.robots.AllowNone", /*tp_name*/ + sizeof(struct __pyx_obj_5reppy_6robots_AllowNone), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5reppy_6robots_Robots, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + "No requests are allowed.", /*tp_doc*/ + __pyx_tp_traverse_5reppy_6robots_Robots, /*tp_traverse*/ + __pyx_tp_clear_5reppy_6robots_Robots, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5reppy_6robots_AllowNone, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + __pyx_pw_5reppy_6robots_9AllowNone_1__init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5reppy_6robots_AllowNone, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyObject *__pyx_tp_new_5reppy_6robots_AllowAll(PyTypeObject *t, PyObject *a, PyObject *k) { + PyObject *o = __pyx_tp_new_5reppy_6robots_Robots(t, a, k); + if (unlikely(!o)) return 0; + return o; +} + +static PyMethodDef __pyx_methods_5reppy_6robots_AllowAll[] = { + {0, 0, 0, 0} +}; + +static PyTypeObject __pyx_type_5reppy_6robots_AllowAll = { + PyVarObject_HEAD_INIT(0, 0) + "reppy.robots.AllowAll", /*tp_name*/ + sizeof(struct __pyx_obj_5reppy_6robots_AllowAll), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc_5reppy_6robots_Robots, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_BASETYPE|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + "All requests are allowed.", /*tp_doc*/ + __pyx_tp_traverse_5reppy_6robots_Robots, /*tp_traverse*/ + __pyx_tp_clear_5reppy_6robots_Robots, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + __pyx_methods_5reppy_6robots_AllowAll, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + __pyx_pw_5reppy_6robots_8AllowAll_1__init__, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new_5reppy_6robots_AllowAll, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[8]; +static int __pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; + +static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { + o = (PyObject*)__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[--__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py]; + memset(o, 0, sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)); + (void) PyObject_INIT(o, t); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyObject *o) { + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { + __pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py++] = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { + PyVarObject_HEAD_INIT(0, 0) + "reppy.robots.__pyx_scope_struct____Pyx_CFunc_object____object___to_py", /*tp_name*/ + sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef __pyx_moduledef = { + #if PY_VERSION_HEX < 0x03020000 + { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, + #else + PyModuleDef_HEAD_INIT, + #endif + "robots", + 0, /* m_doc */ + -1, /* m_size */ + __pyx_methods /* m_methods */, + NULL, /* m_reload */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ +}; +#endif + +static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_n_s_BadStatusCode, __pyx_k_BadStatusCode, sizeof(__pyx_k_BadStatusCode), 0, 0, 1, 1}, + {&__pyx_n_s_ConnectionError, __pyx_k_ConnectionError, sizeof(__pyx_k_ConnectionError), 0, 0, 1, 1}, + {&__pyx_n_s_ConnectionException, __pyx_k_ConnectionException, sizeof(__pyx_k_ConnectionException), 0, 0, 1, 1}, + {&__pyx_n_s_ContentTooLong, __pyx_k_ContentTooLong, sizeof(__pyx_k_ContentTooLong), 0, 0, 1, 1}, + {&__pyx_kp_s_Content_larger_than_s_bytes, __pyx_k_Content_larger_than_s_bytes, sizeof(__pyx_k_Content_larger_than_s_bytes), 0, 0, 1, 0}, + {&__pyx_n_s_DEFAULT_TTL_POLICY, __pyx_k_DEFAULT_TTL_POLICY, sizeof(__pyx_k_DEFAULT_TTL_POLICY), 0, 0, 1, 1}, + {&__pyx_n_s_ExcessiveRedirects, __pyx_k_ExcessiveRedirects, sizeof(__pyx_k_ExcessiveRedirects), 0, 0, 1, 1}, + {&__pyx_n_s_FetchMethod, __pyx_k_FetchMethod, sizeof(__pyx_k_FetchMethod), 0, 0, 1, 1}, + {&__pyx_n_s_FromRobotsMethod, __pyx_k_FromRobotsMethod, sizeof(__pyx_k_FromRobotsMethod), 0, 0, 1, 1}, + {&__pyx_kp_s_Got_i_for_s, __pyx_k_Got_i_for_s, sizeof(__pyx_k_Got_i_for_s), 0, 0, 1, 0}, + {&__pyx_n_s_HeaderWithDefaultPolicy, __pyx_k_HeaderWithDefaultPolicy, sizeof(__pyx_k_HeaderWithDefaultPolicy), 0, 0, 1, 1}, + {&__pyx_n_s_InvalidSchema, __pyx_k_InvalidSchema, sizeof(__pyx_k_InvalidSchema), 0, 0, 1, 1}, + {&__pyx_n_s_InvalidURL, __pyx_k_InvalidURL, sizeof(__pyx_k_InvalidURL), 0, 0, 1, 1}, + {&__pyx_n_s_MalformedUrl, __pyx_k_MalformedUrl, sizeof(__pyx_k_MalformedUrl), 0, 0, 1, 1}, + {&__pyx_n_s_MissingSchema, __pyx_k_MissingSchema, sizeof(__pyx_k_MissingSchema), 0, 0, 1, 1}, + {&__pyx_n_s_PY3, __pyx_k_PY3, sizeof(__pyx_k_PY3), 0, 0, 1, 1}, + {&__pyx_n_s_ParseMethod, __pyx_k_ParseMethod, sizeof(__pyx_k_ParseMethod), 0, 0, 1, 1}, + {&__pyx_n_s_Pyx_CFunc_object____object___t, __pyx_k_Pyx_CFunc_object____object___t, sizeof(__pyx_k_Pyx_CFunc_object____object___t), 0, 0, 1, 1}, + {&__pyx_n_s_RobotsUrlMethod, __pyx_k_RobotsUrlMethod, sizeof(__pyx_k_RobotsUrlMethod), 0, 0, 1, 1}, + {&__pyx_n_s_SSLError, __pyx_k_SSLError, sizeof(__pyx_k_SSLError), 0, 0, 1, 1}, + {&__pyx_n_s_SSLException, __pyx_k_SSLException, sizeof(__pyx_k_SSLException), 0, 0, 1, 1}, + {&__pyx_n_s_TooManyRedirects, __pyx_k_TooManyRedirects, sizeof(__pyx_k_TooManyRedirects), 0, 0, 1, 1}, + {&__pyx_n_s_URLRequired, __pyx_k_URLRequired, sizeof(__pyx_k_URLRequired), 0, 0, 1, 1}, + {&__pyx_kp_b_User_agent_Disallow, __pyx_k_User_agent_Disallow, sizeof(__pyx_k_User_agent_Disallow), 0, 0, 0, 0}, + {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, + {&__pyx_n_s__9, __pyx_k__9, sizeof(__pyx_k__9), 0, 0, 1, 1}, + {&__pyx_kp_b__9, __pyx_k__9, sizeof(__pyx_k__9), 0, 0, 0, 0}, + {&__pyx_n_s_agent, __pyx_k_agent, sizeof(__pyx_k_agent), 0, 0, 1, 1}, + {&__pyx_n_s_amt, __pyx_k_amt, sizeof(__pyx_k_amt), 0, 0, 1, 1}, + {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, + {&__pyx_n_s_cfunc_to_py, __pyx_k_cfunc_to_py, sizeof(__pyx_k_cfunc_to_py), 0, 0, 1, 1}, + {&__pyx_n_s_closing, __pyx_k_closing, sizeof(__pyx_k_closing), 0, 0, 1, 1}, + {&__pyx_n_s_cls, __pyx_k_cls, sizeof(__pyx_k_cls), 0, 0, 1, 1}, + {&__pyx_n_s_content, __pyx_k_content, sizeof(__pyx_k_content), 0, 0, 1, 1}, + {&__pyx_n_s_contextlib, __pyx_k_contextlib, sizeof(__pyx_k_contextlib), 0, 0, 1, 1}, + {&__pyx_n_s_decode, __pyx_k_decode, sizeof(__pyx_k_decode), 0, 0, 1, 1}, + {&__pyx_n_s_decode_content, __pyx_k_decode_content, sizeof(__pyx_k_decode_content), 0, 0, 1, 1}, + {&__pyx_n_s_default, __pyx_k_default, sizeof(__pyx_k_default), 0, 0, 1, 1}, + {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, + {&__pyx_n_s_enter, __pyx_k_enter, sizeof(__pyx_k_enter), 0, 0, 1, 1}, + {&__pyx_n_s_exc, __pyx_k_exc, sizeof(__pyx_k_exc), 0, 0, 1, 1}, + {&__pyx_n_s_exceptions, __pyx_k_exceptions, sizeof(__pyx_k_exceptions), 0, 0, 1, 1}, + {&__pyx_n_s_exit, __pyx_k_exit, sizeof(__pyx_k_exit), 0, 0, 1, 1}, + {&__pyx_n_s_expires, __pyx_k_expires, sizeof(__pyx_k_expires), 0, 0, 1, 1}, + {&__pyx_n_s_fetch, __pyx_k_fetch, sizeof(__pyx_k_fetch), 0, 0, 1, 1}, + {&__pyx_n_s_from_robots, __pyx_k_from_robots, sizeof(__pyx_k_from_robots), 0, 0, 1, 1}, + {&__pyx_n_s_get, __pyx_k_get, sizeof(__pyx_k_get), 0, 0, 1, 1}, + {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, + {&__pyx_n_s_init, __pyx_k_init, sizeof(__pyx_k_init), 0, 0, 1, 1}, + {&__pyx_n_s_kwargs, __pyx_k_kwargs, sizeof(__pyx_k_kwargs), 0, 0, 1, 1}, + {&__pyx_n_s_logger, __pyx_k_logger, sizeof(__pyx_k_logger), 0, 0, 1, 1}, + {&__pyx_n_s_main, __pyx_k_main, sizeof(__pyx_k_main), 0, 0, 1, 1}, + {&__pyx_n_s_map, __pyx_k_map, sizeof(__pyx_k_map), 0, 0, 1, 1}, + {&__pyx_n_s_max_size, __pyx_k_max_size, sizeof(__pyx_k_max_size), 0, 0, 1, 1}, + {&__pyx_n_s_minimum, __pyx_k_minimum, sizeof(__pyx_k_minimum), 0, 0, 1, 1}, + {&__pyx_n_s_name, __pyx_k_name, sizeof(__pyx_k_name), 0, 0, 1, 1}, + {&__pyx_n_s_parse, __pyx_k_parse, sizeof(__pyx_k_parse), 0, 0, 1, 1}, + {&__pyx_n_s_path, __pyx_k_path, sizeof(__pyx_k_path), 0, 0, 1, 1}, + {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, + {&__pyx_n_s_raw, __pyx_k_raw, sizeof(__pyx_k_raw), 0, 0, 1, 1}, + {&__pyx_n_s_read, __pyx_k_read, sizeof(__pyx_k_read), 0, 0, 1, 1}, + {&__pyx_n_s_reppy_robots, __pyx_k_reppy_robots, sizeof(__pyx_k_reppy_robots), 0, 0, 1, 1}, + {&__pyx_n_s_requests, __pyx_k_requests, sizeof(__pyx_k_requests), 0, 0, 1, 1}, + {&__pyx_n_s_requests_exceptions, __pyx_k_requests_exceptions, sizeof(__pyx_k_requests_exceptions), 0, 0, 1, 1}, + {&__pyx_n_s_res, __pyx_k_res, sizeof(__pyx_k_res), 0, 0, 1, 1}, + {&__pyx_n_s_robots, __pyx_k_robots, sizeof(__pyx_k_robots), 0, 0, 1, 1}, + {&__pyx_n_s_robots_url, __pyx_k_robots_url, sizeof(__pyx_k_robots_url), 0, 0, 1, 1}, + {&__pyx_n_s_six, __pyx_k_six, sizeof(__pyx_k_six), 0, 0, 1, 1}, + {&__pyx_n_s_status_code, __pyx_k_status_code, sizeof(__pyx_k_status_code), 0, 0, 1, 1}, + {&__pyx_n_s_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 0, 0, 1, 1}, + {&__pyx_kp_s_stringsource, __pyx_k_stringsource, sizeof(__pyx_k_stringsource), 0, 0, 1, 0}, + {&__pyx_n_s_test, __pyx_k_test, sizeof(__pyx_k_test), 0, 0, 1, 1}, + {&__pyx_n_s_time, __pyx_k_time, sizeof(__pyx_k_time), 0, 0, 1, 1}, + {&__pyx_n_s_ttl, __pyx_k_ttl, sizeof(__pyx_k_ttl), 0, 0, 1, 1}, + {&__pyx_n_s_ttl_policy, __pyx_k_ttl_policy, sizeof(__pyx_k_ttl_policy), 0, 0, 1, 1}, + {&__pyx_n_s_url, __pyx_k_url, sizeof(__pyx_k_url), 0, 0, 1, 1}, + {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, + {&__pyx_n_s_util, __pyx_k_util, sizeof(__pyx_k_util), 0, 0, 1, 1}, + {&__pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_k_vagrant_reppy_robots_pyx, sizeof(__pyx_k_vagrant_reppy_robots_pyx), 0, 0, 1, 0}, + {&__pyx_n_s_value, __pyx_k_value, sizeof(__pyx_k_value), 0, 0, 1, 1}, + {&__pyx_n_s_wrap, __pyx_k_wrap, sizeof(__pyx_k_wrap), 0, 0, 1, 1}, + {0, 0, 0, 0, 0, 0, 0} +}; +static int __Pyx_InitCachedBuiltins(void) { + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 32, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 138, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +static int __Pyx_InitCachedConstants(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "reppy/robots.pyx":24 + * if isinstance(value, bytes): + * return value + * return value.encode('utf-8') # <<<<<<<<<<<<<< + * + * cdef as_string(value): + */ + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 24, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple_); + __Pyx_GIVEREF(__pyx_tuple_); + + /* "reppy/robots.pyx":29 + * if six.PY3: + * if isinstance(value, bytes): + * return value.decode('utf-8') # <<<<<<<<<<<<<< + * return value + * + */ + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 29, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__2); + __Pyx_GIVEREF(__pyx_tuple__2); + + /* "reppy/robots.pyx":78 + * # Limit the size of the request + * kwargs['stream'] = True + * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< + * content = res.raw.read(amt=max_size, decode_content=True) + * # Try to read an additional byte, to see if the response is too big + */ + __pyx_tuple__6 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 78, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__6); + __Pyx_GIVEREF(__pyx_tuple__6); + __pyx_tuple__7 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 78, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__7); + __Pyx_GIVEREF(__pyx_tuple__7); + + /* "cfunc.to_py":65 + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): + * def wrap(object value): # <<<<<<<<<<<<<< + * """wrap(value)""" + * return f(value) + */ + __pyx_tuple__10 = PyTuple_Pack(1, __pyx_n_s_value); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); + __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 65, __pyx_L1_error) + + /* "reppy/robots.pyx":33 + * + * + * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< + * '''Construct an Agent from a CppAgent.''' + * agent = Agent() + */ + __pyx_tuple__12 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(1, 33, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__12); + __Pyx_GIVEREF(__pyx_tuple__12); + __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__12, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(1, 33, __pyx_L1_error) + + /* "reppy/robots.pyx":69 + * + * + * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< + * '''Parse a robots.txt file.''' + * return cls(url, as_bytes(content), expires) + */ + __pyx_tuple__13 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(1, 69, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__13); + __Pyx_GIVEREF(__pyx_tuple__13); + __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 69, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 69, __pyx_L1_error) + + /* "reppy/robots.pyx":73 + * return cls(url, as_bytes(content), expires) + * + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< + * '''Get the robots.txt at the provided URL.''' + * try: + */ + __pyx_tuple__14 = PyTuple_Pack(10, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(1, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__14); + __Pyx_GIVEREF(__pyx_tuple__14); + __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 10, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 73, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 73, __pyx_L1_error) + + /* "reppy/robots.pyx":106 + * raise exceptions.ExcessiveRedirects(exc) + * + * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< + * '''Get the robots.txt URL that corresponds to the provided one.''' + * return as_string(CppRobots.robotsUrl(as_bytes(url))) + */ + __pyx_tuple__15 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__15); + __Pyx_GIVEREF(__pyx_tuple__15); + __pyx_codeobj__8 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 106, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__8)) __PYX_ERR(1, 106, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_InitGlobals(void) { + if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(1, 1, __pyx_L1_error); + __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_int_200 = PyInt_FromLong(200); if (unlikely(!__pyx_int_200)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_int_400 = PyInt_FromLong(400); if (unlikely(!__pyx_int_400)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_int_401 = PyInt_FromLong(401); if (unlikely(!__pyx_int_401)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_int_403 = PyInt_FromLong(403); if (unlikely(!__pyx_int_403)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_int_500 = PyInt_FromLong(500); if (unlikely(!__pyx_int_500)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_int_600 = PyInt_FromLong(600); if (unlikely(!__pyx_int_600)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_int_3600 = PyInt_FromLong(3600); if (unlikely(!__pyx_int_3600)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_int_1048576 = PyInt_FromLong(1048576L); if (unlikely(!__pyx_int_1048576)) __PYX_ERR(1, 1, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} + +#if PY_MAJOR_VERSION < 3 +PyMODINIT_FUNC initrobots(void); /*proto*/ +PyMODINIT_FUNC initrobots(void) +#else +PyMODINIT_FUNC PyInit_robots(void); /*proto*/ +PyMODINIT_FUNC PyInit_robots(void) +#endif +{ + __Pyx_TraceDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_RefNannyDeclarations + #if CYTHON_REFNANNY + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); + if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); + } + #endif + __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_robots(void)", 0); + if (__Pyx_check_binary_version() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(1, 1, __pyx_L1_error) + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_StopAsyncIteration_USED + if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + /*--- Threads initialization code ---*/ + #if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS + #ifdef WITH_THREAD /* Python build with threading support? */ + PyEval_InitThreads(); + #endif + #endif + /*--- Module creation code ---*/ + #if PY_MAJOR_VERSION < 3 + __pyx_m = Py_InitModule4("robots", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); + #else + __pyx_m = PyModule_Create(&__pyx_moduledef); + #endif + if (unlikely(!__pyx_m)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(1, 1, __pyx_L1_error) + Py_INCREF(__pyx_d); + __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(1, 1, __pyx_L1_error) + #if CYTHON_COMPILING_IN_PYPY + Py_INCREF(__pyx_b); + #endif + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(1, 1, __pyx_L1_error); + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitGlobals() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + if (__pyx_module_is_main_reppy__robots) { + if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(1, 1, __pyx_L1_error) + } + #if PY_MAJOR_VERSION >= 3 + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(1, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "reppy.robots")) { + if (unlikely(PyDict_SetItemString(modules, "reppy.robots", __pyx_m) < 0)) __PYX_ERR(1, 1, __pyx_L1_error) + } + } + #endif + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + /*--- Global init code ---*/ + /*--- Variable export code ---*/ + /*--- Function export code ---*/ + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 39, __pyx_L1_error) + __pyx_type_5reppy_6robots_Agent.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 39, __pyx_L1_error) + __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 110, __pyx_L1_error) + __pyx_type_5reppy_6robots_Robots.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 110, __pyx_L1_error) + __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; + __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 159, __pyx_L1_error) + __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 159, __pyx_L1_error) + __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; + __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 166, __pyx_L1_error) + __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 166, __pyx_L1_error) + __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; + if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) + __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; + __pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = &__pyx_scope_struct____Pyx_CFunc_object____object___to_py; + /*--- Type import code ---*/ + /*--- Variable import code ---*/ + /*--- Function import code ---*/ + /*--- Execution code ---*/ + #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) + if (__Pyx_patch_abc() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + #endif + __Pyx_TraceCall("PyMODINIT_FUNC PyInit_robots(void)", __pyx_f[1], 1, 0, __PYX_ERR(1, 1, __pyx_L1_error)); + + /* "reppy/robots.pyx":4 + * # distutils: define_macros=CYTHON_TRACE=1 + * + * from contextlib import closing # <<<<<<<<<<<<<< + * import time + * + */ + __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_n_s_closing); + __Pyx_GIVEREF(__pyx_n_s_closing); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_closing); + __pyx_t_2 = __Pyx_Import(__pyx_n_s_contextlib, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_closing); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_closing, __pyx_t_1) < 0) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "reppy/robots.pyx":5 + * + * from contextlib import closing + * import time # <<<<<<<<<<<<<< + * + * import requests + */ + __Pyx_TraceLine(5,0,__PYX_ERR(1, 5, __pyx_L1_error)) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_time, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_time, __pyx_t_2) < 0) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "reppy/robots.pyx":7 + * import time + * + * import requests # <<<<<<<<<<<<<< + * from requests.exceptions import ( + * SSLError, + */ + __Pyx_TraceLine(7,0,__PYX_ERR(1, 7, __pyx_L1_error)) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_requests, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_requests, __pyx_t_2) < 0) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "reppy/robots.pyx":9 + * import requests + * from requests.exceptions import ( + * SSLError, # <<<<<<<<<<<<<< + * ConnectionError, + * URLRequired, + */ + __Pyx_TraceLine(9,0,__PYX_ERR(1, 9, __pyx_L1_error)) + __pyx_t_2 = PyList_New(7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_n_s_SSLError); + __Pyx_GIVEREF(__pyx_n_s_SSLError); + PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_SSLError); + __Pyx_INCREF(__pyx_n_s_ConnectionError); + __Pyx_GIVEREF(__pyx_n_s_ConnectionError); + PyList_SET_ITEM(__pyx_t_2, 1, __pyx_n_s_ConnectionError); + __Pyx_INCREF(__pyx_n_s_URLRequired); + __Pyx_GIVEREF(__pyx_n_s_URLRequired); + PyList_SET_ITEM(__pyx_t_2, 2, __pyx_n_s_URLRequired); + __Pyx_INCREF(__pyx_n_s_MissingSchema); + __Pyx_GIVEREF(__pyx_n_s_MissingSchema); + PyList_SET_ITEM(__pyx_t_2, 3, __pyx_n_s_MissingSchema); + __Pyx_INCREF(__pyx_n_s_InvalidSchema); + __Pyx_GIVEREF(__pyx_n_s_InvalidSchema); + PyList_SET_ITEM(__pyx_t_2, 4, __pyx_n_s_InvalidSchema); + __Pyx_INCREF(__pyx_n_s_InvalidURL); + __Pyx_GIVEREF(__pyx_n_s_InvalidURL); + PyList_SET_ITEM(__pyx_t_2, 5, __pyx_n_s_InvalidURL); + __Pyx_INCREF(__pyx_n_s_TooManyRedirects); + __Pyx_GIVEREF(__pyx_n_s_TooManyRedirects); + PyList_SET_ITEM(__pyx_t_2, 6, __pyx_n_s_TooManyRedirects); + + /* "reppy/robots.pyx":8 + * + * import requests + * from requests.exceptions import ( # <<<<<<<<<<<<<< + * SSLError, + * ConnectionError, + */ + __Pyx_TraceLine(8,0,__PYX_ERR(1, 8, __pyx_L1_error)) + __pyx_t_1 = __Pyx_Import(__pyx_n_s_requests_exceptions, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_SSLError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_SSLError, __pyx_t_2) < 0) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ConnectionError, __pyx_t_2) < 0) __PYX_ERR(1, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_URLRequired); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_URLRequired, __pyx_t_2) < 0) __PYX_ERR(1, 11, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_MissingSchema, __pyx_t_2) < 0) __PYX_ERR(1, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_InvalidSchema, __pyx_t_2) < 0) __PYX_ERR(1, 13, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_InvalidURL, __pyx_t_2) < 0) __PYX_ERR(1, 14, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_TooManyRedirects, __pyx_t_2) < 0) __PYX_ERR(1, 15, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "reppy/robots.pyx":16 + * InvalidURL, + * TooManyRedirects) + * import six # <<<<<<<<<<<<<< + * + * from .ttl import HeaderWithDefaultPolicy + */ + __Pyx_TraceLine(16,0,__PYX_ERR(1, 16, __pyx_L1_error)) + __pyx_t_1 = __Pyx_Import(__pyx_n_s_six, 0, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 16, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_six, __pyx_t_1) < 0) __PYX_ERR(1, 16, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "reppy/robots.pyx":18 + * import six + * + * from .ttl import HeaderWithDefaultPolicy # <<<<<<<<<<<<<< + * from . import util, logger, exceptions + * + */ + __Pyx_TraceLine(18,0,__PYX_ERR(1, 18, __pyx_L1_error)) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_INCREF(__pyx_n_s_HeaderWithDefaultPolicy); + __Pyx_GIVEREF(__pyx_n_s_HeaderWithDefaultPolicy); + PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_HeaderWithDefaultPolicy); + __pyx_t_2 = __Pyx_Import(__pyx_n_s_ttl, __pyx_t_1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 18, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_HeaderWithDefaultPolicy, __pyx_t_1) < 0) __PYX_ERR(1, 18, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "reppy/robots.pyx":19 + * + * from .ttl import HeaderWithDefaultPolicy + * from . import util, logger, exceptions # <<<<<<<<<<<<<< + * + * cdef as_bytes(value): + */ + __Pyx_TraceLine(19,0,__PYX_ERR(1, 19, __pyx_L1_error)) + __pyx_t_2 = PyList_New(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_n_s_util); + __Pyx_GIVEREF(__pyx_n_s_util); + PyList_SET_ITEM(__pyx_t_2, 0, __pyx_n_s_util); + __Pyx_INCREF(__pyx_n_s_logger); + __Pyx_GIVEREF(__pyx_n_s_logger); + PyList_SET_ITEM(__pyx_t_2, 1, __pyx_n_s_logger); + __Pyx_INCREF(__pyx_n_s_exceptions); + __Pyx_GIVEREF(__pyx_n_s_exceptions); + PyList_SET_ITEM(__pyx_t_2, 2, __pyx_n_s_exceptions); + __pyx_t_1 = __Pyx_Import(__pyx_n_s__9, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_util); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_util, __pyx_t_2) < 0) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_logger); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_logger, __pyx_t_2) < 0) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_exceptions, __pyx_t_2) < 0) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "reppy/robots.pyx":33 + * + * + * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< + * '''Construct an Agent from a CppAgent.''' + * agent = Agent() + */ + __Pyx_TraceLine(33,0,__PYX_ERR(1, 33, __pyx_L1_error)) + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_1FromRobotsMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 33, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FromRobotsMethod, __pyx_t_1) < 0) __PYX_ERR(1, 33, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "reppy/robots.pyx":44 + * cdef CppAgent agent + * + * from_robots = classmethod(FromRobotsMethod) # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_TraceLine(44,0,__PYX_ERR(1, 44, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Agent->tp_dict, __pyx_n_s_from_robots, __pyx_t_2) < 0) __PYX_ERR(1, 44, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + PyType_Modified(__pyx_ptype_5reppy_6robots_Agent); + + /* "reppy/robots.pyx":69 + * + * + * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< + * '''Parse a robots.txt file.''' + * return cls(url, as_bytes(content), expires) + */ + __Pyx_TraceLine(69,0,__PYX_ERR(1, 69, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 69, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(1, 69, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "reppy/robots.pyx":73 + * return cls(url, as_bytes(content), expires) + * + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< + * '''Get the robots.txt at the provided URL.''' + * try: + */ + __Pyx_TraceLine(73,0,__PYX_ERR(1, 73, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 73, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(1, 73, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "reppy/robots.pyx":106 + * raise exceptions.ExcessiveRedirects(exc) + * + * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< + * '''Get the robots.txt URL that corresponds to the provided one.''' + * return as_string(CppRobots.robotsUrl(as_bytes(url))) + */ + __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 106, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 106, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "reppy/robots.pyx":115 + * # The default TTL policy is to cache for 3600 seconds or what's provided in the + * # headers, and a minimum of 600 seconds + * DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) # <<<<<<<<<<<<<< + * + * # Class methods + */ + __Pyx_TraceLine(115,0,__PYX_ERR(1, 115, __pyx_L1_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 115, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 115, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 115, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 115, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 115, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 115, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); + + /* "reppy/robots.pyx":118 + * + * # Class methods + * parse = classmethod(ParseMethod) # <<<<<<<<<<<<<< + * fetch = classmethod(FetchMethod) + * robots_url = classmethod(RobotsUrlMethod) + */ + __Pyx_TraceLine(118,0,__PYX_ERR(1, 118, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 118, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 118, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 118, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); + + /* "reppy/robots.pyx":119 + * # Class methods + * parse = classmethod(ParseMethod) + * fetch = classmethod(FetchMethod) # <<<<<<<<<<<<<< + * robots_url = classmethod(RobotsUrlMethod) + * + */ + __Pyx_TraceLine(119,0,__PYX_ERR(1, 119, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 119, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 119, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 119, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); + + /* "reppy/robots.pyx":120 + * parse = classmethod(ParseMethod) + * fetch = classmethod(FetchMethod) + * robots_url = classmethod(RobotsUrlMethod) # <<<<<<<<<<<<<< + * + * # Data members + */ + __Pyx_TraceLine(120,0,__PYX_ERR(1, 120, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 120, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 120, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 120, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); + + /* "reppy/robots.pyx":1 + * # cython: linetrace=True # <<<<<<<<<<<<<< + * # distutils: define_macros=CYTHON_TRACE=1 + * + */ + __Pyx_TraceLine(1,0,__PYX_ERR(1, 1, __pyx_L1_error)) + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "vector.to_py":67 + * + * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") + * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): # <<<<<<<<<<<<<< + * return [X_to_py(v[i]) for i in range(v.size())] + * + */ + __Pyx_TraceReturn(Py_None, 0); + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + if (__pyx_m) { + if (__pyx_d) { + __Pyx_AddTraceback("init reppy.robots", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + Py_DECREF(__pyx_m); __pyx_m = 0; + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init reppy.robots"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if PY_MAJOR_VERSION < 3 + return; + #else + return __pyx_m; + #endif +} + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule((char *)modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); + if (unlikely(!result)) { + PyErr_Format(PyExc_NameError, +#if PY_MAJOR_VERSION >= 3 + "name '%U' is not defined", name); +#else + "name '%.200s' is not defined", PyString_AS_STRING(name)); +#endif + } + return result; +} + +/* Profile */ +#if CYTHON_PROFILE +static int __Pyx_TraceSetupAndCall(PyCodeObject** code, + PyFrameObject** frame, + const char *funcname, + const char *srcfile, + int firstlineno) { + PyObject *type, *value, *traceback; + int retval; + PyThreadState* tstate = PyThreadState_GET(); + if (*frame == NULL || !CYTHON_PROFILE_REUSE_FRAME) { + if (*code == NULL) { + *code = __Pyx_createFrameCodeObject(funcname, srcfile, firstlineno); + if (*code == NULL) return 0; + } + *frame = PyFrame_New( + tstate, /*PyThreadState *tstate*/ + *code, /*PyCodeObject *code*/ + __pyx_d, /*PyObject *globals*/ + 0 /*PyObject *locals*/ + ); + if (*frame == NULL) return 0; + if (CYTHON_TRACE && (*frame)->f_trace == NULL) { + Py_INCREF(Py_None); + (*frame)->f_trace = Py_None; + } +#if PY_VERSION_HEX < 0x030400B1 + } else { + (*frame)->f_tstate = tstate; +#endif + } + __Pyx_PyFrame_SetLineNumber(*frame, firstlineno); + retval = 1; + tstate->tracing++; + tstate->use_tracing = 0; + PyErr_Fetch(&type, &value, &traceback); + #if CYTHON_TRACE + if (tstate->c_tracefunc) + retval = tstate->c_tracefunc(tstate->c_traceobj, *frame, PyTrace_CALL, NULL) == 0; + if (retval && tstate->c_profilefunc) + #endif + retval = tstate->c_profilefunc(tstate->c_profileobj, *frame, PyTrace_CALL, NULL) == 0; + tstate->use_tracing = (tstate->c_profilefunc || + (CYTHON_TRACE && tstate->c_tracefunc)); + tstate->tracing--; + if (retval) { + PyErr_Restore(type, value, traceback); + return tstate->use_tracing && retval; + } else { + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(traceback); + return -1; + } +} +static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const char *srcfile, int firstlineno) { + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + PyCodeObject *py_code = 0; + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + py_srcfile = PyString_FromString(srcfile); + #else + py_funcname = PyUnicode_FromString(funcname); + py_srcfile = PyUnicode_FromString(srcfile); + #endif + if (!py_funcname | !py_srcfile) goto bad; + py_code = PyCode_New( + 0, + #if PY_MAJOR_VERSION >= 3 + 0, + #endif + 0, + 0, + 0, + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + firstlineno, + __pyx_empty_bytes /*PyObject *lnotab*/ + ); +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return py_code; +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = func->ob_type->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* GetModuleGlobalName */ +static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { + PyObject *result; +#if !CYTHON_AVOID_BORROWED_REFS + result = PyDict_GetItem(__pyx_d, name); + if (likely(result)) { + Py_INCREF(result); + } else { +#else + result = PyObject_GetItem(__pyx_d, name); + if (!result) { + PyErr_Clear(); +#endif + result = __Pyx_GetBuiltinName(name); + } + return result; +} + +/* RaiseArgTupleInvalid */ + static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* RaiseDoubleKeywords */ + static void __Pyx_RaiseDoubleKeywordsError( + const char* func_name, + PyObject* kw_name) +{ + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION >= 3 + "%s() got multiple values for keyword argument '%U'", func_name, kw_name); + #else + "%s() got multiple values for keyword argument '%s'", func_name, + PyString_AsString(kw_name)); + #endif +} + +/* ParseKeywords */ + static int __Pyx_ParseOptionalKeywords( + PyObject *kwds, + PyObject **argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject *key = 0, *value = 0; + Py_ssize_t pos = 0; + PyObject*** name; + PyObject*** first_kw_arg = argnames + num_pos_args; + while (PyDict_Next(kwds, &pos, &key, &value)) { + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + values[name-argnames] = value; + continue; + } + name = first_kw_arg; + #if PY_MAJOR_VERSION < 3 + if (likely(PyString_CheckExact(key)) || likely(PyString_Check(key))) { + while (*name) { + if ((CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**name) == PyString_GET_SIZE(key)) + && _PyString_Eq(**name, key)) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + if ((**argname == key) || ( + (CYTHON_COMPILING_IN_PYPY || PyString_GET_SIZE(**argname) == PyString_GET_SIZE(key)) + && _PyString_Eq(**argname, key))) { + goto arg_passed_twice; + } + argname++; + } + } + } else + #endif + if (likely(PyUnicode_Check(key))) { + while (*name) { + int cmp = (**name == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**name) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**name, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + values[name-argnames] = value; + break; + } + name++; + } + if (*name) continue; + else { + PyObject*** argname = argnames; + while (argname != first_kw_arg) { + int cmp = (**argname == key) ? 0 : + #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 + (PyUnicode_GET_SIZE(**argname) != PyUnicode_GET_SIZE(key)) ? 1 : + #endif + PyUnicode_Compare(**argname, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + argname++; + } + } + } else + goto invalid_keyword_type; + if (kwds2) { + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else { + goto invalid_keyword; + } + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + #if PY_MAJOR_VERSION < 3 + "%.200s() got an unexpected keyword argument '%.200s'", + function_name, PyString_AsString(key)); + #else + "%s() got an unexpected keyword argument '%U'", + function_name, key); + #endif +bad: + return -1; +} + +/* ArgTypeTest */ + static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); +} +static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, + const char *name, int exact) +{ + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + if (none_allowed && obj == Py_None) return 1; + else if (exact) { + if (likely(Py_TYPE(obj) == type)) return 1; + #if PY_MAJOR_VERSION == 2 + else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + #endif + } + else { + if (likely(PyObject_TypeCheck(obj, type))) return 1; + } + __Pyx_RaiseArgumentTypeInvalid(name, obj, type); + return 0; +} + +/* PyFunctionFastCall */ + #if CYTHON_FAST_PYCALL +#include "frameobject.h" +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = PyThreadState_GET(); + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = f->f_localsplus; + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif // CPython < 3.6 +#endif // CYTHON_FAST_PYCALL + +/* PyCFunctionFastCall */ + #if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + PyObject *result; + int flags; + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); +} +#endif // CYTHON_FAST_PYCCALL + +/* PyObjectCallMethO */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallOneArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { +#else + if (likely(PyCFunction_Check(func))) { +#endif + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif + } + } + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +#endif + +/* PyObjectCallNoArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, NULL, 0); + } +#endif +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { +#else + if (likely(PyCFunction_Check(func))) { +#endif + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); + } + } + return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); +} +#endif + +/* PyErrFetchRestore */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} +#endif + +/* RaiseException */ + #if PY_MAJOR_VERSION < 3 +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, + CYTHON_UNUSED PyObject *cause) { + __Pyx_PyThreadState_declare + Py_XINCREF(type); + if (!value || value == Py_None) + value = NULL; + else + Py_INCREF(value); + if (!tb || tb == Py_None) + tb = NULL; + else { + Py_INCREF(tb); + if (!PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto raise_error; + } + } + if (PyType_Check(type)) { +#if CYTHON_COMPILING_IN_PYPY + if (!value) { + Py_INCREF(Py_None); + value = Py_None; + } +#endif + PyErr_NormalizeException(&type, &value, &tb); + } else { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto raise_error; + } + value = type; + type = (PyObject*) Py_TYPE(type); + Py_INCREF(type); + if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto raise_error; + } + } + __Pyx_PyThreadState_assign + __Pyx_ErrRestore(type, value, tb); + return; +raise_error: + Py_XDECREF(value); + Py_XDECREF(type); + Py_XDECREF(tb); + return; +} +#else +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } +#if PY_VERSION_HEX >= 0x03030000 + if (cause) { +#else + if (cause && cause != Py_None) { +#endif + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#else + PyThreadState *tstate = PyThreadState_GET(); + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} +#endif + +/* PyIntBinop */ + #if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { + if (op1 == op2) { + Py_RETURN_TRUE; + } + #if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(op1))) { + const long b = intval; + long a = PyInt_AS_LONG(op1); + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + const long b = intval; + long a; + const digit* digits = ((PyLongObject*)op1)->ob_digit; + const Py_ssize_t size = Py_SIZE(op1); + if (likely(__Pyx_sst_abs(size) <= 1)) { + a = likely(size) ? digits[0] : 0; + if (size == -1) a = -a; + } else { + switch (size) { + case -2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 + default: return PyLong_Type.tp_richcompare(op1, op2, Py_EQ); + #else + default: Py_RETURN_FALSE; + #endif + } + } + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = PyFloat_AS_DOUBLE(op1); + if ((double)a == (double)b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + return PyObject_RichCompare(op1, op2, Py_EQ); +} +#endif + +/* SaveResetException */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +#endif + +/* GetException */ + #if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { +#endif + PyObject *local_type, *local_value, *local_tb; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + #if PY_MAJOR_VERSION >= 3 + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } + #endif + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +} + +/* PyErrExceptionMatches */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err) { + PyObject *exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; + return PyErr_GivenExceptionMatches(exc_type, err); +} +#endif + +/* WriteUnraisableException */ + static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, + CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, + int full_traceback, CYTHON_UNUSED int nogil) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_PyThreadState_declare +#ifdef WITH_THREAD + PyGILState_STATE state; + if (nogil) + state = PyGILState_Ensure(); +#ifdef _MSC_VER + else state = (PyGILState_STATE)-1; +#endif +#endif + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + if (full_traceback) { + Py_XINCREF(old_exc); + Py_XINCREF(old_val); + Py_XINCREF(old_tb); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + PyErr_PrintEx(1); + } + #if PY_MAJOR_VERSION < 3 + ctx = PyString_FromString(name); + #else + ctx = PyUnicode_FromString(name); + #endif + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } +#ifdef WITH_THREAD + if (nogil) + PyGILState_Release(state); +#endif +} + +/* FetchCommonType */ + static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* fake_module; + PyTypeObject* cached_type = NULL; + fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); + if (!fake_module) return NULL; + Py_INCREF(fake_module); + cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + if (cached_type) { + if (!PyType_Check((PyObject*)cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", + type->tp_name); + goto bad; + } + if (cached_type->tp_basicsize != type->tp_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + type->tp_name); + goto bad; + } + } else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + } +done: + Py_DECREF(fake_module); + return cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CythonFunction */ + static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) +{ + if (unlikely(op->func_doc == NULL)) { + if (op->func.m_ml->ml_doc) { +#if PY_MAJOR_VERSION >= 3 + op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); +#else + op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); +#endif + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp = op->func_doc; + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + op->func_doc = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if PY_MAJOR_VERSION >= 3 + op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); +#else + op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + tmp = op->func_name; + Py_INCREF(value); + op->func_name = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op) +{ + Py_INCREF(op->func_qualname); + return op->func_qualname; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; +#if PY_MAJOR_VERSION >= 3 + if (unlikely(value == NULL || !PyUnicode_Check(value))) { +#else + if (unlikely(value == NULL || !PyString_Check(value))) { +#endif + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + tmp = op->func_qualname; + Py_INCREF(value); + op->func_qualname = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) +{ + PyObject *self; + self = m->func_closure; + if (self == NULL) + self = Py_None; + Py_INCREF(self); + return self; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value) +{ + PyObject *tmp; + if (unlikely(value == NULL)) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + tmp = op->func_dict; + Py_INCREF(value); + op->func_dict = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op) +{ + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op) +{ + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value) { + PyObject* tmp; + if (!value) { + value = Py_None; + } else if (value != Py_None && !PyTuple_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + Py_INCREF(value); + tmp = op->defaults_tuple; + op->defaults_tuple = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value) { + PyObject* tmp; + if (!value) { + value = Py_None; + } else if (value != Py_None && !PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + Py_INCREF(value); + tmp = op->defaults_kwdict; + op->defaults_kwdict = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value) { + PyObject* tmp; + if (!value || value == Py_None) { + value = NULL; + } else if (!PyDict_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + tmp = op->func_annotations; + op->func_annotations = value; + Py_XDECREF(tmp); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {(char *) "func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, + {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {(char *) "func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {(char *) "func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {(char *) "func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CyFunctionObject, func.m_module), PY_WRITE_RESTRICTED, 0}, + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromString(m->func.m_ml->ml_name); +#else + return PyString_FromString(m->func.m_ml->ml_name); +#endif +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if PY_VERSION_HEX < 0x030500A0 +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_New(PyTypeObject *type, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + __pyx_CyFunctionObject *op = PyObject_GC_New(__pyx_CyFunctionObject, type); + if (op == NULL) + return NULL; + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; + op->func.m_ml = ml; + op->func.m_self = (PyObject *) op; + Py_XINCREF(closure); + op->func_closure = closure; + Py_XINCREF(module); + op->func.m_module = module; + op->func_dict = NULL; + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; + op->func_classobj = NULL; + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults_pyobjects = 0; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + PyObject_GC_Track(op); + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); + Py_CLEAR(m->func.m_module); + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); + Py_CLEAR(m->func_classobj); + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_XDECREF(pydefaults[i]); + PyObject_Free(m->defaults); + m->defaults = NULL; + } + return 0; +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + PyObject_GC_Del(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + Py_VISIT(m->func_closure); + Py_VISIT(m->func.m_module); + Py_VISIT(m->func_dict); + Py_VISIT(m->func_name); + Py_VISIT(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + Py_VISIT(m->func_code); + Py_VISIT(m->func_classobj); + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + if (m->defaults) { + PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); + int i; + for (i = 0; i < m->defaults_pyobjects; i++) + Py_VISIT(pydefaults[i]); + } + return 0; +} +static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) +{ + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { + Py_INCREF(func); + return func; + } + if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { + if (type == NULL) + type = (PyObject *)(Py_TYPE(obj)); + return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); + } + if (obj == Py_None) + obj = NULL; + return __Pyx_PyMethod_New(func, obj, type); +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_FromFormat("", + op->func_qualname, (void *)op); +#else + return PyString_FromFormat("", + PyString_AsString(op->func_qualname), (void *)op); +#endif +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + Py_ssize_t size; + switch (f->m_ml->ml_flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + size = PyTuple_GET_SIZE(arg); + if (likely(size == 0)) + return (*meth)(self, NULL); + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", + f->m_ml->ml_name, size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { + size = PyTuple_GET_SIZE(arg); + if (likely(size == 1)) { + PyObject *result, *arg0 = PySequence_ITEM(arg, 0); + if (unlikely(!arg0)) return NULL; + result = (*meth)(self, arg0); + Py_DECREF(arg0); + return result; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", + f->m_ml->ml_name, size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags in " + "__Pyx_CyFunction_Call. METH_OLDARGS is no " + "longer supported!"); + return NULL; + } + PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", + f->m_ml->ml_name); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + return __Pyx_CyFunction_CallMethod(func, ((PyCFunctionObject*)func)->m_self, arg, kw); +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; + argc = PyTuple_GET_SIZE(args); + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +static PyTypeObject __pyx_CyFunctionType_type = { + PyVarObject_HEAD_INIT(0, 0) + "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, + (destructor) __Pyx_CyFunction_dealloc, + 0, + 0, + 0, +#if PY_MAJOR_VERSION < 3 + 0, +#else + 0, +#endif + (reprfunc) __Pyx_CyFunction_repr, + 0, + 0, + 0, + 0, + __Pyx_CyFunction_CallAsMethod, + 0, + 0, + 0, + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + 0, + (traverseproc) __Pyx_CyFunction_traverse, + (inquiry) __Pyx_CyFunction_clear, + 0, +#if PY_VERSION_HEX < 0x030500A0 + offsetof(__pyx_CyFunctionObject, func_weakreflist), +#else + offsetof(PyCFunctionObject, m_weakreflist), +#endif + 0, + 0, + __pyx_CyFunction_methods, + __pyx_CyFunction_members, + __pyx_CyFunction_getsets, + 0, + 0, + __Pyx_CyFunction_descr_get, + 0, + offsetof(__pyx_CyFunctionObject, func_dict), + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, +#if PY_VERSION_HEX >= 0x030400a1 + 0, +#endif +}; +static int __pyx_CyFunction_init(void) { + __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); + if (__pyx_CyFunctionType == NULL) { + return -1; + } + return 0; +} +static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_Malloc(size); + if (!m->defaults) + return PyErr_NoMemory(); + memset(m->defaults, 0, size); + m->defaults_pyobjects = pyobjects; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* Import */ + static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *empty_list = 0; + PyObject *module = 0; + PyObject *global_dict = 0; + PyObject *empty_dict = 0; + PyObject *list; + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_import; + py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); + if (!py_import) + goto bad; + #endif + if (from_list) + list = from_list; + else { + empty_list = PyList_New(0); + if (!empty_list) + goto bad; + list = empty_list; + } + global_dict = PyModule_GetDict(__pyx_m); + if (!global_dict) + goto bad; + empty_dict = PyDict_New(); + if (!empty_dict) + goto bad; + { + #if PY_MAJOR_VERSION >= 3 + if (level == -1) { + if (strchr(__Pyx_MODULE_NAME, '.')) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(1); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, 1); + #endif + if (!module) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + #endif + if (!module) { + #if PY_VERSION_HEX < 0x03030000 + PyObject *py_level = PyInt_FromLong(level); + if (!py_level) + goto bad; + module = PyObject_CallFunctionObjArgs(py_import, + name, global_dict, empty_dict, list, py_level, NULL); + Py_DECREF(py_level); + #else + module = PyImport_ImportModuleLevelObject( + name, global_dict, empty_dict, list, level); + #endif + } + } +bad: + #if PY_VERSION_HEX < 0x03030000 + Py_XDECREF(py_import); + #endif + Py_XDECREF(empty_list); + Py_XDECREF(empty_dict); + return module; +} + +/* ImportFrom */ + static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Format(PyExc_ImportError, + #if PY_MAJOR_VERSION < 3 + "cannot import name %.230s", PyString_AS_STRING(name)); + #else + "cannot import name %S", name); + #endif + } + return value; +} + +/* ClassMethod */ + static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { +#if CYTHON_COMPILING_IN_PYPY + if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) { + return PyClassMethod_New(method); + } +#else +#if CYTHON_COMPILING_IN_PYSTON + if (PyMethodDescr_Check(method)) { +#else + static PyTypeObject *methoddescr_type = NULL; + if (methoddescr_type == NULL) { + PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append"); + if (!meth) return NULL; + methoddescr_type = Py_TYPE(meth); + Py_DECREF(meth); + } + if (PyObject_TypeCheck(method, methoddescr_type)) { +#endif + PyMethodDescrObject *descr = (PyMethodDescrObject *)method; + #if PY_VERSION_HEX < 0x03020000 + PyTypeObject *d_type = descr->d_type; + #else + PyTypeObject *d_type = descr->d_common.d_type; + #endif + return PyDescr_NewClassMethod(d_type, descr->d_method); + } +#endif + else if (PyMethod_Check(method)) { + return PyClassMethod_New(PyMethod_GET_FUNCTION(method)); + } + else if (PyCFunction_Check(method)) { + return PyClassMethod_New(method); + } +#ifdef __Pyx_CyFunction_USED + else if (PyObject_TypeCheck(method, __pyx_CyFunctionType)) { + return PyClassMethod_New(method); + } +#endif + PyErr_SetString(PyExc_TypeError, + "Class-level classmethod() can only be called on " + "a method_descriptor or instance method."); + return NULL; +} + +/* CodeObjectCache */ + static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static PyCodeObject *__pyx_find_code_object(int code_line) { + PyCodeObject* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!__pyx_code_cache.entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if (unlikely(pos >= __pyx_code_cache.count) || unlikely(__pyx_code_cache.entries[pos].code_line != code_line)) { + return NULL; + } + code_object = __pyx_code_cache.entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = 64; + __pyx_code_cache.count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(__pyx_code_cache.entries, __pyx_code_cache.count, code_line); + if ((pos < __pyx_code_cache.count) && unlikely(__pyx_code_cache.entries[pos].code_line == code_line)) { + PyCodeObject* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_DECREF(tmp); + return; + } + if (__pyx_code_cache.count == __pyx_code_cache.max_count) { + int new_max = __pyx_code_cache.max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + __pyx_code_cache.entries, (size_t)new_max*sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + __pyx_code_cache.entries = entries; + __pyx_code_cache.max_count = new_max; + } + for (i=__pyx_code_cache.count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + __pyx_code_cache.count++; + Py_INCREF(code_object); +} + +/* AddTraceback */ + #include "compile.h" +#include "frameobject.h" +#include "traceback.h" +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyObject *py_srcfile = 0; + PyObject *py_funcname = 0; + #if PY_MAJOR_VERSION < 3 + py_srcfile = PyString_FromString(filename); + #else + py_srcfile = PyUnicode_FromString(filename); + #endif + if (!py_srcfile) goto bad; + if (c_line) { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #else + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + #endif + } + else { + #if PY_MAJOR_VERSION < 3 + py_funcname = PyString_FromString(funcname); + #else + py_funcname = PyUnicode_FromString(funcname); + #endif + } + if (!py_funcname) goto bad; + py_code = __Pyx_PyCode_New( + 0, + 0, + 0, + 0, + 0, + __pyx_empty_bytes, /*PyObject *code,*/ + __pyx_empty_tuple, /*PyObject *consts,*/ + __pyx_empty_tuple, /*PyObject *names,*/ + __pyx_empty_tuple, /*PyObject *varnames,*/ + __pyx_empty_tuple, /*PyObject *freevars,*/ + __pyx_empty_tuple, /*PyObject *cellvars,*/ + py_srcfile, /*PyObject *filename,*/ + py_funcname, /*PyObject *name,*/ + py_line, + __pyx_empty_bytes /*PyObject *lnotab*/ + ); + Py_DECREF(py_srcfile); + Py_DECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_srcfile); + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + py_code = __pyx_find_code_object(c_line ? c_line : py_line); + if (!py_code) { + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) goto bad; + __pyx_insert_code_object(c_line ? c_line : py_line, py_code); + } + py_frame = PyFrame_New( + PyThreadState_GET(), /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} + +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyInt_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + int one = 1; int little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&value; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); + } +} + +/* CIntFromPyVerify */ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ + static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(size_t) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(size_t, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (size_t) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, digits[0]) + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 2 * PyLong_SHIFT) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 3 * PyLong_SHIFT) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) >= 4 * PyLong_SHIFT) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(size_t) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (size_t) 0; + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) + case -2: + if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(size_t) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(size_t) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(size_t) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } +#endif + if (sizeof(size_t) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + size_t val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (size_t) -1; + } + } else { + size_t val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyInt_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + const long neg_one = (long) -1, const_zero = (long) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(long) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(long, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (long) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case 1: __PYX_VERIFY_RETURN_INT(long, digit, digits[0]) + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 2 * PyLong_SHIFT) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 3 * PyLong_SHIFT) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) >= 4 * PyLong_SHIFT) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(long) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (long) 0; + case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) + case -2: + if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(long) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(long) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(long) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } +#endif + if (sizeof(long) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + long val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (long) -1; + } + } else { + long val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (long) -1; + val = __Pyx_PyInt_As_long(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) -1, const_zero = (int) 0; + const int is_unsigned = neg_one > const_zero; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_Check(x))) { + if (sizeof(int) < sizeof(long)) { + __PYX_VERIFY_RETURN_INT(int, long, PyInt_AS_LONG(x)) + } else { + long val = PyInt_AS_LONG(x); + if (is_unsigned && unlikely(val < 0)) { + goto raise_neg_overflow; + } + return (int) val; + } + } else +#endif + if (likely(PyLong_Check(x))) { + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case 1: __PYX_VERIFY_RETURN_INT(int, digit, digits[0]) + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 2 * PyLong_SHIFT) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 3 * PyLong_SHIFT) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) >= 4 * PyLong_SHIFT) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if (sizeof(int) <= sizeof(unsigned long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)x)->ob_digit; + switch (Py_SIZE(x)) { + case 0: return (int) 0; + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) + case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) + case -2: + if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if (8 * sizeof(int) > 1 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 2 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if (8 * sizeof(int) - 1 > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if (8 * sizeof(int) > 2 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 3 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if (8 * sizeof(int) - 1 > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if (8 * sizeof(int) > 3 * PyLong_SHIFT) { + if (8 * sizeof(unsigned long) > 4 * PyLong_SHIFT) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if (8 * sizeof(int) - 1 > 4 * PyLong_SHIFT) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } +#endif + if (sizeof(int) <= sizeof(long)) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { +#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); +#else + int val; + PyObject *v = __Pyx_PyNumber_IntOrLong(x); + #if PY_MAJOR_VERSION < 3 + if (likely(v) && !PyLong_Check(v)) { + PyObject *tmp = v; + v = PyNumber_Long(tmp); + Py_DECREF(tmp); + } + #endif + if (likely(v)) { + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + int ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); + Py_DECREF(v); + if (likely(!ret)) + return val; + } +#endif + return (int) -1; + } + } else { + int val; + PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); + if (!tmp) return (int) -1; + val = __Pyx_PyInt_As_int(tmp); + Py_DECREF(tmp); + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* CheckBinaryVersion */ + static int __Pyx_check_binary_version(void) { + char ctversion[4], rtversion[4]; + PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); + PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); + if (ctversion[0] != rtversion[0] || ctversion[2] != rtversion[2]) { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compiletime version %s of module '%.100s' " + "does not match runtime version %s", + ctversion, __Pyx_MODULE_NAME, rtversion); + return PyErr_WarnEx(NULL, message, 1); + } + return 0; +} + +/* InitStrings */ + static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + while (t->p) { + #if PY_MAJOR_VERSION < 3 + if (t->is_unicode) { + *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); + } else if (t->intern) { + *t->p = PyString_InternFromString(t->s); + } else { + *t->p = PyString_FromStringAndSize(t->s, t->n - 1); + } + #else + if (t->is_unicode | t->is_str) { + if (t->intern) { + *t->p = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); + } else { + *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + #endif + if (!*t->p) + return -1; + ++t; + } + return 0; +} + +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && +#endif + PyUnicode_Check(o)) { +#if PY_VERSION_HEX < 0x03030000 + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; + } + } + } +#endif + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +#else + if (__Pyx_PyUnicode_READY(o) == -1) return NULL; +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (PyUnicode_IS_ASCII(o)) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif + } else +#endif +#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) + if (PyByteArray_Check(o)) { + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); + } else +#endif + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + const char *name = NULL; + PyObject *res = NULL; +#if PY_MAJOR_VERSION < 3 + if (PyInt_Check(x) || PyLong_Check(x)) +#else + if (PyLong_Check(x)) +#endif + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + #if PY_MAJOR_VERSION < 3 + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Int(x); + } + else if (m && m->nb_long) { + name = "long"; + res = PyNumber_Long(x); + } + #else + if (m && m->nb_int) { + name = "int"; + res = PyNumber_Long(x); + } + #endif +#else + res = PyNumber_Int(x); +#endif + if (res) { +#if PY_MAJOR_VERSION < 3 + if (!PyInt_Check(res) && !PyLong_Check(res)) { +#else + if (!PyLong_Check(res)) { +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type %.200s)", + name, name, Py_TYPE(res)->tp_name); + Py_DECREF(res); + return NULL; + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; +#if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(b))) { + if (sizeof(Py_ssize_t) >= sizeof(long)) + return PyInt_AS_LONG(b); + else + return PyInt_AsSsize_t(x); + } +#endif + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)b)->ob_digit; + const Py_ssize_t size = Py_SIZE(b); + if (likely(__Pyx_sst_abs(size) <= 1)) { + ival = likely(size) ? digits[0] : 0; + if (size == -1) ival = -ival; + return ival; + } else { + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyInt_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { + return PyInt_FromSize_t(ival); +} + + +#endif /* Py_PYTHON_H */ diff --git a/reppy/robots.pxd b/reppy/robots.pxd new file mode 100644 index 0000000..174ac8c --- /dev/null +++ b/reppy/robots.pxd @@ -0,0 +1,38 @@ +# Cython declarations + +from libcpp.string cimport string +from libcpp.vector cimport vector +from libcpp cimport bool + +cdef extern from "rep-cpp/include/directive.h" namespace "Rep": + cpdef cppclass CppDirective "Rep::Directive": + ctypedef size_t priority_t + + CppDirective(const string& line, bool allowed) + priority_t priority() const + bool match(const string& path) const + bool allowed() const + +cdef extern from "rep-cpp/include/agent.h" namespace "Rep": + cpdef cppclass CppAgent "Rep::Agent": + ctypedef float delay_t + + CppAgent() + CppAgent& allow(const string& query) + CppAgent& disallow(const string& query) + CppAgent& delay(delay_t delay) + delay_t delay() const + const vector[CppDirective]& directives() const + bool allowed(const string& path) const + @staticmethod + string escape(const string& query) + +cdef extern from "rep-cpp/include/robots.h" namespace "Rep": + cpdef cppclass CppRobots "Rep::Robots": + CppRobots(const string& content) except +ValueError + CppRobots& operator=(const CppRobots& other) + const vector[string]& sitemaps() const + const CppAgent& agent(const string& name) const + bool allowed(const string& path, const string& name) const + @staticmethod + string robotsUrl(const string& url) diff --git a/reppy/robots.py b/reppy/robots.py deleted file mode 100644 index c79b6da..0000000 --- a/reppy/robots.py +++ /dev/null @@ -1,150 +0,0 @@ -'''A class holding a parsed robots.txt.''' - -from contextlib import closing -import time - -import requests -from requests.exceptions import ( - SSLError, - ConnectionError, - URLRequired, - MissingSchema, - InvalidSchema, - InvalidURL, - TooManyRedirects) - -from .agent import Agent -from .ttl import HeaderWithDefaultPolicy -from . import util, logger, exceptions - - -class Robots(object): - '''A class holding a parsed robots.txt.''' - - # The default TTL policy is to cache for 3600 seconds or what's provided in the - # headers, and a minimum of 600 seconds - DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) - - @classmethod - def fetch(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): - '''Get the robots.txt at the provided URL.''' - try: - # Limit the size of the request - kwargs['stream'] = True - with closing(requests.get(url, *args, **kwargs)) as res: - content = res.raw.read(amt=max_size, decode_content=True) - # Try to read an additional byte, to see if the response is too big - if res.raw.read(amt=1, decode_content=True): - raise exceptions.ContentTooLong( - 'Content larger than %s bytes' % max_size) - - # Get the TTL policy's ruling on the ttl - expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) - - if res.status_code == 200: - return cls.parse(url, content, expires) - elif res.status_code in (401, 403): - return AllowNone(url, expires) - elif res.status_code >= 400 and res.status_code < 500: - return AllowAll(url, expires) - else: - raise exceptions.BadStatusCode( - 'Got %i for %s' % (res.status_code, url), res.status_code) - except SSLError as exc: - raise exceptions.SSLException(exc) - except ConnectionError as exc: - raise exceptions.ConnectionException(exc) - except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: - raise exceptions.MalformedUrl(exc) - except TooManyRedirects as exc: - raise exceptions.ExcessiveRedirects(exc) - - @classmethod - def parse(cls, url, content, expires=None): - '''Parse the provided lines as a robots.txt.''' - sitemaps = [] - agents = {} - agent = None - last_agent = False - for key, value in util.pairs(content): - # Consecutive User-Agent lines mean they all have the same rules - if key == 'user-agent': - if not last_agent: - agent = Agent() - agents[value] = agent - last_agent = True - continue - else: - last_agent = False - - if key == 'sitemap': - sitemaps.append(value) - elif key in ('disallow', 'allow', 'crawl-delay'): - if agent is None: - raise ValueError( - 'Directive "%s" must be preceeded by User-Agent' % key) - - if key == 'disallow': - agent.disallow(value) - elif key == 'allow': - agent.allow(value) - else: - try: - agent.delay = float(value) - except ValueError: - logger.warn('Could not parse crawl delay: %s', value) - else: - logger.warn('Unknown directive "%s"' % key) - - return Robots(url, agents, sitemaps, expires) - - def __init__(self, url, agents, sitemaps, expires=None): - self.url = url - self.agents = agents - self.sitemaps = sitemaps - self.expires = expires - self.default = self.agents.get('*', None) - - @property - def expired(self): - '''True if the current time is past its expiration.''' - return time.time() > self.expires - - @property - def ttl(self): - '''Remaining time for this response to be considered valid.''' - return max(self.expires - time.time(), 0) - - def agent(self, name): - '''Get the rules for the agent with the provided name.''' - return self.agents.get(name.lower(), self.default) - - def url_allowed(self, url, agent): - '''Return true if agent may fetch the path in url.''' - found = self.agent(agent) - if found is None: - return True - return found.url_allowed(url) - - def allowed(self, path, agent): - '''Return true if agent may fetch path.''' - found = self.agent(agent) - if found is None: - return True - return found.allowed(path) - - -class AllowNone(Robots): - '''No requests are allowed.''' - - def __init__(self, url, expires=None): - Robots.__init__(self, url, sitemaps=[], expires=expires, agents={ - '*': Agent().disallow('/') - }) - - -class AllowAll(Robots): - '''All requests are allowed.''' - - def __init__(self, url, expires=None): - Robots.__init__(self, url, {}, [], expires=expires) diff --git a/reppy/robots.pyx b/reppy/robots.pyx new file mode 100644 index 0000000..3f06a8d --- /dev/null +++ b/reppy/robots.pyx @@ -0,0 +1,170 @@ +# cython: linetrace=True +# distutils: define_macros=CYTHON_TRACE=1 + +from contextlib import closing +import time + +import requests +from requests.exceptions import ( + SSLError, + ConnectionError, + URLRequired, + MissingSchema, + InvalidSchema, + InvalidURL, + TooManyRedirects) +import six + +from .ttl import HeaderWithDefaultPolicy +from . import util, logger, exceptions + +cdef as_bytes(value): + if isinstance(value, bytes): + return value + return value.encode('utf-8') + +cdef as_string(value): + if six.PY3: + if isinstance(value, bytes): + return value.decode('utf-8') + return value + + +def FromRobotsMethod(cls, Robots robots, const string& name): + '''Construct an Agent from a CppAgent.''' + agent = Agent() + agent.agent = robots.robots.agent(name) + return agent + +cdef class Agent: + '''Wrapper around rep-cpp's Rep::Agent class.''' + + cdef CppAgent agent + + from_robots = classmethod(FromRobotsMethod) + + @property + def delay(self): + '''The delay associated with this agent.''' + cdef float value = self.agent.delay() + if value > 0: + return value + return None + + def allow(self, path): + '''Allow the provided path.''' + self.agent.allow(as_bytes(path)) + return self + + def disallow(self, path): + '''Disallow the provided path.''' + self.agent.disallow(as_bytes(path)) + return self + + def allowed(self, path): + '''Is the provided URL allowed?''' + return self.agent.allowed(as_bytes(path)) + + +def ParseMethod(cls, url, content, expires=None): + '''Parse a robots.txt file.''' + return cls(url, as_bytes(content), expires) + +def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): + '''Get the robots.txt at the provided URL.''' + try: + # Limit the size of the request + kwargs['stream'] = True + with closing(requests.get(url, *args, **kwargs)) as res: + content = res.raw.read(amt=max_size, decode_content=True) + # Try to read an additional byte, to see if the response is too big + if res.raw.read(amt=1, decode_content=True): + raise exceptions.ContentTooLong( + 'Content larger than %s bytes' % max_size) + + # Get the TTL policy's ruling on the ttl + expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) + + if res.status_code == 200: + return cls.parse(url, content, expires) + elif res.status_code in (401, 403): + return AllowNone(url, expires) + elif res.status_code >= 400 and res.status_code < 500: + return AllowAll(url, expires) + else: + raise exceptions.BadStatusCode( + 'Got %i for %s' % (res.status_code, url), res.status_code) + except SSLError as exc: + raise exceptions.SSLException(exc) + except ConnectionError as exc: + raise exceptions.ConnectionException(exc) + except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: + raise exceptions.MalformedUrl(exc) + except TooManyRedirects as exc: + raise exceptions.ExcessiveRedirects(exc) + +def RobotsUrlMethod(cls, url): + '''Get the robots.txt URL that corresponds to the provided one.''' + return as_string(CppRobots.robotsUrl(as_bytes(url))) + +cdef class Robots: + '''Wrapper around rep-cpp's Rep::Robots class.''' + + # The default TTL policy is to cache for 3600 seconds or what's provided in the + # headers, and a minimum of 600 seconds + DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) + + # Class methods + parse = classmethod(ParseMethod) + fetch = classmethod(FetchMethod) + robots_url = classmethod(RobotsUrlMethod) + + # Data members + cdef CppRobots* robots + cdef object url + cdef object expires + + def __init__(self, url, const string& content, expires=None): + self.url = url + self.robots = new CppRobots(content) + self.expires = expires + + def __dealloc__(self): + del self.robots + + @property + def sitemaps(self): + '''Get all the sitemaps in this robots.txt.''' + return map(as_string, self.robots.sitemaps()) + + def allowed(self, path, name): + '''Is the provided path allowed for the provided agant?''' + return self.robots.allowed(as_bytes(path), as_bytes(name)) + + def agent(self, name): + '''Return the Agent that corresponds to name.''' + return Agent.from_robots(self, as_bytes(name)) + + @property + def expired(self): + '''True if the current time is past its expiration.''' + return time.time() > self.expires + + @property + def ttl(self): + '''Remaining time for this response to be considered valid.''' + return max(self.expires - time.time(), 0) + + +cdef class AllowNone(Robots): + '''No requests are allowed.''' + + def __init__(self, url, expires=None): + Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) + + +cdef class AllowAll(Robots): + '''All requests are allowed.''' + + def __init__(self, url, expires=None): + Robots.__init__(self, url, b'', expires) diff --git a/reppy/util.py b/reppy/util.py index 8f16a4c..d686ed7 100644 --- a/reppy/util.py +++ b/reppy/util.py @@ -4,28 +4,12 @@ import string -def pairs(content): - '''A generator of lowercase, stripped key-value pairs in contents' lines.''' - for line in content.strip().split('\n'): - # Remove any comments - if '#' in line: - line, _ = line.split('#', 1) - - key, _, value = [l.strip() for l in line.strip().partition(':')] - - if not key: - continue - - yield (key.lower(), value) - - def parse_date(string): '''Return a timestamp for the provided datestring, described by RFC 7231.''' parsed = email.utils.parsedate_tz(string) if parsed is None: raise ValueError("Invalid time.") - if parsed[9] is None: - # Default time zone is GMT/UTC - parsed = list(parsed) - parsed[9] = 0 + parsed = list(parsed) + # Default time zone is GMT/UTC + parsed[9] = 0 if parsed[9] is None else parsed[9] return email.utils.mktime_tz(parsed) diff --git a/setup.py b/setup.py index ef42a1a..4b86e61 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,36 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from setuptools import setup +from distutils.core import setup +from distutils.extension import Extension + +ext_files = [ + 'reppy/rep-cpp/src/agent.cpp', + 'reppy/rep-cpp/src/directive.cpp', + 'reppy/rep-cpp/src/robots.cpp', + 'reppy/rep-cpp/deps/url-cpp/src/url.cpp', + 'reppy/rep-cpp/deps/url-cpp/src/utf8.cpp', + 'reppy/rep-cpp/deps/url-cpp/src/punycode.cpp', + 'reppy/rep-cpp/deps/url-cpp/src/psl.cpp' +] + +kwargs = {} + +try: + from Cython.Distutils import build_ext + print('Building from Cython') + ext_files.append('reppy/robots.pyx') + kwargs['cmdclass'] = {'build_ext': build_ext} +except ImportError: + print('Building from C++') + ext_files.append('reppy/robots.cpp') + +ext_modules = [ + Extension('reppy.robots', ext_files, + language='c++', + extra_compile_args=['-std=c++11'], + include_dirs=['reppy/rep-cpp/include', 'reppy/rep-cpp/deps/url-cpp/include']) +] setup( name = 'reppy', @@ -42,6 +71,7 @@ url = 'http://github.com/seomoz/reppy', license = 'MIT', platforms = 'Posix; MacOS X', + ext_modules = ext_modules, packages = [ 'reppy' ], @@ -56,5 +86,7 @@ 'Development Status :: 3 - Alpha', 'Environment :: Web Environment', 'Intended Audience :: Developers', - 'Topic :: Internet :: WWW/HTTP'] + 'Topic :: Internet :: WWW/HTTP' + ], + **kwargs ) diff --git a/tests/test_agent.py b/tests/test_agent.py index c66bea3..c532bd4 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -1,63 +1,101 @@ import unittest -from reppy.agent import Agent +from reppy.robots import Agent, Robots class AgentTest(unittest.TestCase): '''Tests about the Agent.''' - def test_basic(self): - '''Parses a basic example.''' - agent = Agent().allow('/').disallow('/foo') - self.assertEqual(len(agent.directives), 2) + def parse(self, content, name): + '''Parse the robots.txt in content and return the agent of the provided name.''' + return Robots.parse('http://example.com', content).agent(name) + + def test_make_allowed(self): + '''Make an agent that allows a path.''' + agent = Agent().disallow('/path').allow('/path/') + self.assertTrue(agent.allowed('/path/')) + self.assertFalse(agent.allowed('/path')) + + def test_make_disallowed(self): + '''Make an agent that disallows a path.''' + agent = Agent().disallow('/path') + self.assertFalse(agent.allowed('/path')) def test_checks_allowed(self): '''Answers the allowed question.''' - agent = Agent().allow('/path') + agent = self.parse(''' + User-agent: agent + Allow: /path + ''', 'agent') self.assertTrue(agent.allowed('/path')) self.assertTrue(agent.allowed('/elsewhere')) def test_honors_longest_first_priority(self): '''The longest matching rule takes priority.''' - agent = Agent().disallow('/path').allow('/path/exception') + agent = self.parse(''' + User-agent: agent + Disallow: /path + Allow: /path/exception + ''', 'agent') self.assertTrue(agent.allowed('/path/exception')) self.assertFalse(agent.allowed('/path')) def test_robots_txt_allowed(self): '''Robots.txt is always allowed.''' - agent = Agent().disallow('/robots.txt') + agent = self.parse(''' + User-agent: agent + Disallow: /robots.txt + ''', 'agent') self.assertTrue(agent.allowed('/robots.txt')) def test_disallow_none(self): '''Recognizes the "Disallow:" form of "Allow: /"''' - agent = Agent().disallow('') + agent = self.parse(''' + User-agent: agent + Disallow: + ''', 'agent') self.assertTrue(agent.allowed('/anything')) def test_escaped_rule(self): '''Handles the case where the rule is escaped.''' - agent = Agent().disallow('/a%3cd.html') + agent = self.parse(''' + User-agent: agent + Disallow: /a%3cd.html + ''', 'agent') self.assertFalse(agent.allowed('/a') - - -class AllDirectiveTest(unittest.TestCase): - '''Tests about the AllDirective.''' - - def test_matches_everything(self): - '''Matches everything.''' - self.assertTrue( - directive.AllDirective(0, '', True).match('/anything')) - - def test_str(self): - '''Has a resonable string representation.''' - rule = directive.AllDirective(0, '', True) - self.assertEqual( - str(rule), '') - - -class RegexDirectiveTest(unittest.TestCase): - '''Tests about the RegexDirective.''' - - def test_matches_beginning(self): - '''Must match at the beginning to match.''' - self.assertTrue( - directive.RegexDirective(0, '/foo/*', True).match('/foo/bar')) - - def test_does_not_match_middle(self): - '''Does not match on the middle of the query.''' - self.assertFalse( - directive.RegexDirective(0, '/foo/*', True).match('/whiz/foo/bar')) - - def test_any_with_query(self): - '''Matches any path with a query.''' - self.assertTrue( - directive.RegexDirective(0, '/*?', True).match('/path?query')) - - def test_middle_wildcard(self): - '''Matches wildcards in the middle.''' - rule = directive.RegexDirective(0, '/folder*name/', True) - self.assertTrue(rule.match('/folder-middle-name/')) - - def test_matches_end(self): - '''Can match the end of a path.''' - rule = directive.RegexDirective(0, '/*.gif$', True) - self.assertTrue(rule.match('/funny.gif')) - self.assertFalse(rule.match('/funny.gifs/path')) - - def test_many_wildcards(self): - '''Multiple consecutive wildcards are compressed into one.''' - rule = directive.RegexDirective(0, '/********************************.css', True) - self.assertTrue(rule.match('/style.css')) - - def test_multiple_wildcards(self): - '''Multiple wildcards are supported.''' - rule = directive.RegexDirective(0, '/one/*/three/*/five', True) - self.assertTrue(rule.match('/one/two/three/four/five')) - - def test_escaped_rule(self): - '''Handles the case where the rule is escaped.''' - rule = directive.RegexDirective(0, '/a%3cd*', True) - self.assertTrue(rule.match('/a') diff --git a/tests/test_robots.py b/tests/test_robots.py index 578b6a5..604f0bc 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -1,6 +1,8 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function + '''These are unit tests that are derived from the rfc at http://www.robotstxt.org/norobots-rfc.txt''' @@ -9,9 +11,15 @@ import os import unittest -import asis import mock +skip_asis = False +try: + import asis +except ImportError: + print('Skipping asis tests') + skip_asis = True + from reppy import robots @@ -131,7 +139,7 @@ def test_exposes_sitemaps(self): Sitemap: http://a.com/sitemap.xml Sitemap: http://b.com/sitemap.xml ''') - self.assertEqual(robot.sitemaps, [ + self.assertEqual(list(robot.sitemaps), [ 'http://a.com/sitemap.xml', 'http://b.com/sitemap.xml' ]) @@ -139,14 +147,15 @@ def test_case_insensitivity(self): '''Make sure user agent matches are case insensitive''' robot = robots.Robots.parse('http://example.com/robots.txt', ''' User-agent: agent + Disallow: /path ''') - self.assertEqual(robot.agent('agent'), robot.agent('aGeNt')) + self.assertFalse(robot.allowed('/path', 'agent')) + self.assertFalse(robot.allowed('/path', 'aGeNt')) def test_empty(self): '''Makes sure we can parse an empty robots.txt''' robot = robots.Robots.parse('http://example.com/robots.txt', '') - self.assertEqual(robot.agent('agent'), None) - self.assertEqual(robot.sitemaps, []) + self.assertEqual(list(robot.sitemaps), []) self.assertTrue(robot.allowed('/', 'agent')) def test_comments(self): @@ -157,21 +166,13 @@ def test_comments(self): ''') self.assertNotEqual(robot.agent('agent'), None) - def test_url_allowed(self): + def test_accepts_full_url(self): '''Can accept a url string.''' robot = robots.Robots.parse('http://example.com/robots.txt', ''' User-Agent: agent Disallow: / ''') - self.assertFalse(robot.url_allowed('http://example.com/path', 'agent')) - - def test_url_allowed_none_matching(self): - '''Can accept a url string when there are no matching agents.''' - robot = robots.Robots.parse('http://example.com/robots.txt', ''' - User-Agent: other - Disallow: / - ''') - self.assertTrue(robot.url_allowed('http://example.com/path', 'agent')) + self.assertFalse(robot.allowed('http://example.com/path', 'agent')) def test_skip_malformed_line(self): '''If there is no colon in a line, then we must skip it''' @@ -181,42 +182,49 @@ def test_skip_malformed_line(self): ''') self.assertTrue(robot.allowed('/no/colon/in/this/line', 'agent')) + @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_200(self): '''A 200 parses things normally.''' with self.server('test_fetch_status_200'): robot = robots.Robots.fetch('http://localhost:8080/robots.txt') self.assertFalse(robot.allowed('/path', 'agent')) + @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_401(self): '''A 401 gives us an AllowNone Robots.''' with self.server('test_fetch_status_401'): robot = robots.Robots.fetch('http://localhost:8080/robots.txt') self.assertIsInstance(robot, robots.AllowNone) + @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_403(self): '''A 403 gives us an AllowNone Robots.''' with self.server('test_fetch_status_403'): robot = robots.Robots.fetch('http://localhost:8080/robots.txt') self.assertIsInstance(robot, robots.AllowNone) + @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_4XX(self): '''A 4XX gives us an AllowAll Robots.''' with self.server('test_fetch_status_4XX'): robot = robots.Robots.fetch('http://localhost:8080/robots.txt') self.assertIsInstance(robot, robots.AllowAll) + @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_5XX(self): '''A server error raises an exception.''' with self.server('test_fetch_status_5XX'): with self.assertRaises(robots.exceptions.BadStatusCode): robots.Robots.fetch('http://localhost:8080/robots.txt') + @unittest.skipIf(skip_asis, 'Asis not installed') def test_content_too_big(self): '''Raises an exception if the content is too big.''' with self.server('test_content_too_big'): with self.assertRaises(robots.exceptions.ReppyException): robots.Robots.fetch('http://localhost:8080/robots.txt', max_size=5) + @unittest.skipIf(skip_asis, 'Asis not installed') def test_ssl_exception(self): '''Raises a ReppyException on SSL errors.''' with self.server('test_ssl_exception'): @@ -233,12 +241,31 @@ def test_malformed_url(self): with self.assertRaises(robots.exceptions.MalformedUrl): robots.Robots.fetch('gobbledygook') + @unittest.skipIf(skip_asis, 'Asis not installed') def test_excessive_redirects(self): '''Raises a ReppyException on too many redirects.''' with self.server('test_excessive_redirects'): with self.assertRaises(robots.exceptions.ExcessiveRedirects): robots.Robots.fetch('http://localhost:8080/robots.txt') + def test_robots_url_http(self): + '''Works with a http URL.''' + url = 'http://user@example.com:80/path;params?query#fragment' + expected = 'http://example.com/robots.txt' + self.assertEqual(robots.Robots.robots_url(url), expected) + + def test_robots_url_https(self): + '''Works with a https URL.''' + url = 'https://user@example.com:443/path;params?query#fragment' + expected = 'https://example.com/robots.txt' + self.assertEqual(robots.Robots.robots_url(url), expected) + + def test_robots_url_non_default_port(self): + '''Works with a URL with a non-default port.''' + url = 'http://user@example.com:8080/path;params?query#fragment' + expected = 'http://example.com:8080/robots.txt' + self.assertEqual(robots.Robots.robots_url(url), expected) + def test_utf8_bom(self): '''If there's a utf-8 BOM, we should parse it as such''' robot = robots.Robots.parse('http://example.com/robots.txt', @@ -249,8 +276,8 @@ def test_utf8_bom(self): User-Agent: other Disallow: /path ''') - self.assertTrue(robot.url_allowed('http://example.com/path', 'agent')) - self.assertFalse(robot.url_allowed('http://example.com/path', 'other')) + self.assertTrue(robot.allowed('http://example.com/path', 'agent')) + self.assertFalse(robot.allowed('http://example.com/path', 'other')) def test_utf16_bom(self): '''If there's a utf-16 BOM, we should parse it as such''' @@ -262,8 +289,8 @@ def test_utf16_bom(self): User-Agent: other Disallow: /path ''') - self.assertTrue(robot.url_allowed('http://example.com/path', 'agent')) - self.assertFalse(robot.url_allowed('http://example.com/path', 'other')) + self.assertTrue(robot.allowed('http://example.com/path', 'agent')) + self.assertFalse(robot.allowed('http://example.com/path', 'other')) def test_rfc_example(self): '''Tests the example provided by the RFC.''' @@ -352,16 +379,6 @@ def test_allow_robots_txt(self): robot = robots.AllowNone('http://example.com/robots.txt') self.assertTrue(robot.allowed('/robots.txt', 'agent')) - def test_allow_url(self): - '''Allows nothing.''' - robot = robots.AllowNone('http://example.com/robots.txt') - self.assertFalse(robot.url_allowed('http://example.com/', 'agent')) - - def test_allow_robots_txt_url(self): - '''Allows robots.txt url.''' - robot = robots.AllowNone('http://example.com/robots.txt') - self.assertTrue(robot.url_allowed('http://example.com/robots.txt', 'agent')) - class AllowAllTest(unittest.TestCase): '''Tests about the AllowAll Robots class.''' @@ -370,8 +387,3 @@ def test_allow(self): '''Allows nothing.''' robot = robots.AllowAll('http://example.com/robots.txt') self.assertTrue(robot.allowed('/', 'agent')) - - def test_allow_url(self): - '''Allows nothing.''' - robot = robots.AllowAll('http://example.com/robots.txt') - self.assertTrue(robot.url_allowed('http://example.com/', 'agent')) diff --git a/tests/test_util.py b/tests/test_util.py index 324dea9..74e2299 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -3,71 +3,6 @@ from reppy import util -class PairsTest(unittest.TestCase): - '''Tests about how we make pairs.''' - - def test_skips_blanks(self): - '''Skips blank lines.''' - content = ''' - hello - - world - ''' - lines = [ - ('hello', ''), - ('world', '') - ] - self.assertEqual(lines, list(util.pairs(content))) - - def test_strips_keys_and_values(self): - '''Strips leading and trailing whitespace on keys and values.''' - content = ''' - hello: world\t - ''' - lines = [ - ('hello', 'world') - ] - self.assertEqual(lines, list(util.pairs(content))) - - def test_lowercases_keys(self): - '''Makes the key lowercase.''' - content = ''' - HeLLo: World - ''' - lines = [ - ('hello', 'World') - ] - self.assertEqual(lines, list(util.pairs(content))) - - def test_does_not_trip_on_multiple_colons(self): - '''Does not fail when multiple colons are present.''' - content = ''' - hello: world: 2 - ''' - lines = [ - ('hello', 'world: 2') - ] - self.assertEqual(lines, list(util.pairs(content))) - - def test_skips_comments(self): - '''Skips comment lines.''' - content = ''' - # This is a comment. - ''' - lines = [] - self.assertEqual(lines, list(util.pairs(content))) - - def test_omits_comments(self): - '''Removes the comment portion of a line.''' - content = ''' - hello: world # this is a comment - ''' - lines = [ - ('hello', 'world') - ] - self.assertEqual(lines, list(util.pairs(content))) - - class ParseDateTest(unittest.TestCase): '''Tests about parsing dates provided in headers.''' From 5b5d7f757597ce2d10d3619f43f43b2ed2984658 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 3 Nov 2016 11:59:56 -0700 Subject: [PATCH 026/113] Update README. --- README.md | 303 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 180 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index de5a3a6..80123ba 100644 --- a/README.md +++ b/README.md @@ -3,175 +3,232 @@ Robots Exclusion Protocol Parser for Python [![Build Status](https://travis-ci.org/seomoz/reppy.svg?branch=master)](https://travis-ci.org/seomoz/reppy) -This started out of a lack of memoization support in other robots.txt parsers -I've encountered, and the lack of support for `Crawl-delay` and `Sitemap` in -the built-in `robotparser`. - -Features --------- -- Configurable caching of robots.txt files -- Expiration based on the `Expires` and `Cache-Control` headers -- Configurable automatic refetching basing on expiration -- Support for Crawl-delay -- Support for Sitemaps -- Wildcard matching - -Matching -======== -This package supports the -[1996 RFC](http://www.robotstxt.org/norobots-rfc.txt), as well as additional -commonly-implemented features, like wildcard matching, crawl-delay, and -sitemaps. There are varying approaches to matching `Allow` and `Disallow`. One -approach is to use the longest match. Another is to use the most specific. -This package chooses to follow the directive that is longest, the assumption -being that it's the one that is most specific -- a term that is a little -difficult to define in this context. +`Robots.txt` parsing in Python. -Usage +Goals ===== -The easiest way to use `reppy`, you first need a cache object. This object -controls how fetched `robots.txt` are stored and cached, and provides an -interface to issue queries about urls. -```python -from reppy.cache import RobotsCache -# Any args and kwargs provided here are given to `requests.get`. You can use -# this to set request headers and the like -robots = RobotsCache() -# Now ask if a particular url is allowed -robots.allowed('http://example.com/hello', 'my-agent') +- __Fetching__ -- helper utilities for fetching and parsing `robots.txt`s, including + checking `cache-control` and `expires` headers +- __Support for newer features__ -- like `Crawl-Delay` and `Sitemaps` +- __Wildcard matching__ -- without using regexes, no less +- __Performance__ -- with >100k parses per second, >1M URL checks per second once parsed +- __Caching__ -- utilities to help with the caching of `robots.txt` responses + +Installation +============ +`reppy` is available on `pypi`: + +```bash +pip install reppy +``` + +When installing from source, there are submodule dependencies that must also be fetched: + +```bash +git submodule update --init --recursive +make install ``` -By default, it fetches `robots.txt` for you using `requests`. If you're, -interested in getting a `Rules` object (which is a parsed `robots.txt` file), -you can ask for it without it being cached: +Usage +===== + +Checking when pages are allowed +------------------------------- +Two classes answer questions about whether a URL is allowed: `Robots` and +`Agent`: ```python -# This returns a Rules object, which is not cached, but can answer queries -rules = robots.fetch('http://example.com/') -rules.allowed('http://example.com/foo', 'my-agent') +from reppy.robots import Robots + +# This utility uses `requests` to fetch the content +robots = Robots.fetch('http://example.com/robots.txt') +robots.allowed('http://example.com/some/path/', 'my-user-agent') + +# Get the rules for a specific agent +agent = robots.agent('my-user-agent') +agent.allowed('http://example.com/some/path/') ``` -If automatic fetching doesn't suit your needs (perhaps you have your own way of -fetching pages), you can still make use of the cache object. To do this, -you'll need to make your own `Rules` object: +The `Robots` class also exposes properties `expired` and `ttl` to describe how +long the response should be considered valid. A `reppy.ttl` policy is used to +determine what that should be: ```python -from reppy.parser import Rules -# Do some fetching here -# ... -robots.add(Rules('http://example.com/robots.txt', - status_code, # What status code did we get back? - content, # The content of the fetched page - expiration)) # When is this page set to expire? +from reppy.ttl import HeaderWithDefaultPolicy + +# Use the `cache-control` or `expires` headers, defaulting to a 30 minutes and +# ensuring it's at least 10 minutes +policy = HeaderWithDefaultPolicy(default=1800, minimum=600) + +robots = Robots.fetch('http://example.com/robots.txt', ttl_policy=policy) ``` -Expiration ----------- -A `RobotsCache` object can track the expiration times for fetched `robots.txt` -files. This is taken from either the `Cache-Control` or `Expires` headers if -they are present, or defaults to an hour. At some point, I would like this to -be configurable, but I'm still trying to think of the best interface. +Matching Rules and Wildcards +---------------------------- +Both `*` and `$` are supported for wildcard matching. + +This library follows the matching [1996 RFC](http://www.robotstxt.org/norobots-rfc.txt) +describes. In the case where multiple rules match a query, the longest rules wins as +it is presumed to be the most specific. + +Checking sitemaps +----------------- +The `Robots` class also lists the sitemaps that are listed in a `robots.txt` ```python -rules = RobotsCache.find('http://example.com/robots.txt') -# Now long before it expires? -rules.ttl -# When does it expire? -rules.expires -# Has it expired? -rules.expired +# This property holds a list of URL strings of all the sitemaps listed +robots.sitemaps ``` -Caching -======= -The default caching policy is to cache everything until it expires. At some -point, we'll add other caching policies (probably LRU), but you can also extend -the `RobotsCache` object to implement your own. Override the `cache` method and -you're on your way! +Delay +----- +The `Crawl-Delay` directive is per agent and can be accessed through that class. If +none was specified, it's `None`: ```python -class MyCache(RobotsCache): - def cache(self, url, *args, **kwargs): - fetched = self.fetch(url, *args, **kwargs) - self._cache[Utility.hostname(url)] = fetched - # Figure out any cached items that need eviction - return fetched +# What's the delay my-user-agent should use +robots.agent('my-user-agent').delay ``` -You may want to explicitly clear the cache, too, which can be done either with -the `clear` method, or it's done automatically when used as a context manager: +Determining the `robots.txt` URL +-------------------------------- +Given a URL, there's a utility to determine the URL of the corresponding `robots.txt`. +It preserves the scheme and hostname and the port (if it's not the default port for the +scheme). ```python -# Store some results -robots.allowed('http://example.com/foo') -# Now we'll get rid of the cache -robots.clear() +# Get robots.txt URL for http://userinfo@example.com:8080/path;params?query#fragment +# It's http://example.com:8080/robots.txt +Robots.robotsUrl('http://userinfo@example.com:8080/path;params?query#fragment') +``` + +Caching +------- +The caching functionality has temporarily been removed, but will be reintroduced in the +next minor release. -# Now as a context manager -with robots: - robots.allowed('http://example.com/foo') +Development +=========== +A `Vagrantfile` is provided to bootstrap a development environment: -# Now there's nothing cached in robots +```bash +vagrant up ``` -Queries -======= -Allowed / Disallowed --------------------- -Each of these takes a url and a short user agent string (for example, -'my-agent'). +Alternatively, development can be conducted using a `virtualenv`: -```python -robots.allowed('http://example.com/allowed.html', 'my-agent') -# True -robots.disallowed('http://example.com/allowed.html', 'my-agent') -# False +```bash +virtualenv venv +source venv/bin/activate +pip install -r requirements.txt ``` -Alternatively, a rules object provides the same interface: +Tests +===== +Tests may be run in `vagrant`: -```python -rules = robots.find('http://example.com/allowed') -rules.allowed('http://example.com/allowed', 'my-agent') +```bash +make test ``` -Crawl-Delay +Development +=========== + +Environment ----------- -Crawl delay can be specified on a per-agent basis, so when checking the crawl -delay for a site, you must provide an agent. +To launch the `vagrant` image, we only need to +`vagrant up` (though you may have to provide a `--provider` flag): -```python -robots.delay('http://example.com/foo', 'my-agent') +```bash +vagrant up ``` -If there is no crawl delay specified for the provided agent /or/ for the `*` -agent, then `delay` returns None +With a running `vagrant` instance, you can log in and run tests: -Sitemaps --------- -A `robots.txt` file can also specify sitemaps, accessible through a `Rules` -object or a `RobotsCache` object: +```bash +vagrant ssh +cd /vagrant -```python -robots.sitemaps('http://example.com/') +make test ``` -Path-Matching -------------- -Path matching supports both `*` and `$` - Running Tests -============= -In order to run tests, you'll need `nose`, `asis`, `gevent` and `coverage`, all -of which can be installed with `pip`: +------------- +Tests are run with the top-level `Makefile`: ```bash -sudo pip install -r dev-requirements.txt +make test ``` -From there, the tests may be run with: +PRs +=== +These are not all hard-and-fast rules, but in general PRs have the following expectations: + +- __pass Travis__ -- or more generally, whatever CI is used for the particular project +- __be a complete unit__ -- whether a bug fix or feature, it should appear as a complete + unit before consideration. +- __maintain code coverage__ -- some projects may include code coverage requirements as + part of the build as well +- __maintain the established style__ -- this means the existing style of established + projects, the established conventions of the team for a given language on new + projects, and the guidelines of the community of the relevant languages and + frameworks. +- __include failing tests__ -- in the case of bugs, failing tests demonstrating the bug + should be included as one commit, followed by a commit making the test succeed. This + allows us to jump to a world with a bug included, and prove that our test in fact + exercises the bug. +- __be reviewed by one or more developers__ -- not all feedback has to be accepted, but + it should all be considered. +- __avoid 'addressed PR feedback' commits__ -- in general, PR feedback should be rebased + back into the appropriate commits that introduced the change. In cases, where this + is burdensome, PR feedback commits may be used but should still describe the changed + contained therein. + +PR reviews consider the design, organization, and functionality of the submitted code. + +Commits +======= +Certain types of changes should be made in their own commits to improve readability. When +too many different types of changes happen simultaneous to a single commit, the purpose of +each change is muddled. By giving each commit a single logical purpose, it is implicitly +clear why changes in that commit took place. + +- __updating / upgrading dependencies__ -- this is especially true for invocations like + `bundle update` or `berks update`. +- __introducing a new dependency__ -- often preceeded by a commit updating existing + dependencies, this should only include the changes for the new dependency. +- __refactoring__ -- these commits should preserve all the existing functionality and + merely update how it's done. +- __utility components to be used by a new feature__ -- if introducing an auxiliary class + in support of a subsequent commit, add this new class (and its tests) in its own + commit. +- __config changes__ -- when adjusting configuration in isolation +- __formatting / whitespace commits__ -- when adjusting code only for stylistic purposes. + +New Features +------------ +Small new features (where small refers to the size and complexity of the change, not the +impact) are often introduced in a single commit. Larger features or components might be +built up piecewise, with each commit containing a single part of it (and its corresponding +tests). + +Bug Fixes +--------- +In general, bug fixes should come in two-commit pairs: a commit adding a failing test +demonstrating the bug, and a commit making that failing test pass. + +Tagging and Versioning +====================== +Whenever the version included in `setup.py` is changed (and it should be changed when +appropriate using [http://semver.org/](http://semver.org/)), a corresponding tag should +be created with the same version number (formatted `v`). ```bash -make test +git tag -a v0.1.0 -m 'Version 0.1.0 + +This release contains an initial working version of the `crawl` and `parse` +utilities.' + +git push origin ``` From 752f350eb0e161a7dd8d56d8e1646e1399375c05 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 3 Nov 2016 11:07:01 -0700 Subject: [PATCH 027/113] Start testing against python 3 again. --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 85985d7..ac23468 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,10 @@ language: python dist: trusty python: - # - "3.4" - # - "3.3" - "2.7" - # - "pypy" + - "3.3" + - "3.4" + - "3.5" install: - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install -r dev-requirements.txt; fi From e05e3a5210ef06f5dd2101fb22d2b3f01c2a28e5 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Tue, 1 Nov 2016 08:32:14 -0700 Subject: [PATCH 028/113] Version bump to 0.4.0. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4b86e61..80b57be 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( name = 'reppy', - version = '0.3.4', + version = '0.4.0', description = 'Replacement robots.txt Parser', long_description = '''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From 6647ed02592b01ac2ce86ed9511d2423906dd469 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Fri, 4 Nov 2016 18:21:48 -0700 Subject: [PATCH 029/113] Expose .expires on Robots objects. --- reppy/robots.cpp | 152 +++++++++++++++++++++++++++++++++-------------- reppy/robots.pyx | 5 ++ 2 files changed, 114 insertions(+), 43 deletions(-) diff --git a/reppy/robots.cpp b/reppy/robots.cpp index 1fae419..9f9814c 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -676,7 +676,7 @@ struct __pyx_obj_5reppy_6robots_Robots { }; -/* "reppy/robots.pyx":159 +/* "reppy/robots.pyx":164 * * * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< @@ -688,7 +688,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone { }; -/* "reppy/robots.pyx":166 +/* "reppy/robots.pyx":171 * * * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< @@ -1484,6 +1484,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ static PyObject *__pyx_pf_5reppy_6robots_6Robots_4allowed(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_name); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_name); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6robots_AllowNone *__pyx_v_self, PyObject *__pyx_v_url, PyObject *__pyx_v_expires); /* proto */ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6robots_AllowAll *__pyx_v_self, PyObject *__pyx_v_url, PyObject *__pyx_v_expires); /* proto */ @@ -4449,6 +4450,66 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } /* "reppy/robots.pyx":154 + * + * @property + * def expires(self): # <<<<<<<<<<<<<< + * '''The expiration of this robots.txt.''' + * return self.expires + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_7expires_1__get__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_7expires_1__get__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_7expires___get__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__get__", 0); + __Pyx_TraceCall("__get__", __pyx_f[1], 154, 0, __PYX_ERR(1, 154, __pyx_L1_error)); + + /* "reppy/robots.pyx":156 + * def expires(self): + * '''The expiration of this robots.txt.''' + * return self.expires # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_TraceLine(156,0,__PYX_ERR(1, 156, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_self->expires); + __pyx_r = __pyx_v_self->expires; + goto __pyx_L0; + + /* "reppy/robots.pyx":154 + * + * @property + * def expires(self): # <<<<<<<<<<<<<< + * '''The expiration of this robots.txt.''' + * return self.expires + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("reppy.robots.Robots.expires.__get__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":159 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4480,21 +4541,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 154, 0, __PYX_ERR(1, 154, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 159, 0, __PYX_ERR(1, 159, __pyx_L1_error)); - /* "reppy/robots.pyx":156 + /* "reppy/robots.pyx":161 * def ttl(self): * '''Remaining time for this response to be considered valid.''' * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(156,0,__PYX_ERR(1, 156, __pyx_L1_error)) + __Pyx_TraceLine(161,0,__PYX_ERR(1, 161, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 156, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 161, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 156, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 161, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -4508,24 +4569,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 156, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 161, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 156, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 161, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 156, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 161, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 156, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 161, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 156, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 161, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 156, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 161, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 156, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 161, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -4539,7 +4600,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":154 + /* "reppy/robots.pyx":159 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4562,7 +4623,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 return __pyx_r; } -/* "reppy/robots.pyx":162 +/* "reppy/robots.pyx":167 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4603,7 +4664,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 162, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 167, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4618,7 +4679,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 162, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 167, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4641,17 +4702,17 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 162, 0, __PYX_ERR(1, 162, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 167, 0, __PYX_ERR(1, 167, __pyx_L1_error)); - /* "reppy/robots.pyx":163 + /* "reppy/robots.pyx":168 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(163,0,__PYX_ERR(1, 163, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 163, __pyx_L1_error) + __Pyx_TraceLine(168,0,__PYX_ERR(1, 168, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 168, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -4668,7 +4729,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 163, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 168, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -4676,13 +4737,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 163, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 168, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 163, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 168, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -4699,14 +4760,14 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 163, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 168, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":162 + /* "reppy/robots.pyx":167 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4730,7 +4791,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":169 +/* "reppy/robots.pyx":174 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4770,7 +4831,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 169, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 174, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4785,7 +4846,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 169, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 174, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4808,15 +4869,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 169, 0, __PYX_ERR(1, 169, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 174, 0, __PYX_ERR(1, 174, __pyx_L1_error)); - /* "reppy/robots.pyx":170 + /* "reppy/robots.pyx":175 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(170,0,__PYX_ERR(1, 170, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 170, __pyx_L1_error) + __Pyx_TraceLine(175,0,__PYX_ERR(1, 175, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 175, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -4833,7 +4894,7 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 170, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 175, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -4841,13 +4902,13 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 170, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 175, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 170, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 175, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -4864,14 +4925,14 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 170, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 175, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":169 + /* "reppy/robots.pyx":174 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5581,6 +5642,10 @@ static PyObject *__pyx_getprop_5reppy_6robots_6Robots_expired(PyObject *o, CYTHO return __pyx_pw_5reppy_6robots_6Robots_7expired_1__get__(o); } +static PyObject *__pyx_getprop_5reppy_6robots_6Robots_expires(PyObject *o, CYTHON_UNUSED void *x) { + return __pyx_pw_5reppy_6robots_6Robots_7expires_1__get__(o); +} + static PyObject *__pyx_getprop_5reppy_6robots_6Robots_ttl(PyObject *o, CYTHON_UNUSED void *x) { return __pyx_pw_5reppy_6robots_6Robots_3ttl_1__get__(o); } @@ -5594,6 +5659,7 @@ static PyMethodDef __pyx_methods_5reppy_6robots_Robots[] = { static struct PyGetSetDef __pyx_getsets_5reppy_6robots_Robots[] = { {(char *)"sitemaps", __pyx_getprop_5reppy_6robots_6Robots_sitemaps, 0, (char *)"Get all the sitemaps in this robots.txt.", 0}, {(char *)"expired", __pyx_getprop_5reppy_6robots_6Robots_expired, 0, (char *)"True if the current time is past its expiration.", 0}, + {(char *)"expires", __pyx_getprop_5reppy_6robots_6Robots_expires, 0, (char *)"The expiration of this robots.txt.", 0}, {(char *)"ttl", __pyx_getprop_5reppy_6robots_6Robots_ttl, 0, (char *)"Remaining time for this response to be considered valid.", 0}, {0, 0, 0, 0, 0} }; @@ -6207,14 +6273,14 @@ PyMODINIT_FUNC PyInit_robots(void) if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 110, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 159, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 164, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 159, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 164, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 166, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 171, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 166, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 171, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; diff --git a/reppy/robots.pyx b/reppy/robots.pyx index 3f06a8d..d820155 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -150,6 +150,11 @@ cdef class Robots: '''True if the current time is past its expiration.''' return time.time() > self.expires + @property + def expires(self): + '''The expiration of this robots.txt.''' + return self.expires + @property def ttl(self): '''Remaining time for this response to be considered valid.''' From 7ddfac926cd25ee149e7f6ada1bbfe5e4061a208 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Fri, 4 Nov 2016 18:39:35 -0700 Subject: [PATCH 030/113] Add cachetools as a dependency. --- dev-requirements-py3.txt | 1 + dev-requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/dev-requirements-py3.txt b/dev-requirements-py3.txt index 0a2c807..b8318ea 100644 --- a/dev-requirements-py3.txt +++ b/dev-requirements-py3.txt @@ -1,6 +1,7 @@ colorama==0.3.7 coverage==4.0.3 Cython==0.25.1 +gevent==1.1.1 mock==2.0.0 nose==1.3.7 pbr==1.9.1 diff --git a/dev-requirements.txt b/dev-requirements.txt index ef26086..b81d8df 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,6 @@ asis==0.3.0 bottle==0.12.9 +cachetools==2.0.0 colorama==0.3.7 coverage==4.0.3 cython==0.25.1 From 3e0e86ddb55ef468cd57729c59db0f19a2fde78a Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Mon, 7 Nov 2016 13:02:51 -0800 Subject: [PATCH 031/113] Remove dependency on asis, unify dev-requirements{,-py3}.txt --- .travis.yml | 5 +---- dev-requirements-py3.txt | 14 ------------ dev-requirements.txt | 12 +++-------- setup.py | 1 - tests/test_robots.py | 42 +++++++++++------------------------- tests/util.py | 46 ++++++++++++++++++++++++++++++++++++++++ tox.ini | 2 +- 7 files changed, 63 insertions(+), 59 deletions(-) delete mode 100644 dev-requirements-py3.txt create mode 100644 tests/util.py diff --git a/.travis.yml b/.travis.yml index ac23468..087ae8b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,6 @@ python: - "3.4" - "3.5" -install: - - if [[ $TRAVIS_PYTHON_VERSION == 2* ]]; then travis_retry pip install -r dev-requirements.txt; fi - - if [[ $TRAVIS_PYTHON_VERSION == 3* ]]; then travis_retry pip install -r dev-requirements-py3.txt; fi - - if [[ $TRAVIS_PYTHON_VERSION == 'pypy' ]]; then travis_retry pip install -r dev-requirements-py3.txt; fi +install: pip install -r dev-requirements.txt script: make test diff --git a/dev-requirements-py3.txt b/dev-requirements-py3.txt deleted file mode 100644 index b8318ea..0000000 --- a/dev-requirements-py3.txt +++ /dev/null @@ -1,14 +0,0 @@ -colorama==0.3.7 -coverage==4.0.3 -Cython==0.25.1 -gevent==1.1.1 -mock==2.0.0 -nose==1.3.7 -pbr==1.9.1 -publicsuffix==1.1.0 -python-dateutil==2.5.3 -python-termstyle==0.1.10 -rednose==1.1.1 -requests==2.10.0 -six==1.10.0 -url==0.3.0 diff --git a/dev-requirements.txt b/dev-requirements.txt index b81d8df..2a40a3e 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,20 +1,14 @@ -asis==0.3.0 -bottle==0.12.9 cachetools==2.0.0 colorama==0.3.7 coverage==4.0.3 -cython==0.25.1 +Cython==0.25.1 funcsigs==1.0.2 -gevent==1.1.1 -greenlet==0.4.9 mock==2.0.0 nose==1.3.7 -pbr==1.9.1 -publicsuffix==1.1.0 +pbr==1.10.0 python-dateutil==2.5.3 python-termstyle==0.1.10 rednose==1.1.1 requests==2.10.0 +requests-mock==1.1.0 six==1.10.0 -url==0.3.0 -wheel==0.24.0 diff --git a/setup.py b/setup.py index 80b57be..064a378 100644 --- a/setup.py +++ b/setup.py @@ -77,7 +77,6 @@ ], install_requires = [ 'python-dateutil>=1.5, !=2.0', - 'url>=0.2.0', 'requests', 'six' ], diff --git a/tests/test_robots.py b/tests/test_robots.py index 604f0bc..e76e438 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -12,27 +12,17 @@ import unittest import mock - -skip_asis = False -try: - import asis -except ImportError: - print('Skipping asis tests') - skip_asis = True +import requests_mock +from requests.exceptions import SSLError from reppy import robots +from .util import requests_fixtures + class RobotsTest(unittest.TestCase): '''Tests about our Robots class.''' - @contextlib.contextmanager - def server(self, *segments): - '''Run an asis server with the provided path.''' - path = os.path.join('tests', 'asis', *segments) - with asis.Server(path, server='gevent', port=8080).greenlet(): - yield - def test_expired(self): '''Returns true if expired.''' with mock.patch.object(robots.time, 'time', return_value=10): @@ -182,52 +172,45 @@ def test_skip_malformed_line(self): ''') self.assertTrue(robot.allowed('/no/colon/in/this/line', 'agent')) - @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_200(self): '''A 200 parses things normally.''' - with self.server('test_fetch_status_200'): + with requests_fixtures('test_fetch_status_200'): robot = robots.Robots.fetch('http://localhost:8080/robots.txt') self.assertFalse(robot.allowed('/path', 'agent')) - @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_401(self): '''A 401 gives us an AllowNone Robots.''' - with self.server('test_fetch_status_401'): + with requests_fixtures('test_fetch_status_401'): robot = robots.Robots.fetch('http://localhost:8080/robots.txt') self.assertIsInstance(robot, robots.AllowNone) - @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_403(self): '''A 403 gives us an AllowNone Robots.''' - with self.server('test_fetch_status_403'): + with requests_fixtures('test_fetch_status_403'): robot = robots.Robots.fetch('http://localhost:8080/robots.txt') self.assertIsInstance(robot, robots.AllowNone) - @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_4XX(self): '''A 4XX gives us an AllowAll Robots.''' - with self.server('test_fetch_status_4XX'): + with requests_fixtures('test_fetch_status_4XX'): robot = robots.Robots.fetch('http://localhost:8080/robots.txt') self.assertIsInstance(robot, robots.AllowAll) - @unittest.skipIf(skip_asis, 'Asis not installed') def test_fetch_status_5XX(self): '''A server error raises an exception.''' - with self.server('test_fetch_status_5XX'): + with requests_fixtures('test_fetch_status_5XX'): with self.assertRaises(robots.exceptions.BadStatusCode): robots.Robots.fetch('http://localhost:8080/robots.txt') - @unittest.skipIf(skip_asis, 'Asis not installed') def test_content_too_big(self): '''Raises an exception if the content is too big.''' - with self.server('test_content_too_big'): + with requests_fixtures('test_content_too_big'): with self.assertRaises(robots.exceptions.ReppyException): robots.Robots.fetch('http://localhost:8080/robots.txt', max_size=5) - @unittest.skipIf(skip_asis, 'Asis not installed') def test_ssl_exception(self): '''Raises a ReppyException on SSL errors.''' - with self.server('test_ssl_exception'): + with mock.patch.object(robots.requests, 'get', side_effect=SSLError('Kaboom')): with self.assertRaises(robots.exceptions.SSLException): robots.Robots.fetch('https://localhost:8080/robots.txt') @@ -241,10 +224,9 @@ def test_malformed_url(self): with self.assertRaises(robots.exceptions.MalformedUrl): robots.Robots.fetch('gobbledygook') - @unittest.skipIf(skip_asis, 'Asis not installed') def test_excessive_redirects(self): '''Raises a ReppyException on too many redirects.''' - with self.server('test_excessive_redirects'): + with requests_fixtures('test_excessive_redirects'): with self.assertRaises(robots.exceptions.ExcessiveRedirects): robots.Robots.fetch('http://localhost:8080/robots.txt') diff --git a/tests/util.py b/tests/util.py new file mode 100644 index 0000000..6ab980f --- /dev/null +++ b/tests/util.py @@ -0,0 +1,46 @@ +'''Testing utilities.''' + +import contextlib +import json +import os + +import six + +import requests_mock + + +@contextlib.contextmanager +def requests_fixtures(*segments): + '''Mock the paths provided in the fixture.''' + # This reads in each of the asis files + path = os.path.join('tests', 'asis', *segments) + with requests_mock.mock() as mock: + for name in os.listdir(path): + with open(os.path.join(path, name), 'rb') as fin: + content = iter(fin) + + # Read in the status line + line = next(content) + _, status, reason = line.split(b' ', 2) + + # Read in the headers + headers = {} + for line in content: + if not line.strip(): + break + + key, _, value = line.partition(b': ') + headers[key.strip()] = value.strip() + + # In python 3 mode, headers need to be text + if six.PY3: + headers = { + k.decode('utf-8'): v.decode('utf-8') for k, v in headers.items()} + + mock.get( + '/%s' % name, + status_code=int(status), + reason=reason, + headers=headers, + content=b'\n'.join(content)) + yield diff --git a/tox.ini b/tox.ini index 03bef21..62fbd0b 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ commands= [testenv:py33,py34,pypy] deps= - -rdev-requirements-py3.txt + -rdev-requirements.txt commands= nose From ab74001942b4acc5bc2cf6cbcf0251c3b2379dbb Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Mon, 7 Nov 2016 13:02:21 -0800 Subject: [PATCH 032/113] AgentCache and RobotsCache. --- README.md | 55 ++++++- reppy/cache/__init__.py | 108 +++++++++++++ reppy/cache/policy.py | 34 +++++ requirements.txt | 1 + setup.py | 8 +- tests/asis/test_agent_allowed/robots.txt | 6 + tests/asis/test_caches_agent/robots.txt | 6 + tests/asis/test_caches_robots/robots.txt | 6 + .../test_returns_a_robots_object/robots.txt | 5 + .../test_returns_an_agent_object/robots.txt | 5 + tests/asis/test_robots_allowed/robots.txt | 6 + tests/test_cache/__init__.py | 0 tests/test_cache/test_cache.py | 143 ++++++++++++++++++ tests/test_cache/test_policy.py | 59 ++++++++ 14 files changed, 438 insertions(+), 4 deletions(-) create mode 100644 reppy/cache/__init__.py create mode 100644 reppy/cache/policy.py create mode 100644 tests/asis/test_agent_allowed/robots.txt create mode 100644 tests/asis/test_caches_agent/robots.txt create mode 100644 tests/asis/test_caches_robots/robots.txt create mode 100644 tests/asis/test_returns_a_robots_object/robots.txt create mode 100644 tests/asis/test_returns_an_agent_object/robots.txt create mode 100644 tests/asis/test_robots_allowed/robots.txt create mode 100644 tests/test_cache/__init__.py create mode 100644 tests/test_cache/test_cache.py create mode 100644 tests/test_cache/test_policy.py diff --git a/README.md b/README.md index 80123ba..7644e0b 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,15 @@ policy = HeaderWithDefaultPolicy(default=1800, minimum=600) robots = Robots.fetch('http://example.com/robots.txt', ttl_policy=policy) ``` +Customizing fetch +----------------- +The `fetch` method accepts `*args` and `**kwargs` that are passed on to `requests.get`, +allowing you to customize the way the `fetch` is executed: + +```python +robots = Robots.fetch('http://example.com/robots.txt', headers={...}) +``` + Matching Rules and Wildcards ---------------------------- Both `*` and `$` are supported for wildcard matching. @@ -104,9 +113,49 @@ Robots.robotsUrl('http://userinfo@example.com:8080/path;params?query#fragment') ``` Caching -------- -The caching functionality has temporarily been removed, but will be reintroduced in the -next minor release. +======= +There are two cache classes provided -- `RobotsCache`, which caches entire `reppy.Robots` +objects, and `AgentCache`, which only caches the `reppy.Agent` relevant to a client. These +caches duck-type the class that they cache for the purposes of checking if a URL is +allowed: + +```python +from reppy.cache import RobotsCache +cache = RobotsCache(capacity=100) +cache.allowed('http://example.com/foo/bar', 'my-user-agent') + +from reppy.cache import AgentCache +cache = AgentCache(agent='my-user-agent', capacity=100) +cache.allowed('http://example.com/foo/bar') +``` + +Like `reppy.Robots.fetch`, the cache constructory accepts a `ttl_policy` to inform the +expiration of the fetched `Robots` objects, as well as `*args` and `**kwargs` to be passed +to `reppy.Robots.fetch`. + +Caching Failures +---------------- +There's a piece of classic caching advice: "don't cache failures." However, this is not +always appropriate in certain circumstances. For example, if the failure is a timeout, +clients may want to cache this result so that every check doesn't take a very long time. + +To this end, the `cache` module provides a notion of a cache policy. It determines what +to do in the case of an exception. The default is to cache a form of a disallowed response +for 10 minutes, but you can configure it as you see fit: + +```python +# Do not cache failures (note the `ttl=0`): +from reppy.cache.policy import ReraiseExceptionPolicy +cache = AgentCache('my-user-agent', cache_policy=ReraiseExceptionPolicy(ttl=0)) + +# Cache and reraise failures for 10 minutes (note the `ttl=600`): +cache = AgentCache('my-user-agent', cache_policy=ReraiseExceptionPolicy(ttl=600)) + +# Treat failures as being disallowed +cache = AgentCache( + 'my-user-agent', + cache_policy=DefaultObjectPolicy(ttl=600, lambda _: Agent().disallow('/'))) +``` Development =========== diff --git a/reppy/cache/__init__.py b/reppy/cache/__init__.py new file mode 100644 index 0000000..d440b50 --- /dev/null +++ b/reppy/cache/__init__.py @@ -0,0 +1,108 @@ +'''A robots.txt cache.''' + +from functools import partial +import threading +import time + +from cachetools import LRUCache + +from .policy import DefaultObjectPolicy, ReraiseExceptionPolicy +from ..robots import Robots, AllowNone, Agent + +class ExpiringObject(object): + '''Wrap an object that expires over time.''' + + def __init__(self, factory): + self.factory = factory + self.lock = threading.Lock() + self.obj = None + self.expires = 0 + self.exception = None + + def get(self): + '''Get the wrapped object.''' + if (self.obj is None) or (time.time() >= self.expires): + with self.lock: + self.expires, self.obj = self.factory() + if isinstance(self.obj, BaseException): + self.exception = self.obj + else: + self.exception = None + + if self.exception: + raise self.exception + else: + return self.obj + + +class BaseCache(object): + '''A base cache class.''' + + DEFAULT_CACHE_POLICY = ReraiseExceptionPolicy(ttl=600) + DEFAULT_TTL_POLICY = Robots.DEFAULT_TTL_POLICY + + def __init__(self, capacity, cache_policy=None, ttl_policy=None, *args, **kwargs): + self.cache_policy = cache_policy or self.DEFAULT_CACHE_POLICY + self.ttl_policy = ttl_policy or self.DEFAULT_TTL_POLICY + self.cache = LRUCache(maxsize=capacity, missing=self.missing) + self.args = args + self.kwargs = kwargs + + def get(self, url): + '''Get the entity that corresponds to URL.''' + return self.cache[Robots.robots_url(url)].get() + + def factory(self, url): + ''' + Return (expiration, obj) corresponding to provided url, exercising the + cache_policy as necessary. + ''' + try: + return self.fetch(url) + except BaseException as exc: + return self.cache_policy.exception(url, exc) + + def fetch(self, url): + '''Return (expiration, obj) corresponding to provided url.''' + raise NotImplementedError('BaseCache does not implement fetch.') + + def missing(self, url): + '''Invoked on cache misses.''' + return ExpiringObject(partial(self.factory, url)) + + +class RobotsCache(BaseCache): + '''A cache of Robots objects.''' + + DEFAULT_CACHE_POLICY = DefaultObjectPolicy(ttl=600, factory=AllowNone) + + def allowed(self, url, agent): + '''Return true if the provided URL is allowed to agent.''' + return self.get(url).allowed(url, agent) + + def fetch(self, url): + '''Return (expiration, Robots) for the robots.txt at the provided URL.''' + robots = Robots.fetch( + url, ttl_policy=self.ttl_policy, *self.args, **self.kwargs) + return (robots.expires, robots) + + +class AgentCache(BaseCache): + '''A cache of Agent objects.''' + + DEFAULT_CACHE_POLICY = DefaultObjectPolicy( + ttl=600, factory=lambda url: Agent().disallow('/')) + + def __init__(self, agent, *args, **kwargs): + BaseCache.__init__(self, *args, **kwargs) + self.agent = agent + + def allowed(self, url): + '''Return true if the provided URL is allowed to self.agent.''' + return self.get(url).allowed(url) + + def fetch(self, url): + '''Return (expiration, Agent) for the robots.txt at the provided URL.''' + robots = Robots.fetch( + url, ttl_policy=self.ttl_policy, *self.args, **self.kwargs) + return (robots.expires, robots.agent(self.agent)) diff --git a/reppy/cache/policy.py b/reppy/cache/policy.py new file mode 100644 index 0000000..b519f48 --- /dev/null +++ b/reppy/cache/policy.py @@ -0,0 +1,34 @@ +'''Policies for caching.''' + +import time + + +class CachePolicyBase(object): + '''Policy for caching.''' + + def exception(self, url, exception): + '''What to return when there's an exception.''' + raise NotImplementedError('CachePolicyBase does not implement exception.') + + +class DefaultObjectPolicy(object): + '''Return a default object on exception.''' + + def __init__(self, ttl, factory): + self.ttl = ttl + self.factory = factory + + def exception(self, url, exception): + '''What to return when there's an exception.''' + return (time.time() + self.ttl, self.factory(url)) + + +class ReraiseExceptionPolicy(object): + '''Reraise the exception.''' + + def __init__(self, ttl): + self.ttl = ttl + + def exception(self, url, exception): + '''What to return when there's an exception.''' + return (time.time() + self.ttl, exception) diff --git a/requirements.txt b/requirements.txt index b771e7a..3fa38b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +cachetools==2.0.0 url==0.3.0 requests==2.10.0 six==1.10.0 diff --git a/setup.py b/setup.py index 064a378..4f7489f 100644 --- a/setup.py +++ b/setup.py @@ -73,9 +73,15 @@ platforms = 'Posix; MacOS X', ext_modules = ext_modules, packages = [ - 'reppy' + 'reppy', + 'reppy.cache' ], + package_dir = { + 'reppy': 'reppy', + 'reppy.cache': 'reppy/cache' + }, install_requires = [ + 'cachetools', 'python-dateutil>=1.5, !=2.0', 'requests', 'six' diff --git a/tests/asis/test_agent_allowed/robots.txt b/tests/asis/test_agent_allowed/robots.txt new file mode 100644 index 0000000..75dac68 --- /dev/null +++ b/tests/asis/test_agent_allowed/robots.txt @@ -0,0 +1,6 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: /disallowed +Allow: /allowed diff --git a/tests/asis/test_caches_agent/robots.txt b/tests/asis/test_caches_agent/robots.txt new file mode 100644 index 0000000..75dac68 --- /dev/null +++ b/tests/asis/test_caches_agent/robots.txt @@ -0,0 +1,6 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: /disallowed +Allow: /allowed diff --git a/tests/asis/test_caches_robots/robots.txt b/tests/asis/test_caches_robots/robots.txt new file mode 100644 index 0000000..75dac68 --- /dev/null +++ b/tests/asis/test_caches_robots/robots.txt @@ -0,0 +1,6 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: /disallowed +Allow: /allowed diff --git a/tests/asis/test_returns_a_robots_object/robots.txt b/tests/asis/test_returns_a_robots_object/robots.txt new file mode 100644 index 0000000..1d914e6 --- /dev/null +++ b/tests/asis/test_returns_a_robots_object/robots.txt @@ -0,0 +1,5 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: / diff --git a/tests/asis/test_returns_an_agent_object/robots.txt b/tests/asis/test_returns_an_agent_object/robots.txt new file mode 100644 index 0000000..1d914e6 --- /dev/null +++ b/tests/asis/test_returns_an_agent_object/robots.txt @@ -0,0 +1,5 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: / diff --git a/tests/asis/test_robots_allowed/robots.txt b/tests/asis/test_robots_allowed/robots.txt new file mode 100644 index 0000000..75dac68 --- /dev/null +++ b/tests/asis/test_robots_allowed/robots.txt @@ -0,0 +1,6 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: /disallowed +Allow: /allowed diff --git a/tests/test_cache/__init__.py b/tests/test_cache/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cache/test_cache.py b/tests/test_cache/test_cache.py new file mode 100644 index 0000000..3a27854 --- /dev/null +++ b/tests/test_cache/test_cache.py @@ -0,0 +1,143 @@ +from __future__ import print_function + +'''Tests about our caching utilities.''' + +import contextlib +import os +import unittest + +import requests_mock +import mock + +from reppy import cache + +from ..util import requests_fixtures + + +class TestExpiringObject(unittest.TestCase): + '''Tests about ExpiringObject.''' + + def test_uses_factory(self): + '''Uses the result from the factory.''' + factory = mock.Mock(return_value=(10, 'result')) + obj = cache.ExpiringObject(factory) + self.assertEqual(obj.get(), 'result') + self.assertEqual(obj.expires, 10) + + def test_memoizes_cached_result(self): + '''Memoizes what's returned by the factory.''' + factory = mock.Mock(return_value=(10, 'result')) + obj = cache.ExpiringObject(factory) + with mock.patch.object(cache.time, 'time', return_value=0): + for _ in range(10): + obj.get() + self.assertEqual(factory.call_count, 1) + + def test_reraise_exception(self): + '''If the provided factory returns an exception, reraise it.''' + factory = mock.Mock(return_value=(10, ValueError('Kaboom!'))) + obj = cache.ExpiringObject(factory) + with self.assertRaises(ValueError): + obj.get() + self.assertEqual(obj.expires, 10) + + +class TestBaseCache(unittest.TestCase): + '''Tests about BaseCache.''' + + def test_does_not_implement_missing(self): + '''Does not implement the missing method.''' + with self.assertRaises(NotImplementedError): + cache.BaseCache(10).fetch('http://example.com/robots.txt') + + +class TestRobotsCache(unittest.TestCase): + '''Tests about RobotsCache.''' + + def setUp(self): + self.cache = cache.RobotsCache(10) + + def test_returns_a_robots_object(self): + '''Returns a Robots object.''' + with requests_fixtures('test_returns_a_robots_object'): + self.assertIsInstance( + self.cache.get('http://example.com/blog'), cache.Robots) + + def test_returns_allow_none_on_failure(self): + '''Returns a AllowNone object on exception.''' + self.assertIsInstance( + self.cache.get('http://does-not-resolve/'), cache.AllowNone) + + def test_uses_default_expiration_on_failure(self): + '''When we get AllowNone, it uses the default expiration.''' + with mock.patch.object(self.cache.cache_policy, 'ttl', 17): + with mock.patch.object(cache.time, 'time', return_value=0): + self.cache.get('http://does-not-resolve/') + self.assertEqual( + self.cache.cache['http://does-not-resolve/robots.txt'].expires, 17) + + def test_robots_allowed(self): + '''Can check for allowed.''' + with requests_fixtures('test_robots_allowed'): + self.assertFalse( + self.cache.allowed('http://example.com/disallowed', 'agent')) + self.assertTrue( + self.cache.allowed('http://example.com/allowed', 'agent')) + + def test_caches_robots(self): + '''Caches robots responses.''' + with requests_fixtures('test_caches_robots'): + self.cache.get('http://example.com/') + + # The fact that these pass without the `requests_fixtures` block demonstrates + # that these don't result in a second fetch. + self.assertFalse( + self.cache.allowed('http://example.com/disallowed', 'agent')) + self.assertTrue( + self.cache.allowed('http://example.com/allowed', 'agent')) + + +class TestAgentCache(unittest.TestCase): + '''Tests about AgentCache.''' + + def setUp(self): + self.cache = cache.AgentCache('agent', 10) + + def test_returns_an_agent_object(self): + '''Returns a Robots object.''' + with requests_fixtures('test_returns_an_agent_object'): + self.assertIsInstance( + self.cache.get('http://example.com/blog'), cache.Agent) + + def test_allows_none_on_failure(self): + '''Nothing is allowed on failure.''' + self.assertFalse( + self.cache.get('http://does-not-resolve/').allowed('/path')) + + def test_uses_default_expiration_on_failure(self): + '''On fetch failure, it uses the default expiration.''' + with mock.patch.object(self.cache.cache_policy, 'ttl', 17): + with mock.patch.object(cache.time, 'time', return_value=0): + self.cache.get('http://does-not-resolve/') + self.assertEqual( + self.cache.cache['http://does-not-resolve/robots.txt'].expires, 17) + + def test_agent_allowed(self): + '''Can check for allowed.''' + with requests_fixtures('test_agent_allowed'): + self.assertFalse( + self.cache.allowed('http://example.com/disallowed')) + self.assertTrue( + self.cache.allowed('http://example.com/allowed')) + + def test_caches_agent(self): + '''Caches agent responses.''' + with requests_fixtures('test_caches_agent'): + self.cache.get('http://example.com/') + + # The fact that these pass without the `requests_fixtures` block demonstrates + # that these don't result in a second fetch. + self.assertFalse( + self.cache.allowed('http://example.com/disallowed')) + self.assertTrue( + self.cache.allowed('http://example.com/allowed')) diff --git a/tests/test_cache/test_policy.py b/tests/test_cache/test_policy.py new file mode 100644 index 0000000..23f1a06 --- /dev/null +++ b/tests/test_cache/test_policy.py @@ -0,0 +1,59 @@ +'''Tests about our caching policies.''' + +import unittest + +import mock + +from reppy.cache import policy + + +class TestCachePolicyBase(unittest.TestCase): + '''Tests about CachePolicyBase.''' + + def test_exception_not_implemented(self): + '''Does not implement the exception method.''' + with self.assertRaises(NotImplementedError): + policy.CachePolicyBase().exception( + 'http://example.com/', ValueError('Kaboom')) + + +class TestDefaultObjectPolicy(unittest.TestCase): + '''Tests about DefaultObjectPolicy.''' + + def setUp(self): + self.factory = mock.Mock() + self.ttl = 17 + self.policy = policy.DefaultObjectPolicy(self.ttl, self.factory) + + def test_uses_ttl(self): + '''Uses the provided TTL.''' + with mock.patch.object(policy.time, 'time', return_value=0): + expiration, _ = self.policy.exception( + 'http://example.com/', ValueError('Kaboom')) + self.assertEqual(expiration, self.ttl) + + def test_uses_factory(self): + '''Uses the provided factory.''' + _, value = self.policy.exception('http://example.com/', ValueError('Kaboom')) + self.assertEqual(value, self.factory.return_value) + + +class TestReraiseExceptionPolicy(unittest.TestCase): + '''Tests about ReraiseExceptionPolicy.''' + + def setUp(self): + self.ttl = 17 + self.policy = policy.ReraiseExceptionPolicy(self.ttl) + + def test_uses_ttl(self): + '''Uses the provided TTL.''' + with mock.patch.object(policy.time, 'time', return_value=0): + expiration, _ = self.policy.exception( + 'http://example.com/', ValueError('Kaboom')) + self.assertEqual(expiration, self.ttl) + + def test_returns_exception(self): + '''Returns whatever exception was passed in.''' + exception = ValueError('Kaboom') + _, value = self.policy.exception('http://example.com/', exception) + self.assertEqual(value, exception) From 36464520cc8cc75633361072030e8ab63be31138 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Fri, 4 Nov 2016 18:39:49 -0700 Subject: [PATCH 033/113] Version bump to 0.4.1. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4f7489f..b63dc08 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( name = 'reppy', - version = '0.4.0', + version = '0.4.1', description = 'Replacement robots.txt Parser', long_description = '''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From 2713b5c4a8db663c7d2e833422dcf53d6c82a4e2 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Mon, 7 Nov 2016 16:23:19 -0800 Subject: [PATCH 034/113] MANIFEST.in includes extension source files. --- MANIFEST.in | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a778bd9 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include reppy/rep-cpp/deps/url-cpp/src/* +include reppy/rep-cpp/deps/url-cpp/include/* +include reppy/rep-cpp/src/* +include reppy/rep-cpp/include/* +include reppy/robots.pyx +include reppy/robots.pxd +include reppy/robots.cpp From d0f8458bfab29dcfc3661dc0e85307afdb90b47a Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 10 Nov 2016 13:19:03 -0800 Subject: [PATCH 035/113] Failing test about query-only directives. --- tests/test_agent.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_agent.py b/tests/test_agent.py index c532bd4..ff35c8f 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -99,3 +99,21 @@ def test_accepts_full_url(self): Disallow: /path;params?query ''', 'agent') self.assertFalse(agent.allowed('http://exmaple.com/path;params?query')) + + def test_query_only(self): + '''Recognized query-only rules.''' + agent = self.parse(''' + User-agent: agent + Disallow: /? + ''', 'agent') + self.assertFalse(agent.allowed('/?')) + self.assertTrue(agent.allowed('/')) + + def test_params_only(self): + '''Recognized params-only rules.''' + agent = self.parse(''' + User-agent: agent + Disallow: /; + ''', 'agent') + self.assertFalse(agent.allowed('/;')) + self.assertTrue(agent.allowed('/')) From 087ee05a45b427b26366dd573d0d39509d8c6984 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 10 Nov 2016 13:22:26 -0800 Subject: [PATCH 036/113] Passing test about query-only directives. --- reppy/rep-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index d59ee14..2439eb6 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit d59ee1486e78e8dea53dca38acd0d874117b96da +Subproject commit 2439eb64a3fba37981be6f9e2a48934f2176bb82 From d8bf6c5c252d20eea6caf1b18c592668b1d691ad Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Thu, 10 Nov 2016 13:25:21 -0800 Subject: [PATCH 037/113] Version bump to 0.4.2. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b63dc08..62a9922 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( name = 'reppy', - version = '0.4.1', + version = '0.4.2', description = 'Replacement robots.txt Parser', long_description = '''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From abbb22203eaff703bf63289de37d655134499923 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 16 Nov 2016 08:00:50 -0800 Subject: [PATCH 038/113] Failing test about robots.txt agent case insensitivity. --- tests/test_robots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_robots.py b/tests/test_robots.py index e76e438..fd05433 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -136,7 +136,7 @@ def test_exposes_sitemaps(self): def test_case_insensitivity(self): '''Make sure user agent matches are case insensitive''' robot = robots.Robots.parse('http://example.com/robots.txt', ''' - User-agent: agent + User-agent: Agent Disallow: /path ''') self.assertFalse(robot.allowed('/path', 'agent')) From 419135c774dea818dc232a2e056cc1c0b7114c02 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 16 Nov 2016 08:01:48 -0800 Subject: [PATCH 039/113] Passing test about robots.txt agent case insensitivity. Rebase me when https://github.com/seomoz/rep-cpp/pull/8 is merged. --- reppy/rep-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index 2439eb6..e13fbe2 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit 2439eb64a3fba37981be6f9e2a48934f2176bb82 +Subproject commit e13fbe2e2844d7d5ad4e01087f9b4b888589a1e8 From e9e24230475d26c0edef3c91a30f2fd4243dbb1a Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 16 Nov 2016 08:03:52 -0800 Subject: [PATCH 040/113] Version bump to 0.4.3. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 62a9922..237be36 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( name = 'reppy', - version = '0.4.2', + version = '0.4.3', description = 'Replacement robots.txt Parser', long_description = '''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From 2d163b7bd3ffe547c5b701f9e0ea09c8a820ac41 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 7 Dec 2016 09:23:59 -0800 Subject: [PATCH 041/113] Add missing .python-version. This was actually causing the vagrant provisioning to fail. --- .python-version | 1 + 1 file changed, 1 insertion(+) create mode 100644 .python-version diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..4712731 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +2.7.12 From 6a5f92cf0f221a6f95327f5aacd3f01fd59cd239 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 7 Dec 2016 10:31:24 -0800 Subject: [PATCH 042/113] Fix docs to use right method name. Resolves #39. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7644e0b..2c28db1 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ scheme). ```python # Get robots.txt URL for http://userinfo@example.com:8080/path;params?query#fragment # It's http://example.com:8080/robots.txt -Robots.robotsUrl('http://userinfo@example.com:8080/path;params?query#fragment') +Robots.robots_url('http://userinfo@example.com:8080/path;params?query#fragment') ``` Caching From 1aaf69e020d0335a6c696c306a327f4fbad7b629 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Tue, 17 Jan 2017 17:09:01 -0800 Subject: [PATCH 043/113] Drop trailing whitespace. Also, add blank line at top-level for PEP-8 compliance. --- reppy/cache/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reppy/cache/__init__.py b/reppy/cache/__init__.py index d440b50..2f45297 100644 --- a/reppy/cache/__init__.py +++ b/reppy/cache/__init__.py @@ -9,6 +9,7 @@ from .policy import DefaultObjectPolicy, ReraiseExceptionPolicy from ..robots import Robots, AllowNone, Agent + class ExpiringObject(object): '''Wrap an object that expires over time.''' @@ -33,7 +34,7 @@ def get(self): raise self.exception else: return self.obj - + class BaseCache(object): '''A base cache class.''' From b1dc7e616010e9915720467f5348909e91dc54a6 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Thu, 19 Jan 2017 13:27:42 -0800 Subject: [PATCH 044/113] Bump rep-cpp to latest version. --- reppy/rep-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index e13fbe2..752058d 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit e13fbe2e2844d7d5ad4e01087f9b4b888589a1e8 +Subproject commit 752058d57a32b7ff2098b6e511feecc8cdd7507e From b3cf04076a7a7ff181f8113fd040ecd5baffeb7f Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Tue, 7 Feb 2017 10:13:55 -0800 Subject: [PATCH 045/113] Fix handling of Cache-Control header. The header is named *Cache-Control*, not *Cache_Control*. --- reppy/ttl.py | 2 +- tests/test_ttl.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/reppy/ttl.py b/reppy/ttl.py index 0f6ae8f..c8ac49f 100644 --- a/reppy/ttl.py +++ b/reppy/ttl.py @@ -29,7 +29,7 @@ def ttl(self, response): '''Get the ttl from headers.''' # If max-age is specified in Cache-Control, use it and ignore any # Expires header, as per RFC2616 Sec. 13.2.4. - cache_control = response.headers.get('cache_control') + cache_control = response.headers.get('cache-control') if cache_control is not None: for directive in cache_control.split(','): name, _, value = directive.lower().partition('=') diff --git a/tests/test_ttl.py b/tests/test_ttl.py index ec88898..3ec31b8 100644 --- a/tests/test_ttl.py +++ b/tests/test_ttl.py @@ -27,7 +27,7 @@ class HeaderWithDefaultPolicyTest(unittest.TestCase): def test_no_store(self): '''Returns the minimum when no-store present.''' response = mock.Mock(headers={ - 'cache_control': 'no-store' + 'cache-control': 'no-store' }) policy = ttl.HeaderWithDefaultPolicy(20, 10) self.assertEqual(policy.ttl(response), 10) @@ -35,7 +35,7 @@ def test_no_store(self): def test_must_revalidate(self): '''Returns the minimum when must-revalidate present.''' response = mock.Mock(headers={ - 'cache_control': 'must-revalidate' + 'cache-control': 'must-revalidate' }) policy = ttl.HeaderWithDefaultPolicy(20, 10) self.assertEqual(policy.ttl(response), 10) @@ -43,7 +43,7 @@ def test_must_revalidate(self): def test_no_cache(self): '''Returns the minimum when no-cache present.''' response = mock.Mock(headers={ - 'cache_control': 'no-cache' + 'cache-control': 'no-cache' }) policy = ttl.HeaderWithDefaultPolicy(20, 10) self.assertEqual(policy.ttl(response), 10) @@ -51,7 +51,7 @@ def test_no_cache(self): def test_s_maxage(self): '''Returns the parsed s-maxage.''' response = mock.Mock(headers={ - 'cache_control': 's-maxage=15' + 'cache-control': 's-maxage=15' }) policy = ttl.HeaderWithDefaultPolicy(20, 10) self.assertEqual(policy.ttl(response), 15) @@ -59,7 +59,7 @@ def test_s_maxage(self): def test_max_age(self): '''Returns the parsed max-age.''' response = mock.Mock(headers={ - 'cache_control': 'max-age=15' + 'cache-control': 'max-age=15' }) policy = ttl.HeaderWithDefaultPolicy(20, 10) self.assertEqual(policy.ttl(response), 15) @@ -67,7 +67,7 @@ def test_max_age(self): def test_default_for_malformed_maxage(self): '''Returns the default when maxage cannot be parsed.''' response = mock.Mock(headers={ - 'cache_control': 'max-age=not-a-number' + 'cache-control': 'max-age=not-a-number' }) policy = ttl.HeaderWithDefaultPolicy(20, 10) self.assertEqual(policy.ttl(response), 20) @@ -75,7 +75,7 @@ def test_default_for_malformed_maxage(self): def test_multiple_cache_control(self): '''Can walk through multiple cache control configs.''' response = mock.Mock(headers={ - 'cache_control': 'foo, max-age=15' + 'cache-control': 'foo, max-age=15' }) policy = ttl.HeaderWithDefaultPolicy(20, 10) self.assertEqual(policy.ttl(response), 15) @@ -125,7 +125,7 @@ def test_malformed_expires(self): def test_cache_control_precedence(self): '''Cache control is used before expires.''' response = mock.Mock(headers={ - 'cache_control': 'max-age=30', + 'cache-control': 'max-age=30', 'expires': 'Thu, 13 Oct 2016 15:50:54 GMT', 'date': 'Thu, 13 Oct 2016 15:49:54 GMT' }) From 60b3367d7c8effd329822cd92e66ca1c39cf6bbe Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Tue, 7 Feb 2017 10:16:08 -0800 Subject: [PATCH 046/113] Bump reppy version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 237be36..c3e0196 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( name = 'reppy', - version = '0.4.3', + version = '0.4.4', description = 'Replacement robots.txt Parser', long_description = '''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From 1ce8080b16385cc4383d557afea8437842bf1f68 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Tue, 7 Feb 2017 10:51:47 -0800 Subject: [PATCH 047/113] Update docs to include `--tags` option to push release tags. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2c28db1..4ae1f1a 100644 --- a/README.md +++ b/README.md @@ -279,5 +279,5 @@ git tag -a v0.1.0 -m 'Version 0.1.0 This release contains an initial working version of the `crawl` and `parse` utilities.' -git push origin +git push --tags origin ``` From 42345b67bafbd43dacb746554597940ab39274af Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 8 Feb 2017 10:57:31 -0800 Subject: [PATCH 048/113] Drop unused import. --- reppy/util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reppy/util.py b/reppy/util.py index d686ed7..8cbf9d0 100644 --- a/reppy/util.py +++ b/reppy/util.py @@ -1,7 +1,6 @@ '''Utility functions.''' import email -import string def parse_date(string): From 6d0379b3fd1177a9843477283e785bc13aa06c4b Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 8 Feb 2017 11:30:24 -0800 Subject: [PATCH 049/113] Bump rep-cpp version. --- reppy/rep-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index 752058d..9197fa7 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit 752058d57a32b7ff2098b6e511feecc8cdd7507e +Subproject commit 9197fa75becea7ef01a2381a6e11b8a4cbea447c From f407443b44402b08d1387aa1e4924434493d5feb Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 8 Feb 2017 13:12:47 -0800 Subject: [PATCH 050/113] Bump reppy version. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c3e0196..154fd59 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( name = 'reppy', - version = '0.4.4', + version = '0.4.5', description = 'Replacement robots.txt Parser', long_description = '''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From aa1f7d2c2f3c7cab3948351077e7801339242d84 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 8 Feb 2017 13:32:46 -0800 Subject: [PATCH 051/113] Fix test from rep-cpp change. --- tests/test_robots.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_robots.py b/tests/test_robots.py index fd05433..abc94e9 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -41,12 +41,15 @@ def test_ttl(self): robot = robots.Robots.parse('http://example.com/robots.txt', '', expires=15) self.assertEqual(robot.ttl, 5) - def test_disallow_first(self): - '''Handles a missing User-Agent line.''' - with self.assertRaises(ValueError): - robot = robots.Robots.parse('http://example.com/robots.txt', ''' - Disallow: /path - ''') + def test_no_leading_user_agent(self): + '''Treats missing User-Agent as default user agent''' + robot = robots.Robots.parse('http://example.com/robots.txt', ''' + Disallow: /path + Allow: /path/exception + Crawl-delay: 7 + ''') + self.assertNotEqual(robot.agent('agent'), None) + self.assertEquals(robot.agent('agent').delay, 7) def test_malformed_crawl_delay(self): '''Handles a malformed delay.''' From 36f37e86cea323232ba183eed048d1f87b502258 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 8 Feb 2017 14:36:54 -0800 Subject: [PATCH 052/113] Add additional assertions to test case. --- tests/test_robots.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_robots.py b/tests/test_robots.py index abc94e9..b569780 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -49,6 +49,9 @@ def test_no_leading_user_agent(self): Crawl-delay: 7 ''') self.assertNotEqual(robot.agent('agent'), None) + self.assertTrue(robot.allowed('/path/exception', 'agent')) + self.assertFalse(robot.allowed('/path', 'agent')) + self.assertTrue(robot.allowed('/', 'agent')) self.assertEquals(robot.agent('agent').delay, 7) def test_malformed_crawl_delay(self): From aa52ce949e7d785008d68d065e8453d013cd1216 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 8 Feb 2017 16:15:12 -0800 Subject: [PATCH 053/113] Drop unused imports. --- tests/test_robots.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_robots.py b/tests/test_robots.py index b569780..9b918a5 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -7,12 +7,9 @@ http://www.robotstxt.org/norobots-rfc.txt''' import codecs -import contextlib -import os import unittest import mock -import requests_mock from requests.exceptions import SSLError from reppy import robots From 0b239258cb8299b3fba6e6b4a3b1f53c9e1d0494 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 13 Feb 2017 13:23:41 -0800 Subject: [PATCH 054/113] Add failing test for invalid port. --- tests/test_robots.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_robots.py b/tests/test_robots.py index 9b918a5..6886e6e 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -251,6 +251,12 @@ def test_robots_url_non_default_port(self): expected = 'http://example.com:8080/robots.txt' self.assertEqual(robots.Robots.robots_url(url), expected) + def test_robots_url_invalid_port(self): + '''Raises exception when given an invalid port.''' + url = 'http://:::cnn.com/' + with self.assertRaises(ValueError): + robots.Robots.robots_url(url) + def test_utf8_bom(self): '''If there's a utf-8 BOM, we should parse it as such''' robot = robots.Robots.parse('http://example.com/robots.txt', From a310c28ef204c7d3653d42991a93b6402b9f8e0d Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 13 Feb 2017 13:33:47 -0800 Subject: [PATCH 055/113] Handle invalid port numbers and other exceptions from robotsUrl. Fixes #46. --- reppy/rep-cpp | 2 +- reppy/robots.cpp | 21 ++++++++++++++------- reppy/robots.pxd | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index 9197fa7..e461f36 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit 9197fa75becea7ef01a2381a6e11b8a4cbea447c +Subproject commit e461f36071f82e3addcc23ed6b9865a566cd42e7 diff --git a/reppy/robots.cpp b/reppy/robots.cpp index 9f9814c..4ff99da 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -3784,7 +3784,8 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; - PyObject *__pyx_t_3 = NULL; + std::string __pyx_t_3; + PyObject *__pyx_t_4 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__8) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 106, 0, __PYX_ERR(1, 106, __pyx_L1_error)); @@ -3802,13 +3803,19 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 108, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(Rep::Robots::robotsUrl(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 108, __pyx_L1_error) + try { + __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); + } catch(...) { + try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } + __PYX_ERR(1, 108, __pyx_L1_error) + } + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 108, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 108, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 108, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_3; - __pyx_t_3 = 0; + __pyx_r = __pyx_t_4; + __pyx_t_4 = 0; goto __pyx_L0; /* "reppy/robots.pyx":106 @@ -3822,7 +3829,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; diff --git a/reppy/robots.pxd b/reppy/robots.pxd index 174ac8c..06aa29f 100644 --- a/reppy/robots.pxd +++ b/reppy/robots.pxd @@ -35,4 +35,4 @@ cdef extern from "rep-cpp/include/robots.h" namespace "Rep": const CppAgent& agent(const string& name) const bool allowed(const string& path, const string& name) const @staticmethod - string robotsUrl(const string& url) + string robotsUrl(const string& url) except +ValueError From 524b19719d115fa1e2c474eff797ad53c04c63c7 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 13 Feb 2017 13:37:58 -0800 Subject: [PATCH 056/113] Bump to version 0.4.6. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 154fd59..3f230b9 100644 --- a/setup.py +++ b/setup.py @@ -54,7 +54,7 @@ setup( name = 'reppy', - version = '0.4.5', + version = '0.4.6', description = 'Replacement robots.txt Parser', long_description = '''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From 1e4c934a561a1a6e852c4257d0874eca1449d2c0 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 13 Feb 2017 13:52:23 -0800 Subject: [PATCH 057/113] PEP-8 changes for setup.py. --- setup.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index 3f230b9..f3b8049 100644 --- a/setup.py +++ b/setup.py @@ -46,17 +46,20 @@ ext_files.append('reppy/robots.cpp') ext_modules = [ - Extension('reppy.robots', ext_files, + Extension( + 'reppy.robots', ext_files, language='c++', extra_compile_args=['-std=c++11'], - include_dirs=['reppy/rep-cpp/include', 'reppy/rep-cpp/deps/url-cpp/include']) + include_dirs=[ + 'reppy/rep-cpp/include', + 'reppy/rep-cpp/deps/url-cpp/include']) ] setup( - name = 'reppy', - version = '0.4.6', - description = 'Replacement robots.txt Parser', - long_description = '''Replaces the built-in robotsparser with a + name='reppy', + version='0.4.6', + description='Replacement robots.txt Parser', + long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like Sitemaps, Allow, and Crawl-delay. Main features: @@ -66,27 +69,27 @@ - Configurable user agent for fetching robots.txt - Automatic refetching basing on expiration ''', - author = 'Dan Lecocq', - author_email = 'dan@moz.com', - url = 'http://github.com/seomoz/reppy', - license = 'MIT', - platforms = 'Posix; MacOS X', - ext_modules = ext_modules, - packages = [ + author='Dan Lecocq', + author_email='dan@moz.com', + url='http://github.com/seomoz/reppy', + license='MIT', + platforms='Posix; MacOS X', + ext_modules=ext_modules, + packages=[ 'reppy', 'reppy.cache' ], - package_dir = { + package_dir={ 'reppy': 'reppy', 'reppy.cache': 'reppy/cache' }, - install_requires = [ + install_requires=[ 'cachetools', 'python-dateutil>=1.5, !=2.0', 'requests', 'six' ], - classifiers = [ + classifiers=[ 'License :: OSI Approved :: MIT License', 'Development Status :: 3 - Alpha', 'Environment :: Web Environment', From db9ef9534c96b11e98ee831e9da9e31c76d0e02c Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 3 Mar 2017 09:47:54 -0800 Subject: [PATCH 058/113] Automatically activate virtualenv on `vagrant ssh`. Also, automatically change to correct directory to build code. --- README.md | 2 -- scripts/vagrant/provision.sh | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4ae1f1a..575f4c6 100644 --- a/README.md +++ b/README.md @@ -197,8 +197,6 @@ With a running `vagrant` instance, you can log in and run tests: ```bash vagrant ssh -cd /vagrant - make test ``` diff --git a/scripts/vagrant/provision.sh b/scripts/vagrant/provision.sh index a8c3db9..a380701 100755 --- a/scripts/vagrant/provision.sh +++ b/scripts/vagrant/provision.sh @@ -42,4 +42,9 @@ pushd /vagrant # Lastly, our dependencies pip install -r requirements.txt + echo ' + cd /vagrant + # Activate virtualenv + . /vagrant/venv/bin/activate + ' >> ~/.bash_profile popd From e68ae726f91a918c76198a761c463d2091bf2d49 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 3 Mar 2017 12:08:42 -0800 Subject: [PATCH 059/113] Add pip install -r dev-requirements.txt These are needed for a proper working build environment. --- scripts/vagrant/provision.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/vagrant/provision.sh b/scripts/vagrant/provision.sh index a380701..80d5643 100755 --- a/scripts/vagrant/provision.sh +++ b/scripts/vagrant/provision.sh @@ -41,6 +41,7 @@ pushd /vagrant # Lastly, our dependencies pip install -r requirements.txt + pip install -r dev-requirements.txt echo ' cd /vagrant From 2ac89f05cb7ded042502ac91a14ecd0a4b429427 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 3 Mar 2017 13:04:57 -0800 Subject: [PATCH 060/113] Drop url-py as dependency. It is no longer directly needed. Instead reppy depends on rep-cpp which depends directly on url-cpp rather than on url-py. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3fa38b9..5798789 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ cachetools==2.0.0 -url==0.3.0 requests==2.10.0 six==1.10.0 python-dateutil==2.5.3 From ab8ee6ad2e9901991d37cbf9f3615440036c2399 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 3 Mar 2017 14:55:39 -0800 Subject: [PATCH 061/113] Add MANIFEST to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 5471fd0..f04eb3b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ build/ dist/ reppy.egg-info/ +MANIFEST # Dev artifacts venv* From 459c96237c3e0f4b6f50b9389aa29b9cd964d048 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 3 Mar 2017 14:57:28 -0800 Subject: [PATCH 062/113] Tell PyPi where README is located. --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 00703b7..2ae5961 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,6 @@ +[metadata] +description-file = README.md + [nosetests] verbosity=2 rednose=1 From c5113dbe6177493c1c51e02307b47db1325a654a Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 16 Jun 2017 16:00:52 -0700 Subject: [PATCH 063/113] Bump rep-cpp version. --- reppy/rep-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index e461f36..d28a77b 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit e461f36071f82e3addcc23ed6b9865a566cd42e7 +Subproject commit d28a77b551a4b5e4c69648cd4249ced067faf73c From badeca99f8c4c715afdc43cf161df00a1d268572 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 16 Jun 2017 16:10:05 -0700 Subject: [PATCH 064/113] Add str() methods to C++ objects. --- reppy/robots.cpp | 315 +++++++++++++++++++++++++++++------------------ reppy/robots.pxd | 3 + reppy/robots.pyx | 3 + 3 files changed, 200 insertions(+), 121 deletions(-) diff --git a/reppy/robots.cpp b/reppy/robots.cpp index 4ff99da..c76a189 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -676,7 +676,7 @@ struct __pyx_obj_5reppy_6robots_Robots { }; -/* "reppy/robots.pyx":164 +/* "reppy/robots.pyx":167 * * * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< @@ -688,7 +688,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone { }; -/* "reppy/robots.pyx":171 +/* "reppy/robots.pyx":174 * * * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< @@ -1479,10 +1479,11 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, PyObject *__pyx_v_url); /* proto */ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_url, std::string __pyx_v_content, PyObject *__pyx_v_expires); /* proto */ -static void __pyx_pf_5reppy_6robots_6Robots_2__dealloc__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ +static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_6Robots_4allowed(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_name); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_name); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_name); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_name); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ @@ -3960,7 +3961,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob * self.robots = new CppRobots(content) * self.expires = expires # <<<<<<<<<<<<<< * - * def __dealloc__(self): + * def __str__(self): */ __Pyx_TraceLine(130,0,__PYX_ERR(1, 130, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_expires); @@ -3992,40 +3993,104 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob /* "reppy/robots.pyx":132 * self.expires = expires * + * def __str__(self): # <<<<<<<<<<<<<< + * return self.robots.str() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_3__str__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_3__str__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__str__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_2__str__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__str__", 0); + __Pyx_TraceCall("__str__", __pyx_f[1], 132, 0, __PYX_ERR(1, 132, __pyx_L1_error)); + + /* "reppy/robots.pyx":133 + * + * def __str__(self): + * return self.robots.str() # <<<<<<<<<<<<<< + * + * def __dealloc__(self): + */ + __Pyx_TraceLine(133,0,__PYX_ERR(1, 133, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 133, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":132 + * self.expires = expires + * + * def __str__(self): # <<<<<<<<<<<<<< + * return self.robots.str() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Robots.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":135 + * return self.robots.str() + * * def __dealloc__(self): # <<<<<<<<<<<<<< * del self.robots * */ /* Python wrapper */ -static void __pyx_pw_5reppy_6robots_6Robots_3__dealloc__(PyObject *__pyx_v_self); /*proto*/ -static void __pyx_pw_5reppy_6robots_6Robots_3__dealloc__(PyObject *__pyx_v_self) { +static void __pyx_pw_5reppy_6robots_6Robots_5__dealloc__(PyObject *__pyx_v_self); /*proto*/ +static void __pyx_pw_5reppy_6robots_6Robots_5__dealloc__(PyObject *__pyx_v_self) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__ (wrapper)", 0); - __pyx_pf_5reppy_6robots_6Robots_2__dealloc__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self)); + __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); } -static void __pyx_pf_5reppy_6robots_6Robots_2__dealloc__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self) { +static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self) { __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - __Pyx_TraceCall("__dealloc__", __pyx_f[1], 132, 0, __PYX_ERR(1, 132, __pyx_L1_error)); + __Pyx_TraceCall("__dealloc__", __pyx_f[1], 135, 0, __PYX_ERR(1, 135, __pyx_L1_error)); - /* "reppy/robots.pyx":133 + /* "reppy/robots.pyx":136 * * def __dealloc__(self): * del self.robots # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(133,0,__PYX_ERR(1, 133, __pyx_L1_error)) + __Pyx_TraceLine(136,0,__PYX_ERR(1, 136, __pyx_L1_error)) delete __pyx_v_self->robots; - /* "reppy/robots.pyx":132 - * self.expires = expires + /* "reppy/robots.pyx":135 + * return self.robots.str() * * def __dealloc__(self): # <<<<<<<<<<<<<< * del self.robots @@ -4041,7 +4106,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_2__dealloc__(struct __pyx_obj_5reppy __Pyx_RefNannyFinishContext(); } -/* "reppy/robots.pyx":136 +/* "reppy/robots.pyx":139 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4070,22 +4135,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 136, 0, __PYX_ERR(1, 136, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 139, 0, __PYX_ERR(1, 139, __pyx_L1_error)); - /* "reppy/robots.pyx":138 + /* "reppy/robots.pyx":141 * def sitemaps(self): * '''Get all the sitemaps in this robots.txt.''' * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< * * def allowed(self, path, name): */ - __Pyx_TraceLine(138,0,__PYX_ERR(1, 138, __pyx_L1_error)) + __Pyx_TraceLine(141,0,__PYX_ERR(1, 141, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 138, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 138, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 138, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -4093,14 +4158,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 138, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":136 + /* "reppy/robots.pyx":139 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4122,7 +4187,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ return __pyx_r; } -/* "reppy/robots.pyx":140 +/* "reppy/robots.pyx":143 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -4131,9 +4196,9 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_6Robots_5allowed(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5reppy_6robots_6Robots_4allowed[] = "Is the provided path allowed for the provided agant?"; -static PyObject *__pyx_pw_5reppy_6robots_6Robots_5allowed(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { +static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static char __pyx_doc_5reppy_6robots_6Robots_6allowed[] = "Is the provided path allowed for the provided agant?"; +static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_path = 0; PyObject *__pyx_v_name = 0; PyObject *__pyx_r = 0; @@ -4159,11 +4224,11 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_5allowed(PyObject *__pyx_v_self case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 140, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 143, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 140, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 143, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4176,20 +4241,20 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_5allowed(PyObject *__pyx_v_self } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 140, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 143, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; - __pyx_r = __pyx_pf_5reppy_6robots_6Robots_4allowed(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self), __pyx_v_path, __pyx_v_name); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_6allowed(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self), __pyx_v_path, __pyx_v_name); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_6Robots_4allowed(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_name) { +static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_path, PyObject *__pyx_v_name) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations @@ -4197,32 +4262,32 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_4allowed(struct __pyx_obj_5repp std::string __pyx_t_2; std::string __pyx_t_3; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[1], 140, 0, __PYX_ERR(1, 140, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[1], 143, 0, __PYX_ERR(1, 143, __pyx_L1_error)); - /* "reppy/robots.pyx":142 + /* "reppy/robots.pyx":145 * def allowed(self, path, name): * '''Is the provided path allowed for the provided agant?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) # <<<<<<<<<<<<<< * * def agent(self, name): */ - __Pyx_TraceLine(142,0,__PYX_ERR(1, 142, __pyx_L1_error)) + __Pyx_TraceLine(145,0,__PYX_ERR(1, 145, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 142, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 142, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 145, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 142, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 142, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 145, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 142, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":140 + /* "reppy/robots.pyx":143 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -4242,7 +4307,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_4allowed(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":144 +/* "reppy/robots.pyx":147 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -4251,20 +4316,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_4allowed(struct __pyx_obj_5repp */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_6Robots_7agent(PyObject *__pyx_v_self, PyObject *__pyx_v_name); /*proto*/ -static char __pyx_doc_5reppy_6robots_6Robots_6agent[] = "Return the Agent that corresponds to name."; -static PyObject *__pyx_pw_5reppy_6robots_6Robots_7agent(PyObject *__pyx_v_self, PyObject *__pyx_v_name) { +static PyObject *__pyx_pw_5reppy_6robots_6Robots_9agent(PyObject *__pyx_v_self, PyObject *__pyx_v_name); /*proto*/ +static char __pyx_doc_5reppy_6robots_6Robots_8agent[] = "Return the Agent that corresponds to name."; +static PyObject *__pyx_pw_5reppy_6robots_6Robots_9agent(PyObject *__pyx_v_self, PyObject *__pyx_v_name) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("agent (wrapper)", 0); - __pyx_r = __pyx_pf_5reppy_6robots_6Robots_6agent(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self), ((PyObject *)__pyx_v_name)); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_8agent(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self), ((PyObject *)__pyx_v_name)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_name) { +static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_name) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations @@ -4275,20 +4340,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_ int __pyx_t_5; PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("agent", 0); - __Pyx_TraceCall("agent", __pyx_f[1], 144, 0, __PYX_ERR(1, 144, __pyx_L1_error)); + __Pyx_TraceCall("agent", __pyx_f[1], 147, 0, __PYX_ERR(1, 147, __pyx_L1_error)); - /* "reppy/robots.pyx":146 + /* "reppy/robots.pyx":149 * def agent(self, name): * '''Return the Agent that corresponds to name.''' * return Agent.from_robots(self, as_bytes(name)) # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(146,0,__PYX_ERR(1, 146, __pyx_L1_error)) + __Pyx_TraceLine(149,0,__PYX_ERR(1, 149, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; __pyx_t_5 = 0; @@ -4305,7 +4370,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 149, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -4314,14 +4379,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 149, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -4332,7 +4397,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_ __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -4341,7 +4406,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":144 + /* "reppy/robots.pyx":147 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -4365,7 +4430,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6agent(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":149 +/* "reppy/robots.pyx":152 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4394,20 +4459,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 149, 0, __PYX_ERR(1, 149, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 152, 0, __PYX_ERR(1, 152, __pyx_L1_error)); - /* "reppy/robots.pyx":151 + /* "reppy/robots.pyx":154 * def expired(self): * '''True if the current time is past its expiration.''' * return time.time() > self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(151,0,__PYX_ERR(1, 151, __pyx_L1_error)) + __Pyx_TraceLine(154,0,__PYX_ERR(1, 154, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 151, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 151, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -4421,20 +4486,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } } if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 151, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 151, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 154, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 151, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":149 + /* "reppy/robots.pyx":152 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4456,7 +4521,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":154 +/* "reppy/robots.pyx":157 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4482,22 +4547,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 154, 0, __PYX_ERR(1, 154, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 157, 0, __PYX_ERR(1, 157, __pyx_L1_error)); - /* "reppy/robots.pyx":156 + /* "reppy/robots.pyx":159 * def expires(self): * '''The expiration of this robots.txt.''' * return self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(156,0,__PYX_ERR(1, 156, __pyx_L1_error)) + __Pyx_TraceLine(159,0,__PYX_ERR(1, 159, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_self->expires); __pyx_r = __pyx_v_self->expires; goto __pyx_L0; - /* "reppy/robots.pyx":154 + /* "reppy/robots.pyx":157 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4516,7 +4581,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":159 +/* "reppy/robots.pyx":162 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4548,21 +4613,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 159, 0, __PYX_ERR(1, 159, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 162, 0, __PYX_ERR(1, 162, __pyx_L1_error)); - /* "reppy/robots.pyx":161 + /* "reppy/robots.pyx":164 * def ttl(self): * '''Remaining time for this response to be considered valid.''' * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(161,0,__PYX_ERR(1, 161, __pyx_L1_error)) + __Pyx_TraceLine(164,0,__PYX_ERR(1, 164, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 161, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 161, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -4576,24 +4641,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 161, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 161, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 164, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 161, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 161, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 161, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 161, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 161, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -4607,7 +4672,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":159 + /* "reppy/robots.pyx":162 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4630,7 +4695,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 return __pyx_r; } -/* "reppy/robots.pyx":167 +/* "reppy/robots.pyx":170 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4671,7 +4736,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 167, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 170, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4686,7 +4751,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 167, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 170, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4709,17 +4774,17 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 167, 0, __PYX_ERR(1, 167, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 170, 0, __PYX_ERR(1, 170, __pyx_L1_error)); - /* "reppy/robots.pyx":168 + /* "reppy/robots.pyx":171 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(168,0,__PYX_ERR(1, 168, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 168, __pyx_L1_error) + __Pyx_TraceLine(171,0,__PYX_ERR(1, 171, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 171, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -4736,7 +4801,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 168, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 171, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -4744,13 +4809,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 168, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 171, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 168, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 171, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -4767,14 +4832,14 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 168, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 171, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":167 + /* "reppy/robots.pyx":170 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4798,7 +4863,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":174 +/* "reppy/robots.pyx":177 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4838,7 +4903,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 174, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 177, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4853,7 +4918,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 174, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 177, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4876,15 +4941,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 174, 0, __PYX_ERR(1, 174, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 177, 0, __PYX_ERR(1, 177, __pyx_L1_error)); - /* "reppy/robots.pyx":175 + /* "reppy/robots.pyx":178 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(175,0,__PYX_ERR(1, 175, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 175, __pyx_L1_error) + __Pyx_TraceLine(178,0,__PYX_ERR(1, 178, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 178, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -4901,7 +4966,7 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 175, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -4909,13 +4974,13 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 175, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 175, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 178, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -4932,14 +4997,14 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 175, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":174 + /* "reppy/robots.pyx":177 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5608,7 +5673,7 @@ static void __pyx_tp_dealloc_5reppy_6robots_Robots(PyObject *o) { PyObject *etype, *eval, *etb; PyErr_Fetch(&etype, &eval, &etb); ++Py_REFCNT(o); - __pyx_pw_5reppy_6robots_6Robots_3__dealloc__(o); + __pyx_pw_5reppy_6robots_6Robots_5__dealloc__(o); --Py_REFCNT(o); PyErr_Restore(etype, eval, etb); } @@ -5658,8 +5723,8 @@ static PyObject *__pyx_getprop_5reppy_6robots_6Robots_ttl(PyObject *o, CYTHON_UN } static PyMethodDef __pyx_methods_5reppy_6robots_Robots[] = { - {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_5allowed, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_6Robots_4allowed}, - {"agent", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_7agent, METH_O, __pyx_doc_5reppy_6robots_6Robots_6agent}, + {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_7allowed, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_6Robots_6allowed}, + {"agent", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_9agent, METH_O, __pyx_doc_5reppy_6robots_6Robots_8agent}, {0, 0, 0, 0} }; @@ -5692,7 +5757,7 @@ static PyTypeObject __pyx_type_5reppy_6robots_Robots = { 0, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ - 0, /*tp_str*/ + __pyx_pw_5reppy_6robots_6Robots_3__str__, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ @@ -5760,7 +5825,11 @@ static PyTypeObject __pyx_type_5reppy_6robots_AllowNone = { 0, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ + #if CYTHON_COMPILING_IN_PYPY + __pyx_pw_5reppy_6robots_6Robots_3__str__, /*tp_str*/ + #else 0, /*tp_str*/ + #endif 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ @@ -5828,7 +5897,11 @@ static PyTypeObject __pyx_type_5reppy_6robots_AllowAll = { 0, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ + #if CYTHON_COMPILING_IN_PYPY + __pyx_pw_5reppy_6robots_6Robots_3__str__, /*tp_str*/ + #else 0, /*tp_str*/ + #endif 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ @@ -6054,8 +6127,8 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { - __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 32, __pyx_L1_error) - __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 138, __pyx_L1_error) + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 141, __pyx_L1_error) __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) return 0; __pyx_L1_error:; @@ -6280,14 +6353,14 @@ PyMODINIT_FUNC PyInit_robots(void) if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 110, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 164, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 167, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 164, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 167, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 171, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 174, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 171, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 174, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; diff --git a/reppy/robots.pxd b/reppy/robots.pxd index 06aa29f..5af14da 100644 --- a/reppy/robots.pxd +++ b/reppy/robots.pxd @@ -12,6 +12,7 @@ cdef extern from "rep-cpp/include/directive.h" namespace "Rep": priority_t priority() const bool match(const string& path) const bool allowed() const + string str() const cdef extern from "rep-cpp/include/agent.h" namespace "Rep": cpdef cppclass CppAgent "Rep::Agent": @@ -24,6 +25,7 @@ cdef extern from "rep-cpp/include/agent.h" namespace "Rep": delay_t delay() const const vector[CppDirective]& directives() const bool allowed(const string& path) const + string str() const @staticmethod string escape(const string& query) @@ -34,5 +36,6 @@ cdef extern from "rep-cpp/include/robots.h" namespace "Rep": const vector[string]& sitemaps() const const CppAgent& agent(const string& name) const bool allowed(const string& path, const string& name) const + string str() const @staticmethod string robotsUrl(const string& url) except +ValueError diff --git a/reppy/robots.pyx b/reppy/robots.pyx index d820155..be218ec 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -129,6 +129,9 @@ cdef class Robots: self.robots = new CppRobots(content) self.expires = expires + def __str__(self): + return self.robots.str() + def __dealloc__(self): del self.robots From 5a60622c435914d2c8cf6750bdd6ea342abe825a Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 19 Jun 2017 16:45:27 -0700 Subject: [PATCH 065/113] Release 0.4.7. Also, update package metadata while I'm at it. --- setup.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index f3b8049..0bbe8e9 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright (c) 2011 SEOmoz +# Copyright (c) 2011-2017 SEOmoz, Inc. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.6', + version='0.4.7', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like @@ -67,10 +67,10 @@ - Expiration taken from the `Expires` header - Batch queries - Configurable user agent for fetching robots.txt -- Automatic refetching basing on expiration +- Automatic refetching based on expiration ''', - author='Dan Lecocq', - author_email='dan@moz.com', + maintainer='Brandon Forehand', + maintainer_email='brandon@moz.com', url='http://github.com/seomoz/reppy', license='MIT', platforms='Posix; MacOS X', @@ -91,10 +91,14 @@ ], classifiers=[ 'License :: OSI Approved :: MIT License', - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Environment :: Web Environment', 'Intended Audience :: Developers', - 'Topic :: Internet :: WWW/HTTP' + 'Topic :: Internet :: WWW/HTTP', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5' ], **kwargs ) From 3fffce7b635b69fb3e4402f0c780e5e3c593cdb8 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 19 Jun 2017 17:46:00 -0700 Subject: [PATCH 066/113] Prefer setuptools to distutils. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0bbe8e9..3896800 100644 --- a/setup.py +++ b/setup.py @@ -21,8 +21,8 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from distutils.core import setup -from distutils.extension import Extension +from setuptools import setup +from setuptools.extension import Extension ext_files = [ 'reppy/rep-cpp/src/agent.cpp', From d90ccc61c121b3a7ad0f74c41f45d47b5f2af410 Mon Sep 17 00:00:00 2001 From: Lindsey Reno Date: Thu, 31 Aug 2017 14:12:38 -0700 Subject: [PATCH 067/113] Add str() method to Agent. --- reppy/robots.cpp | 961 +++++++++++++++++++++++++---------------------- reppy/robots.pyx | 3 + 2 files changed, 516 insertions(+), 448 deletions(-) diff --git a/reppy/robots.cpp b/reppy/robots.cpp index c76a189..062904a 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -661,7 +661,7 @@ struct __pyx_obj_5reppy_6robots_Agent { }; -/* "reppy/robots.pyx":110 +/* "reppy/robots.pyx":113 * return as_string(CppRobots.robotsUrl(as_bytes(url))) * * cdef class Robots: # <<<<<<<<<<<<<< @@ -676,7 +676,7 @@ struct __pyx_obj_5reppy_6robots_Robots { }; -/* "reppy/robots.pyx":167 +/* "reppy/robots.pyx":170 * * * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< @@ -688,7 +688,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone { }; -/* "reppy/robots.pyx":174 +/* "reppy/robots.pyx":177 * * * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< @@ -1471,10 +1471,11 @@ static PyObject *__pyx_kp_s_vagrant_reppy_robots_pyx; static PyObject *__pyx_n_s_value; static PyObject *__pyx_n_s_wrap; static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_robots, std::string __pyx_v_name); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_5Agent_allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_5Agent_2disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_5Agent_4allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_content, PyObject *__pyx_v_expires); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, PyObject *__pyx_v_url); /* proto */ @@ -1878,7 +1879,71 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":47 +/* "reppy/robots.pyx":46 + * from_robots = classmethod(FromRobotsMethod) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return self.agent.str() + * + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_1__str__(PyObject *__pyx_v_self); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_1__str__(PyObject *__pyx_v_self) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__str__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent___str__(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__str__", 0); + __Pyx_TraceCall("__str__", __pyx_f[1], 46, 0, __PYX_ERR(1, 46, __pyx_L1_error)); + + /* "reppy/robots.pyx":47 + * + * def __str__(self): + * return self.agent.str() # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_TraceLine(47,0,__PYX_ERR(1, 47, __pyx_L1_error)) + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->agent.str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* "reppy/robots.pyx":46 + * from_robots = classmethod(FromRobotsMethod) + * + * def __str__(self): # <<<<<<<<<<<<<< + * return self.agent.str() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Agent.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":50 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -1907,45 +1972,45 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ int __pyx_t_1; PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 47, 0, __PYX_ERR(1, 47, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 50, 0, __PYX_ERR(1, 50, __pyx_L1_error)); - /* "reppy/robots.pyx":49 + /* "reppy/robots.pyx":52 * def delay(self): * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() # <<<<<<<<<<<<<< * if value > 0: * return value */ - __Pyx_TraceLine(49,0,__PYX_ERR(1, 49, __pyx_L1_error)) + __Pyx_TraceLine(52,0,__PYX_ERR(1, 52, __pyx_L1_error)) __pyx_v_value = __pyx_v_self->agent.delay(); - /* "reppy/robots.pyx":50 + /* "reppy/robots.pyx":53 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< * return value * return None */ - __Pyx_TraceLine(50,0,__PYX_ERR(1, 50, __pyx_L1_error)) + __Pyx_TraceLine(53,0,__PYX_ERR(1, 53, __pyx_L1_error)) __pyx_t_1 = ((__pyx_v_value > 0.0) != 0); if (__pyx_t_1) { - /* "reppy/robots.pyx":51 + /* "reppy/robots.pyx":54 * cdef float value = self.agent.delay() * if value > 0: * return value # <<<<<<<<<<<<<< * return None * */ - __Pyx_TraceLine(51,0,__PYX_ERR(1, 51, __pyx_L1_error)) + __Pyx_TraceLine(54,0,__PYX_ERR(1, 54, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 51, __pyx_L1_error) + __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 54, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":50 + /* "reppy/robots.pyx":53 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< @@ -1954,20 +2019,20 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ */ } - /* "reppy/robots.pyx":52 + /* "reppy/robots.pyx":55 * if value > 0: * return value * return None # <<<<<<<<<<<<<< * * def allow(self, path): */ - __Pyx_TraceLine(52,0,__PYX_ERR(1, 52, __pyx_L1_error)) + __Pyx_TraceLine(55,0,__PYX_ERR(1, 55, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; goto __pyx_L0; - /* "reppy/robots.pyx":47 + /* "reppy/robots.pyx":50 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -1987,7 +2052,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ return __pyx_r; } -/* "reppy/robots.pyx":54 +/* "reppy/robots.pyx":57 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -1996,56 +2061,56 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_1allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ -static char __pyx_doc_5reppy_6robots_5Agent_allow[] = "Allow the provided path."; -static PyObject *__pyx_pw_5reppy_6robots_5Agent_1allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pw_5reppy_6robots_5Agent_3allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ +static char __pyx_doc_5reppy_6robots_5Agent_2allow[] = "Allow the provided path."; +static PyObject *__pyx_pw_5reppy_6robots_5Agent_3allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("allow (wrapper)", 0); - __pyx_r = __pyx_pf_5reppy_6robots_5Agent_allow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_2allow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_5Agent_allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allow", 0); - __Pyx_TraceCall("allow", __pyx_f[1], 54, 0, __PYX_ERR(1, 54, __pyx_L1_error)); + __Pyx_TraceCall("allow", __pyx_f[1], 57, 0, __PYX_ERR(1, 57, __pyx_L1_error)); - /* "reppy/robots.pyx":56 + /* "reppy/robots.pyx":59 * def allow(self, path): * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(56,0,__PYX_ERR(1, 56, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 56, __pyx_L1_error) + __Pyx_TraceLine(59,0,__PYX_ERR(1, 59, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 59, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 56, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 59, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.allow(__pyx_t_2); - /* "reppy/robots.pyx":57 + /* "reppy/robots.pyx":60 * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def disallow(self, path): */ - __Pyx_TraceLine(57,0,__PYX_ERR(1, 57, __pyx_L1_error)) + __Pyx_TraceLine(60,0,__PYX_ERR(1, 60, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":54 + /* "reppy/robots.pyx":57 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -2065,7 +2130,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_allow(struct __pyx_obj_5reppy_6r return __pyx_r; } -/* "reppy/robots.pyx":59 +/* "reppy/robots.pyx":62 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2074,56 +2139,56 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_allow(struct __pyx_obj_5reppy_6r */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_3disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ -static char __pyx_doc_5reppy_6robots_5Agent_2disallow[] = "Disallow the provided path."; -static PyObject *__pyx_pw_5reppy_6robots_5Agent_3disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pw_5reppy_6robots_5Agent_5disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ +static char __pyx_doc_5reppy_6robots_5Agent_4disallow[] = "Disallow the provided path."; +static PyObject *__pyx_pw_5reppy_6robots_5Agent_5disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("disallow (wrapper)", 0); - __pyx_r = __pyx_pf_5reppy_6robots_5Agent_2disallow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_4disallow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_5Agent_2disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("disallow", 0); - __Pyx_TraceCall("disallow", __pyx_f[1], 59, 0, __PYX_ERR(1, 59, __pyx_L1_error)); + __Pyx_TraceCall("disallow", __pyx_f[1], 62, 0, __PYX_ERR(1, 62, __pyx_L1_error)); - /* "reppy/robots.pyx":61 + /* "reppy/robots.pyx":64 * def disallow(self, path): * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(61,0,__PYX_ERR(1, 61, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_TraceLine(64,0,__PYX_ERR(1, 64, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 64, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 64, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.disallow(__pyx_t_2); - /* "reppy/robots.pyx":62 + /* "reppy/robots.pyx":65 * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def allowed(self, path): */ - __Pyx_TraceLine(62,0,__PYX_ERR(1, 62, __pyx_L1_error)) + __Pyx_TraceLine(65,0,__PYX_ERR(1, 65, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":59 + /* "reppy/robots.pyx":62 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2143,7 +2208,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2disallow(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":64 +/* "reppy/robots.pyx":67 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2152,48 +2217,48 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2disallow(struct __pyx_obj_5repp */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_5allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ -static char __pyx_doc_5reppy_6robots_5Agent_4allowed[] = "Is the provided URL allowed?"; -static PyObject *__pyx_pw_5reppy_6robots_5Agent_5allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pw_5reppy_6robots_5Agent_7allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ +static char __pyx_doc_5reppy_6robots_5Agent_6allowed[] = "Is the provided URL allowed?"; +static PyObject *__pyx_pw_5reppy_6robots_5Agent_7allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("allowed (wrapper)", 0); - __pyx_r = __pyx_pf_5reppy_6robots_5Agent_4allowed(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_6allowed(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_5Agent_4allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[1], 64, 0, __PYX_ERR(1, 64, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[1], 67, 0, __PYX_ERR(1, 67, __pyx_L1_error)); - /* "reppy/robots.pyx":66 + /* "reppy/robots.pyx":69 * def allowed(self, path): * '''Is the provided URL allowed?''' * return self.agent.allowed(as_bytes(path)) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(66,0,__PYX_ERR(1, 66, __pyx_L1_error)) + __Pyx_TraceLine(69,0,__PYX_ERR(1, 69, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 69, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 66, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 69, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 66, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 69, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":64 + /* "reppy/robots.pyx":67 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2213,7 +2278,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4allowed(struct __pyx_obj_5reppy return __pyx_r; } -/* "reppy/robots.pyx":69 +/* "reppy/robots.pyx":72 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2256,12 +2321,12 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(1, 69, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(1, 72, __pyx_L3_error) } case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(1, 69, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(1, 72, __pyx_L3_error) } case 3: if (kw_args > 0) { @@ -2270,7 +2335,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(1, 69, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(1, 72, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2289,7 +2354,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 69, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 72, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.ParseMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2314,18 +2379,18 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_6 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__4) __Pyx_RefNannySetupContext("ParseMethod", 0); - __Pyx_TraceCall("ParseMethod", __pyx_f[1], 69, 0, __PYX_ERR(1, 69, __pyx_L1_error)); + __Pyx_TraceCall("ParseMethod", __pyx_f[1], 72, 0, __PYX_ERR(1, 72, __pyx_L1_error)); - /* "reppy/robots.pyx":71 + /* "reppy/robots.pyx":74 * def ParseMethod(cls, url, content, expires=None): * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) # <<<<<<<<<<<<<< * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): */ - __Pyx_TraceLine(71,0,__PYX_ERR(1, 71, __pyx_L1_error)) + __Pyx_TraceLine(74,0,__PYX_ERR(1, 74, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 71, __pyx_L1_error) + __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 74, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_v_cls); __pyx_t_3 = __pyx_v_cls; __pyx_t_4 = NULL; @@ -2343,7 +2408,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 71, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 74, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -2352,14 +2417,14 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 71, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 74, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 71, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 74, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -2373,7 +2438,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_5, __pyx_v_expires); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 71, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 74, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -2382,7 +2447,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":69 + /* "reppy/robots.pyx":72 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2406,7 +2471,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":73 +/* "reppy/robots.pyx":76 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -2465,7 +2530,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(1, 73, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(1, 76, __pyx_L3_error) } case 2: if (kw_args > 0) { @@ -2480,7 +2545,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } if (unlikely(kw_args > 0)) { const Py_ssize_t used_pos_args = (pos_args < 4) ? pos_args : 4; - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(1, 73, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(1, 76, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2502,7 +2567,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 73, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 76, __pyx_L3_error) __pyx_L3_error:; __Pyx_DECREF(__pyx_v_args); __pyx_v_args = 0; __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; @@ -2547,16 +2612,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_18 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__5) __Pyx_RefNannySetupContext("FetchMethod", 0); - __Pyx_TraceCall("FetchMethod", __pyx_f[1], 73, 0, __PYX_ERR(1, 73, __pyx_L1_error)); + __Pyx_TraceCall("FetchMethod", __pyx_f[1], 76, 0, __PYX_ERR(1, 76, __pyx_L1_error)); - /* "reppy/robots.pyx":75 + /* "reppy/robots.pyx":78 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_TraceLine(75,0,__PYX_ERR(1, 75, __pyx_L3_error)) + __Pyx_TraceLine(78,0,__PYX_ERR(1, 78, __pyx_L3_error)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign @@ -2566,41 +2631,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "reppy/robots.pyx":77 + /* "reppy/robots.pyx":80 * try: * # Limit the size of the request * kwargs['stream'] = True # <<<<<<<<<<<<<< * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) */ - __Pyx_TraceLine(77,0,__PYX_ERR(1, 77, __pyx_L3_error)) - if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 77, __pyx_L3_error) + __Pyx_TraceLine(80,0,__PYX_ERR(1, 80, __pyx_L3_error)) + if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 80, __pyx_L3_error) - /* "reppy/robots.pyx":78 + /* "reppy/robots.pyx":81 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __Pyx_TraceLine(78,0,__PYX_ERR(1, 78, __pyx_L3_error)) + __Pyx_TraceLine(81,0,__PYX_ERR(1, 81, __pyx_L3_error)) /*with:*/ { - __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_url); - __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -2615,14 +2680,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -2631,28 +2696,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { - __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); __pyx_t_6 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_7, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_7, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } } __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 78, __pyx_L3_error) + __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 78, __pyx_L11_error) + __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 81, __pyx_L11_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { @@ -2665,10 +2730,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 78, __pyx_L11_error) + __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L11_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { - __pyx_t_5 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 78, __pyx_L11_error) + __pyx_t_5 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L11_error) } __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -2687,78 +2752,78 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_res = __pyx_t_7; __pyx_t_7 = 0; - /* "reppy/robots.pyx":79 + /* "reppy/robots.pyx":82 * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) # <<<<<<<<<<<<<< * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): */ - __Pyx_TraceLine(79,0,__PYX_ERR(1, 79, __pyx_L17_error)) - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 79, __pyx_L17_error) + __Pyx_TraceLine(82,0,__PYX_ERR(1, 82, __pyx_L17_error)) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 82, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 79, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 79, __pyx_L17_error) + __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 82, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 79, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 79, __pyx_L17_error) - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 79, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 82, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 82, __pyx_L17_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 82, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_v_content = __pyx_t_5; __pyx_t_5 = 0; - /* "reppy/robots.pyx":81 + /* "reppy/robots.pyx":84 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) */ - __Pyx_TraceLine(81,0,__PYX_ERR(1, 81, __pyx_L17_error)) - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L17_error) + __Pyx_TraceLine(84,0,__PYX_ERR(1, 84, __pyx_L17_error)) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 84, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 81, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 84, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = PyDict_New(); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L17_error) + __pyx_t_5 = PyDict_New(); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 84, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); - if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 81, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 81, __pyx_L17_error) - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 84, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 84, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 84, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 81, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 84, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_13) { - /* "reppy/robots.pyx":82 + /* "reppy/robots.pyx":85 * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( # <<<<<<<<<<<<<< * 'Content larger than %s bytes' % max_size) * */ - __Pyx_TraceLine(82,0,__PYX_ERR(1, 82, __pyx_L17_error)) - __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 82, __pyx_L17_error) + __Pyx_TraceLine(85,0,__PYX_ERR(1, 85, __pyx_L17_error)) + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 82, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 85, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - /* "reppy/robots.pyx":83 + /* "reppy/robots.pyx":86 * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) # <<<<<<<<<<<<<< * * # Get the TTL policy's ruling on the ttl */ - __Pyx_TraceLine(83,0,__PYX_ERR(1, 83, __pyx_L17_error)) - __pyx_t_5 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 83, __pyx_L17_error) + __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L17_error)) + __pyx_t_5 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 86, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -2771,14 +2836,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L17_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_4); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_5}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; @@ -2787,20 +2852,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_5}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 82, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 85, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_5); __pyx_t_5 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -2808,9 +2873,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 82, __pyx_L17_error) + __PYX_ERR(1, 85, __pyx_L17_error) - /* "reppy/robots.pyx":81 + /* "reppy/robots.pyx":84 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< @@ -2819,28 +2884,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":86 + /* "reppy/robots.pyx":89 * * # Get the TTL policy's ruling on the ttl * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) # <<<<<<<<<<<<<< * * if res.status_code == 200: */ - __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L17_error)) - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_TraceLine(89,0,__PYX_ERR(1, 89, __pyx_L17_error)) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 89, __pyx_L17_error) if (!__pyx_t_13) { } else { __Pyx_INCREF(__pyx_v_ttl_policy); __pyx_t_7 = __pyx_v_ttl_policy; goto __pyx_L26_bool_binop_done; } - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 86, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_8); __pyx_t_7 = __pyx_t_8; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __pyx_L26_bool_binop_done:; - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 86, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = NULL; @@ -2854,13 +2919,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_v_res); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_v_res); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_4); } else @@ -2868,19 +2933,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_4); } else #endif { - __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 86, __pyx_L17_error) + __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_v_res); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } @@ -2889,33 +2954,33 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_expires = __pyx_t_4; __pyx_t_4 = 0; - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":91 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< * return cls.parse(url, content, expires) * elif res.status_code in (401, 403): */ - __Pyx_TraceLine(88,0,__PYX_ERR(1, 88, __pyx_L17_error)) - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 88, __pyx_L17_error) + __Pyx_TraceLine(91,0,__PYX_ERR(1, 91, __pyx_L17_error)) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 91, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = __Pyx_PyInt_EqObjC(__pyx_t_4, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 88, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyInt_EqObjC(__pyx_t_4, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 91, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 88, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 91, __pyx_L17_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_13) { - /* "reppy/robots.pyx":89 + /* "reppy/robots.pyx":92 * * if res.status_code == 200: * return cls.parse(url, content, expires) # <<<<<<<<<<<<<< * elif res.status_code in (401, 403): * return AllowNone(url, expires) */ - __Pyx_TraceLine(89,0,__PYX_ERR(1, 89, __pyx_L17_error)) + __Pyx_TraceLine(92,0,__PYX_ERR(1, 92, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 92, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = NULL; __pyx_t_14 = 0; @@ -2932,7 +2997,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[4] = {__pyx_t_5, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_8); } else @@ -2940,13 +3005,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[4] = {__pyx_t_5, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_8); } else #endif { - __pyx_t_7 = PyTuple_New(3+__pyx_t_14); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_7 = PyTuple_New(3+__pyx_t_14); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 92, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); if (__pyx_t_5) { __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); __pyx_t_5 = NULL; @@ -2960,7 +3025,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_14, __pyx_v_expires); - __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_7, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_7, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } @@ -2969,7 +3034,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_8 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":91 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< @@ -2978,28 +3043,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":93 * if res.status_code == 200: * return cls.parse(url, content, expires) * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: */ - __Pyx_TraceLine(90,0,__PYX_ERR(1, 90, __pyx_L17_error)) - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_TraceLine(93,0,__PYX_ERR(1, 93, __pyx_L17_error)) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 90, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 90, __pyx_L17_error) + __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (!__pyx_t_15) { } else { __pyx_t_13 = __pyx_t_15; goto __pyx_L29_bool_binop_done; } - __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 90, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 90, __pyx_L17_error) + __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_13 = __pyx_t_15; __pyx_L29_bool_binop_done:; @@ -3007,16 +3072,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_15 = (__pyx_t_13 != 0); if (__pyx_t_15) { - /* "reppy/robots.pyx":91 + /* "reppy/robots.pyx":94 * return cls.parse(url, content, expires) * elif res.status_code in (401, 403): * return AllowNone(url, expires) # <<<<<<<<<<<<<< * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) */ - __Pyx_TraceLine(91,0,__PYX_ERR(1, 91, __pyx_L17_error)) + __Pyx_TraceLine(94,0,__PYX_ERR(1, 94, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 94, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); @@ -3024,14 +3089,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_v_expires); - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 94, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":93 * if res.status_code == 200: * return cls.parse(url, content, expires) * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< @@ -3040,45 +3105,45 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":92 + /* "reppy/robots.pyx":95 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< * return AllowAll(url, expires) * else: */ - __Pyx_TraceLine(92,0,__PYX_ERR(1, 92, __pyx_L17_error)) - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 92, __pyx_L17_error) + __Pyx_TraceLine(95,0,__PYX_ERR(1, 95, __pyx_L17_error)) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 95, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = PyObject_RichCompare(__pyx_t_4, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_8 = PyObject_RichCompare(__pyx_t_4, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 95, __pyx_L17_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_13) { } else { __pyx_t_15 = __pyx_t_13; goto __pyx_L31_bool_binop_done; } - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_4 = PyObject_RichCompare(__pyx_t_8, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_4 = PyObject_RichCompare(__pyx_t_8, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 95, __pyx_L17_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 95, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_15 = __pyx_t_13; __pyx_L31_bool_binop_done:; if (__pyx_t_15) { - /* "reppy/robots.pyx":93 + /* "reppy/robots.pyx":96 * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) # <<<<<<<<<<<<<< * else: * raise exceptions.BadStatusCode( */ - __Pyx_TraceLine(93,0,__PYX_ERR(1, 93, __pyx_L17_error)) + __Pyx_TraceLine(96,0,__PYX_ERR(1, 96, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); @@ -3086,14 +3151,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_v_expires); - __pyx_t_8 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_4, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_4, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":92 + /* "reppy/robots.pyx":95 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< @@ -3102,32 +3167,32 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":95 + /* "reppy/robots.pyx":98 * return AllowAll(url, expires) * else: * raise exceptions.BadStatusCode( # <<<<<<<<<<<<<< * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: */ - __Pyx_TraceLine(95,0,__PYX_ERR(1, 95, __pyx_L17_error)) + __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L17_error)) /*else*/ { - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "reppy/robots.pyx":96 + /* "reppy/robots.pyx":99 * else: * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) # <<<<<<<<<<<<<< * except SSLError as exc: * raise exceptions.SSLException(exc) */ - __Pyx_TraceLine(96,0,__PYX_ERR(1, 96, __pyx_L17_error)) - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 96, __pyx_L17_error) + __Pyx_TraceLine(99,0,__PYX_ERR(1, 99, __pyx_L17_error)) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); @@ -3135,10 +3200,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_url); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = NULL; __pyx_t_14 = 0; @@ -3155,7 +3220,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_4, __pyx_t_5}; - __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3165,7 +3230,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_4, __pyx_t_5}; - __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3173,7 +3238,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } else #endif { - __pyx_t_16 = PyTuple_New(2+__pyx_t_14); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_16 = PyTuple_New(2+__pyx_t_14); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_6) { __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; @@ -3184,17 +3249,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_14, __pyx_t_5); __pyx_t_4 = 0; __pyx_t_5 = 0; - __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_8, 0, 0, 0); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __PYX_ERR(1, 95, __pyx_L17_error) + __PYX_ERR(1, 98, __pyx_L17_error) } - /* "reppy/robots.pyx":78 + /* "reppy/robots.pyx":81 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< @@ -3212,20 +3277,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; /*except:*/ { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 78, __pyx_L19_except_error) + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 81, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_16); - __pyx_t_5 = PyTuple_Pack(3, __pyx_t_8, __pyx_t_7, __pyx_t_16); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 78, __pyx_L19_except_error) + __pyx_t_5 = PyTuple_Pack(3, __pyx_t_8, __pyx_t_7, __pyx_t_16); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_5, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 78, __pyx_L19_except_error) + if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 81, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_17); __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_17); __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; - if (__pyx_t_15 < 0) __PYX_ERR(1, 78, __pyx_L19_except_error) + if (__pyx_t_15 < 0) __PYX_ERR(1, 81, __pyx_L19_except_error) __pyx_t_13 = ((!(__pyx_t_15 != 0)) != 0); if (__pyx_t_13) { __Pyx_GIVEREF(__pyx_t_8); @@ -3233,7 +3298,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGIVEREF(__pyx_t_16); __Pyx_ErrRestoreWithState(__pyx_t_8, __pyx_t_7, __pyx_t_16); __pyx_t_8 = 0; __pyx_t_7 = 0; __pyx_t_16 = 0; - __PYX_ERR(1, 78, __pyx_L19_except_error) + __PYX_ERR(1, 81, __pyx_L19_except_error) } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3267,7 +3332,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__6, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 78, __pyx_L3_error) + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; } @@ -3279,7 +3344,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__7, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 78, __pyx_L3_error) + if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 81, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } @@ -3296,7 +3361,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L36:; } - /* "reppy/robots.pyx":75 + /* "reppy/robots.pyx":78 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * try: # <<<<<<<<<<<<<< @@ -3317,38 +3382,38 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; - /* "reppy/robots.pyx":97 + /* "reppy/robots.pyx":100 * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: # <<<<<<<<<<<<<< * raise exceptions.SSLException(exc) * except ConnectionError as exc: */ - __Pyx_TraceLine(97,0,__PYX_ERR(1, 97, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 97, __pyx_L5_except_error) + __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 100, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; if (__pyx_t_14) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_7, &__pyx_t_8) < 0) __PYX_ERR(1, 97, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_7, &__pyx_t_8) < 0) __PYX_ERR(1, 100, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_7); __pyx_v_exc = __pyx_t_7; - /* "reppy/robots.pyx":98 + /* "reppy/robots.pyx":101 * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: * raise exceptions.SSLException(exc) # <<<<<<<<<<<<<< * except ConnectionError as exc: * raise exceptions.ConnectionException(exc) */ - __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L5_except_error)) - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __Pyx_TraceLine(101,0,__PYX_ERR(1, 101, __pyx_L5_except_error)) + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 101, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = NULL; @@ -3362,13 +3427,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_4) { - __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_5); } else @@ -3376,19 +3441,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_5); } else #endif { - __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 101, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_4); __pyx_t_4 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 98, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; } @@ -3396,41 +3461,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_Raise(__pyx_t_5, 0, 0, 0); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 98, __pyx_L5_except_error) + __PYX_ERR(1, 101, __pyx_L5_except_error) } - /* "reppy/robots.pyx":99 + /* "reppy/robots.pyx":102 * except SSLError as exc: * raise exceptions.SSLException(exc) * except ConnectionError as exc: # <<<<<<<<<<<<<< * raise exceptions.ConnectionException(exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: */ - __Pyx_TraceLine(99,0,__PYX_ERR(1, 99, __pyx_L5_except_error)) - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 99, __pyx_L5_except_error) + __Pyx_TraceLine(102,0,__PYX_ERR(1, 102, __pyx_L5_except_error)) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 102, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_14) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 99, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 102, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_16); __Pyx_INCREF(__pyx_t_7); __pyx_v_exc = __pyx_t_7; - /* "reppy/robots.pyx":100 + /* "reppy/robots.pyx":103 * raise exceptions.SSLException(exc) * except ConnectionError as exc: * raise exceptions.ConnectionException(exc) # <<<<<<<<<<<<<< * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * raise exceptions.MalformedUrl(exc) */ - __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L5_except_error)) - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __Pyx_TraceLine(103,0,__PYX_ERR(1, 103, __pyx_L5_except_error)) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 103, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_18 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __pyx_t_18 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 103, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_t_6 = NULL; @@ -3444,13 +3509,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_18, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_18, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_18)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_5); } else @@ -3458,19 +3523,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_18)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_5); } else #endif { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 103, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_exc); - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_18, __pyx_t_4, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_18, __pyx_t_4, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } @@ -3478,24 +3543,24 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; __Pyx_Raise(__pyx_t_5, 0, 0, 0); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 100, __pyx_L5_except_error) + __PYX_ERR(1, 103, __pyx_L5_except_error) } - /* "reppy/robots.pyx":101 + /* "reppy/robots.pyx":104 * except ConnectionError as exc: * raise exceptions.ConnectionException(exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: # <<<<<<<<<<<<<< * raise exceptions.MalformedUrl(exc) * except TooManyRedirects as exc: */ - __Pyx_TraceLine(101,0,__PYX_ERR(1, 101, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 104, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 104, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 104, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_5); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -3504,24 +3569,24 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_14) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_7) < 0) __PYX_ERR(1, 101, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_7) < 0) __PYX_ERR(1, 104, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_7); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":102 + /* "reppy/robots.pyx":105 * raise exceptions.ConnectionException(exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * raise exceptions.MalformedUrl(exc) # <<<<<<<<<<<<<< * except TooManyRedirects as exc: * raise exceptions.ExcessiveRedirects(exc) */ - __Pyx_TraceLine(102,0,__PYX_ERR(1, 102, __pyx_L5_except_error)) - __pyx_t_18 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __Pyx_TraceLine(105,0,__PYX_ERR(1, 105, __pyx_L5_except_error)) + __pyx_t_18 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; __pyx_t_18 = NULL; @@ -3535,13 +3600,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_18) { - __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; __Pyx_GOTREF(__pyx_t_16); } else @@ -3549,19 +3614,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; __Pyx_GOTREF(__pyx_t_16); } else #endif { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_GIVEREF(__pyx_t_18); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_18); __pyx_t_18 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_v_exc); - __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -3569,41 +3634,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_Raise(__pyx_t_16, 0, 0, 0); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __PYX_ERR(1, 102, __pyx_L5_except_error) + __PYX_ERR(1, 105, __pyx_L5_except_error) } - /* "reppy/robots.pyx":103 + /* "reppy/robots.pyx":106 * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * raise exceptions.MalformedUrl(exc) * except TooManyRedirects as exc: # <<<<<<<<<<<<<< * raise exceptions.ExcessiveRedirects(exc) * */ - __Pyx_TraceLine(103,0,__PYX_ERR(1, 103, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 103, __pyx_L5_except_error) + __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 106, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (__pyx_t_14) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_5) < 0) __PYX_ERR(1, 103, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_5) < 0) __PYX_ERR(1, 106, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_5); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":104 + /* "reppy/robots.pyx":107 * raise exceptions.MalformedUrl(exc) * except TooManyRedirects as exc: * raise exceptions.ExcessiveRedirects(exc) # <<<<<<<<<<<<<< * * def RobotsUrlMethod(cls, url): */ - __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L5_except_error)) - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_TraceLine(107,0,__PYX_ERR(1, 107, __pyx_L5_except_error)) + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = NULL; @@ -3617,13 +3682,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_4) { - __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_16); } else @@ -3631,19 +3696,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_16); } else #endif { - __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_4); __pyx_t_4 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); - __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; } @@ -3651,12 +3716,12 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_Raise(__pyx_t_16, 0, 0, 0); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __PYX_ERR(1, 104, __pyx_L5_except_error) + __PYX_ERR(1, 107, __pyx_L5_except_error) } goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "reppy/robots.pyx":75 + /* "reppy/robots.pyx":78 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * try: # <<<<<<<<<<<<<< @@ -3679,7 +3744,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L10_try_end:; } - /* "reppy/robots.pyx":73 + /* "reppy/robots.pyx":76 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -3711,7 +3776,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":106 +/* "reppy/robots.pyx":109 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -3749,11 +3814,11 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 106, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 109, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 106, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 109, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -3766,7 +3831,7 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 106, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 109, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3789,37 +3854,37 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject PyObject *__pyx_t_4 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__8) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); - __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 106, 0, __PYX_ERR(1, 106, __pyx_L1_error)); + __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 109, 0, __PYX_ERR(1, 109, __pyx_L1_error)); - /* "reppy/robots.pyx":108 + /* "reppy/robots.pyx":111 * def RobotsUrlMethod(cls, url): * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) # <<<<<<<<<<<<<< * * cdef class Robots: */ - __Pyx_TraceLine(108,0,__PYX_ERR(1, 108, __pyx_L1_error)) + __Pyx_TraceLine(111,0,__PYX_ERR(1, 111, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 108, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 108, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 108, __pyx_L1_error) + __PYX_ERR(1, 111, __pyx_L1_error) } - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 108, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 108, __pyx_L1_error) + __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 111, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":106 + /* "reppy/robots.pyx":109 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -3840,7 +3905,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":127 +/* "reppy/robots.pyx":130 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -3879,7 +3944,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 127, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 130, __pyx_L3_error) } case 2: if (kw_args > 0) { @@ -3888,7 +3953,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 127, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 130, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -3900,12 +3965,12 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } __pyx_v_url = values[0]; - __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 127, __pyx_L3_error) + __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 130, __pyx_L3_error) __pyx_v_expires = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 127, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 130, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3924,53 +3989,53 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob __Pyx_RefNannyDeclarations Rep::Robots *__pyx_t_1; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 127, 0, __PYX_ERR(1, 127, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 130, 0, __PYX_ERR(1, 130, __pyx_L1_error)); - /* "reppy/robots.pyx":128 + /* "reppy/robots.pyx":131 * * def __init__(self, url, const string& content, expires=None): * self.url = url # <<<<<<<<<<<<<< * self.robots = new CppRobots(content) * self.expires = expires */ - __Pyx_TraceLine(128,0,__PYX_ERR(1, 128, __pyx_L1_error)) + __Pyx_TraceLine(131,0,__PYX_ERR(1, 131, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); __Pyx_GOTREF(__pyx_v_self->url); __Pyx_DECREF(__pyx_v_self->url); __pyx_v_self->url = __pyx_v_url; - /* "reppy/robots.pyx":129 + /* "reppy/robots.pyx":132 * def __init__(self, url, const string& content, expires=None): * self.url = url * self.robots = new CppRobots(content) # <<<<<<<<<<<<<< * self.expires = expires * */ - __Pyx_TraceLine(129,0,__PYX_ERR(1, 129, __pyx_L1_error)) + __Pyx_TraceLine(132,0,__PYX_ERR(1, 132, __pyx_L1_error)) try { __pyx_t_1 = new Rep::Robots(__pyx_v_content); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 129, __pyx_L1_error) + __PYX_ERR(1, 132, __pyx_L1_error) } __pyx_v_self->robots = __pyx_t_1; - /* "reppy/robots.pyx":130 + /* "reppy/robots.pyx":133 * self.url = url * self.robots = new CppRobots(content) * self.expires = expires # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(130,0,__PYX_ERR(1, 130, __pyx_L1_error)) + __Pyx_TraceLine(133,0,__PYX_ERR(1, 133, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); __Pyx_GOTREF(__pyx_v_self->expires); __Pyx_DECREF(__pyx_v_self->expires); __pyx_v_self->expires = __pyx_v_expires; - /* "reppy/robots.pyx":127 + /* "reppy/robots.pyx":130 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -3990,7 +4055,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob return __pyx_r; } -/* "reppy/robots.pyx":132 +/* "reppy/robots.pyx":135 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4017,24 +4082,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[1], 132, 0, __PYX_ERR(1, 132, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[1], 135, 0, __PYX_ERR(1, 135, __pyx_L1_error)); - /* "reppy/robots.pyx":133 + /* "reppy/robots.pyx":136 * * def __str__(self): * return self.robots.str() # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __Pyx_TraceLine(133,0,__PYX_ERR(1, 133, __pyx_L1_error)) + __Pyx_TraceLine(136,0,__PYX_ERR(1, 136, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 133, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":132 + /* "reppy/robots.pyx":135 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4054,7 +4119,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":135 +/* "reppy/robots.pyx":138 * return self.robots.str() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4077,19 +4142,19 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - __Pyx_TraceCall("__dealloc__", __pyx_f[1], 135, 0, __PYX_ERR(1, 135, __pyx_L1_error)); + __Pyx_TraceCall("__dealloc__", __pyx_f[1], 138, 0, __PYX_ERR(1, 138, __pyx_L1_error)); - /* "reppy/robots.pyx":136 + /* "reppy/robots.pyx":139 * * def __dealloc__(self): * del self.robots # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(136,0,__PYX_ERR(1, 136, __pyx_L1_error)) + __Pyx_TraceLine(139,0,__PYX_ERR(1, 139, __pyx_L1_error)) delete __pyx_v_self->robots; - /* "reppy/robots.pyx":135 + /* "reppy/robots.pyx":138 * return self.robots.str() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4106,7 +4171,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_RefNannyFinishContext(); } -/* "reppy/robots.pyx":139 +/* "reppy/robots.pyx":142 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4135,22 +4200,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 139, 0, __PYX_ERR(1, 139, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 142, 0, __PYX_ERR(1, 142, __pyx_L1_error)); - /* "reppy/robots.pyx":141 + /* "reppy/robots.pyx":144 * def sitemaps(self): * '''Get all the sitemaps in this robots.txt.''' * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< * * def allowed(self, path, name): */ - __Pyx_TraceLine(141,0,__PYX_ERR(1, 141, __pyx_L1_error)) + __Pyx_TraceLine(144,0,__PYX_ERR(1, 144, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 141, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 141, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 141, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -4158,14 +4223,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 141, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":139 + /* "reppy/robots.pyx":142 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4187,7 +4252,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ return __pyx_r; } -/* "reppy/robots.pyx":143 +/* "reppy/robots.pyx":146 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -4224,11 +4289,11 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 143, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 146, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 143, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 146, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4241,7 +4306,7 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 143, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 146, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4262,32 +4327,32 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp std::string __pyx_t_2; std::string __pyx_t_3; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[1], 143, 0, __PYX_ERR(1, 143, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[1], 146, 0, __PYX_ERR(1, 146, __pyx_L1_error)); - /* "reppy/robots.pyx":145 + /* "reppy/robots.pyx":148 * def allowed(self, path, name): * '''Is the provided path allowed for the provided agant?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) # <<<<<<<<<<<<<< * * def agent(self, name): */ - __Pyx_TraceLine(145,0,__PYX_ERR(1, 145, __pyx_L1_error)) + __Pyx_TraceLine(148,0,__PYX_ERR(1, 148, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 145, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 145, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 148, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 145, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 145, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 148, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 145, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":143 + /* "reppy/robots.pyx":146 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -4307,7 +4372,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":147 +/* "reppy/robots.pyx":150 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -4340,20 +4405,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ int __pyx_t_5; PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("agent", 0); - __Pyx_TraceCall("agent", __pyx_f[1], 147, 0, __PYX_ERR(1, 147, __pyx_L1_error)); + __Pyx_TraceCall("agent", __pyx_f[1], 150, 0, __PYX_ERR(1, 150, __pyx_L1_error)); - /* "reppy/robots.pyx":149 + /* "reppy/robots.pyx":152 * def agent(self, name): * '''Return the Agent that corresponds to name.''' * return Agent.from_robots(self, as_bytes(name)) # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(149,0,__PYX_ERR(1, 149, __pyx_L1_error)) + __Pyx_TraceLine(152,0,__PYX_ERR(1, 152, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 149, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 149, __pyx_L1_error) + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; __pyx_t_5 = 0; @@ -4370,7 +4435,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 149, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 152, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -4379,14 +4444,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 149, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 152, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 149, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -4397,7 +4462,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 149, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -4406,7 +4471,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":147 + /* "reppy/robots.pyx":150 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -4430,7 +4495,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":152 +/* "reppy/robots.pyx":155 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4459,20 +4524,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 152, 0, __PYX_ERR(1, 152, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 155, 0, __PYX_ERR(1, 155, __pyx_L1_error)); - /* "reppy/robots.pyx":154 + /* "reppy/robots.pyx":157 * def expired(self): * '''True if the current time is past its expiration.''' * return time.time() > self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(154,0,__PYX_ERR(1, 154, __pyx_L1_error)) + __Pyx_TraceLine(157,0,__PYX_ERR(1, 157, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 157, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 157, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -4486,20 +4551,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } } if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 157, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 157, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 157, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":152 + /* "reppy/robots.pyx":155 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4521,7 +4586,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":157 +/* "reppy/robots.pyx":160 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4547,22 +4612,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 157, 0, __PYX_ERR(1, 157, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 160, 0, __PYX_ERR(1, 160, __pyx_L1_error)); - /* "reppy/robots.pyx":159 + /* "reppy/robots.pyx":162 * def expires(self): * '''The expiration of this robots.txt.''' * return self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(159,0,__PYX_ERR(1, 159, __pyx_L1_error)) + __Pyx_TraceLine(162,0,__PYX_ERR(1, 162, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_self->expires); __pyx_r = __pyx_v_self->expires; goto __pyx_L0; - /* "reppy/robots.pyx":157 + /* "reppy/robots.pyx":160 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4581,7 +4646,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":162 +/* "reppy/robots.pyx":165 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4613,21 +4678,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 162, 0, __PYX_ERR(1, 162, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 165, 0, __PYX_ERR(1, 165, __pyx_L1_error)); - /* "reppy/robots.pyx":164 + /* "reppy/robots.pyx":167 * def ttl(self): * '''Remaining time for this response to be considered valid.''' * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(164,0,__PYX_ERR(1, 164, __pyx_L1_error)) + __Pyx_TraceLine(167,0,__PYX_ERR(1, 167, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -4641,24 +4706,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 167, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -4672,7 +4737,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":162 + /* "reppy/robots.pyx":165 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4695,7 +4760,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 return __pyx_r; } -/* "reppy/robots.pyx":170 +/* "reppy/robots.pyx":173 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4736,7 +4801,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 170, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 173, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4751,7 +4816,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 170, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 173, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4774,17 +4839,17 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 170, 0, __PYX_ERR(1, 170, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 173, 0, __PYX_ERR(1, 173, __pyx_L1_error)); - /* "reppy/robots.pyx":171 + /* "reppy/robots.pyx":174 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(171,0,__PYX_ERR(1, 171, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 171, __pyx_L1_error) + __Pyx_TraceLine(174,0,__PYX_ERR(1, 174, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -4801,7 +4866,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 171, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -4809,13 +4874,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 171, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 171, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -4832,14 +4897,14 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 171, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":170 + /* "reppy/robots.pyx":173 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4863,7 +4928,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":177 +/* "reppy/robots.pyx":180 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4903,7 +4968,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 177, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 180, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4918,7 +4983,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 177, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 180, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4941,15 +5006,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 177, 0, __PYX_ERR(1, 177, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 180, 0, __PYX_ERR(1, 180, __pyx_L1_error)); - /* "reppy/robots.pyx":178 + /* "reppy/robots.pyx":181 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(178,0,__PYX_ERR(1, 178, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 178, __pyx_L1_error) + __Pyx_TraceLine(181,0,__PYX_ERR(1, 181, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -4966,7 +5031,7 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -4974,13 +5039,13 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -4997,14 +5062,14 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":177 + /* "reppy/robots.pyx":180 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5577,9 +5642,9 @@ static PyObject *__pyx_getprop_5reppy_6robots_5Agent_delay(PyObject *o, CYTHON_U } static PyMethodDef __pyx_methods_5reppy_6robots_Agent[] = { - {"allow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_1allow, METH_O, __pyx_doc_5reppy_6robots_5Agent_allow}, - {"disallow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_3disallow, METH_O, __pyx_doc_5reppy_6robots_5Agent_2disallow}, - {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_5allowed, METH_O, __pyx_doc_5reppy_6robots_5Agent_4allowed}, + {"allow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_3allow, METH_O, __pyx_doc_5reppy_6robots_5Agent_2allow}, + {"disallow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_5disallow, METH_O, __pyx_doc_5reppy_6robots_5Agent_4disallow}, + {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_7allowed, METH_O, __pyx_doc_5reppy_6robots_5Agent_6allowed}, {0, 0, 0, 0} }; @@ -5609,7 +5674,7 @@ static PyTypeObject __pyx_type_5reppy_6robots_Agent = { 0, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ - 0, /*tp_str*/ + __pyx_pw_5reppy_6robots_5Agent_1__str__, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ @@ -6128,7 +6193,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { }; static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) - __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 141, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 144, __pyx_L1_error) __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) return 0; __pyx_L1_error:; @@ -6161,17 +6226,17 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); - /* "reppy/robots.pyx":78 + /* "reppy/robots.pyx":81 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __pyx_tuple__6 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 78, __pyx_L1_error) + __pyx_tuple__6 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 81, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__6); __Pyx_GIVEREF(__pyx_tuple__6); - __pyx_tuple__7 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 78, __pyx_L1_error) + __pyx_tuple__7 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 81, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); @@ -6199,41 +6264,41 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(__pyx_tuple__12); __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__12, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(1, 33, __pyx_L1_error) - /* "reppy/robots.pyx":69 + /* "reppy/robots.pyx":72 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __pyx_tuple__13 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(1, 69, __pyx_L1_error) + __pyx_tuple__13 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(1, 72, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__13); __Pyx_GIVEREF(__pyx_tuple__13); - __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 69, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 69, __pyx_L1_error) + __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 72, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 72, __pyx_L1_error) - /* "reppy/robots.pyx":73 + /* "reppy/robots.pyx":76 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * try: */ - __pyx_tuple__14 = PyTuple_Pack(10, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(1, 73, __pyx_L1_error) + __pyx_tuple__14 = PyTuple_Pack(10, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__14); __Pyx_GIVEREF(__pyx_tuple__14); - __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 10, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 73, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 73, __pyx_L1_error) + __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 10, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 76, __pyx_L1_error) - /* "reppy/robots.pyx":106 + /* "reppy/robots.pyx":109 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __pyx_tuple__15 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 106, __pyx_L1_error) + __pyx_tuple__15 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 109, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__15); __Pyx_GIVEREF(__pyx_tuple__15); - __pyx_codeobj__8 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 106, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__8)) __PYX_ERR(1, 106, __pyx_L1_error) + __pyx_codeobj__8 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 109, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__8)) __PYX_ERR(1, 109, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -6348,19 +6413,19 @@ PyMODINIT_FUNC PyInit_robots(void) __pyx_type_5reppy_6robots_Agent.tp_print = 0; if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 39, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; - if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 110, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 113, __pyx_L1_error) __pyx_type_5reppy_6robots_Robots.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 110, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 113, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 167, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 170, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 167, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 170, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 174, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 177, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 174, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 177, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; @@ -6584,7 +6649,7 @@ PyMODINIT_FUNC PyInit_robots(void) * * from_robots = classmethod(FromRobotsMethod) # <<<<<<<<<<<<<< * - * @property + * def __str__(self): */ __Pyx_TraceLine(44,0,__PYX_ERR(1, 44, __pyx_L1_error)) __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) @@ -6596,115 +6661,115 @@ PyMODINIT_FUNC PyInit_robots(void) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Agent); - /* "reppy/robots.pyx":69 + /* "reppy/robots.pyx":72 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __Pyx_TraceLine(69,0,__PYX_ERR(1, 69, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 69, __pyx_L1_error) + __Pyx_TraceLine(72,0,__PYX_ERR(1, 72, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 72, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(1, 69, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(1, 72, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":73 + /* "reppy/robots.pyx":76 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * try: */ - __Pyx_TraceLine(73,0,__PYX_ERR(1, 73, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 73, __pyx_L1_error) + __Pyx_TraceLine(76,0,__PYX_ERR(1, 76, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(1, 73, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":106 + /* "reppy/robots.pyx":109 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 106, __pyx_L1_error) + __Pyx_TraceLine(109,0,__PYX_ERR(1, 109, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 106, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 109, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":115 + /* "reppy/robots.pyx":118 * # The default TTL policy is to cache for 3600 seconds or what's provided in the * # headers, and a minimum of 600 seconds * DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) # <<<<<<<<<<<<<< * * # Class methods */ - __Pyx_TraceLine(115,0,__PYX_ERR(1, 115, __pyx_L1_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 115, __pyx_L1_error) + __Pyx_TraceLine(118,0,__PYX_ERR(1, 118, __pyx_L1_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 118, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 115, __pyx_L1_error) + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 118, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 115, __pyx_L1_error) - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 115, __pyx_L1_error) - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 115, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 118, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 118, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 118, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 115, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 118, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":118 + /* "reppy/robots.pyx":121 * * # Class methods * parse = classmethod(ParseMethod) # <<<<<<<<<<<<<< * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) */ - __Pyx_TraceLine(118,0,__PYX_ERR(1, 118, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 118, __pyx_L1_error) + __Pyx_TraceLine(121,0,__PYX_ERR(1, 121, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 121, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 118, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 121, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 118, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 121, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":119 + /* "reppy/robots.pyx":122 * # Class methods * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) # <<<<<<<<<<<<<< * robots_url = classmethod(RobotsUrlMethod) * */ - __Pyx_TraceLine(119,0,__PYX_ERR(1, 119, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 119, __pyx_L1_error) + __Pyx_TraceLine(122,0,__PYX_ERR(1, 122, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 119, __pyx_L1_error) + __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 119, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 122, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":120 + /* "reppy/robots.pyx":123 * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) # <<<<<<<<<<<<<< * * # Data members */ - __Pyx_TraceLine(120,0,__PYX_ERR(1, 120, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 120, __pyx_L1_error) + __Pyx_TraceLine(123,0,__PYX_ERR(1, 123, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 120, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 120, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 123, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); diff --git a/reppy/robots.pyx b/reppy/robots.pyx index be218ec..60751e0 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -43,6 +43,9 @@ cdef class Agent: from_robots = classmethod(FromRobotsMethod) + def __str__(self): + return self.agent.str() + @property def delay(self): '''The delay associated with this agent.''' From f40167ee014de22085b7f0865f2d520d10e616be Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 6 Sep 2017 13:28:07 -0700 Subject: [PATCH 068/113] Fix typo in docstring. --- reppy/rep-cpp | 2 +- reppy/robots.pyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index d28a77b..9ed3d8f 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit d28a77b551a4b5e4c69648cd4249ced067faf73c +Subproject commit 9ed3d8f0c4bdec590347511d68f25db2e2eed370 diff --git a/reppy/robots.pyx b/reppy/robots.pyx index 60751e0..4fae8d6 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -144,7 +144,7 @@ cdef class Robots: return map(as_string, self.robots.sitemaps()) def allowed(self, path, name): - '''Is the provided path allowed for the provided agant?''' + '''Is the provided path allowed for the provided agent?''' return self.robots.allowed(as_bytes(path), as_bytes(name)) def agent(self, name): From 93eccb64c1a69306e2be5703fc8e92713a37ced1 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 8 Sep 2017 10:36:29 -0700 Subject: [PATCH 069/113] Ignore Allow/Disallow directives for external URLs. Integrate changes from seomoz/rep-cpp#28 and bring in other most recent changes from rep-cpp. --- reppy/rep-cpp | 2 +- reppy/robots.cpp | 964 +++++++++++++++++++++++------------------------ reppy/robots.pxd | 8 +- reppy/robots.pyx | 15 +- 4 files changed, 490 insertions(+), 499 deletions(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index 9ed3d8f..ee70c28 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit 9ed3d8f0c4bdec590347511d68f25db2e2eed370 +Subproject commit ee70c280c574a8cf6c896c5ae333adf9dbe2be6d diff --git a/reppy/robots.cpp b/reppy/robots.cpp index 062904a..480eced 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -648,7 +648,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone; struct __pyx_obj_5reppy_6robots_AllowAll; struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; -/* "reppy/robots.pyx":39 +/* "reppy/robots.pyx":43 * return agent * * cdef class Agent: # <<<<<<<<<<<<<< @@ -661,7 +661,7 @@ struct __pyx_obj_5reppy_6robots_Agent { }; -/* "reppy/robots.pyx":113 +/* "reppy/robots.pyx":117 * return as_string(CppRobots.robotsUrl(as_bytes(url))) * * cdef class Robots: # <<<<<<<<<<<<<< @@ -671,12 +671,11 @@ struct __pyx_obj_5reppy_6robots_Agent { struct __pyx_obj_5reppy_6robots_Robots { PyObject_HEAD Rep::Robots *robots; - PyObject *url; PyObject *expires; }; -/* "reppy/robots.pyx":170 +/* "reppy/robots.pyx":177 * * * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< @@ -688,7 +687,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone { }; -/* "reppy/robots.pyx":177 +/* "reppy/robots.pyx":184 * * * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< @@ -1826,8 +1825,8 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject * def FromRobotsMethod(cls, Robots robots, const string& name): * '''Construct an Agent from a CppAgent.''' * agent = Agent() # <<<<<<<<<<<<<< - * agent.agent = robots.robots.agent(name) - * return agent + * # This is somewhat inefficient due to the copying, but it is + * # required to be copied because we often toss the containing */ __Pyx_TraceLine(35,0,__PYX_ERR(1, 35, __pyx_L1_error)) __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 35, __pyx_L1_error) @@ -1835,24 +1834,24 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject __pyx_v_agent = ((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":36 - * '''Construct an Agent from a CppAgent.''' - * agent = Agent() + /* "reppy/robots.pyx":40 + * # Robots object as a temporary thus we'd leave the underlying + * # Agent object dangling without a full copy. * agent.agent = robots.robots.agent(name) # <<<<<<<<<<<<<< * return agent * */ - __Pyx_TraceLine(36,0,__PYX_ERR(1, 36, __pyx_L1_error)) + __Pyx_TraceLine(40,0,__PYX_ERR(1, 40, __pyx_L1_error)) __pyx_v_agent->agent = __pyx_v_robots->robots->agent(__pyx_v_name); - /* "reppy/robots.pyx":37 - * agent = Agent() + /* "reppy/robots.pyx":41 + * # Agent object dangling without a full copy. * agent.agent = robots.robots.agent(name) * return agent # <<<<<<<<<<<<<< * * cdef class Agent: */ - __Pyx_TraceLine(37,0,__PYX_ERR(1, 37, __pyx_L1_error)) + __Pyx_TraceLine(41,0,__PYX_ERR(1, 41, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_agent)); __pyx_r = ((PyObject *)__pyx_v_agent); @@ -1879,7 +1878,7 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":46 +/* "reppy/robots.pyx":50 * from_robots = classmethod(FromRobotsMethod) * * def __str__(self): # <<<<<<<<<<<<<< @@ -1906,24 +1905,24 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[1], 46, 0, __PYX_ERR(1, 46, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[1], 50, 0, __PYX_ERR(1, 50, __pyx_L1_error)); - /* "reppy/robots.pyx":47 + /* "reppy/robots.pyx":51 * * def __str__(self): * return self.agent.str() # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(47,0,__PYX_ERR(1, 47, __pyx_L1_error)) + __Pyx_TraceLine(51,0,__PYX_ERR(1, 51, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->agent.str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 47, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->agent.str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 51, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":46 + /* "reppy/robots.pyx":50 * from_robots = classmethod(FromRobotsMethod) * * def __str__(self): # <<<<<<<<<<<<<< @@ -1943,7 +1942,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":50 +/* "reppy/robots.pyx":54 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -1972,45 +1971,45 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ int __pyx_t_1; PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 50, 0, __PYX_ERR(1, 50, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 54, 0, __PYX_ERR(1, 54, __pyx_L1_error)); - /* "reppy/robots.pyx":52 + /* "reppy/robots.pyx":56 * def delay(self): * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() # <<<<<<<<<<<<<< * if value > 0: * return value */ - __Pyx_TraceLine(52,0,__PYX_ERR(1, 52, __pyx_L1_error)) + __Pyx_TraceLine(56,0,__PYX_ERR(1, 56, __pyx_L1_error)) __pyx_v_value = __pyx_v_self->agent.delay(); - /* "reppy/robots.pyx":53 + /* "reppy/robots.pyx":57 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< * return value * return None */ - __Pyx_TraceLine(53,0,__PYX_ERR(1, 53, __pyx_L1_error)) + __Pyx_TraceLine(57,0,__PYX_ERR(1, 57, __pyx_L1_error)) __pyx_t_1 = ((__pyx_v_value > 0.0) != 0); if (__pyx_t_1) { - /* "reppy/robots.pyx":54 + /* "reppy/robots.pyx":58 * cdef float value = self.agent.delay() * if value > 0: * return value # <<<<<<<<<<<<<< * return None * */ - __Pyx_TraceLine(54,0,__PYX_ERR(1, 54, __pyx_L1_error)) + __Pyx_TraceLine(58,0,__PYX_ERR(1, 58, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 54, __pyx_L1_error) + __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 58, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":53 + /* "reppy/robots.pyx":57 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< @@ -2019,20 +2018,20 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ */ } - /* "reppy/robots.pyx":55 + /* "reppy/robots.pyx":59 * if value > 0: * return value * return None # <<<<<<<<<<<<<< * * def allow(self, path): */ - __Pyx_TraceLine(55,0,__PYX_ERR(1, 55, __pyx_L1_error)) + __Pyx_TraceLine(59,0,__PYX_ERR(1, 59, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; goto __pyx_L0; - /* "reppy/robots.pyx":50 + /* "reppy/robots.pyx":54 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -2052,7 +2051,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ return __pyx_r; } -/* "reppy/robots.pyx":57 +/* "reppy/robots.pyx":61 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -2081,36 +2080,36 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allow", 0); - __Pyx_TraceCall("allow", __pyx_f[1], 57, 0, __PYX_ERR(1, 57, __pyx_L1_error)); + __Pyx_TraceCall("allow", __pyx_f[1], 61, 0, __PYX_ERR(1, 61, __pyx_L1_error)); - /* "reppy/robots.pyx":59 + /* "reppy/robots.pyx":63 * def allow(self, path): * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(59,0,__PYX_ERR(1, 59, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 59, __pyx_L1_error) + __Pyx_TraceLine(63,0,__PYX_ERR(1, 63, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 63, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 59, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 63, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.allow(__pyx_t_2); - /* "reppy/robots.pyx":60 + /* "reppy/robots.pyx":64 * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def disallow(self, path): */ - __Pyx_TraceLine(60,0,__PYX_ERR(1, 60, __pyx_L1_error)) + __Pyx_TraceLine(64,0,__PYX_ERR(1, 64, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":57 + /* "reppy/robots.pyx":61 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -2130,7 +2129,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":62 +/* "reppy/robots.pyx":66 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2159,36 +2158,36 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("disallow", 0); - __Pyx_TraceCall("disallow", __pyx_f[1], 62, 0, __PYX_ERR(1, 62, __pyx_L1_error)); + __Pyx_TraceCall("disallow", __pyx_f[1], 66, 0, __PYX_ERR(1, 66, __pyx_L1_error)); - /* "reppy/robots.pyx":64 + /* "reppy/robots.pyx":68 * def disallow(self, path): * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(64,0,__PYX_ERR(1, 64, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 64, __pyx_L1_error) + __Pyx_TraceLine(68,0,__PYX_ERR(1, 68, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 68, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 64, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 68, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.disallow(__pyx_t_2); - /* "reppy/robots.pyx":65 + /* "reppy/robots.pyx":69 * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def allowed(self, path): */ - __Pyx_TraceLine(65,0,__PYX_ERR(1, 65, __pyx_L1_error)) + __Pyx_TraceLine(69,0,__PYX_ERR(1, 69, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":62 + /* "reppy/robots.pyx":66 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2208,7 +2207,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":67 +/* "reppy/robots.pyx":71 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2237,28 +2236,28 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[1], 67, 0, __PYX_ERR(1, 67, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[1], 71, 0, __PYX_ERR(1, 71, __pyx_L1_error)); - /* "reppy/robots.pyx":69 + /* "reppy/robots.pyx":73 * def allowed(self, path): * '''Is the provided URL allowed?''' * return self.agent.allowed(as_bytes(path)) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(69,0,__PYX_ERR(1, 69, __pyx_L1_error)) + __Pyx_TraceLine(73,0,__PYX_ERR(1, 73, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 69, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 73, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 69, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 73, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 69, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 73, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":67 + /* "reppy/robots.pyx":71 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2278,7 +2277,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy return __pyx_r; } -/* "reppy/robots.pyx":72 +/* "reppy/robots.pyx":76 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2321,12 +2320,12 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(1, 72, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(1, 76, __pyx_L3_error) } case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(1, 72, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(1, 76, __pyx_L3_error) } case 3: if (kw_args > 0) { @@ -2335,7 +2334,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(1, 72, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(1, 76, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2354,7 +2353,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 72, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 76, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.ParseMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2379,18 +2378,18 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_6 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__4) __Pyx_RefNannySetupContext("ParseMethod", 0); - __Pyx_TraceCall("ParseMethod", __pyx_f[1], 72, 0, __PYX_ERR(1, 72, __pyx_L1_error)); + __Pyx_TraceCall("ParseMethod", __pyx_f[1], 76, 0, __PYX_ERR(1, 76, __pyx_L1_error)); - /* "reppy/robots.pyx":74 + /* "reppy/robots.pyx":78 * def ParseMethod(cls, url, content, expires=None): * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) # <<<<<<<<<<<<<< * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): */ - __Pyx_TraceLine(74,0,__PYX_ERR(1, 74, __pyx_L1_error)) + __Pyx_TraceLine(78,0,__PYX_ERR(1, 78, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 74, __pyx_L1_error) + __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 78, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_v_cls); __pyx_t_3 = __pyx_v_cls; __pyx_t_4 = NULL; @@ -2408,7 +2407,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 74, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 78, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -2417,14 +2416,14 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 74, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 78, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 74, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 78, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -2438,7 +2437,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_5, __pyx_v_expires); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 74, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 78, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -2447,7 +2446,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":72 + /* "reppy/robots.pyx":76 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2471,7 +2470,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":76 +/* "reppy/robots.pyx":80 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -2530,7 +2529,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(1, 76, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(1, 80, __pyx_L3_error) } case 2: if (kw_args > 0) { @@ -2545,7 +2544,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } if (unlikely(kw_args > 0)) { const Py_ssize_t used_pos_args = (pos_args < 4) ? pos_args : 4; - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(1, 76, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(1, 80, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2567,7 +2566,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 76, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 80, __pyx_L3_error) __pyx_L3_error:; __Pyx_DECREF(__pyx_v_args); __pyx_v_args = 0; __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; @@ -2612,16 +2611,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_18 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__5) __Pyx_RefNannySetupContext("FetchMethod", 0); - __Pyx_TraceCall("FetchMethod", __pyx_f[1], 76, 0, __PYX_ERR(1, 76, __pyx_L1_error)); + __Pyx_TraceCall("FetchMethod", __pyx_f[1], 80, 0, __PYX_ERR(1, 80, __pyx_L1_error)); - /* "reppy/robots.pyx":78 + /* "reppy/robots.pyx":82 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_TraceLine(78,0,__PYX_ERR(1, 78, __pyx_L3_error)) + __Pyx_TraceLine(82,0,__PYX_ERR(1, 82, __pyx_L3_error)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign @@ -2631,41 +2630,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGOTREF(__pyx_t_3); /*try:*/ { - /* "reppy/robots.pyx":80 + /* "reppy/robots.pyx":84 * try: * # Limit the size of the request * kwargs['stream'] = True # <<<<<<<<<<<<<< * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) */ - __Pyx_TraceLine(80,0,__PYX_ERR(1, 80, __pyx_L3_error)) - if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 80, __pyx_L3_error) + __Pyx_TraceLine(84,0,__PYX_ERR(1, 84, __pyx_L3_error)) + if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 84, __pyx_L3_error) - /* "reppy/robots.pyx":81 + /* "reppy/robots.pyx":85 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __Pyx_TraceLine(81,0,__PYX_ERR(1, 81, __pyx_L3_error)) + __Pyx_TraceLine(85,0,__PYX_ERR(1, 85, __pyx_L3_error)) /*with:*/ { - __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_url); - __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -2680,14 +2679,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -2696,28 +2695,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { - __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); __pyx_t_6 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_7, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_7, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } } __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 81, __pyx_L3_error) + __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 81, __pyx_L11_error) + __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 85, __pyx_L11_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { @@ -2730,10 +2729,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L11_error) + __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L11_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { - __pyx_t_5 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L11_error) + __pyx_t_5 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L11_error) } __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -2752,78 +2751,78 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_res = __pyx_t_7; __pyx_t_7 = 0; - /* "reppy/robots.pyx":82 + /* "reppy/robots.pyx":86 * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) # <<<<<<<<<<<<<< * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): */ - __Pyx_TraceLine(82,0,__PYX_ERR(1, 82, __pyx_L17_error)) - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 82, __pyx_L17_error) + __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L17_error)) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 86, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 82, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 82, __pyx_L17_error) + __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 86, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 82, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 82, __pyx_L17_error) - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 82, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 86, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 86, __pyx_L17_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 86, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_v_content = __pyx_t_5; __pyx_t_5 = 0; - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":88 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) */ - __Pyx_TraceLine(84,0,__PYX_ERR(1, 84, __pyx_L17_error)) - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 84, __pyx_L17_error) + __Pyx_TraceLine(88,0,__PYX_ERR(1, 88, __pyx_L17_error)) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 88, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 84, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 88, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = PyDict_New(); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 84, __pyx_L17_error) + __pyx_t_5 = PyDict_New(); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 88, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); - if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 84, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 84, __pyx_L17_error) - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 84, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 88, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 88, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 88, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 84, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 88, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (__pyx_t_13) { - /* "reppy/robots.pyx":85 + /* "reppy/robots.pyx":89 * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( # <<<<<<<<<<<<<< * 'Content larger than %s bytes' % max_size) * */ - __Pyx_TraceLine(85,0,__PYX_ERR(1, 85, __pyx_L17_error)) - __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L17_error) + __Pyx_TraceLine(89,0,__PYX_ERR(1, 89, __pyx_L17_error)) + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 85, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - /* "reppy/robots.pyx":86 + /* "reppy/robots.pyx":90 * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) # <<<<<<<<<<<<<< * * # Get the TTL policy's ruling on the ttl */ - __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L17_error)) - __pyx_t_5 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_TraceLine(90,0,__PYX_ERR(1, 90, __pyx_L17_error)) + __pyx_t_5 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 90, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -2836,14 +2835,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_4); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_5}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; @@ -2852,20 +2851,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_5}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 85, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_5); __pyx_t_5 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -2873,9 +2872,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_4, 0, 0, 0); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 85, __pyx_L17_error) + __PYX_ERR(1, 89, __pyx_L17_error) - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":88 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< @@ -2884,28 +2883,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":89 + /* "reppy/robots.pyx":93 * * # Get the TTL policy's ruling on the ttl * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) # <<<<<<<<<<<<<< * * if res.status_code == 200: */ - __Pyx_TraceLine(89,0,__PYX_ERR(1, 89, __pyx_L17_error)) - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 89, __pyx_L17_error) + __Pyx_TraceLine(93,0,__PYX_ERR(1, 93, __pyx_L17_error)) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 93, __pyx_L17_error) if (!__pyx_t_13) { } else { __Pyx_INCREF(__pyx_v_ttl_policy); __pyx_t_7 = __pyx_v_ttl_policy; goto __pyx_L26_bool_binop_done; } - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_8); __pyx_t_7 = __pyx_t_8; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __pyx_L26_bool_binop_done:; - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = NULL; @@ -2919,13 +2918,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_v_res); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_v_res); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_4); } else @@ -2933,19 +2932,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_4); } else #endif { - __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_v_res); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } @@ -2954,33 +2953,33 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_expires = __pyx_t_4; __pyx_t_4 = 0; - /* "reppy/robots.pyx":91 + /* "reppy/robots.pyx":95 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< * return cls.parse(url, content, expires) * elif res.status_code in (401, 403): */ - __Pyx_TraceLine(91,0,__PYX_ERR(1, 91, __pyx_L17_error)) - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 91, __pyx_L17_error) + __Pyx_TraceLine(95,0,__PYX_ERR(1, 95, __pyx_L17_error)) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 95, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = __Pyx_PyInt_EqObjC(__pyx_t_4, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyInt_EqObjC(__pyx_t_4, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 95, __pyx_L17_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_13) { - /* "reppy/robots.pyx":92 + /* "reppy/robots.pyx":96 * * if res.status_code == 200: * return cls.parse(url, content, expires) # <<<<<<<<<<<<<< * elif res.status_code in (401, 403): * return AllowNone(url, expires) */ - __Pyx_TraceLine(92,0,__PYX_ERR(1, 92, __pyx_L17_error)) + __Pyx_TraceLine(96,0,__PYX_ERR(1, 96, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_5 = NULL; __pyx_t_14 = 0; @@ -2997,7 +2996,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[4] = {__pyx_t_5, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_8); } else @@ -3005,13 +3004,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[4] = {__pyx_t_5, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_GOTREF(__pyx_t_8); } else #endif { - __pyx_t_7 = PyTuple_New(3+__pyx_t_14); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_7 = PyTuple_New(3+__pyx_t_14); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); if (__pyx_t_5) { __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); __pyx_t_5 = NULL; @@ -3025,7 +3024,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_14, __pyx_v_expires); - __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_7, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 92, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_7, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } @@ -3034,7 +3033,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_8 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":91 + /* "reppy/robots.pyx":95 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< @@ -3043,28 +3042,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":93 + /* "reppy/robots.pyx":97 * if res.status_code == 200: * return cls.parse(url, content, expires) * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: */ - __Pyx_TraceLine(93,0,__PYX_ERR(1, 93, __pyx_L17_error)) - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L17_error) + __Pyx_TraceLine(97,0,__PYX_ERR(1, 97, __pyx_L17_error)) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 93, __pyx_L17_error) + __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; if (!__pyx_t_15) { } else { __pyx_t_13 = __pyx_t_15; goto __pyx_L29_bool_binop_done; } - __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 93, __pyx_L17_error) + __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_13 = __pyx_t_15; __pyx_L29_bool_binop_done:; @@ -3072,16 +3071,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_15 = (__pyx_t_13 != 0); if (__pyx_t_15) { - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":98 * return cls.parse(url, content, expires) * elif res.status_code in (401, 403): * return AllowNone(url, expires) # <<<<<<<<<<<<<< * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) */ - __Pyx_TraceLine(94,0,__PYX_ERR(1, 94, __pyx_L17_error)) + __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 94, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); @@ -3089,14 +3088,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_v_expires); - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 94, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":93 + /* "reppy/robots.pyx":97 * if res.status_code == 200: * return cls.parse(url, content, expires) * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< @@ -3105,45 +3104,45 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":95 + /* "reppy/robots.pyx":99 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< * return AllowAll(url, expires) * else: */ - __Pyx_TraceLine(95,0,__PYX_ERR(1, 95, __pyx_L17_error)) - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_TraceLine(99,0,__PYX_ERR(1, 99, __pyx_L17_error)) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = PyObject_RichCompare(__pyx_t_4, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_8 = PyObject_RichCompare(__pyx_t_4, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_13) { } else { __pyx_t_15 = __pyx_t_13; goto __pyx_L31_bool_binop_done; } - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_4 = PyObject_RichCompare(__pyx_t_8, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_4 = PyObject_RichCompare(__pyx_t_8, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 99, __pyx_L17_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_15 = __pyx_t_13; __pyx_L31_bool_binop_done:; if (__pyx_t_15) { - /* "reppy/robots.pyx":96 + /* "reppy/robots.pyx":100 * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) # <<<<<<<<<<<<<< * else: * raise exceptions.BadStatusCode( */ - __Pyx_TraceLine(96,0,__PYX_ERR(1, 96, __pyx_L17_error)) + __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 100, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); @@ -3151,14 +3150,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_v_expires); - __pyx_t_8 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_4, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_4, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 100, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_r = __pyx_t_8; __pyx_t_8 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":95 + /* "reppy/robots.pyx":99 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< @@ -3167,32 +3166,32 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":98 + /* "reppy/robots.pyx":102 * return AllowAll(url, expires) * else: * raise exceptions.BadStatusCode( # <<<<<<<<<<<<<< * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: */ - __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L17_error)) + __Pyx_TraceLine(102,0,__PYX_ERR(1, 102, __pyx_L17_error)) /*else*/ { - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 102, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 102, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - /* "reppy/robots.pyx":99 + /* "reppy/robots.pyx":103 * else: * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) # <<<<<<<<<<<<<< * except SSLError as exc: * raise exceptions.SSLException(exc) */ - __Pyx_TraceLine(99,0,__PYX_ERR(1, 99, __pyx_L17_error)) - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 99, __pyx_L17_error) + __Pyx_TraceLine(103,0,__PYX_ERR(1, 103, __pyx_L17_error)) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 103, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 99, __pyx_L17_error) + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); @@ -3200,10 +3199,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_url); __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 99, __pyx_L17_error) + __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 103, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 99, __pyx_L17_error) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_6 = NULL; __pyx_t_14 = 0; @@ -3220,7 +3219,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_4, __pyx_t_5}; - __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 102, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3230,7 +3229,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_4, __pyx_t_5}; - __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 102, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3238,7 +3237,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } else #endif { - __pyx_t_16 = PyTuple_New(2+__pyx_t_14); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_16 = PyTuple_New(2+__pyx_t_14); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_6) { __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; @@ -3249,17 +3248,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_14, __pyx_t_5); __pyx_t_4 = 0; __pyx_t_5 = 0; - __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 102, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_8, 0, 0, 0); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __PYX_ERR(1, 98, __pyx_L17_error) + __PYX_ERR(1, 102, __pyx_L17_error) } - /* "reppy/robots.pyx":81 + /* "reppy/robots.pyx":85 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< @@ -3277,20 +3276,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; /*except:*/ { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 81, __pyx_L19_except_error) + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 85, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_16); - __pyx_t_5 = PyTuple_Pack(3, __pyx_t_8, __pyx_t_7, __pyx_t_16); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 81, __pyx_L19_except_error) + __pyx_t_5 = PyTuple_Pack(3, __pyx_t_8, __pyx_t_7, __pyx_t_16); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_5, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 81, __pyx_L19_except_error) + if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 85, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_17); __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_17); __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; - if (__pyx_t_15 < 0) __PYX_ERR(1, 81, __pyx_L19_except_error) + if (__pyx_t_15 < 0) __PYX_ERR(1, 85, __pyx_L19_except_error) __pyx_t_13 = ((!(__pyx_t_15 != 0)) != 0); if (__pyx_t_13) { __Pyx_GIVEREF(__pyx_t_8); @@ -3298,7 +3297,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGIVEREF(__pyx_t_16); __Pyx_ErrRestoreWithState(__pyx_t_8, __pyx_t_7, __pyx_t_16); __pyx_t_8 = 0; __pyx_t_7 = 0; __pyx_t_16 = 0; - __PYX_ERR(1, 81, __pyx_L19_except_error) + __PYX_ERR(1, 85, __pyx_L19_except_error) } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3332,7 +3331,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__6, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 81, __pyx_L3_error) + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; } @@ -3344,7 +3343,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__7, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 81, __pyx_L3_error) + if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 85, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } @@ -3361,7 +3360,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L36:; } - /* "reppy/robots.pyx":78 + /* "reppy/robots.pyx":82 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * try: # <<<<<<<<<<<<<< @@ -3382,38 +3381,38 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; - /* "reppy/robots.pyx":100 + /* "reppy/robots.pyx":104 * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: # <<<<<<<<<<<<<< * raise exceptions.SSLException(exc) * except ConnectionError as exc: */ - __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 100, __pyx_L5_except_error) + __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; if (__pyx_t_14) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_7, &__pyx_t_8) < 0) __PYX_ERR(1, 100, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_7, &__pyx_t_8) < 0) __PYX_ERR(1, 104, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_7); __pyx_v_exc = __pyx_t_7; - /* "reppy/robots.pyx":101 + /* "reppy/robots.pyx":105 * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: * raise exceptions.SSLException(exc) # <<<<<<<<<<<<<< * except ConnectionError as exc: * raise exceptions.ConnectionException(exc) */ - __Pyx_TraceLine(101,0,__PYX_ERR(1, 101, __pyx_L5_except_error)) - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __Pyx_TraceLine(105,0,__PYX_ERR(1, 105, __pyx_L5_except_error)) + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = NULL; @@ -3427,13 +3426,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_4) { - __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_5); } else @@ -3441,19 +3440,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_5); } else #endif { - __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_4); __pyx_t_4 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 101, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 105, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; } @@ -3461,41 +3460,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_Raise(__pyx_t_5, 0, 0, 0); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 101, __pyx_L5_except_error) + __PYX_ERR(1, 105, __pyx_L5_except_error) } - /* "reppy/robots.pyx":102 + /* "reppy/robots.pyx":106 * except SSLError as exc: * raise exceptions.SSLException(exc) * except ConnectionError as exc: # <<<<<<<<<<<<<< * raise exceptions.ConnectionException(exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: */ - __Pyx_TraceLine(102,0,__PYX_ERR(1, 102, __pyx_L5_except_error)) - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 102, __pyx_L5_except_error) + __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L5_except_error)) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 106, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_14) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 102, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 106, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_16); __Pyx_INCREF(__pyx_t_7); __pyx_v_exc = __pyx_t_7; - /* "reppy/robots.pyx":103 + /* "reppy/robots.pyx":107 * raise exceptions.SSLException(exc) * except ConnectionError as exc: * raise exceptions.ConnectionException(exc) # <<<<<<<<<<<<<< * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * raise exceptions.MalformedUrl(exc) */ - __Pyx_TraceLine(103,0,__PYX_ERR(1, 103, __pyx_L5_except_error)) - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 103, __pyx_L5_except_error) + __Pyx_TraceLine(107,0,__PYX_ERR(1, 107, __pyx_L5_except_error)) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_18 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 103, __pyx_L5_except_error) + __pyx_t_18 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_t_6 = NULL; @@ -3509,13 +3508,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_18, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_18, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_18)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_5); } else @@ -3523,19 +3522,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_18)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_5); } else #endif { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 103, __pyx_L5_except_error) + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_exc); - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_18, __pyx_t_4, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_18, __pyx_t_4, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } @@ -3543,24 +3542,24 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; __Pyx_Raise(__pyx_t_5, 0, 0, 0); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 103, __pyx_L5_except_error) + __PYX_ERR(1, 107, __pyx_L5_except_error) } - /* "reppy/robots.pyx":104 + /* "reppy/robots.pyx":108 * except ConnectionError as exc: * raise exceptions.ConnectionException(exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: # <<<<<<<<<<<<<< * raise exceptions.MalformedUrl(exc) * except TooManyRedirects as exc: */ - __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_TraceLine(108,0,__PYX_ERR(1, 108, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 108, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 108, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 108, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 108, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_5); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -3569,24 +3568,24 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_14) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_7) < 0) __PYX_ERR(1, 104, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_7) < 0) __PYX_ERR(1, 108, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_5); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_7); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":105 + /* "reppy/robots.pyx":109 * raise exceptions.ConnectionException(exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * raise exceptions.MalformedUrl(exc) # <<<<<<<<<<<<<< * except TooManyRedirects as exc: * raise exceptions.ExcessiveRedirects(exc) */ - __Pyx_TraceLine(105,0,__PYX_ERR(1, 105, __pyx_L5_except_error)) - __pyx_t_18 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 105, __pyx_L5_except_error) + __Pyx_TraceLine(109,0,__PYX_ERR(1, 109, __pyx_L5_except_error)) + __pyx_t_18 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 109, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 105, __pyx_L5_except_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 109, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; __pyx_t_18 = NULL; @@ -3600,13 +3599,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_18) { - __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 105, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 109, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 105, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 109, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; __Pyx_GOTREF(__pyx_t_16); } else @@ -3614,19 +3613,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 105, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 109, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; __Pyx_GOTREF(__pyx_t_16); } else #endif { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 105, __pyx_L5_except_error) + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 109, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_GIVEREF(__pyx_t_18); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_18); __pyx_t_18 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_v_exc); - __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 105, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 109, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -3634,41 +3633,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_Raise(__pyx_t_16, 0, 0, 0); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __PYX_ERR(1, 105, __pyx_L5_except_error) + __PYX_ERR(1, 109, __pyx_L5_except_error) } - /* "reppy/robots.pyx":106 + /* "reppy/robots.pyx":110 * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * raise exceptions.MalformedUrl(exc) * except TooManyRedirects as exc: # <<<<<<<<<<<<<< * raise exceptions.ExcessiveRedirects(exc) * */ - __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 106, __pyx_L5_except_error) + __Pyx_TraceLine(110,0,__PYX_ERR(1, 110, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 110, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (__pyx_t_14) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_5) < 0) __PYX_ERR(1, 106, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_5) < 0) __PYX_ERR(1, 110, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_5); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":107 + /* "reppy/robots.pyx":111 * raise exceptions.MalformedUrl(exc) * except TooManyRedirects as exc: * raise exceptions.ExcessiveRedirects(exc) # <<<<<<<<<<<<<< * * def RobotsUrlMethod(cls, url): */ - __Pyx_TraceLine(107,0,__PYX_ERR(1, 107, __pyx_L5_except_error)) - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __Pyx_TraceLine(111,0,__PYX_ERR(1, 111, __pyx_L5_except_error)) + __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 111, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_4); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 111, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = NULL; @@ -3682,13 +3681,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_4) { - __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 111, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 111, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_16); } else @@ -3696,19 +3695,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 111, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_16); } else #endif { - __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 111, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_4); __pyx_t_4 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); - __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 111, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; } @@ -3716,12 +3715,12 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_Raise(__pyx_t_16, 0, 0, 0); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __PYX_ERR(1, 107, __pyx_L5_except_error) + __PYX_ERR(1, 111, __pyx_L5_except_error) } goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "reppy/robots.pyx":78 + /* "reppy/robots.pyx":82 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * try: # <<<<<<<<<<<<<< @@ -3744,7 +3743,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L10_try_end:; } - /* "reppy/robots.pyx":76 + /* "reppy/robots.pyx":80 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -3776,7 +3775,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":109 +/* "reppy/robots.pyx":113 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -3814,11 +3813,11 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 109, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 113, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 109, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 113, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -3831,7 +3830,7 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 109, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 113, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3854,37 +3853,37 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject PyObject *__pyx_t_4 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__8) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); - __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 109, 0, __PYX_ERR(1, 109, __pyx_L1_error)); + __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 113, 0, __PYX_ERR(1, 113, __pyx_L1_error)); - /* "reppy/robots.pyx":111 + /* "reppy/robots.pyx":115 * def RobotsUrlMethod(cls, url): * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) # <<<<<<<<<<<<<< * * cdef class Robots: */ - __Pyx_TraceLine(111,0,__PYX_ERR(1, 111, __pyx_L1_error)) + __Pyx_TraceLine(115,0,__PYX_ERR(1, 115, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 115, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 111, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 115, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 111, __pyx_L1_error) + __PYX_ERR(1, 115, __pyx_L1_error) } - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 115, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 111, __pyx_L1_error) + __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 115, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":109 + /* "reppy/robots.pyx":113 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -3905,12 +3904,12 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":130 +/* "reppy/robots.pyx":133 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< - * self.url = url - * self.robots = new CppRobots(content) + * self.robots = new CppRobots(content, as_bytes(url)) + * self.expires = expires */ /* Python wrapper */ @@ -3944,7 +3943,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 130, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 133, __pyx_L3_error) } case 2: if (kw_args > 0) { @@ -3953,7 +3952,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 130, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 133, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -3965,12 +3964,12 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } __pyx_v_url = values[0]; - __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 130, __pyx_L3_error) + __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 133, __pyx_L3_error) __pyx_v_expires = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 130, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 133, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3987,66 +3986,59 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob int __pyx_r; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations - Rep::Robots *__pyx_t_1; + PyObject *__pyx_t_1 = NULL; + std::string __pyx_t_2; + Rep::Robots *__pyx_t_3; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 130, 0, __PYX_ERR(1, 130, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 133, 0, __PYX_ERR(1, 133, __pyx_L1_error)); - /* "reppy/robots.pyx":131 + /* "reppy/robots.pyx":134 * * def __init__(self, url, const string& content, expires=None): - * self.url = url # <<<<<<<<<<<<<< - * self.robots = new CppRobots(content) - * self.expires = expires - */ - __Pyx_TraceLine(131,0,__PYX_ERR(1, 131, __pyx_L1_error)) - __Pyx_INCREF(__pyx_v_url); - __Pyx_GIVEREF(__pyx_v_url); - __Pyx_GOTREF(__pyx_v_self->url); - __Pyx_DECREF(__pyx_v_self->url); - __pyx_v_self->url = __pyx_v_url; - - /* "reppy/robots.pyx":132 - * def __init__(self, url, const string& content, expires=None): - * self.url = url - * self.robots = new CppRobots(content) # <<<<<<<<<<<<<< + * self.robots = new CppRobots(content, as_bytes(url)) # <<<<<<<<<<<<<< * self.expires = expires * */ - __Pyx_TraceLine(132,0,__PYX_ERR(1, 132, __pyx_L1_error)) + __Pyx_TraceLine(134,0,__PYX_ERR(1, 134, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 134, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 134, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { - __pyx_t_1 = new Rep::Robots(__pyx_v_content); + __pyx_t_3 = new Rep::Robots(__pyx_v_content, __pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 132, __pyx_L1_error) + __PYX_ERR(1, 134, __pyx_L1_error) } - __pyx_v_self->robots = __pyx_t_1; + __pyx_v_self->robots = __pyx_t_3; - /* "reppy/robots.pyx":133 - * self.url = url - * self.robots = new CppRobots(content) + /* "reppy/robots.pyx":135 + * def __init__(self, url, const string& content, expires=None): + * self.robots = new CppRobots(content, as_bytes(url)) * self.expires = expires # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(133,0,__PYX_ERR(1, 133, __pyx_L1_error)) + __Pyx_TraceLine(135,0,__PYX_ERR(1, 135, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); __Pyx_GOTREF(__pyx_v_self->expires); __Pyx_DECREF(__pyx_v_self->expires); __pyx_v_self->expires = __pyx_v_expires; - /* "reppy/robots.pyx":130 + /* "reppy/robots.pyx":133 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< - * self.url = url - * self.robots = new CppRobots(content) + * self.robots = new CppRobots(content, as_bytes(url)) + * self.expires = expires */ /* function exit code */ __pyx_r = 0; goto __pyx_L0; __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = -1; __pyx_L0:; @@ -4055,7 +4047,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob return __pyx_r; } -/* "reppy/robots.pyx":135 +/* "reppy/robots.pyx":137 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4082,24 +4074,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[1], 135, 0, __PYX_ERR(1, 135, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[1], 137, 0, __PYX_ERR(1, 137, __pyx_L1_error)); - /* "reppy/robots.pyx":136 + /* "reppy/robots.pyx":138 * * def __str__(self): * return self.robots.str() # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __Pyx_TraceLine(136,0,__PYX_ERR(1, 136, __pyx_L1_error)) + __Pyx_TraceLine(138,0,__PYX_ERR(1, 138, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 136, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":135 + /* "reppy/robots.pyx":137 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4119,7 +4111,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":138 +/* "reppy/robots.pyx":140 * return self.robots.str() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4142,19 +4134,19 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - __Pyx_TraceCall("__dealloc__", __pyx_f[1], 138, 0, __PYX_ERR(1, 138, __pyx_L1_error)); + __Pyx_TraceCall("__dealloc__", __pyx_f[1], 140, 0, __PYX_ERR(1, 140, __pyx_L1_error)); - /* "reppy/robots.pyx":139 + /* "reppy/robots.pyx":141 * * def __dealloc__(self): * del self.robots # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(139,0,__PYX_ERR(1, 139, __pyx_L1_error)) + __Pyx_TraceLine(141,0,__PYX_ERR(1, 141, __pyx_L1_error)) delete __pyx_v_self->robots; - /* "reppy/robots.pyx":138 + /* "reppy/robots.pyx":140 * return self.robots.str() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4171,7 +4163,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_RefNannyFinishContext(); } -/* "reppy/robots.pyx":142 +/* "reppy/robots.pyx":144 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4200,22 +4192,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 142, 0, __PYX_ERR(1, 142, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 144, 0, __PYX_ERR(1, 144, __pyx_L1_error)); - /* "reppy/robots.pyx":144 + /* "reppy/robots.pyx":146 * def sitemaps(self): * '''Get all the sitemaps in this robots.txt.''' * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< * * def allowed(self, path, name): */ - __Pyx_TraceLine(144,0,__PYX_ERR(1, 144, __pyx_L1_error)) + __Pyx_TraceLine(146,0,__PYX_ERR(1, 146, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 144, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 144, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 144, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -4223,14 +4215,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 144, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":142 + /* "reppy/robots.pyx":144 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4252,17 +4244,17 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ return __pyx_r; } -/* "reppy/robots.pyx":146 +/* "reppy/robots.pyx":148 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< - * '''Is the provided path allowed for the provided agant?''' + * '''Is the provided path allowed for the provided agent?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) */ /* Python wrapper */ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static char __pyx_doc_5reppy_6robots_6Robots_6allowed[] = "Is the provided path allowed for the provided agant?"; +static char __pyx_doc_5reppy_6robots_6Robots_6allowed[] = "Is the provided path allowed for the provided agent?"; static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_path = 0; PyObject *__pyx_v_name = 0; @@ -4289,11 +4281,11 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 146, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 148, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 146, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 148, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4306,7 +4298,7 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 146, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 148, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4327,36 +4319,36 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp std::string __pyx_t_2; std::string __pyx_t_3; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[1], 146, 0, __PYX_ERR(1, 146, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[1], 148, 0, __PYX_ERR(1, 148, __pyx_L1_error)); - /* "reppy/robots.pyx":148 + /* "reppy/robots.pyx":150 * def allowed(self, path, name): - * '''Is the provided path allowed for the provided agant?''' + * '''Is the provided path allowed for the provided agent?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) # <<<<<<<<<<<<<< * * def agent(self, name): */ - __Pyx_TraceLine(148,0,__PYX_ERR(1, 148, __pyx_L1_error)) + __Pyx_TraceLine(150,0,__PYX_ERR(1, 150, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 148, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 148, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 150, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 148, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 148, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 150, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 148, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":146 + /* "reppy/robots.pyx":148 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< - * '''Is the provided path allowed for the provided agant?''' + * '''Is the provided path allowed for the provided agent?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) */ @@ -4372,17 +4364,17 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":150 +/* "reppy/robots.pyx":152 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< - * '''Return the Agent that corresponds to name.''' - * return Agent.from_robots(self, as_bytes(name)) + * '''Return the Agent that corresponds to name. + * */ /* Python wrapper */ static PyObject *__pyx_pw_5reppy_6robots_6Robots_9agent(PyObject *__pyx_v_self, PyObject *__pyx_v_name); /*proto*/ -static char __pyx_doc_5reppy_6robots_6Robots_8agent[] = "Return the Agent that corresponds to name."; +static char __pyx_doc_5reppy_6robots_6Robots_8agent[] = "Return the Agent that corresponds to name.\n\n Note modifications to the returned Agent will not be reflected\n in this Robots object because it is a *copy*, not the original\n Agent object.\n "; static PyObject *__pyx_pw_5reppy_6robots_6Robots_9agent(PyObject *__pyx_v_self, PyObject *__pyx_v_name) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations @@ -4405,20 +4397,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ int __pyx_t_5; PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("agent", 0); - __Pyx_TraceCall("agent", __pyx_f[1], 150, 0, __PYX_ERR(1, 150, __pyx_L1_error)); + __Pyx_TraceCall("agent", __pyx_f[1], 152, 0, __PYX_ERR(1, 152, __pyx_L1_error)); - /* "reppy/robots.pyx":152 - * def agent(self, name): - * '''Return the Agent that corresponds to name.''' + /* "reppy/robots.pyx":159 + * Agent object. + * ''' * return Agent.from_robots(self, as_bytes(name)) # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(152,0,__PYX_ERR(1, 152, __pyx_L1_error)) + __Pyx_TraceLine(159,0,__PYX_ERR(1, 159, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 152, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 159, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 152, __pyx_L1_error) + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 159, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; __pyx_t_5 = 0; @@ -4435,7 +4427,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 152, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 159, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -4444,14 +4436,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 152, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 159, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 152, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 159, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -4462,7 +4454,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 152, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 159, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -4471,12 +4463,12 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":150 + /* "reppy/robots.pyx":152 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< - * '''Return the Agent that corresponds to name.''' - * return Agent.from_robots(self, as_bytes(name)) + * '''Return the Agent that corresponds to name. + * */ /* function exit code */ @@ -4495,7 +4487,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":155 +/* "reppy/robots.pyx":162 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4524,20 +4516,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 155, 0, __PYX_ERR(1, 155, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 162, 0, __PYX_ERR(1, 162, __pyx_L1_error)); - /* "reppy/robots.pyx":157 + /* "reppy/robots.pyx":164 * def expired(self): * '''True if the current time is past its expiration.''' * return time.time() > self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(157,0,__PYX_ERR(1, 157, __pyx_L1_error)) + __Pyx_TraceLine(164,0,__PYX_ERR(1, 164, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -4551,20 +4543,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } } if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 157, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":155 + /* "reppy/robots.pyx":162 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4586,7 +4578,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":160 +/* "reppy/robots.pyx":167 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4612,22 +4604,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 160, 0, __PYX_ERR(1, 160, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 167, 0, __PYX_ERR(1, 167, __pyx_L1_error)); - /* "reppy/robots.pyx":162 + /* "reppy/robots.pyx":169 * def expires(self): * '''The expiration of this robots.txt.''' * return self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(162,0,__PYX_ERR(1, 162, __pyx_L1_error)) + __Pyx_TraceLine(169,0,__PYX_ERR(1, 169, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_self->expires); __pyx_r = __pyx_v_self->expires; goto __pyx_L0; - /* "reppy/robots.pyx":160 + /* "reppy/robots.pyx":167 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4646,7 +4638,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":165 +/* "reppy/robots.pyx":172 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4678,21 +4670,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 165, 0, __PYX_ERR(1, 165, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 172, 0, __PYX_ERR(1, 172, __pyx_L1_error)); - /* "reppy/robots.pyx":167 + /* "reppy/robots.pyx":174 * def ttl(self): * '''Remaining time for this response to be considered valid.''' * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(167,0,__PYX_ERR(1, 167, __pyx_L1_error)) + __Pyx_TraceLine(174,0,__PYX_ERR(1, 174, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -4706,24 +4698,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 174, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 174, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -4737,7 +4729,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":165 + /* "reppy/robots.pyx":172 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4760,7 +4752,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 return __pyx_r; } -/* "reppy/robots.pyx":173 +/* "reppy/robots.pyx":180 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4801,7 +4793,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 173, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 180, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4816,7 +4808,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 173, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 180, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4839,17 +4831,17 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 173, 0, __PYX_ERR(1, 173, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 180, 0, __PYX_ERR(1, 180, __pyx_L1_error)); - /* "reppy/robots.pyx":174 + /* "reppy/robots.pyx":181 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(174,0,__PYX_ERR(1, 174, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 174, __pyx_L1_error) + __Pyx_TraceLine(181,0,__PYX_ERR(1, 181, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -4866,7 +4858,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -4874,13 +4866,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -4897,14 +4889,14 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":173 + /* "reppy/robots.pyx":180 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4928,7 +4920,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":180 +/* "reppy/robots.pyx":187 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4968,7 +4960,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 180, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 187, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4983,7 +4975,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 180, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 187, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5006,15 +4998,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 180, 0, __PYX_ERR(1, 180, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 187, 0, __PYX_ERR(1, 187, __pyx_L1_error)); - /* "reppy/robots.pyx":181 + /* "reppy/robots.pyx":188 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(181,0,__PYX_ERR(1, 181, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 181, __pyx_L1_error) + __Pyx_TraceLine(188,0,__PYX_ERR(1, 188, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -5031,7 +5023,7 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -5039,13 +5031,13 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 181, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -5062,14 +5054,14 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":180 + /* "reppy/robots.pyx":187 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5721,7 +5713,6 @@ static PyObject *__pyx_tp_new_5reppy_6robots_Robots(PyTypeObject *t, CYTHON_UNUS } if (unlikely(!o)) return 0; p = ((struct __pyx_obj_5reppy_6robots_Robots *)o); - p->url = Py_None; Py_INCREF(Py_None); p->expires = Py_None; Py_INCREF(Py_None); return o; } @@ -5742,7 +5733,6 @@ static void __pyx_tp_dealloc_5reppy_6robots_Robots(PyObject *o) { --Py_REFCNT(o); PyErr_Restore(etype, eval, etb); } - Py_CLEAR(p->url); Py_CLEAR(p->expires); (*Py_TYPE(o)->tp_free)(o); } @@ -5750,9 +5740,6 @@ static void __pyx_tp_dealloc_5reppy_6robots_Robots(PyObject *o) { static int __pyx_tp_traverse_5reppy_6robots_Robots(PyObject *o, visitproc v, void *a) { int e; struct __pyx_obj_5reppy_6robots_Robots *p = (struct __pyx_obj_5reppy_6robots_Robots *)o; - if (p->url) { - e = (*v)(p->url, a); if (e) return e; - } if (p->expires) { e = (*v)(p->expires, a); if (e) return e; } @@ -5762,9 +5749,6 @@ static int __pyx_tp_traverse_5reppy_6robots_Robots(PyObject *o, visitproc v, voi static int __pyx_tp_clear_5reppy_6robots_Robots(PyObject *o) { PyObject* tmp; struct __pyx_obj_5reppy_6robots_Robots *p = (struct __pyx_obj_5reppy_6robots_Robots *)o; - tmp = ((PyObject*)p->url); - p->url = Py_None; Py_INCREF(Py_None); - Py_XDECREF(tmp); tmp = ((PyObject*)p->expires); p->expires = Py_None; Py_INCREF(Py_None); Py_XDECREF(tmp); @@ -6193,7 +6177,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { }; static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) - __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 144, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 146, __pyx_L1_error) __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) return 0; __pyx_L1_error:; @@ -6226,17 +6210,17 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); - /* "reppy/robots.pyx":81 + /* "reppy/robots.pyx":85 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __pyx_tuple__6 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 81, __pyx_L1_error) + __pyx_tuple__6 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__6); __Pyx_GIVEREF(__pyx_tuple__6); - __pyx_tuple__7 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 81, __pyx_L1_error) + __pyx_tuple__7 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); @@ -6264,41 +6248,41 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(__pyx_tuple__12); __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__12, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(1, 33, __pyx_L1_error) - /* "reppy/robots.pyx":72 + /* "reppy/robots.pyx":76 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __pyx_tuple__13 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(1, 72, __pyx_L1_error) + __pyx_tuple__13 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__13); __Pyx_GIVEREF(__pyx_tuple__13); - __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 72, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 72, __pyx_L1_error) + __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 76, __pyx_L1_error) - /* "reppy/robots.pyx":76 + /* "reppy/robots.pyx":80 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * try: */ - __pyx_tuple__14 = PyTuple_Pack(10, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_tuple__14 = PyTuple_Pack(10, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__14); __Pyx_GIVEREF(__pyx_tuple__14); - __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 10, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 10, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 80, __pyx_L1_error) - /* "reppy/robots.pyx":109 + /* "reppy/robots.pyx":113 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __pyx_tuple__15 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 109, __pyx_L1_error) + __pyx_tuple__15 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 113, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__15); __Pyx_GIVEREF(__pyx_tuple__15); - __pyx_codeobj__8 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 109, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__8)) __PYX_ERR(1, 109, __pyx_L1_error) + __pyx_codeobj__8 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 113, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__8)) __PYX_ERR(1, 113, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -6409,23 +6393,23 @@ PyMODINIT_FUNC PyInit_robots(void) /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 39, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 43, __pyx_L1_error) __pyx_type_5reppy_6robots_Agent.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 39, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 43, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; - if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 113, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 117, __pyx_L1_error) __pyx_type_5reppy_6robots_Robots.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 113, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 117, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 170, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 177, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 170, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 177, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 177, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 184, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 177, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 184, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; @@ -6644,132 +6628,132 @@ PyMODINIT_FUNC PyInit_robots(void) if (PyDict_SetItem(__pyx_d, __pyx_n_s_FromRobotsMethod, __pyx_t_1) < 0) __PYX_ERR(1, 33, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":44 + /* "reppy/robots.pyx":48 * cdef CppAgent agent * * from_robots = classmethod(FromRobotsMethod) # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(44,0,__PYX_ERR(1, 44, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) + __Pyx_TraceLine(48,0,__PYX_ERR(1, 48, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 44, __pyx_L1_error) + __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Agent->tp_dict, __pyx_n_s_from_robots, __pyx_t_2) < 0) __PYX_ERR(1, 44, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Agent->tp_dict, __pyx_n_s_from_robots, __pyx_t_2) < 0) __PYX_ERR(1, 48, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Agent); - /* "reppy/robots.pyx":72 + /* "reppy/robots.pyx":76 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __Pyx_TraceLine(72,0,__PYX_ERR(1, 72, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 72, __pyx_L1_error) + __Pyx_TraceLine(76,0,__PYX_ERR(1, 76, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(1, 72, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(1, 76, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":76 + /* "reppy/robots.pyx":80 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * try: */ - __Pyx_TraceLine(76,0,__PYX_ERR(1, 76, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 76, __pyx_L1_error) + __Pyx_TraceLine(80,0,__PYX_ERR(1, 80, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(1, 76, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":109 + /* "reppy/robots.pyx":113 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __Pyx_TraceLine(109,0,__PYX_ERR(1, 109, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L1_error) + __Pyx_TraceLine(113,0,__PYX_ERR(1, 113, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 113, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 109, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 113, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":118 + /* "reppy/robots.pyx":122 * # The default TTL policy is to cache for 3600 seconds or what's provided in the * # headers, and a minimum of 600 seconds * DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) # <<<<<<<<<<<<<< * * # Class methods */ - __Pyx_TraceLine(118,0,__PYX_ERR(1, 118, __pyx_L1_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 118, __pyx_L1_error) + __Pyx_TraceLine(122,0,__PYX_ERR(1, 122, __pyx_L1_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 118, __pyx_L1_error) + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 118, __pyx_L1_error) - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 118, __pyx_L1_error) - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 118, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 122, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 122, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 122, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 118, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 122, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":121 + /* "reppy/robots.pyx":125 * * # Class methods * parse = classmethod(ParseMethod) # <<<<<<<<<<<<<< * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) */ - __Pyx_TraceLine(121,0,__PYX_ERR(1, 121, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 121, __pyx_L1_error) + __Pyx_TraceLine(125,0,__PYX_ERR(1, 125, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 125, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 121, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 125, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 121, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 125, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":122 + /* "reppy/robots.pyx":126 * # Class methods * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) # <<<<<<<<<<<<<< * robots_url = classmethod(RobotsUrlMethod) * */ - __Pyx_TraceLine(122,0,__PYX_ERR(1, 122, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 122, __pyx_L1_error) + __Pyx_TraceLine(126,0,__PYX_ERR(1, 126, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 126, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 122, __pyx_L1_error) + __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 126, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 122, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 126, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":123 + /* "reppy/robots.pyx":127 * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) # <<<<<<<<<<<<<< * * # Data members */ - __Pyx_TraceLine(123,0,__PYX_ERR(1, 123, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 123, __pyx_L1_error) + __Pyx_TraceLine(127,0,__PYX_ERR(1, 127, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 127, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 123, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 127, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 123, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 127, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); diff --git a/reppy/robots.pxd b/reppy/robots.pxd index 5af14da..ee9beb7 100644 --- a/reppy/robots.pxd +++ b/reppy/robots.pxd @@ -9,6 +9,7 @@ cdef extern from "rep-cpp/include/directive.h" namespace "Rep": ctypedef size_t priority_t CppDirective(const string& line, bool allowed) + CppDirective(const CppDirective& rhs) priority_t priority() const bool match(const string& path) const bool allowed() const @@ -19,6 +20,7 @@ cdef extern from "rep-cpp/include/agent.h" namespace "Rep": ctypedef float delay_t CppAgent() + CppAgent(const string& host) CppAgent& allow(const string& query) CppAgent& disallow(const string& query) CppAgent& delay(delay_t delay) @@ -26,15 +28,13 @@ cdef extern from "rep-cpp/include/agent.h" namespace "Rep": const vector[CppDirective]& directives() const bool allowed(const string& path) const string str() const - @staticmethod - string escape(const string& query) cdef extern from "rep-cpp/include/robots.h" namespace "Rep": cpdef cppclass CppRobots "Rep::Robots": CppRobots(const string& content) except +ValueError - CppRobots& operator=(const CppRobots& other) + CppRobots(const string& content, const string& base_url) except +ValueError const vector[string]& sitemaps() const - const CppAgent& agent(const string& name) const + CppAgent& agent(const string& name) const bool allowed(const string& path, const string& name) const string str() const @staticmethod diff --git a/reppy/robots.pyx b/reppy/robots.pyx index 4fae8d6..15bb789 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -33,6 +33,10 @@ cdef as_string(value): def FromRobotsMethod(cls, Robots robots, const string& name): '''Construct an Agent from a CppAgent.''' agent = Agent() + # This is somewhat inefficient due to the copying, but it is + # required to be copied because we often toss the containing + # Robots object as a temporary thus we'd leave the underlying + # Agent object dangling without a full copy. agent.agent = robots.robots.agent(name) return agent @@ -124,12 +128,10 @@ cdef class Robots: # Data members cdef CppRobots* robots - cdef object url cdef object expires def __init__(self, url, const string& content, expires=None): - self.url = url - self.robots = new CppRobots(content) + self.robots = new CppRobots(content, as_bytes(url)) self.expires = expires def __str__(self): @@ -148,7 +150,12 @@ cdef class Robots: return self.robots.allowed(as_bytes(path), as_bytes(name)) def agent(self, name): - '''Return the Agent that corresponds to name.''' + '''Return the Agent that corresponds to name. + + Note modifications to the returned Agent will not be reflected + in this Robots object because it is a *copy*, not the original + Agent object. + ''' return Agent.from_robots(self, as_bytes(name)) @property From d70138bd3c3d64df365e6aba2fee0a4a8f6bb36e Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 8 Sep 2017 13:49:31 -0700 Subject: [PATCH 070/113] Shorten test case names. --- tests/test_agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_agent.py b/tests/test_agent.py index ff35c8f..b048390 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -57,7 +57,7 @@ def test_disallow_none(self): self.assertTrue(agent.allowed('/anything')) def test_escaped_rule(self): - '''Handles the case where the rule is escaped.''' + '''Handles an escaped rule.''' agent = self.parse(''' User-agent: agent Disallow: /a%3cd.html @@ -66,7 +66,7 @@ def test_escaped_rule(self): self.assertFalse(agent.allowed('/a%3cd.html')) def test_unescaped_rule(self): - '''Handles the case where the rule is unescaped.''' + '''Handles an unescaped rule.''' agent = self.parse(''' User-agent: agent Disallow: /a Date: Fri, 8 Sep 2017 13:52:34 -0700 Subject: [PATCH 071/113] Fix circular dependency on reppy/robots.so Previously, when rebuilding with make, it would report the following warning: make: Circular reppy/robots.so <- reppy/robots.so dependency dropped. This is because reppy/robots.so itself matches the pattern reppy/%.*, so by restricting the dependency to reppy/%.py*, it won't have a self-dependency. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5f816b1..ead7f97 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ test: reppy/robots.so nosetests --with-coverage tests -reppy/%.so: reppy/%.* reppy/rep-cpp/src/* reppy/rep-cpp/include/* reppy/rep-cpp/deps/url-cpp/include/* reppy/rep-cpp/deps/url-cpp/src/* +reppy/%.so: reppy/%.py* reppy/rep-cpp/src/* reppy/rep-cpp/include/* reppy/rep-cpp/deps/url-cpp/include/* reppy/rep-cpp/deps/url-cpp/src/* python setup.py build_ext --inplace install: From a7549f21557951d45c1279e6691cd8cd017674f4 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 8 Sep 2017 13:55:27 -0700 Subject: [PATCH 072/113] Remove `reppy/*.so` during `make clean`. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ead7f97..022a60f 100644 --- a/Makefile +++ b/Makefile @@ -15,5 +15,5 @@ dev-requirements-py3: pip freeze | grep -v -e reppy > dev-requirements-py3.txt clean: - rm -rf build dist *.egg-info + rm -rf build dist *.egg-info reppy/*.so find . -name '*.pyc' | xargs --no-run-if-empty rm From 12b45f238a2f3cb66ccd1e6e86eddb39ce6bae64 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 8 Sep 2017 14:40:30 -0700 Subject: [PATCH 073/113] Add fetch hook support. This will allow Aardwolf to peek at the response and the full Robots object to collect additional information during crawls. --- reppy/robots.cpp | 1634 ++++++++++------- reppy/robots.pyx | 10 +- tests/asis/test_after_parse_hook/robots.txt | 6 + .../asis/test_after_response_hook/robots.txt | 6 + tests/test_robots.py | 24 + 5 files changed, 978 insertions(+), 702 deletions(-) create mode 100644 tests/asis/test_after_parse_hook/robots.txt create mode 100644 tests/asis/test_after_response_hook/robots.txt diff --git a/reppy/robots.cpp b/reppy/robots.cpp index 480eced..0336fde 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -661,7 +661,7 @@ struct __pyx_obj_5reppy_6robots_Agent { }; -/* "reppy/robots.pyx":117 +/* "reppy/robots.pyx":125 * return as_string(CppRobots.robotsUrl(as_bytes(url))) * * cdef class Robots: # <<<<<<<<<<<<<< @@ -675,7 +675,7 @@ struct __pyx_obj_5reppy_6robots_Robots { }; -/* "reppy/robots.pyx":177 +/* "reppy/robots.pyx":185 * * * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< @@ -687,7 +687,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone { }; -/* "reppy/robots.pyx":184 +/* "reppy/robots.pyx":192 * * * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< @@ -1308,13 +1308,14 @@ int __pyx_module_is_main_reppy__robots = 0; static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_map; static PyObject *__pyx_builtin_range; -static const char __pyx_k__9[] = ""; static const char __pyx_k_PY3[] = "PY3"; +static const char __pyx_k__11[] = ""; static const char __pyx_k_amt[] = "amt"; static const char __pyx_k_cls[] = "cls"; static const char __pyx_k_exc[] = "exc"; static const char __pyx_k_get[] = "get"; static const char __pyx_k_map[] = "map"; +static const char __pyx_k_pop[] = "pop"; static const char __pyx_k_raw[] = "raw"; static const char __pyx_k_res[] = "res"; static const char __pyx_k_six[] = "six"; @@ -1379,10 +1380,12 @@ static const char __pyx_k_ConnectionError[] = "ConnectionError"; static const char __pyx_k_RobotsUrlMethod[] = "RobotsUrlMethod"; static const char __pyx_k_FromRobotsMethod[] = "FromRobotsMethod"; static const char __pyx_k_TooManyRedirects[] = "TooManyRedirects"; +static const char __pyx_k_after_parse_hook[] = "after_parse_hook"; static const char __pyx_k_DEFAULT_TTL_POLICY[] = "DEFAULT_TTL_POLICY"; static const char __pyx_k_ExcessiveRedirects[] = "ExcessiveRedirects"; static const char __pyx_k_ConnectionException[] = "ConnectionException"; static const char __pyx_k_User_agent_Disallow[] = "User-agent: *\nDisallow: /"; +static const char __pyx_k_after_response_hook[] = "after_response_hook"; static const char __pyx_k_requests_exceptions[] = "requests.exceptions"; static const char __pyx_k_HeaderWithDefaultPolicy[] = "HeaderWithDefaultPolicy"; static const char __pyx_k_vagrant_reppy_robots_pyx[] = "/vagrant/reppy/robots.pyx"; @@ -1413,8 +1416,10 @@ static PyObject *__pyx_n_s_TooManyRedirects; static PyObject *__pyx_n_s_URLRequired; static PyObject *__pyx_kp_b_User_agent_Disallow; static PyObject *__pyx_n_s_ValueError; -static PyObject *__pyx_n_s__9; -static PyObject *__pyx_kp_b__9; +static PyObject *__pyx_n_s__11; +static PyObject *__pyx_kp_b__11; +static PyObject *__pyx_n_s_after_parse_hook; +static PyObject *__pyx_n_s_after_response_hook; static PyObject *__pyx_n_s_agent; static PyObject *__pyx_n_s_amt; static PyObject *__pyx_n_s_args; @@ -1446,6 +1451,7 @@ static PyObject *__pyx_n_s_minimum; static PyObject *__pyx_n_s_name; static PyObject *__pyx_n_s_parse; static PyObject *__pyx_n_s_path; +static PyObject *__pyx_n_s_pop; static PyObject *__pyx_n_s_range; static PyObject *__pyx_n_s_raw; static PyObject *__pyx_n_s_read; @@ -1508,16 +1514,18 @@ static PyObject *__pyx_tuple_; static PyObject *__pyx_tuple__2; static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; -static PyObject *__pyx_tuple__10; +static PyObject *__pyx_tuple__8; +static PyObject *__pyx_tuple__9; static PyObject *__pyx_tuple__12; -static PyObject *__pyx_tuple__13; static PyObject *__pyx_tuple__14; static PyObject *__pyx_tuple__15; +static PyObject *__pyx_tuple__16; +static PyObject *__pyx_tuple__17; static PyObject *__pyx_codeobj__3; static PyObject *__pyx_codeobj__4; static PyObject *__pyx_codeobj__5; -static PyObject *__pyx_codeobj__8; -static PyObject *__pyx_codeobj__11; +static PyObject *__pyx_codeobj__10; +static PyObject *__pyx_codeobj__13; /* "reppy/robots.pyx":21 * from . import util, logger, exceptions @@ -2475,7 +2483,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' - * try: + * after_response_hook = kwargs.pop('after_response_hook', None) */ /* Python wrapper */ @@ -2584,9 +2592,12 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs) { + PyObject *__pyx_v_after_response_hook = NULL; + PyObject *__pyx_v_after_parse_hook = NULL; PyObject *__pyx_v_res = NULL; PyObject *__pyx_v_content = NULL; PyObject *__pyx_v_expires = NULL; + PyObject *__pyx_v_robots = NULL; PyObject *__pyx_v_exc = NULL; PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations @@ -2616,107 +2627,139 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ /* "reppy/robots.pyx":82 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' + * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< + * after_parse_hook = kwargs.pop('after_parse_hook', None) + * try: + */ + __Pyx_TraceLine(82,0,__PYX_ERR(1, 82, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_after_response_hook = __pyx_t_2; + __pyx_t_2 = 0; + + /* "reppy/robots.pyx":83 + * '''Get the robots.txt at the provided URL.''' + * after_response_hook = kwargs.pop('after_response_hook', None) + * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< + * try: + * # Limit the size of the request + */ + __Pyx_TraceLine(83,0,__PYX_ERR(1, 83, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 83, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 83, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_after_parse_hook = __pyx_t_1; + __pyx_t_1 = 0; + + /* "reppy/robots.pyx":84 + * after_response_hook = kwargs.pop('after_response_hook', None) + * after_parse_hook = kwargs.pop('after_parse_hook', None) * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_TraceLine(82,0,__PYX_ERR(1, 82, __pyx_L3_error)) + __Pyx_TraceLine(84,0,__PYX_ERR(1, 84, __pyx_L3_error)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign - __Pyx_ExceptionSave(&__pyx_t_1, &__pyx_t_2, &__pyx_t_3); - __Pyx_XGOTREF(__pyx_t_1); - __Pyx_XGOTREF(__pyx_t_2); + __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_4, &__pyx_t_5); __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_5); /*try:*/ { - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":86 * try: * # Limit the size of the request * kwargs['stream'] = True # <<<<<<<<<<<<<< * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) */ - __Pyx_TraceLine(84,0,__PYX_ERR(1, 84, __pyx_L3_error)) - if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 84, __pyx_L3_error) + __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L3_error)) + if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 86, __pyx_L3_error) - /* "reppy/robots.pyx":85 + /* "reppy/robots.pyx":87 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __Pyx_TraceLine(85,0,__PYX_ERR(1, 85, __pyx_L3_error)) + __Pyx_TraceLine(87,0,__PYX_ERR(1, 87, __pyx_L3_error)) /*with:*/ { - __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L3_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 85, __pyx_L3_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 87, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 85, __pyx_L3_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 85, __pyx_L3_error) + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_url); - __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 85, __pyx_L3_error) + __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 85, __pyx_L3_error) + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __pyx_t_8 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_5))) { - __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_5); + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_2); if (likely(__pyx_t_8)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); __Pyx_INCREF(__pyx_t_8); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_5, function); + __Pyx_DECREF_SET(__pyx_t_2, function); } } if (!__pyx_t_8) { - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_6); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_5)) { + if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_GOTREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_GOTREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { - __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 85, __pyx_L3_error) + __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); __pyx_t_6 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_7, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L3_error) - __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 87, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } } - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 85, __pyx_L3_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_4, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 85, __pyx_L11_error) + __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 87, __pyx_L11_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { @@ -2729,16 +2772,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L11_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 87, __pyx_L11_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { - __pyx_t_5 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L11_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 87, __pyx_L11_error) } - __Pyx_GOTREF(__pyx_t_5); + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_t_5; - __pyx_t_5 = 0; - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_7 = __pyx_t_2; + __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /*try:*/ { { __Pyx_PyThreadState_declare @@ -2751,79 +2794,79 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_res = __pyx_t_7; __pyx_t_7 = 0; - /* "reppy/robots.pyx":86 + /* "reppy/robots.pyx":88 * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) # <<<<<<<<<<<<<< * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): */ - __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L17_error)) - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 86, __pyx_L17_error) + __Pyx_TraceLine(88,0,__PYX_ERR(1, 88, __pyx_L17_error)) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 88, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 86, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 86, __pyx_L17_error) + __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 88, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 86, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 86, __pyx_L17_error) - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 86, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 88, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 88, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 88, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_v_content = __pyx_t_5; - __pyx_t_5 = 0; + __pyx_v_content = __pyx_t_2; + __pyx_t_2 = 0; - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":90 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) */ - __Pyx_TraceLine(88,0,__PYX_ERR(1, 88, __pyx_L17_error)) - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 88, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 88, __pyx_L17_error) + __Pyx_TraceLine(90,0,__PYX_ERR(1, 90, __pyx_L17_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 90, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = PyDict_New(); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 88, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_5); - if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 88, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 88, __pyx_L17_error) - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 88, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 90, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 90, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 88, __pyx_L17_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_13) { - /* "reppy/robots.pyx":89 + /* "reppy/robots.pyx":91 * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( # <<<<<<<<<<<<<< * 'Content larger than %s bytes' % max_size) * */ - __Pyx_TraceLine(89,0,__PYX_ERR(1, 89, __pyx_L17_error)) - __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 89, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 89, __pyx_L17_error) + __Pyx_TraceLine(91,0,__PYX_ERR(1, 91, __pyx_L17_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 91, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 91, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":92 * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) # <<<<<<<<<<<<<< * - * # Get the TTL policy's ruling on the ttl + * if after_response_hook is not None: */ - __Pyx_TraceLine(90,0,__PYX_ERR(1, 90, __pyx_L17_error)) - __pyx_t_5 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 90, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_5); + __Pyx_TraceLine(92,0,__PYX_ERR(1, 92, __pyx_L17_error)) + __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 92, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_7); @@ -2835,46 +2878,46 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 91, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { - PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_5}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 91, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { - PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_5}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 91, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 89, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 91, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; - __Pyx_GIVEREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_5); - __pyx_t_5 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 89, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_2); + __pyx_t_2 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 91, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } } __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_t_4, 0, 0, 0); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __PYX_ERR(1, 89, __pyx_L17_error) + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 91, __pyx_L17_error) - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":90 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< @@ -2883,281 +2926,442 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":93 + /* "reppy/robots.pyx":94 + * 'Content larger than %s bytes' % max_size) + * + * if after_response_hook is not None: # <<<<<<<<<<<<<< + * after_response_hook(res) + * + */ + __Pyx_TraceLine(94,0,__PYX_ERR(1, 94, __pyx_L17_error)) + __pyx_t_13 = (__pyx_v_after_response_hook != Py_None); + __pyx_t_14 = (__pyx_t_13 != 0); + if (__pyx_t_14) { + + /* "reppy/robots.pyx":95 + * + * if after_response_hook is not None: + * after_response_hook(res) # <<<<<<<<<<<<<< + * + * # Get the TTL policy's ruling on the ttl + */ + __Pyx_TraceLine(95,0,__PYX_ERR(1, 95, __pyx_L17_error)) + __Pyx_INCREF(__pyx_v_after_response_hook); + __pyx_t_7 = __pyx_v_after_response_hook; __pyx_t_8 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_7); + if (likely(__pyx_t_8)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); + __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_7, function); + } + } + if (!__pyx_t_8) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_7)) { + PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { + PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + { + __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_8); __pyx_t_8 = NULL; + __Pyx_INCREF(__pyx_v_res); + __Pyx_GIVEREF(__pyx_v_res); + PyTuple_SET_ITEM(__pyx_t_2, 0+1, __pyx_v_res); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 95, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } + } + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "reppy/robots.pyx":94 + * 'Content larger than %s bytes' % max_size) + * + * if after_response_hook is not None: # <<<<<<<<<<<<<< + * after_response_hook(res) + * + */ + } + + /* "reppy/robots.pyx":98 * * # Get the TTL policy's ruling on the ttl * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) # <<<<<<<<<<<<<< * * if res.status_code == 200: */ - __Pyx_TraceLine(93,0,__PYX_ERR(1, 93, __pyx_L17_error)) - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 93, __pyx_L17_error) - if (!__pyx_t_13) { + __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L17_error)) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 98, __pyx_L17_error) + if (!__pyx_t_14) { } else { __Pyx_INCREF(__pyx_v_ttl_policy); __pyx_t_7 = __pyx_v_ttl_policy; - goto __pyx_L26_bool_binop_done; + goto __pyx_L27_bool_binop_done; } - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_INCREF(__pyx_t_8); - __pyx_t_7 = __pyx_t_8; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_L26_bool_binop_done:; - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_8); + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 98, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_t_2); + __pyx_t_7 = __pyx_t_2; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_L27_bool_binop_done:; + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 98, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_8))) { - __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_8); + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_2); if (likely(__pyx_t_7)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); __Pyx_INCREF(__pyx_t_7); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_8, function); + __Pyx_DECREF_SET(__pyx_t_2, function); } } if (!__pyx_t_7) { - __pyx_t_4 = __Pyx_PyObject_CallOneArg(__pyx_t_8, __pyx_v_res); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 98, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_8)) { + if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_GOTREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_t_1); } else #endif #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_GOTREF(__pyx_t_4); + __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 93, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_7); __pyx_t_7 = NULL; + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); - PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_v_res); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_5, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 93, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_res); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 98, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } } - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_v_expires = __pyx_t_4; - __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_expires = __pyx_t_1; + __pyx_t_1 = 0; - /* "reppy/robots.pyx":95 + /* "reppy/robots.pyx":100 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< - * return cls.parse(url, content, expires) - * elif res.status_code in (401, 403): + * robots = cls.parse(url, content, expires) + * if after_parse_hook is not None: */ - __Pyx_TraceLine(95,0,__PYX_ERR(1, 95, __pyx_L17_error)) - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 95, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = __Pyx_PyInt_EqObjC(__pyx_t_4, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 95, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 95, __pyx_L17_error) - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - if (__pyx_t_13) { + __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L17_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 100, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 100, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 100, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_14) { - /* "reppy/robots.pyx":96 + /* "reppy/robots.pyx":101 * * if res.status_code == 200: - * return cls.parse(url, content, expires) # <<<<<<<<<<<<<< - * elif res.status_code in (401, 403): - * return AllowNone(url, expires) - */ - __Pyx_TraceLine(96,0,__PYX_ERR(1, 96, __pyx_L17_error)) - __Pyx_XDECREF(__pyx_r); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 96, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = NULL; - __pyx_t_14 = 0; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { - __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); - if (likely(__pyx_t_5)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); - __Pyx_INCREF(__pyx_t_5); + * robots = cls.parse(url, content, expires) # <<<<<<<<<<<<<< + * if after_parse_hook is not None: + * after_parse_hook(robots) + */ + __Pyx_TraceLine(101,0,__PYX_ERR(1, 101, __pyx_L17_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_8 = NULL; + __pyx_t_15 = 0; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_1))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_1); + if (likely(__pyx_t_8)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_1); + __Pyx_INCREF(__pyx_t_8); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_4, function); - __pyx_t_14 = 1; + __Pyx_DECREF_SET(__pyx_t_1, function); + __pyx_t_15 = 1; } } #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_4)) { - PyObject *__pyx_temp[4] = {__pyx_t_5, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 96, __pyx_L17_error) - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_GOTREF(__pyx_t_8); + if (PyFunction_Check(__pyx_t_1)) { + PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 101, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_GOTREF(__pyx_t_2); } else #endif #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { - PyObject *__pyx_temp[4] = {__pyx_t_5, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_14, 3+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 96, __pyx_L17_error) - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { + PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 101, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_7 = PyTuple_New(3+__pyx_t_14); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 101, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - if (__pyx_t_5) { - __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); __pyx_t_5 = NULL; + if (__pyx_t_8) { + __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; } __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_7, 0+__pyx_t_14, __pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_7, 0+__pyx_t_15, __pyx_v_url); __Pyx_INCREF(__pyx_v_content); __Pyx_GIVEREF(__pyx_v_content); - PyTuple_SET_ITEM(__pyx_t_7, 1+__pyx_t_14, __pyx_v_content); + PyTuple_SET_ITEM(__pyx_t_7, 1+__pyx_t_15, __pyx_v_content); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); - PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_14, __pyx_v_expires); - __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_7, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 96, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_8); + PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_15, __pyx_v_expires); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 101, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_r = __pyx_t_8; - __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_robots = __pyx_t_2; + __pyx_t_2 = 0; + + /* "reppy/robots.pyx":102 + * if res.status_code == 200: + * robots = cls.parse(url, content, expires) + * if after_parse_hook is not None: # <<<<<<<<<<<<<< + * after_parse_hook(robots) + * return robots + */ + __Pyx_TraceLine(102,0,__PYX_ERR(1, 102, __pyx_L17_error)) + __pyx_t_14 = (__pyx_v_after_parse_hook != Py_None); + __pyx_t_13 = (__pyx_t_14 != 0); + if (__pyx_t_13) { + + /* "reppy/robots.pyx":103 + * robots = cls.parse(url, content, expires) + * if after_parse_hook is not None: + * after_parse_hook(robots) # <<<<<<<<<<<<<< + * return robots + * elif res.status_code in (401, 403): + */ + __Pyx_TraceLine(103,0,__PYX_ERR(1, 103, __pyx_L17_error)) + __Pyx_INCREF(__pyx_v_after_parse_hook); + __pyx_t_1 = __pyx_v_after_parse_hook; __pyx_t_7 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_1))) { + __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_1); + if (likely(__pyx_t_7)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_1); + __Pyx_INCREF(__pyx_t_7); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_1, function); + } + } + if (!__pyx_t_7) { + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_1)) { + PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_GOTREF(__pyx_t_2); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { + PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L17_error) + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_GOTREF(__pyx_t_2); + } else + #endif + { + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 103, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; + __Pyx_INCREF(__pyx_v_robots); + __Pyx_GIVEREF(__pyx_v_robots); + PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_robots); + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + } + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "reppy/robots.pyx":102 + * if res.status_code == 200: + * robots = cls.parse(url, content, expires) + * if after_parse_hook is not None: # <<<<<<<<<<<<<< + * after_parse_hook(robots) + * return robots + */ + } + + /* "reppy/robots.pyx":104 + * if after_parse_hook is not None: + * after_parse_hook(robots) + * return robots # <<<<<<<<<<<<<< + * elif res.status_code in (401, 403): + * return AllowNone(url, expires) + */ + __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L17_error)) + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_robots); + __pyx_r = __pyx_v_robots; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":95 + /* "reppy/robots.pyx":100 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< - * return cls.parse(url, content, expires) - * elif res.status_code in (401, 403): + * robots = cls.parse(url, content, expires) + * if after_parse_hook is not None: */ } - /* "reppy/robots.pyx":97 - * if res.status_code == 200: - * return cls.parse(url, content, expires) + /* "reppy/robots.pyx":105 + * after_parse_hook(robots) + * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: */ - __Pyx_TraceLine(97,0,__PYX_ERR(1, 97, __pyx_L17_error)) - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 97, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 97, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 97, __pyx_L17_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - if (!__pyx_t_15) { + __Pyx_TraceLine(105,0,__PYX_ERR(1, 105, __pyx_L17_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 105, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 105, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 105, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (!__pyx_t_14) { } else { - __pyx_t_13 = __pyx_t_15; - goto __pyx_L29_bool_binop_done; + __pyx_t_13 = __pyx_t_14; + goto __pyx_L31_bool_binop_done; } - __pyx_t_4 = __Pyx_PyInt_EqObjC(__pyx_t_8, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 97, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_15 < 0)) __PYX_ERR(1, 97, __pyx_L17_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_13 = __pyx_t_15; - __pyx_L29_bool_binop_done:; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_15 = (__pyx_t_13 != 0); - if (__pyx_t_15) { + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 105, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 105, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_13 = __pyx_t_14; + __pyx_L31_bool_binop_done:; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_14 = (__pyx_t_13 != 0); + if (__pyx_t_14) { - /* "reppy/robots.pyx":98 - * return cls.parse(url, content, expires) + /* "reppy/robots.pyx":106 + * return robots * elif res.status_code in (401, 403): * return AllowNone(url, expires) # <<<<<<<<<<<<<< * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) */ - __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L17_error)) + __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_8 = PyTuple_New(2); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_8); + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 106, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_url); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); - PyTuple_SET_ITEM(__pyx_t_8, 1, __pyx_v_expires); - __pyx_t_4 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_8, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 98, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_r = __pyx_t_4; - __pyx_t_4 = 0; + PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_expires); + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 106, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":97 - * if res.status_code == 200: - * return cls.parse(url, content, expires) + /* "reppy/robots.pyx":105 + * after_parse_hook(robots) + * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: */ } - /* "reppy/robots.pyx":99 + /* "reppy/robots.pyx":107 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< * return AllowAll(url, expires) * else: */ - __Pyx_TraceLine(99,0,__PYX_ERR(1, 99, __pyx_L17_error)) - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 99, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_8 = PyObject_RichCompare(__pyx_t_4, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_8); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 99, __pyx_L17_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_8); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 99, __pyx_L17_error) - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_TraceLine(107,0,__PYX_ERR(1, 107, __pyx_L17_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 107, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 107, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_13) { } else { - __pyx_t_15 = __pyx_t_13; - goto __pyx_L31_bool_binop_done; + __pyx_t_14 = __pyx_t_13; + goto __pyx_L33_bool_binop_done; } - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 99, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_4 = PyObject_RichCompare(__pyx_t_8, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 99, __pyx_L17_error) - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 99, __pyx_L17_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_15 = __pyx_t_13; - __pyx_L31_bool_binop_done:; - if (__pyx_t_15) { - - /* "reppy/robots.pyx":100 + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 107, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 107, __pyx_L17_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_14 = __pyx_t_13; + __pyx_L33_bool_binop_done:; + if (__pyx_t_14) { + + /* "reppy/robots.pyx":108 * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) # <<<<<<<<<<<<<< * else: * raise exceptions.BadStatusCode( */ - __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L17_error)) + __Pyx_TraceLine(108,0,__PYX_ERR(1, 108, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_4 = PyTuple_New(2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 100, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 108, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_url); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); - PyTuple_SET_ITEM(__pyx_t_4, 1, __pyx_v_expires); - __pyx_t_8 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_4, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 100, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_r = __pyx_t_8; - __pyx_t_8 = 0; + PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_expires); + __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 108, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":99 + /* "reppy/robots.pyx":107 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< @@ -3166,99 +3370,99 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":102 + /* "reppy/robots.pyx":110 * return AllowAll(url, expires) * else: * raise exceptions.BadStatusCode( # <<<<<<<<<<<<<< * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: */ - __Pyx_TraceLine(102,0,__PYX_ERR(1, 102, __pyx_L17_error)) + __Pyx_TraceLine(110,0,__PYX_ERR(1, 110, __pyx_L17_error)) /*else*/ { - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 102, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 102, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 110, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 110, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":103 + /* "reppy/robots.pyx":111 * else: * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) # <<<<<<<<<<<<<< * except SSLError as exc: * raise exceptions.SSLException(exc) */ - __Pyx_TraceLine(103,0,__PYX_ERR(1, 103, __pyx_L17_error)) - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 103, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); + __Pyx_TraceLine(111,0,__PYX_ERR(1, 111, __pyx_L17_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 111, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_url); - __pyx_t_4 = 0; - __pyx_t_4 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_5); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 103, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 103, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_5); + PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_v_url); + __pyx_t_1 = 0; + __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 111, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; - __pyx_t_14 = 0; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { - __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_7); + __pyx_t_15 = 0; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_8))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_8); if (likely(__pyx_t_6)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); __Pyx_INCREF(__pyx_t_6); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_7, function); - __pyx_t_14 = 1; + __Pyx_DECREF_SET(__pyx_t_8, function); + __pyx_t_15 = 1; } } #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_7)) { - PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_4, __pyx_t_5}; - __pyx_t_8 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 102, __pyx_L17_error) + if (PyFunction_Check(__pyx_t_8)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 110, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } else #endif #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { - PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_4, __pyx_t_5}; - __pyx_t_8 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-__pyx_t_14, 2+__pyx_t_14); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 102, __pyx_L17_error) + if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 110, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_8); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } else #endif { - __pyx_t_16 = PyTuple_New(2+__pyx_t_14); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 102, __pyx_L17_error) + __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 110, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_6) { __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; } - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_16, 0+__pyx_t_14, __pyx_t_4); - __Pyx_GIVEREF(__pyx_t_5); - PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_14, __pyx_t_5); - __pyx_t_4 = 0; - __pyx_t_5 = 0; - __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_16, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 102, __pyx_L17_error) - __Pyx_GOTREF(__pyx_t_8); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_16, 0+__pyx_t_15, __pyx_t_1); + __Pyx_GIVEREF(__pyx_t_7); + PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_15, __pyx_t_7); + __pyx_t_1 = 0; + __pyx_t_7 = 0; + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 110, __pyx_L17_error) + __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_Raise(__pyx_t_8, 0, 0, 0); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __PYX_ERR(1, 102, __pyx_L17_error) + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(1, 110, __pyx_L17_error) } - /* "reppy/robots.pyx":85 + /* "reppy/robots.pyx":87 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< @@ -3269,38 +3473,38 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L17_error:; __Pyx_PyThreadState_assign __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; /*except:*/ { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 85, __pyx_L19_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(1, 87, __pyx_L19_except_error) + __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); - __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_16); - __pyx_t_5 = PyTuple_Pack(3, __pyx_t_8, __pyx_t_7, __pyx_t_16); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 85, __pyx_L19_except_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_5, NULL); + __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 87, __pyx_L19_except_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_7, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 85, __pyx_L19_except_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 87, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_17); - __pyx_t_15 = __Pyx_PyObject_IsTrue(__pyx_t_17); + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_17); __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; - if (__pyx_t_15 < 0) __PYX_ERR(1, 85, __pyx_L19_except_error) - __pyx_t_13 = ((!(__pyx_t_15 != 0)) != 0); + if (__pyx_t_14 < 0) __PYX_ERR(1, 87, __pyx_L19_except_error) + __pyx_t_13 = ((!(__pyx_t_14 != 0)) != 0); if (__pyx_t_13) { + __Pyx_GIVEREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_8); - __Pyx_GIVEREF(__pyx_t_7); __Pyx_XGIVEREF(__pyx_t_16); - __Pyx_ErrRestoreWithState(__pyx_t_8, __pyx_t_7, __pyx_t_16); - __pyx_t_8 = 0; __pyx_t_7 = 0; __pyx_t_16 = 0; - __PYX_ERR(1, 85, __pyx_L19_except_error) + __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_8, __pyx_t_16); + __pyx_t_2 = 0; __pyx_t_8 = 0; __pyx_t_16 = 0; + __PYX_ERR(1, 87, __pyx_L19_except_error) } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; goto __pyx_L18_exception_handled; } @@ -3329,9 +3533,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ /*finally:*/ { /*normal exit:*/{ if (__pyx_t_9) { - __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__6, NULL); + __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__8, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 85, __pyx_L3_error) + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; } @@ -3341,9 +3545,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_12 = __pyx_r; __pyx_r = 0; if (__pyx_t_9) { - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__7, NULL); + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__9, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 85, __pyx_L3_error) + if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 87, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } @@ -3353,148 +3557,148 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } __pyx_L16:; } - goto __pyx_L36; + goto __pyx_L38; __pyx_L11_error:; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; goto __pyx_L3_error; - __pyx_L36:; + __pyx_L38:; } - /* "reppy/robots.pyx":82 - * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): - * '''Get the robots.txt at the provided URL.''' + /* "reppy/robots.pyx":84 + * after_response_hook = kwargs.pop('after_response_hook', None) + * after_parse_hook = kwargs.pop('after_parse_hook', None) * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True */ } - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; goto __pyx_L10_try_end; __pyx_L3_error:; __Pyx_PyThreadState_assign __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; - /* "reppy/robots.pyx":104 + /* "reppy/robots.pyx":112 * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: # <<<<<<<<<<<<<< * raise exceptions.SSLException(exc) * except ConnectionError as exc: */ - __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 104, __pyx_L5_except_error) + __Pyx_TraceLine(112,0,__PYX_ERR(1, 112, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 112, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); + __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - if (__pyx_t_14) { + if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_7, &__pyx_t_8) < 0) __PYX_ERR(1, 104, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(1, 112, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_8); - __Pyx_INCREF(__pyx_t_7); - __pyx_v_exc = __pyx_t_7; + __Pyx_GOTREF(__pyx_t_2); + __Pyx_INCREF(__pyx_t_8); + __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":105 + /* "reppy/robots.pyx":113 * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: * raise exceptions.SSLException(exc) # <<<<<<<<<<<<<< * except ConnectionError as exc: * raise exceptions.ConnectionException(exc) */ - __Pyx_TraceLine(105,0,__PYX_ERR(1, 105, __pyx_L5_except_error)) - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 105, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 105, __pyx_L5_except_error) + __Pyx_TraceLine(113,0,__PYX_ERR(1, 113, __pyx_L5_except_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 113, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 113, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = NULL; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6); - if (likely(__pyx_t_4)) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); - __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx_t_1); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_6, function); } } - if (!__pyx_t_4) { - __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 105, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_5); + if (!__pyx_t_1) { + __pyx_t_7 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 113, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_6)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 105, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GOTREF(__pyx_t_5); + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_v_exc}; + __pyx_t_7 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 113, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_7); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 105, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GOTREF(__pyx_t_5); + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_v_exc}; + __pyx_t_7 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 113, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_7); } else #endif { - __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 105, __pyx_L5_except_error) + __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 113, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); - __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_1); __pyx_t_1 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 105, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 113, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; } } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_Raise(__pyx_t_5, 0, 0, 0); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 105, __pyx_L5_except_error) + __Pyx_Raise(__pyx_t_7, 0, 0, 0); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __PYX_ERR(1, 113, __pyx_L5_except_error) } - /* "reppy/robots.pyx":106 + /* "reppy/robots.pyx":114 * except SSLError as exc: * raise exceptions.SSLException(exc) * except ConnectionError as exc: # <<<<<<<<<<<<<< * raise exceptions.ConnectionException(exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: */ - __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L5_except_error)) - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 106, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - if (__pyx_t_14) { + __Pyx_TraceLine(114,0,__PYX_ERR(1, 114, __pyx_L5_except_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 114, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_2); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_7, &__pyx_t_16) < 0) __PYX_ERR(1, 106, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(1, 114, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); - __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_16); - __Pyx_INCREF(__pyx_t_7); - __pyx_v_exc = __pyx_t_7; + __Pyx_INCREF(__pyx_t_8); + __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":107 + /* "reppy/robots.pyx":115 * raise exceptions.SSLException(exc) * except ConnectionError as exc: * raise exceptions.ConnectionException(exc) # <<<<<<<<<<<<<< * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * raise exceptions.MalformedUrl(exc) */ - __Pyx_TraceLine(107,0,__PYX_ERR(1, 107, __pyx_L5_except_error)) - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __Pyx_TraceLine(115,0,__PYX_ERR(1, 115, __pyx_L5_except_error)) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 115, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_18 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __pyx_t_18 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 115, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_t_6 = NULL; @@ -3508,206 +3712,206 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_5 = __Pyx_PyObject_CallOneArg(__pyx_t_18, __pyx_v_exc); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_5); + __pyx_t_7 = __Pyx_PyObject_CallOneArg(__pyx_t_18, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 115, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_18)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __pyx_t_7 = __Pyx_PyFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 115, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_5); + __Pyx_GOTREF(__pyx_t_7); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_18)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; - __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L5_except_error) + __pyx_t_7 = __Pyx_PyCFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 115, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_5); + __Pyx_GOTREF(__pyx_t_7); } else #endif { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 107, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_6); __pyx_t_6 = NULL; + __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 115, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); - PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_exc); - __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_18, __pyx_t_4, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 107, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_v_exc); + __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_18, __pyx_t_1, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 115, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; } } __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; - __Pyx_Raise(__pyx_t_5, 0, 0, 0); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - __PYX_ERR(1, 107, __pyx_L5_except_error) + __Pyx_Raise(__pyx_t_7, 0, 0, 0); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __PYX_ERR(1, 115, __pyx_L5_except_error) } - /* "reppy/robots.pyx":108 + /* "reppy/robots.pyx":116 * except ConnectionError as exc: * raise exceptions.ConnectionException(exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: # <<<<<<<<<<<<<< * raise exceptions.MalformedUrl(exc) * except TooManyRedirects as exc: */ - __Pyx_TraceLine(108,0,__PYX_ERR(1, 108, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 108, __pyx_L5_except_error) + __Pyx_TraceLine(116,0,__PYX_ERR(1, 116, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 116, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 108, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 108, __pyx_L5_except_error) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 116, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_5 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 108, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_5); - __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_5); + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 116, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 116, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_2) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; - if (__pyx_t_14) { + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_7) < 0) __PYX_ERR(1, 108, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GOTREF(__pyx_t_8); + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(1, 116, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __Pyx_INCREF(__pyx_t_8); - __pyx_v_exc = __pyx_t_8; + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_t_2); + __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":109 + /* "reppy/robots.pyx":117 * raise exceptions.ConnectionException(exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * raise exceptions.MalformedUrl(exc) # <<<<<<<<<<<<<< * except TooManyRedirects as exc: * raise exceptions.ExcessiveRedirects(exc) */ - __Pyx_TraceLine(109,0,__PYX_ERR(1, 109, __pyx_L5_except_error)) - __pyx_t_18 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 109, __pyx_L5_except_error) + __Pyx_TraceLine(117,0,__PYX_ERR(1, 117, __pyx_L5_except_error)) + __pyx_t_18 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 117, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 109, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_4); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 117, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; __pyx_t_18 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_4))) { - __pyx_t_18 = PyMethod_GET_SELF(__pyx_t_4); + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_1))) { + __pyx_t_18 = PyMethod_GET_SELF(__pyx_t_1); if (likely(__pyx_t_18)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_1); __Pyx_INCREF(__pyx_t_18); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_4, function); + __Pyx_DECREF_SET(__pyx_t_1, function); } } if (!__pyx_t_18) { - __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 109, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 117, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); } else { #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_4)) { + if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 109, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 117, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; __Pyx_GOTREF(__pyx_t_16); } else #endif #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { + if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 109, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 117, __pyx_L5_except_error) __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; __Pyx_GOTREF(__pyx_t_16); } else #endif { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 109, __pyx_L5_except_error) + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 117, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_GIVEREF(__pyx_t_18); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_18); __pyx_t_18 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_v_exc); - __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 109, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_6, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 117, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } } - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_Raise(__pyx_t_16, 0, 0, 0); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __PYX_ERR(1, 109, __pyx_L5_except_error) + __PYX_ERR(1, 117, __pyx_L5_except_error) } - /* "reppy/robots.pyx":110 + /* "reppy/robots.pyx":118 * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * raise exceptions.MalformedUrl(exc) * except TooManyRedirects as exc: # <<<<<<<<<<<<<< * raise exceptions.ExcessiveRedirects(exc) * */ - __Pyx_TraceLine(110,0,__PYX_ERR(1, 110, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 110, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_14 = __Pyx_PyErr_ExceptionMatches(__pyx_t_7); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (__pyx_t_14) { + __Pyx_TraceLine(118,0,__PYX_ERR(1, 118, __pyx_L5_except_error)) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 118, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_8, &__pyx_t_5) < 0) __PYX_ERR(1, 110, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(1, 118, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); - __Pyx_GOTREF(__pyx_t_5); - __Pyx_INCREF(__pyx_t_8); - __pyx_v_exc = __pyx_t_8; + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_t_7); + __Pyx_INCREF(__pyx_t_2); + __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":111 + /* "reppy/robots.pyx":119 * raise exceptions.MalformedUrl(exc) * except TooManyRedirects as exc: * raise exceptions.ExcessiveRedirects(exc) # <<<<<<<<<<<<<< * * def RobotsUrlMethod(cls, url): */ - __Pyx_TraceLine(111,0,__PYX_ERR(1, 111, __pyx_L5_except_error)) - __pyx_t_4 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 111, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_4); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 111, __pyx_L5_except_error) + __Pyx_TraceLine(119,0,__PYX_ERR(1, 119, __pyx_L5_except_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 119, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_6); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = NULL; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_6); - if (likely(__pyx_t_4)) { + __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); - __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx_t_1); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_6, function); } } - if (!__pyx_t_4) { - __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 111, __pyx_L5_except_error) + if (!__pyx_t_1) { + __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_6)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 111, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_v_exc}; + __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 119, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GOTREF(__pyx_t_16); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 111, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_v_exc}; + __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 119, __pyx_L5_except_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GOTREF(__pyx_t_16); } else #endif { - __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 111, __pyx_L5_except_error) + __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_18); - __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_1); __pyx_t_1 = NULL; __Pyx_INCREF(__pyx_v_exc); __Pyx_GIVEREF(__pyx_v_exc); PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); - __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 111, __pyx_L5_except_error) + __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; } @@ -3715,30 +3919,30 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_Raise(__pyx_t_16, 0, 0, 0); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __PYX_ERR(1, 111, __pyx_L5_except_error) + __PYX_ERR(1, 119, __pyx_L5_except_error) } goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "reppy/robots.pyx":82 - * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): - * '''Get the robots.txt at the provided URL.''' + /* "reppy/robots.pyx":84 + * after_response_hook = kwargs.pop('after_response_hook', None) + * after_parse_hook = kwargs.pop('after_parse_hook', None) * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True */ __Pyx_PyThreadState_assign - __Pyx_XGIVEREF(__pyx_t_1); - __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); - __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); goto __pyx_L1_error; __pyx_L7_try_return:; __Pyx_PyThreadState_assign - __Pyx_XGIVEREF(__pyx_t_1); - __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); - __Pyx_ExceptionReset(__pyx_t_1, __pyx_t_2, __pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); goto __pyx_L0; __pyx_L10_try_end:; } @@ -3748,15 +3952,15 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' - * try: + * after_response_hook = kwargs.pop('after_response_hook', None) */ /* function exit code */ __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_4); - __Pyx_XDECREF(__pyx_t_5); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); @@ -3765,9 +3969,12 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; + __Pyx_XDECREF(__pyx_v_after_response_hook); + __Pyx_XDECREF(__pyx_v_after_parse_hook); __Pyx_XDECREF(__pyx_v_res); __Pyx_XDECREF(__pyx_v_content); __Pyx_XDECREF(__pyx_v_expires); + __Pyx_XDECREF(__pyx_v_robots); __Pyx_XDECREF(__pyx_v_exc); __Pyx_XGIVEREF(__pyx_r); __Pyx_TraceReturn(__pyx_r, 0); @@ -3775,7 +3982,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":113 +/* "reppy/robots.pyx":121 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -3813,11 +4020,11 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 113, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 121, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 113, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 121, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -3830,7 +4037,7 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 113, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 121, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3851,39 +4058,39 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject std::string __pyx_t_2; std::string __pyx_t_3; PyObject *__pyx_t_4 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__8) + __Pyx_TraceFrameInit(__pyx_codeobj__10) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); - __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 113, 0, __PYX_ERR(1, 113, __pyx_L1_error)); + __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 121, 0, __PYX_ERR(1, 121, __pyx_L1_error)); - /* "reppy/robots.pyx":115 + /* "reppy/robots.pyx":123 * def RobotsUrlMethod(cls, url): * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) # <<<<<<<<<<<<<< * * cdef class Robots: */ - __Pyx_TraceLine(115,0,__PYX_ERR(1, 115, __pyx_L1_error)) + __Pyx_TraceLine(123,0,__PYX_ERR(1, 123, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 115, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 115, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 123, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 115, __pyx_L1_error) + __PYX_ERR(1, 123, __pyx_L1_error) } - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 115, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 115, __pyx_L1_error) + __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 123, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":113 + /* "reppy/robots.pyx":121 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -3904,7 +4111,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":133 +/* "reppy/robots.pyx":141 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -3943,7 +4150,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 133, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 141, __pyx_L3_error) } case 2: if (kw_args > 0) { @@ -3952,7 +4159,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 133, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 141, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -3964,12 +4171,12 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } __pyx_v_url = values[0]; - __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 133, __pyx_L3_error) + __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 141, __pyx_L3_error) __pyx_v_expires = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 133, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 141, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3990,43 +4197,43 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob std::string __pyx_t_2; Rep::Robots *__pyx_t_3; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 133, 0, __PYX_ERR(1, 133, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 141, 0, __PYX_ERR(1, 141, __pyx_L1_error)); - /* "reppy/robots.pyx":134 + /* "reppy/robots.pyx":142 * * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) # <<<<<<<<<<<<<< * self.expires = expires * */ - __Pyx_TraceLine(134,0,__PYX_ERR(1, 134, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 134, __pyx_L1_error) + __Pyx_TraceLine(142,0,__PYX_ERR(1, 142, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 142, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 134, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 142, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = new Rep::Robots(__pyx_v_content, __pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 134, __pyx_L1_error) + __PYX_ERR(1, 142, __pyx_L1_error) } __pyx_v_self->robots = __pyx_t_3; - /* "reppy/robots.pyx":135 + /* "reppy/robots.pyx":143 * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) * self.expires = expires # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(135,0,__PYX_ERR(1, 135, __pyx_L1_error)) + __Pyx_TraceLine(143,0,__PYX_ERR(1, 143, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); __Pyx_GOTREF(__pyx_v_self->expires); __Pyx_DECREF(__pyx_v_self->expires); __pyx_v_self->expires = __pyx_v_expires; - /* "reppy/robots.pyx":133 + /* "reppy/robots.pyx":141 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -4047,7 +4254,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob return __pyx_r; } -/* "reppy/robots.pyx":137 +/* "reppy/robots.pyx":145 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4074,24 +4281,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[1], 137, 0, __PYX_ERR(1, 137, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[1], 145, 0, __PYX_ERR(1, 145, __pyx_L1_error)); - /* "reppy/robots.pyx":138 + /* "reppy/robots.pyx":146 * * def __str__(self): * return self.robots.str() # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __Pyx_TraceLine(138,0,__PYX_ERR(1, 138, __pyx_L1_error)) + __Pyx_TraceLine(146,0,__PYX_ERR(1, 146, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 138, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":137 + /* "reppy/robots.pyx":145 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4111,7 +4318,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":140 +/* "reppy/robots.pyx":148 * return self.robots.str() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4134,19 +4341,19 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - __Pyx_TraceCall("__dealloc__", __pyx_f[1], 140, 0, __PYX_ERR(1, 140, __pyx_L1_error)); + __Pyx_TraceCall("__dealloc__", __pyx_f[1], 148, 0, __PYX_ERR(1, 148, __pyx_L1_error)); - /* "reppy/robots.pyx":141 + /* "reppy/robots.pyx":149 * * def __dealloc__(self): * del self.robots # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(141,0,__PYX_ERR(1, 141, __pyx_L1_error)) + __Pyx_TraceLine(149,0,__PYX_ERR(1, 149, __pyx_L1_error)) delete __pyx_v_self->robots; - /* "reppy/robots.pyx":140 + /* "reppy/robots.pyx":148 * return self.robots.str() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4163,7 +4370,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_RefNannyFinishContext(); } -/* "reppy/robots.pyx":144 +/* "reppy/robots.pyx":152 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4192,22 +4399,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 144, 0, __PYX_ERR(1, 144, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 152, 0, __PYX_ERR(1, 152, __pyx_L1_error)); - /* "reppy/robots.pyx":146 + /* "reppy/robots.pyx":154 * def sitemaps(self): * '''Get all the sitemaps in this robots.txt.''' * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< * * def allowed(self, path, name): */ - __Pyx_TraceLine(146,0,__PYX_ERR(1, 146, __pyx_L1_error)) + __Pyx_TraceLine(154,0,__PYX_ERR(1, 154, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -4215,14 +4422,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":144 + /* "reppy/robots.pyx":152 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4244,7 +4451,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ return __pyx_r; } -/* "reppy/robots.pyx":148 +/* "reppy/robots.pyx":156 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -4281,11 +4488,11 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 148, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 156, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 148, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 156, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4298,7 +4505,7 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 148, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 156, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4319,32 +4526,32 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp std::string __pyx_t_2; std::string __pyx_t_3; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[1], 148, 0, __PYX_ERR(1, 148, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[1], 156, 0, __PYX_ERR(1, 156, __pyx_L1_error)); - /* "reppy/robots.pyx":150 + /* "reppy/robots.pyx":158 * def allowed(self, path, name): * '''Is the provided path allowed for the provided agent?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) # <<<<<<<<<<<<<< * * def agent(self, name): */ - __Pyx_TraceLine(150,0,__PYX_ERR(1, 150, __pyx_L1_error)) + __Pyx_TraceLine(158,0,__PYX_ERR(1, 158, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 150, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 158, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 150, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 158, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 150, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 158, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 150, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 158, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 150, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 158, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":148 + /* "reppy/robots.pyx":156 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -4364,7 +4571,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":152 +/* "reppy/robots.pyx":160 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -4397,20 +4604,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ int __pyx_t_5; PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("agent", 0); - __Pyx_TraceCall("agent", __pyx_f[1], 152, 0, __PYX_ERR(1, 152, __pyx_L1_error)); + __Pyx_TraceCall("agent", __pyx_f[1], 160, 0, __PYX_ERR(1, 160, __pyx_L1_error)); - /* "reppy/robots.pyx":159 + /* "reppy/robots.pyx":167 * Agent object. * ''' * return Agent.from_robots(self, as_bytes(name)) # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(159,0,__PYX_ERR(1, 159, __pyx_L1_error)) + __Pyx_TraceLine(167,0,__PYX_ERR(1, 167, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 159, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 159, __pyx_L1_error) + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; __pyx_t_5 = 0; @@ -4427,7 +4634,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 159, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -4436,14 +4643,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 159, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 159, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -4454,7 +4661,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 159, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -4463,7 +4670,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":152 + /* "reppy/robots.pyx":160 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -4487,7 +4694,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":162 +/* "reppy/robots.pyx":170 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4516,20 +4723,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 162, 0, __PYX_ERR(1, 162, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 170, 0, __PYX_ERR(1, 170, __pyx_L1_error)); - /* "reppy/robots.pyx":164 + /* "reppy/robots.pyx":172 * def expired(self): * '''True if the current time is past its expiration.''' * return time.time() > self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(164,0,__PYX_ERR(1, 164, __pyx_L1_error)) + __Pyx_TraceLine(172,0,__PYX_ERR(1, 172, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 172, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 172, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -4543,20 +4750,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } } if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 172, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 172, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 172, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":162 + /* "reppy/robots.pyx":170 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4578,7 +4785,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":167 +/* "reppy/robots.pyx":175 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4604,22 +4811,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 167, 0, __PYX_ERR(1, 167, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 175, 0, __PYX_ERR(1, 175, __pyx_L1_error)); - /* "reppy/robots.pyx":169 + /* "reppy/robots.pyx":177 * def expires(self): * '''The expiration of this robots.txt.''' * return self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(169,0,__PYX_ERR(1, 169, __pyx_L1_error)) + __Pyx_TraceLine(177,0,__PYX_ERR(1, 177, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_self->expires); __pyx_r = __pyx_v_self->expires; goto __pyx_L0; - /* "reppy/robots.pyx":167 + /* "reppy/robots.pyx":175 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4638,7 +4845,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":172 +/* "reppy/robots.pyx":180 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4670,21 +4877,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 172, 0, __PYX_ERR(1, 172, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 180, 0, __PYX_ERR(1, 180, __pyx_L1_error)); - /* "reppy/robots.pyx":174 + /* "reppy/robots.pyx":182 * def ttl(self): * '''Remaining time for this response to be considered valid.''' * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(174,0,__PYX_ERR(1, 174, __pyx_L1_error)) + __Pyx_TraceLine(182,0,__PYX_ERR(1, 182, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 182, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 182, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -4698,24 +4905,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 182, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 182, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 182, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 182, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 182, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 182, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 174, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 182, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -4729,7 +4936,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":172 + /* "reppy/robots.pyx":180 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4752,7 +4959,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 return __pyx_r; } -/* "reppy/robots.pyx":180 +/* "reppy/robots.pyx":188 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4793,7 +5000,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 180, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 188, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4808,7 +5015,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 180, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 188, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4831,17 +5038,17 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 180, 0, __PYX_ERR(1, 180, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 188, 0, __PYX_ERR(1, 188, __pyx_L1_error)); - /* "reppy/robots.pyx":181 + /* "reppy/robots.pyx":189 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(181,0,__PYX_ERR(1, 181, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 181, __pyx_L1_error) + __Pyx_TraceLine(189,0,__PYX_ERR(1, 189, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -4858,7 +5065,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 189, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -4866,13 +5073,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 189, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 181, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -4889,14 +5096,14 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":180 + /* "reppy/robots.pyx":188 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4920,7 +5127,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":187 +/* "reppy/robots.pyx":195 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -4960,7 +5167,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 187, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 195, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4975,7 +5182,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 187, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 195, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4998,15 +5205,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 187, 0, __PYX_ERR(1, 187, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 195, 0, __PYX_ERR(1, 195, __pyx_L1_error)); - /* "reppy/robots.pyx":188 + /* "reppy/robots.pyx":196 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(188,0,__PYX_ERR(1, 188, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 188, __pyx_L1_error) + __Pyx_TraceLine(196,0,__PYX_ERR(1, 196, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 196, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -5022,22 +5229,22 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 188, __pyx_L1_error) + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__11, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 196, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__9, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 188, __pyx_L1_error) + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__11, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 196, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 196, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -5048,20 +5255,20 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_4, __pyx_v_url); - __Pyx_INCREF(__pyx_kp_b__9); - __Pyx_GIVEREF(__pyx_kp_b__9); - PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b__9); + __Pyx_INCREF(__pyx_kp_b__11); + __Pyx_GIVEREF(__pyx_kp_b__11); + PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b__11); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 196, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":187 + /* "reppy/robots.pyx":195 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5500,7 +5707,7 @@ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyO * return f(value) */ __Pyx_TraceLine(65,0,__PYX_ERR(2, 65, __pyx_L1_error)) - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__13)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_wrap = __pyx_t_1; __pyx_t_1 = 0; @@ -6117,8 +6324,10 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_URLRequired, __pyx_k_URLRequired, sizeof(__pyx_k_URLRequired), 0, 0, 1, 1}, {&__pyx_kp_b_User_agent_Disallow, __pyx_k_User_agent_Disallow, sizeof(__pyx_k_User_agent_Disallow), 0, 0, 0, 0}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, - {&__pyx_n_s__9, __pyx_k__9, sizeof(__pyx_k__9), 0, 0, 1, 1}, - {&__pyx_kp_b__9, __pyx_k__9, sizeof(__pyx_k__9), 0, 0, 0, 0}, + {&__pyx_n_s__11, __pyx_k__11, sizeof(__pyx_k__11), 0, 0, 1, 1}, + {&__pyx_kp_b__11, __pyx_k__11, sizeof(__pyx_k__11), 0, 0, 0, 0}, + {&__pyx_n_s_after_parse_hook, __pyx_k_after_parse_hook, sizeof(__pyx_k_after_parse_hook), 0, 0, 1, 1}, + {&__pyx_n_s_after_response_hook, __pyx_k_after_response_hook, sizeof(__pyx_k_after_response_hook), 0, 0, 1, 1}, {&__pyx_n_s_agent, __pyx_k_agent, sizeof(__pyx_k_agent), 0, 0, 1, 1}, {&__pyx_n_s_amt, __pyx_k_amt, sizeof(__pyx_k_amt), 0, 0, 1, 1}, {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, @@ -6150,6 +6359,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_name, __pyx_k_name, sizeof(__pyx_k_name), 0, 0, 1, 1}, {&__pyx_n_s_parse, __pyx_k_parse, sizeof(__pyx_k_parse), 0, 0, 1, 1}, {&__pyx_n_s_path, __pyx_k_path, sizeof(__pyx_k_path), 0, 0, 1, 1}, + {&__pyx_n_s_pop, __pyx_k_pop, sizeof(__pyx_k_pop), 0, 0, 1, 1}, {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, {&__pyx_n_s_raw, __pyx_k_raw, sizeof(__pyx_k_raw), 0, 0, 1, 1}, {&__pyx_n_s_read, __pyx_k_read, sizeof(__pyx_k_read), 0, 0, 1, 1}, @@ -6177,7 +6387,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { }; static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) - __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 154, __pyx_L1_error) __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) return 0; __pyx_L1_error:; @@ -6210,19 +6420,41 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); - /* "reppy/robots.pyx":85 + /* "reppy/robots.pyx":82 + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): + * '''Get the robots.txt at the provided URL.''' + * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< + * after_parse_hook = kwargs.pop('after_parse_hook', None) + * try: + */ + __pyx_tuple__6 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__6); + __Pyx_GIVEREF(__pyx_tuple__6); + + /* "reppy/robots.pyx":83 + * '''Get the robots.txt at the provided URL.''' + * after_response_hook = kwargs.pop('after_response_hook', None) + * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< + * try: + * # Limit the size of the request + */ + __pyx_tuple__7 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 83, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__7); + __Pyx_GIVEREF(__pyx_tuple__7); + + /* "reppy/robots.pyx":87 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __pyx_tuple__6 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 85, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__6); - __Pyx_GIVEREF(__pyx_tuple__6); - __pyx_tuple__7 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 85, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__7); - __Pyx_GIVEREF(__pyx_tuple__7); + __pyx_tuple__8 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(1, 87, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__8); + __Pyx_GIVEREF(__pyx_tuple__8); + __pyx_tuple__9 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(1, 87, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__9); + __Pyx_GIVEREF(__pyx_tuple__9); /* "cfunc.to_py":65 * @cname("__Pyx_CFunc_object____object___to_py") @@ -6231,10 +6463,10 @@ static int __Pyx_InitCachedConstants(void) { * """wrap(value)""" * return f(value) */ - __pyx_tuple__10 = PyTuple_Pack(1, __pyx_n_s_value); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); - __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 65, __pyx_L1_error) + __pyx_tuple__12 = PyTuple_Pack(1, __pyx_n_s_value); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__12); + __Pyx_GIVEREF(__pyx_tuple__12); + __pyx_codeobj__13 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__12, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__13)) __PYX_ERR(2, 65, __pyx_L1_error) /* "reppy/robots.pyx":33 * @@ -6243,10 +6475,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Construct an Agent from a CppAgent.''' * agent = Agent() */ - __pyx_tuple__12 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(1, 33, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__12); - __Pyx_GIVEREF(__pyx_tuple__12); - __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__12, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(1, 33, __pyx_L1_error) + __pyx_tuple__14 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(1, 33, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__14); + __Pyx_GIVEREF(__pyx_tuple__14); + __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(1, 33, __pyx_L1_error) /* "reppy/robots.pyx":76 * @@ -6255,34 +6487,34 @@ static int __Pyx_InitCachedConstants(void) { * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __pyx_tuple__13 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(1, 76, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__13); - __Pyx_GIVEREF(__pyx_tuple__13); - __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__13, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_tuple__15 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__15); + __Pyx_GIVEREF(__pyx_tuple__15); + __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 76, __pyx_L1_error) /* "reppy/robots.pyx":80 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' - * try: + * after_response_hook = kwargs.pop('after_response_hook', None) */ - __pyx_tuple__14 = PyTuple_Pack(10, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(1, 80, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__14); - __Pyx_GIVEREF(__pyx_tuple__14); - __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 10, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 80, __pyx_L1_error) + __pyx_tuple__16 = PyTuple_Pack(13, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 80, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__16); + __Pyx_GIVEREF(__pyx_tuple__16); + __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 13, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__16, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 80, __pyx_L1_error) - /* "reppy/robots.pyx":113 + /* "reppy/robots.pyx":121 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __pyx_tuple__15 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 113, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__15); - __Pyx_GIVEREF(__pyx_tuple__15); - __pyx_codeobj__8 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 113, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__8)) __PYX_ERR(1, 113, __pyx_L1_error) + __pyx_tuple__17 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 121, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__17); + __Pyx_GIVEREF(__pyx_tuple__17); + __pyx_codeobj__10 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__17, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 121, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__10)) __PYX_ERR(1, 121, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -6397,19 +6629,19 @@ PyMODINIT_FUNC PyInit_robots(void) __pyx_type_5reppy_6robots_Agent.tp_print = 0; if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 43, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; - if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 117, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 125, __pyx_L1_error) __pyx_type_5reppy_6robots_Robots.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 117, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 125, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 177, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 185, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 177, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 185, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 184, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 192, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 184, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 192, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; @@ -6598,7 +6830,7 @@ PyMODINIT_FUNC PyInit_robots(void) __Pyx_INCREF(__pyx_n_s_exceptions); __Pyx_GIVEREF(__pyx_n_s_exceptions); PyList_SET_ITEM(__pyx_t_2, 2, __pyx_n_s_exceptions); - __pyx_t_1 = __Pyx_Import(__pyx_n_s__9, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 19, __pyx_L1_error) + __pyx_t_1 = __Pyx_Import(__pyx_n_s__11, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_util); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) @@ -6663,7 +6895,7 @@ PyMODINIT_FUNC PyInit_robots(void) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' - * try: + * after_response_hook = kwargs.pop('after_response_hook', None) */ __Pyx_TraceLine(80,0,__PYX_ERR(1, 80, __pyx_L1_error)) __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 80, __pyx_L1_error) @@ -6671,89 +6903,89 @@ PyMODINIT_FUNC PyInit_robots(void) if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":113 + /* "reppy/robots.pyx":121 * raise exceptions.ExcessiveRedirects(exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __Pyx_TraceLine(113,0,__PYX_ERR(1, 113, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 113, __pyx_L1_error) + __Pyx_TraceLine(121,0,__PYX_ERR(1, 121, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 121, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 113, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 121, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":122 + /* "reppy/robots.pyx":130 * # The default TTL policy is to cache for 3600 seconds or what's provided in the * # headers, and a minimum of 600 seconds * DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) # <<<<<<<<<<<<<< * * # Class methods */ - __Pyx_TraceLine(122,0,__PYX_ERR(1, 122, __pyx_L1_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 122, __pyx_L1_error) + __Pyx_TraceLine(130,0,__PYX_ERR(1, 130, __pyx_L1_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 130, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 122, __pyx_L1_error) + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 130, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 122, __pyx_L1_error) - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 122, __pyx_L1_error) - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 122, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 130, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 130, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 130, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 122, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 130, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":125 + /* "reppy/robots.pyx":133 * * # Class methods * parse = classmethod(ParseMethod) # <<<<<<<<<<<<<< * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) */ - __Pyx_TraceLine(125,0,__PYX_ERR(1, 125, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 125, __pyx_L1_error) + __Pyx_TraceLine(133,0,__PYX_ERR(1, 133, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 133, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 125, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 133, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 125, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 133, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":126 + /* "reppy/robots.pyx":134 * # Class methods * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) # <<<<<<<<<<<<<< * robots_url = classmethod(RobotsUrlMethod) * */ - __Pyx_TraceLine(126,0,__PYX_ERR(1, 126, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 126, __pyx_L1_error) + __Pyx_TraceLine(134,0,__PYX_ERR(1, 134, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 134, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 126, __pyx_L1_error) + __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 134, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 126, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 134, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":127 + /* "reppy/robots.pyx":135 * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) # <<<<<<<<<<<<<< * * # Data members */ - __Pyx_TraceLine(127,0,__PYX_ERR(1, 127, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 127, __pyx_L1_error) + __Pyx_TraceLine(135,0,__PYX_ERR(1, 135, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 135, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 127, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 135, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 127, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 135, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); diff --git a/reppy/robots.pyx b/reppy/robots.pyx index 15bb789..e7f5389 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -79,6 +79,8 @@ def ParseMethod(cls, url, content, expires=None): def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): '''Get the robots.txt at the provided URL.''' + after_response_hook = kwargs.pop('after_response_hook', None) + after_parse_hook = kwargs.pop('after_parse_hook', None) try: # Limit the size of the request kwargs['stream'] = True @@ -89,11 +91,17 @@ def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): raise exceptions.ContentTooLong( 'Content larger than %s bytes' % max_size) + if after_response_hook is not None: + after_response_hook(res) + # Get the TTL policy's ruling on the ttl expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) if res.status_code == 200: - return cls.parse(url, content, expires) + robots = cls.parse(url, content, expires) + if after_parse_hook is not None: + after_parse_hook(robots) + return robots elif res.status_code in (401, 403): return AllowNone(url, expires) elif res.status_code >= 400 and res.status_code < 500: diff --git a/tests/asis/test_after_parse_hook/robots.txt b/tests/asis/test_after_parse_hook/robots.txt new file mode 100644 index 0000000..75dac68 --- /dev/null +++ b/tests/asis/test_after_parse_hook/robots.txt @@ -0,0 +1,6 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: /disallowed +Allow: /allowed diff --git a/tests/asis/test_after_response_hook/robots.txt b/tests/asis/test_after_response_hook/robots.txt new file mode 100644 index 0000000..75dac68 --- /dev/null +++ b/tests/asis/test_after_response_hook/robots.txt @@ -0,0 +1,6 @@ +HTTP/1.0 200 OK +Content-Type: text/plain + +User-Agent: * +Disallow: /disallowed +Allow: /allowed diff --git a/tests/test_robots.py b/tests/test_robots.py index 6886e6e..91d1374 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -356,6 +356,30 @@ def test_rfc_example(self): self.assertFalse(robot.allowed('/%7Ejim/jim.html', 'anything')) self.assertTrue(robot.allowed('/%7Emak/mak.html', 'anything')) + def test_after_response_hook(self): + '''Calls after_response_hook when response is received''' + state = {"called": False} + + def hook(response): + state["called"] = True + self.assertEquals(response.status_code, 200) + with requests_fixtures('test_after_response_hook'): + robots.Robots.fetch( + 'http://example.com/robots.txt', after_response_hook=hook) + self.assertTrue(state["called"]) + + def test_after_parse_hook(self): + '''Calls after_parse_hook after parsing robots.txt''' + state = {"called": False} + + def hook(robots): + state["called"] = True + self.assertFalse(robots.allowed('/disallowed', 'me')) + with requests_fixtures('test_after_parse_hook'): + robots.Robots.fetch( + 'http://example.com/robots.txt', after_parse_hook=hook) + self.assertTrue(state["called"]) + class AllowNoneTest(unittest.TestCase): '''Tests about the AllowNone Robots class.''' From c245fedefa6586fc3eaff10700b01f497f287519 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 8 Sep 2017 15:28:58 -0700 Subject: [PATCH 074/113] Release version 0.4.8. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3896800..5a99fa0 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.7', + version='0.4.8', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From 67dce37348033c4f6066aaf91903f5a74fe1e6bc Mon Sep 17 00:00:00 2001 From: Steve Lewis Date: Mon, 9 Oct 2017 08:30:19 -0700 Subject: [PATCH 075/113] Add new py3 version to CI (#77) * Add new py3 version to CI I noticed when submitting a PR for another project to add support for respecting robots.txt files that reppy was not building under python 3.7-dev and nightly builds. To ensure proper visibility for coming versions, I am proposing additional versions for the testing matrix. * Allow failure in Python 3.7-dev Addresses instability in an emerging Python version while still providing insight. * Correcting allowed failures syntax for travis ci --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 087ae8b..80cedab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,12 @@ python: - "3.3" - "3.4" - "3.5" + - "3.6" + - "3.7-dev" + +matrix: + allow_failures: + - python: "3.7-dev" install: pip install -r dev-requirements.txt From f56a4b71c9ead3cdcf1c57f9a75aa24543c1e778 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 23 Oct 2017 10:34:40 -0700 Subject: [PATCH 076/113] Fix tox.ini to partially work. This was my last attempt at getting tox to work correctly. It partially works, but fails in py33,py34,py35 despite those working in Travis. --- tox.ini | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/tox.ini b/tox.ini index 62fbd0b..aa76a27 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,6 @@ [tox] -envlist=py27,py33,py34,pypy - -[testenv:py27] -deps= - -rdev-requirements.txt -commands= - nose - -[testenv:py33,py34,pypy] -deps= - -rdev-requirements.txt -commands= - nose +envlist=py27,py33,py34,py35 +[testenv] +deps= -rdev-requirements.txt +commands=nosetests [] From bdf492fd0b50747584d3200c357053db074cf8f2 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 23 Oct 2017 10:37:15 -0700 Subject: [PATCH 077/113] Drop tox.ini since it doesn't fully work. The Travis build currently builds all versions of Python that we care about, so we don't need local support for tox. Fixes #42. --- tox.ini | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 tox.ini diff --git a/tox.ini b/tox.ini deleted file mode 100644 index aa76a27..0000000 --- a/tox.ini +++ /dev/null @@ -1,6 +0,0 @@ -[tox] -envlist=py27,py33,py34,py35 - -[testenv] -deps= -rdev-requirements.txt -commands=nosetests [] From ec1c823215bd3fddad17a5ddefe33cb782b9d46c Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 23 Oct 2017 10:50:03 -0700 Subject: [PATCH 078/113] Drop unused make target. This is no longer required since 3e0e86d. --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 022a60f..042e6ae 100644 --- a/Makefile +++ b/Makefile @@ -11,9 +11,6 @@ install: dev-requirements: pip freeze | grep -v -e reppy > dev-requirements.txt -dev-requirements-py3: - pip freeze | grep -v -e reppy > dev-requirements-py3.txt - clean: rm -rf build dist *.egg-info reppy/*.so find . -name '*.pyc' | xargs --no-run-if-empty rm From 9d8c9cdcb2dad26f2e0ccb0209d6649a0bfb6cc5 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Tue, 14 Nov 2017 16:44:01 -0800 Subject: [PATCH 079/113] Ensure after_response_hook is called on errors. --- reppy/robots.cpp | 2388 ++++++++++++++++++++++++------------------ reppy/robots.pyx | 14 +- setup.py | 2 +- tests/test_robots.py | 14 + 4 files changed, 1387 insertions(+), 1031 deletions(-) diff --git a/reppy/robots.cpp b/reppy/robots.cpp index 0336fde..ef3c3d5 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -646,6 +646,7 @@ struct __pyx_obj_5reppy_6robots_Agent; struct __pyx_obj_5reppy_6robots_Robots; struct __pyx_obj_5reppy_6robots_AllowNone; struct __pyx_obj_5reppy_6robots_AllowAll; +struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod; struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; /* "reppy/robots.pyx":43 @@ -661,7 +662,7 @@ struct __pyx_obj_5reppy_6robots_Agent { }; -/* "reppy/robots.pyx":125 +/* "reppy/robots.pyx":131 * return as_string(CppRobots.robotsUrl(as_bytes(url))) * * cdef class Robots: # <<<<<<<<<<<<<< @@ -675,7 +676,7 @@ struct __pyx_obj_5reppy_6robots_Robots { }; -/* "reppy/robots.pyx":185 +/* "reppy/robots.pyx":191 * * * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< @@ -687,7 +688,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone { }; -/* "reppy/robots.pyx":192 +/* "reppy/robots.pyx":198 * * * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< @@ -699,6 +700,20 @@ struct __pyx_obj_5reppy_6robots_AllowAll { }; +/* "reppy/robots.pyx":80 + * return cls(url, as_bytes(content), expires) + * + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< + * '''Get the robots.txt at the provided URL.''' + * after_response_hook = kwargs.pop('after_response_hook', None) + */ +struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod { + PyObject_HEAD + PyObject *__pyx_v_after_response_hook; + PyObject *__pyx_v_url; +}; + + /* "cfunc.to_py":64 * * @cname("__Pyx_CFunc_object____object___to_py") @@ -1049,37 +1064,25 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject /* PyObjectCallOneArg.proto */ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); -/* PyObjectLookupSpecial.proto */ -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 -static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { - PyObject *res; - PyTypeObject *tp = Py_TYPE(obj); +/* None.proto */ +static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname); + +/* PyObjectSetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_SetAttrStr(o,n,NULL) +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_setattro)) + return tp->tp_setattro(obj, attr_name, value); #if PY_MAJOR_VERSION < 3 - if (unlikely(PyInstance_Check(obj))) - return __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (likely(tp->tp_setattr)) + return tp->tp_setattr(obj, PyString_AS_STRING(attr_name), value); #endif - res = _PyType_Lookup(tp, attr_name); - if (likely(res)) { - descrgetfunc f = Py_TYPE(res)->tp_descr_get; - if (!f) { - Py_INCREF(res); - } else { - res = f(res, obj, (PyObject *)tp); - } - } else { - PyErr_SetObject(PyExc_AttributeError, attr_name); - } - return res; + return PyObject_SetAttr(obj, attr_name, value); } #else -#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) -#endif - -/* PyObjectCallNoArg.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); -#else -#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) +#define __Pyx_PyObject_DelAttrStr(o,n) PyObject_DelAttr(o,n) +#define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v) #endif /* PyThreadStateGet.proto */ @@ -1109,46 +1112,6 @@ static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject /* RaiseException.proto */ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); -/* PyIntBinop.proto */ -#if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); -#else -#define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ - PyObject_RichCompare(op1, op2, Py_EQ) - #endif - -/* SaveResetException.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); -#else -#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) -#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) -#endif - -/* GetException.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) -static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); -#endif - -/* PyErrExceptionMatches.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) -static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); -#else -#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) -#endif - -/* WriteUnraisableException.proto */ -static void __Pyx_WriteUnraisable(const char *name, int clineno, - int lineno, const char *filename, - int full_traceback, int nogil); - /* FetchCommonType.proto */ static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); @@ -1206,6 +1169,79 @@ static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, PyObject *dict); static int __pyx_CyFunction_init(void); +/* PyObjectLookupSpecial.proto */ +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 +static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); +#if PY_MAJOR_VERSION < 3 + if (unlikely(PyInstance_Check(obj))) + return __Pyx_PyObject_GetAttrStr(obj, attr_name); +#endif + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#else +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* PyObjectCallNoArg.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); +#else +#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) +#endif + +/* PyIntBinop.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); +#else +#define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ + PyObject_RichCompare(op1, op2, Py_EQ) + #endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* WriteUnraisableException.proto */ +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil); + /* ListCompAppend.proto */ #if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { @@ -1290,6 +1326,7 @@ static PyTypeObject *__pyx_ptype_5reppy_6robots_Agent = 0; static PyTypeObject *__pyx_ptype_5reppy_6robots_Robots = 0; static PyTypeObject *__pyx_ptype_5reppy_6robots_AllowNone = 0; static PyTypeObject *__pyx_ptype_5reppy_6robots_AllowAll = 0; +static PyTypeObject *__pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod = 0; static PyTypeObject *__pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *); /*proto*/ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *); /*proto*/ @@ -1309,7 +1346,7 @@ static PyObject *__pyx_builtin_ValueError; static PyObject *__pyx_builtin_map; static PyObject *__pyx_builtin_range; static const char __pyx_k_PY3[] = "PY3"; -static const char __pyx_k__11[] = ""; +static const char __pyx_k__13[] = ""; static const char __pyx_k_amt[] = "amt"; static const char __pyx_k_cls[] = "cls"; static const char __pyx_k_exc[] = "exc"; @@ -1333,7 +1370,9 @@ static const char __pyx_k_time[] = "time"; static const char __pyx_k_util[] = "util"; static const char __pyx_k_wrap[] = "wrap"; static const char __pyx_k_agent[] = "agent"; +static const char __pyx_k_cause[] = "cause"; static const char __pyx_k_enter[] = "__enter__"; +static const char __pyx_k_etype[] = "etype"; static const char __pyx_k_fetch[] = "fetch"; static const char __pyx_k_parse[] = "parse"; static const char __pyx_k_range[] = "range"; @@ -1351,6 +1390,7 @@ static const char __pyx_k_content[] = "content"; static const char __pyx_k_default[] = "default"; static const char __pyx_k_expires[] = "expires"; static const char __pyx_k_minimum[] = "minimum"; +static const char __pyx_k_wrapped[] = "wrapped"; static const char __pyx_k_SSLError[] = "SSLError"; static const char __pyx_k_max_size[] = "max_size"; static const char __pyx_k_requests[] = "requests"; @@ -1376,6 +1416,7 @@ static const char __pyx_k_InvalidSchema[] = "InvalidSchema"; static const char __pyx_k_MissingSchema[] = "MissingSchema"; static const char __pyx_k_ContentTooLong[] = "ContentTooLong"; static const char __pyx_k_decode_content[] = "decode_content"; +static const char __pyx_k_wrap_exception[] = "wrap_exception"; static const char __pyx_k_ConnectionError[] = "ConnectionError"; static const char __pyx_k_RobotsUrlMethod[] = "RobotsUrlMethod"; static const char __pyx_k_FromRobotsMethod[] = "FromRobotsMethod"; @@ -1391,6 +1432,7 @@ static const char __pyx_k_HeaderWithDefaultPolicy[] = "HeaderWithDefaultPolicy"; static const char __pyx_k_vagrant_reppy_robots_pyx[] = "/vagrant/reppy/robots.pyx"; static const char __pyx_k_Content_larger_than_s_bytes[] = "Content larger than %s bytes"; static const char __pyx_k_Pyx_CFunc_object____object___t[] = "__Pyx_CFunc_object____object___to_py..wrap"; +static const char __pyx_k_FetchMethod_locals_wrap_exceptio[] = "FetchMethod..wrap_exception"; static PyObject *__pyx_n_s_BadStatusCode; static PyObject *__pyx_n_s_ConnectionError; static PyObject *__pyx_n_s_ConnectionException; @@ -1399,6 +1441,7 @@ static PyObject *__pyx_kp_s_Content_larger_than_s_bytes; static PyObject *__pyx_n_s_DEFAULT_TTL_POLICY; static PyObject *__pyx_n_s_ExcessiveRedirects; static PyObject *__pyx_n_s_FetchMethod; +static PyObject *__pyx_n_s_FetchMethod_locals_wrap_exceptio; static PyObject *__pyx_n_s_FromRobotsMethod; static PyObject *__pyx_kp_s_Got_i_for_s; static PyObject *__pyx_n_s_HeaderWithDefaultPolicy; @@ -1416,13 +1459,14 @@ static PyObject *__pyx_n_s_TooManyRedirects; static PyObject *__pyx_n_s_URLRequired; static PyObject *__pyx_kp_b_User_agent_Disallow; static PyObject *__pyx_n_s_ValueError; -static PyObject *__pyx_n_s__11; -static PyObject *__pyx_kp_b__11; +static PyObject *__pyx_n_s__13; +static PyObject *__pyx_kp_b__13; static PyObject *__pyx_n_s_after_parse_hook; static PyObject *__pyx_n_s_after_response_hook; static PyObject *__pyx_n_s_agent; static PyObject *__pyx_n_s_amt; static PyObject *__pyx_n_s_args; +static PyObject *__pyx_n_s_cause; static PyObject *__pyx_n_s_cfunc_to_py; static PyObject *__pyx_n_s_closing; static PyObject *__pyx_n_s_cls; @@ -1433,6 +1477,7 @@ static PyObject *__pyx_n_s_decode_content; static PyObject *__pyx_n_s_default; static PyObject *__pyx_n_s_encode; static PyObject *__pyx_n_s_enter; +static PyObject *__pyx_n_s_etype; static PyObject *__pyx_n_s_exc; static PyObject *__pyx_n_s_exceptions; static PyObject *__pyx_n_s_exit; @@ -1475,6 +1520,8 @@ static PyObject *__pyx_n_s_util; static PyObject *__pyx_kp_s_vagrant_reppy_robots_pyx; static PyObject *__pyx_n_s_value; static PyObject *__pyx_n_s_wrap; +static PyObject *__pyx_n_s_wrap_exception; +static PyObject *__pyx_n_s_wrapped; static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_robots, std::string __pyx_v_name); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ @@ -1482,6 +1529,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_content, PyObject *__pyx_v_expires); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject *__pyx_self, PyObject *__pyx_v_etype, PyObject *__pyx_v_cause); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, PyObject *__pyx_v_url); /* proto */ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, PyObject *__pyx_v_url, std::string __pyx_v_content, PyObject *__pyx_v_expires); /* proto */ @@ -1500,6 +1548,7 @@ static PyObject *__pyx_tp_new_5reppy_6robots_Agent(PyTypeObject *t, PyObject *a, static PyObject *__pyx_tp_new_5reppy_6robots_Robots(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5reppy_6robots_AllowNone(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5reppy_6robots_AllowAll(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static PyObject *__pyx_tp_new_5reppy_6robots___pyx_scope_struct__FetchMethod(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_int_1; static PyObject *__pyx_int_200; @@ -1515,17 +1564,19 @@ static PyObject *__pyx_tuple__2; static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__7; static PyObject *__pyx_tuple__8; -static PyObject *__pyx_tuple__9; -static PyObject *__pyx_tuple__12; +static PyObject *__pyx_tuple__10; +static PyObject *__pyx_tuple__11; static PyObject *__pyx_tuple__14; -static PyObject *__pyx_tuple__15; static PyObject *__pyx_tuple__16; static PyObject *__pyx_tuple__17; +static PyObject *__pyx_tuple__18; +static PyObject *__pyx_tuple__19; static PyObject *__pyx_codeobj__3; static PyObject *__pyx_codeobj__4; static PyObject *__pyx_codeobj__5; -static PyObject *__pyx_codeobj__10; -static PyObject *__pyx_codeobj__13; +static PyObject *__pyx_codeobj__9; +static PyObject *__pyx_codeobj__12; +static PyObject *__pyx_codeobj__15; /* "reppy/robots.pyx":21 * from . import util, logger, exceptions @@ -2591,14 +2642,77 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs) { - PyObject *__pyx_v_after_response_hook = NULL; - PyObject *__pyx_v_after_parse_hook = NULL; - PyObject *__pyx_v_res = NULL; - PyObject *__pyx_v_content = NULL; - PyObject *__pyx_v_expires = NULL; - PyObject *__pyx_v_robots = NULL; - PyObject *__pyx_v_exc = NULL; +/* "reppy/robots.pyx":84 + * after_response_hook = kwargs.pop('after_response_hook', None) + * after_parse_hook = kwargs.pop('after_parse_hook', None) + * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< + * wrapped = etype(cause) + * wrapped.url = url + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ +static PyMethodDef __pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception = {"wrap_exception", (PyCFunction)__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception, METH_VARARGS|METH_KEYWORDS, 0}; +static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { + PyObject *__pyx_v_etype = 0; + PyObject *__pyx_v_cause = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("wrap_exception (wrapper)", 0); + { + static PyObject **__pyx_pyargnames[] = {&__pyx_n_s_etype,&__pyx_n_s_cause,0}; + PyObject* values[2] = {0,0}; + if (unlikely(__pyx_kwds)) { + Py_ssize_t kw_args; + const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); + switch (pos_args) { + case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + kw_args = PyDict_Size(__pyx_kwds); + switch (pos_args) { + case 0: + if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_etype)) != 0)) kw_args--; + else goto __pyx_L5_argtuple_error; + case 1: + if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cause)) != 0)) kw_args--; + else { + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(1, 84, __pyx_L3_error) + } + } + if (unlikely(kw_args > 0)) { + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "wrap_exception") < 0)) __PYX_ERR(1, 84, __pyx_L3_error) + } + } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + } + __pyx_v_etype = values[0]; + __pyx_v_cause = values[1]; + } + goto __pyx_L4_argument_unpacking_done; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 84, __pyx_L3_error) + __pyx_L3_error:; + __Pyx_AddTraceback("reppy.robots.FetchMethod.wrap_exception", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + __pyx_r = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_self, __pyx_v_etype, __pyx_v_cause); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject *__pyx_self, PyObject *__pyx_v_etype, PyObject *__pyx_v_cause) { + struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *__pyx_cur_scope; + struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *__pyx_outer_scope; + PyObject *__pyx_v_wrapped = NULL; PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations @@ -2606,30 +2720,248 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; - PyObject *__pyx_t_5 = NULL; - PyObject *__pyx_t_6 = NULL; - PyObject *__pyx_t_7 = NULL; - PyObject *__pyx_t_8 = NULL; - PyObject *__pyx_t_9 = NULL; - PyObject *__pyx_t_10 = NULL; - PyObject *__pyx_t_11 = NULL; - PyObject *__pyx_t_12 = NULL; - int __pyx_t_13; - int __pyx_t_14; - int __pyx_t_15; - PyObject *__pyx_t_16 = NULL; - PyObject *__pyx_t_17 = NULL; - PyObject *__pyx_t_18 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__5) - __Pyx_RefNannySetupContext("FetchMethod", 0); - __Pyx_TraceCall("FetchMethod", __pyx_f[1], 80, 0, __PYX_ERR(1, 80, __pyx_L1_error)); + int __pyx_t_5; + int __pyx_t_6; + __Pyx_RefNannySetupContext("wrap_exception", 0); + __pyx_outer_scope = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *) __Pyx_CyFunction_GetClosure(__pyx_self); + __pyx_cur_scope = __pyx_outer_scope; + __Pyx_TraceCall("wrap_exception", __pyx_f[1], 84, 0, __PYX_ERR(1, 84, __pyx_L1_error)); - /* "reppy/robots.pyx":82 - * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): - * '''Get the robots.txt at the provided URL.''' - * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< + /* "reppy/robots.pyx":85 * after_parse_hook = kwargs.pop('after_parse_hook', None) - * try: + * def wrap_exception(etype, cause): + * wrapped = etype(cause) # <<<<<<<<<<<<<< + * wrapped.url = url + * if after_response_hook is not None: + */ + __Pyx_TraceLine(85,0,__PYX_ERR(1, 85, __pyx_L1_error)) + __Pyx_INCREF(__pyx_v_etype); + __pyx_t_2 = __pyx_v_etype; __pyx_t_3 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_3)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + } + } + if (!__pyx_t_3) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 85, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 85, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 85, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + { + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_INCREF(__pyx_v_cause); + __Pyx_GIVEREF(__pyx_v_cause); + PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_cause); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 85, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_wrapped = __pyx_t_1; + __pyx_t_1 = 0; + + /* "reppy/robots.pyx":86 + * def wrap_exception(etype, cause): + * wrapped = etype(cause) + * wrapped.url = url # <<<<<<<<<<<<<< + * if after_response_hook is not None: + * after_response_hook(wrapped) + */ + __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_url)) { __Pyx_RaiseClosureNameError("url"); __PYX_ERR(1, 86, __pyx_L1_error) } + if (__Pyx_PyObject_SetAttrStr(__pyx_v_wrapped, __pyx_n_s_url, __pyx_cur_scope->__pyx_v_url) < 0) __PYX_ERR(1, 86, __pyx_L1_error) + + /* "reppy/robots.pyx":87 + * wrapped = etype(cause) + * wrapped.url = url + * if after_response_hook is not None: # <<<<<<<<<<<<<< + * after_response_hook(wrapped) + * raise wrapped + */ + __Pyx_TraceLine(87,0,__PYX_ERR(1, 87, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(1, 87, __pyx_L1_error) } + __pyx_t_5 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); + __pyx_t_6 = (__pyx_t_5 != 0); + if (__pyx_t_6) { + + /* "reppy/robots.pyx":88 + * wrapped.url = url + * if after_response_hook is not None: + * after_response_hook(wrapped) # <<<<<<<<<<<<<< + * raise wrapped + * try: + */ + __Pyx_TraceLine(88,0,__PYX_ERR(1, 88, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(1, 88, __pyx_L1_error) } + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); + __pyx_t_2 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_4 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_4)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_2, function); + } + } + if (!__pyx_t_4) { + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + { + __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 88, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_v_wrapped); + __Pyx_GIVEREF(__pyx_v_wrapped); + PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_v_wrapped); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + + /* "reppy/robots.pyx":87 + * wrapped = etype(cause) + * wrapped.url = url + * if after_response_hook is not None: # <<<<<<<<<<<<<< + * after_response_hook(wrapped) + * raise wrapped + */ + } + + /* "reppy/robots.pyx":89 + * if after_response_hook is not None: + * after_response_hook(wrapped) + * raise wrapped # <<<<<<<<<<<<<< + * try: + * # Limit the size of the request + */ + __Pyx_TraceLine(89,0,__PYX_ERR(1, 89, __pyx_L1_error)) + __Pyx_Raise(__pyx_v_wrapped, 0, 0, 0); + __PYX_ERR(1, 89, __pyx_L1_error) + + /* "reppy/robots.pyx":84 + * after_response_hook = kwargs.pop('after_response_hook', None) + * after_parse_hook = kwargs.pop('after_parse_hook', None) + * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< + * wrapped = etype(cause) + * wrapped.url = url + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("reppy.robots.FetchMethod.wrap_exception", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XDECREF(__pyx_v_wrapped); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":80 + * return cls(url, as_bytes(content), expires) + * + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< + * '''Get the robots.txt at the provided URL.''' + * after_response_hook = kwargs.pop('after_response_hook', None) + */ + +static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs) { + struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *__pyx_cur_scope; + PyObject *__pyx_v_after_parse_hook = NULL; + PyObject *__pyx_v_wrap_exception = 0; + PyObject *__pyx_v_res = NULL; + PyObject *__pyx_v_content = NULL; + PyObject *__pyx_v_expires = NULL; + PyObject *__pyx_v_robots = NULL; + PyObject *__pyx_v_exc = NULL; + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + int __pyx_t_13; + int __pyx_t_14; + int __pyx_t_15; + PyObject *__pyx_t_16 = NULL; + PyObject *__pyx_t_17 = NULL; + __Pyx_TraceFrameInit(__pyx_codeobj__5) + __Pyx_RefNannySetupContext("FetchMethod", 0); + __pyx_cur_scope = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)__pyx_tp_new_5reppy_6robots___pyx_scope_struct__FetchMethod(__pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod, __pyx_empty_tuple, NULL); + if (unlikely(!__pyx_cur_scope)) { + __pyx_cur_scope = ((struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(1, 80, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); + } + __Pyx_TraceCall("FetchMethod", __pyx_f[1], 80, 0, __PYX_ERR(1, 80, __pyx_L1_error)); + __pyx_cur_scope->__pyx_v_url = __pyx_v_url; + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); + + /* "reppy/robots.pyx":82 + * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): + * '''Get the robots.txt at the provided URL.''' + * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< + * after_parse_hook = kwargs.pop('after_parse_hook', None) + * def wrap_exception(etype, cause): */ __Pyx_TraceLine(82,0,__PYX_ERR(1, 82, __pyx_L1_error)) __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 82, __pyx_L1_error) @@ -2637,15 +2969,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 82, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_after_response_hook = __pyx_t_2; + __Pyx_GIVEREF(__pyx_t_2); + __pyx_cur_scope->__pyx_v_after_response_hook = __pyx_t_2; __pyx_t_2 = 0; /* "reppy/robots.pyx":83 * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< - * try: - * # Limit the size of the request + * def wrap_exception(etype, cause): + * wrapped = etype(cause) */ __Pyx_TraceLine(83,0,__PYX_ERR(1, 83, __pyx_L1_error)) __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 83, __pyx_L1_error) @@ -2659,11 +2992,24 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ /* "reppy/robots.pyx":84 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) + * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< + * wrapped = etype(cause) + * wrapped.url = url + */ + __Pyx_TraceLine(84,0,__PYX_ERR(1, 84, __pyx_L1_error)) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__9)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 84, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_v_wrap_exception = __pyx_t_1; + __pyx_t_1 = 0; + + /* "reppy/robots.pyx":90 + * after_response_hook(wrapped) + * raise wrapped * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_TraceLine(84,0,__PYX_ERR(1, 84, __pyx_L3_error)) + __Pyx_TraceLine(90,0,__PYX_ERR(1, 90, __pyx_L3_error)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign @@ -2673,41 +3019,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGOTREF(__pyx_t_5); /*try:*/ { - /* "reppy/robots.pyx":86 + /* "reppy/robots.pyx":92 * try: * # Limit the size of the request * kwargs['stream'] = True # <<<<<<<<<<<<<< * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) */ - __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L3_error)) - if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 86, __pyx_L3_error) + __Pyx_TraceLine(92,0,__PYX_ERR(1, 92, __pyx_L3_error)) + if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 92, __pyx_L3_error) - /* "reppy/robots.pyx":87 + /* "reppy/robots.pyx":93 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __Pyx_TraceLine(87,0,__PYX_ERR(1, 87, __pyx_L3_error)) + __Pyx_TraceLine(93,0,__PYX_ERR(1, 93, __pyx_L3_error)) /*with:*/ { - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); - __Pyx_INCREF(__pyx_v_url); - __Pyx_GIVEREF(__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_url); - __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 87, __pyx_L3_error) + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_cur_scope->__pyx_v_url); + __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -2722,14 +3068,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -2738,28 +3084,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { - __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); __pyx_t_6 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 87, __pyx_L3_error) + __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 87, __pyx_L11_error) + __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 93, __pyx_L11_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { @@ -2772,10 +3118,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (__pyx_t_6) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 87, __pyx_L11_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 93, __pyx_L11_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 87, __pyx_L11_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 93, __pyx_L11_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -2794,78 +3140,78 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_res = __pyx_t_7; __pyx_t_7 = 0; - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":94 * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) # <<<<<<<<<<<<<< * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): */ - __Pyx_TraceLine(88,0,__PYX_ERR(1, 88, __pyx_L17_error)) - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 88, __pyx_L17_error) + __Pyx_TraceLine(94,0,__PYX_ERR(1, 94, __pyx_L17_error)) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 94, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 94, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 88, __pyx_L17_error) + __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 94, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 88, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 88, __pyx_L17_error) - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 88, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 94, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 94, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 94, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_v_content = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":96 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) */ - __Pyx_TraceLine(90,0,__PYX_ERR(1, 90, __pyx_L17_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 90, __pyx_L17_error) + __Pyx_TraceLine(96,0,__PYX_ERR(1, 96, __pyx_L17_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 90, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 90, __pyx_L17_error) + __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 90, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 90, __pyx_L17_error) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 90, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 96, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 90, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 96, __pyx_L17_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_13) { - /* "reppy/robots.pyx":91 + /* "reppy/robots.pyx":97 * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( # <<<<<<<<<<<<<< * 'Content larger than %s bytes' % max_size) * */ - __Pyx_TraceLine(91,0,__PYX_ERR(1, 91, __pyx_L17_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 91, __pyx_L17_error) + __Pyx_TraceLine(97,0,__PYX_ERR(1, 97, __pyx_L17_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":92 + /* "reppy/robots.pyx":98 * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) # <<<<<<<<<<<<<< * * if after_response_hook is not None: */ - __Pyx_TraceLine(92,0,__PYX_ERR(1, 92, __pyx_L17_error)) - __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 92, __pyx_L17_error) + __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L17_error)) + __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 98, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -2878,14 +3224,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -2894,20 +3240,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_2); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 91, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 97, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -2915,9 +3261,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(1, 91, __pyx_L17_error) + __PYX_ERR(1, 97, __pyx_L17_error) - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":96 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< @@ -2926,28 +3272,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":100 * 'Content larger than %s bytes' % max_size) * * if after_response_hook is not None: # <<<<<<<<<<<<<< * after_response_hook(res) * */ - __Pyx_TraceLine(94,0,__PYX_ERR(1, 94, __pyx_L17_error)) - __pyx_t_13 = (__pyx_v_after_response_hook != Py_None); + __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L17_error)) + __pyx_t_13 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { - /* "reppy/robots.pyx":95 + /* "reppy/robots.pyx":101 * * if after_response_hook is not None: * after_response_hook(res) # <<<<<<<<<<<<<< * * # Get the TTL policy's ruling on the ttl */ - __Pyx_TraceLine(95,0,__PYX_ERR(1, 95, __pyx_L17_error)) - __Pyx_INCREF(__pyx_v_after_response_hook); - __pyx_t_7 = __pyx_v_after_response_hook; __pyx_t_8 = NULL; + __Pyx_TraceLine(101,0,__PYX_ERR(1, 101, __pyx_L17_error)) + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); + __pyx_t_7 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_8 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_7); if (likely(__pyx_t_8)) { @@ -2958,13 +3304,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -2972,19 +3318,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 101, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_2, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 95, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } @@ -2992,7 +3338,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":100 * 'Content larger than %s bytes' % max_size) * * if after_response_hook is not None: # <<<<<<<<<<<<<< @@ -3001,28 +3347,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":98 + /* "reppy/robots.pyx":104 * * # Get the TTL policy's ruling on the ttl * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) # <<<<<<<<<<<<<< * * if res.status_code == 200: */ - __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L17_error)) - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 98, __pyx_L17_error) + __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L17_error)) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 104, __pyx_L17_error) if (!__pyx_t_14) { } else { __Pyx_INCREF(__pyx_v_ttl_policy); __pyx_t_7 = __pyx_v_ttl_policy; goto __pyx_L27_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 104, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_2); __pyx_t_7 = __pyx_t_2; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_L27_bool_binop_done:; - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 104, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = NULL; @@ -3036,13 +3382,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 104, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 104, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3050,19 +3396,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 104, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 104, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 98, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 104, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3071,32 +3417,32 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_expires = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":100 + /* "reppy/robots.pyx":106 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: */ - __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L17_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 100, __pyx_L17_error) + __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L17_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 106, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 100, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 106, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 100, __pyx_L17_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 106, __pyx_L17_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_14) { - /* "reppy/robots.pyx":101 + /* "reppy/robots.pyx":107 * * if res.status_code == 200: * robots = cls.parse(url, content, expires) # <<<<<<<<<<<<<< * if after_parse_hook is not None: * after_parse_hook(robots) */ - __Pyx_TraceLine(101,0,__PYX_ERR(1, 101, __pyx_L17_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) + __Pyx_TraceLine(107,0,__PYX_ERR(1, 107, __pyx_L17_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 107, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_8 = NULL; __pyx_t_15 = 0; @@ -3112,36 +3458,36 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { - PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 101, __pyx_L17_error) + PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { - PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 101, __pyx_L17_error) + PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 101, __pyx_L17_error) + __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 107, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); if (__pyx_t_8) { __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; } - __Pyx_INCREF(__pyx_v_url); - __Pyx_GIVEREF(__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_7, 0+__pyx_t_15, __pyx_v_url); + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_7, 0+__pyx_t_15, __pyx_cur_scope->__pyx_v_url); __Pyx_INCREF(__pyx_v_content); __Pyx_GIVEREF(__pyx_v_content); PyTuple_SET_ITEM(__pyx_t_7, 1+__pyx_t_15, __pyx_v_content); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_15, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 101, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } @@ -3149,26 +3495,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_robots = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":102 + /* "reppy/robots.pyx":108 * if res.status_code == 200: * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: # <<<<<<<<<<<<<< * after_parse_hook(robots) * return robots */ - __Pyx_TraceLine(102,0,__PYX_ERR(1, 102, __pyx_L17_error)) + __Pyx_TraceLine(108,0,__PYX_ERR(1, 108, __pyx_L17_error)) __pyx_t_14 = (__pyx_v_after_parse_hook != Py_None); __pyx_t_13 = (__pyx_t_14 != 0); if (__pyx_t_13) { - /* "reppy/robots.pyx":103 + /* "reppy/robots.pyx":109 * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: * after_parse_hook(robots) # <<<<<<<<<<<<<< * return robots * elif res.status_code in (401, 403): */ - __Pyx_TraceLine(103,0,__PYX_ERR(1, 103, __pyx_L17_error)) + __Pyx_TraceLine(109,0,__PYX_ERR(1, 109, __pyx_L17_error)) __Pyx_INCREF(__pyx_v_after_parse_hook); __pyx_t_1 = __pyx_v_after_parse_hook; __pyx_t_7 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_1))) { @@ -3181,13 +3527,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else @@ -3195,19 +3541,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 103, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 109, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_robots); __Pyx_GIVEREF(__pyx_v_robots); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_robots); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 103, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3215,7 +3561,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":102 + /* "reppy/robots.pyx":108 * if res.status_code == 200: * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: # <<<<<<<<<<<<<< @@ -3224,20 +3570,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":104 + /* "reppy/robots.pyx":110 * if after_parse_hook is not None: * after_parse_hook(robots) * return robots # <<<<<<<<<<<<<< * elif res.status_code in (401, 403): * return AllowNone(url, expires) */ - __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L17_error)) + __Pyx_TraceLine(110,0,__PYX_ERR(1, 110, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_robots); __pyx_r = __pyx_v_robots; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":100 + /* "reppy/robots.pyx":106 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< @@ -3246,28 +3592,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":105 + /* "reppy/robots.pyx":111 * after_parse_hook(robots) * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: */ - __Pyx_TraceLine(105,0,__PYX_ERR(1, 105, __pyx_L17_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 105, __pyx_L17_error) + __Pyx_TraceLine(111,0,__PYX_ERR(1, 111, __pyx_L17_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 111, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 105, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 105, __pyx_L17_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 111, __pyx_L17_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (!__pyx_t_14) { } else { __pyx_t_13 = __pyx_t_14; goto __pyx_L31_bool_binop_done; } - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 105, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 105, __pyx_L17_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 111, __pyx_L17_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_13 = __pyx_t_14; __pyx_L31_bool_binop_done:; @@ -3275,31 +3621,31 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { - /* "reppy/robots.pyx":106 + /* "reppy/robots.pyx":112 * return robots * elif res.status_code in (401, 403): * return AllowNone(url, expires) # <<<<<<<<<<<<<< * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) */ - __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L17_error)) + __Pyx_TraceLine(112,0,__PYX_ERR(1, 112, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 106, __pyx_L17_error) + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 112, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); - __Pyx_INCREF(__pyx_v_url); - __Pyx_GIVEREF(__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_url); + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_cur_scope->__pyx_v_url); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 106, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 112, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":105 + /* "reppy/robots.pyx":111 * after_parse_hook(robots) * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< @@ -3308,60 +3654,60 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":107 + /* "reppy/robots.pyx":113 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< * return AllowAll(url, expires) * else: */ - __Pyx_TraceLine(107,0,__PYX_ERR(1, 107, __pyx_L17_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 107, __pyx_L17_error) + __Pyx_TraceLine(113,0,__PYX_ERR(1, 113, __pyx_L17_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 113, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) + __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 113, __pyx_L17_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 107, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 113, __pyx_L17_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_13) { } else { __pyx_t_14 = __pyx_t_13; goto __pyx_L33_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 113, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 107, __pyx_L17_error) + __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 113, __pyx_L17_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 107, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 113, __pyx_L17_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_14 = __pyx_t_13; __pyx_L33_bool_binop_done:; if (__pyx_t_14) { - /* "reppy/robots.pyx":108 + /* "reppy/robots.pyx":114 * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) # <<<<<<<<<<<<<< * else: * raise exceptions.BadStatusCode( */ - __Pyx_TraceLine(108,0,__PYX_ERR(1, 108, __pyx_L17_error)) + __Pyx_TraceLine(114,0,__PYX_ERR(1, 114, __pyx_L17_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 108, __pyx_L17_error) + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 114, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_INCREF(__pyx_v_url); - __Pyx_GIVEREF(__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_url); + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_cur_scope->__pyx_v_url); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 108, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 114, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L21_try_return; - /* "reppy/robots.pyx":107 + /* "reppy/robots.pyx":113 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< @@ -3370,43 +3716,43 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":110 + /* "reppy/robots.pyx":116 * return AllowAll(url, expires) * else: * raise exceptions.BadStatusCode( # <<<<<<<<<<<<<< * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: */ - __Pyx_TraceLine(110,0,__PYX_ERR(1, 110, __pyx_L17_error)) + __Pyx_TraceLine(116,0,__PYX_ERR(1, 116, __pyx_L17_error)) /*else*/ { - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 110, __pyx_L17_error) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 116, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 110, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 116, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":111 + /* "reppy/robots.pyx":117 * else: * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) # <<<<<<<<<<<<<< * except SSLError as exc: - * raise exceptions.SSLException(exc) + * wrap_exception(exceptions.SSLException, exc) */ - __Pyx_TraceLine(111,0,__PYX_ERR(1, 111, __pyx_L17_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L17_error) + __Pyx_TraceLine(117,0,__PYX_ERR(1, 117, __pyx_L17_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 117, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 111, __pyx_L17_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 117, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); - __Pyx_INCREF(__pyx_v_url); - __Pyx_GIVEREF(__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_v_url); + __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); + __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_cur_scope->__pyx_v_url); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 117, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 111, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 117, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; __pyx_t_15 = 0; @@ -3423,7 +3769,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 110, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 116, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -3433,7 +3779,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 110, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 116, __pyx_L17_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -3441,7 +3787,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } else #endif { - __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 110, __pyx_L17_error) + __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 116, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_6) { __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; @@ -3452,17 +3798,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_15, __pyx_t_7); __pyx_t_1 = 0; __pyx_t_7 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 110, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 116, __pyx_L17_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 110, __pyx_L17_error) + __PYX_ERR(1, 116, __pyx_L17_error) } - /* "reppy/robots.pyx":87 + /* "reppy/robots.pyx":93 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< @@ -3480,20 +3826,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; /*except:*/ { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(1, 87, __pyx_L19_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(1, 93, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); - __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 87, __pyx_L19_except_error) + __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 93, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_7, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 87, __pyx_L19_except_error) + if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 93, __pyx_L19_except_error) __Pyx_GOTREF(__pyx_t_17); __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_17); __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; - if (__pyx_t_14 < 0) __PYX_ERR(1, 87, __pyx_L19_except_error) + if (__pyx_t_14 < 0) __PYX_ERR(1, 93, __pyx_L19_except_error) __pyx_t_13 = ((!(__pyx_t_14 != 0)) != 0); if (__pyx_t_13) { __Pyx_GIVEREF(__pyx_t_2); @@ -3501,7 +3847,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGIVEREF(__pyx_t_16); __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_8, __pyx_t_16); __pyx_t_2 = 0; __pyx_t_8 = 0; __pyx_t_16 = 0; - __PYX_ERR(1, 87, __pyx_L19_except_error) + __PYX_ERR(1, 93, __pyx_L19_except_error) } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -3533,9 +3879,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ /*finally:*/ { /*normal exit:*/{ if (__pyx_t_9) { - __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__8, NULL); + __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__10, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 87, __pyx_L3_error) + if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; } @@ -3545,9 +3891,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_12 = __pyx_r; __pyx_r = 0; if (__pyx_t_9) { - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__9, NULL); + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__11, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 87, __pyx_L3_error) + if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } @@ -3564,9 +3910,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L38:; } - /* "reppy/robots.pyx":84 - * after_response_hook = kwargs.pop('after_response_hook', None) - * after_parse_hook = kwargs.pop('after_parse_hook', None) + /* "reppy/robots.pyx":90 + * after_response_hook(wrapped) + * raise wrapped * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True @@ -3585,185 +3931,109 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; - /* "reppy/robots.pyx":112 + /* "reppy/robots.pyx":118 * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: # <<<<<<<<<<<<<< - * raise exceptions.SSLException(exc) + * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: */ - __Pyx_TraceLine(112,0,__PYX_ERR(1, 112, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 112, __pyx_L5_except_error) + __Pyx_TraceLine(118,0,__PYX_ERR(1, 118, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 118, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(1, 112, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(1, 118, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":113 + /* "reppy/robots.pyx":119 * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: - * raise exceptions.SSLException(exc) # <<<<<<<<<<<<<< + * wrap_exception(exceptions.SSLException, exc) # <<<<<<<<<<<<<< * except ConnectionError as exc: - * raise exceptions.ConnectionException(exc) + * wrap_exception(exceptions.ConnectionException, exc) */ - __Pyx_TraceLine(113,0,__PYX_ERR(1, 113, __pyx_L5_except_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 113, __pyx_L5_except_error) + __Pyx_TraceLine(119,0,__PYX_ERR(1, 119, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 119, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 113, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 119, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { - __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); - if (likely(__pyx_t_1)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); - __Pyx_INCREF(__pyx_t_1); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_6, function); - } - } - if (!__pyx_t_1) { - __pyx_t_7 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 113, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_6)) { - PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_v_exc}; - __pyx_t_7 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 113, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_GOTREF(__pyx_t_7); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { - PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_v_exc}; - __pyx_t_7 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 113, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_GOTREF(__pyx_t_7); - } else - #endif - { - __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 113, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_18); - __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_1); __pyx_t_1 = NULL; - __Pyx_INCREF(__pyx_v_exc); - __Pyx_GIVEREF(__pyx_v_exc); - PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); - __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 113, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; - } - } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_Raise(__pyx_t_7, 0, 0, 0); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __PYX_ERR(1, 113, __pyx_L5_except_error) + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":114 + /* "reppy/robots.pyx":120 * except SSLError as exc: - * raise exceptions.SSLException(exc) + * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: # <<<<<<<<<<<<<< - * raise exceptions.ConnectionException(exc) + * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: */ - __Pyx_TraceLine(114,0,__PYX_ERR(1, 114, __pyx_L5_except_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 114, __pyx_L5_except_error) + __Pyx_TraceLine(120,0,__PYX_ERR(1, 120, __pyx_L5_except_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 120, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(1, 114, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(1, 120, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":115 - * raise exceptions.SSLException(exc) + /* "reppy/robots.pyx":121 + * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: - * raise exceptions.ConnectionException(exc) # <<<<<<<<<<<<<< + * wrap_exception(exceptions.ConnectionException, exc) # <<<<<<<<<<<<<< * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: - * raise exceptions.MalformedUrl(exc) - */ - __Pyx_TraceLine(115,0,__PYX_ERR(1, 115, __pyx_L5_except_error)) - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 115, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_6); - __pyx_t_18 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 115, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_18); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_18))) { - __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_18); - if (likely(__pyx_t_6)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_18); - __Pyx_INCREF(__pyx_t_6); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_18, function); - } - } - if (!__pyx_t_6) { - __pyx_t_7 = __Pyx_PyObject_CallOneArg(__pyx_t_18, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 115, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_18)) { - PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; - __pyx_t_7 = __Pyx_PyFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 115, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_7); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_18)) { - PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_v_exc}; - __pyx_t_7 = __Pyx_PyCFunction_FastCall(__pyx_t_18, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 115, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_7); - } else - #endif - { - __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 115, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; - __Pyx_INCREF(__pyx_v_exc); - __Pyx_GIVEREF(__pyx_v_exc); - PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_v_exc); - __pyx_t_7 = __Pyx_PyObject_Call(__pyx_t_18, __pyx_t_1, NULL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 115, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - } - } - __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; - __Pyx_Raise(__pyx_t_7, 0, 0, 0); + * wrap_exception(exceptions.MalformedUrl, exc) + */ + __Pyx_TraceLine(121,0,__PYX_ERR(1, 121, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 121, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 121, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 121, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __PYX_ERR(1, 115, __pyx_L5_except_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":116 + /* "reppy/robots.pyx":122 * except ConnectionError as exc: - * raise exceptions.ConnectionException(exc) + * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: # <<<<<<<<<<<<<< - * raise exceptions.MalformedUrl(exc) + * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: */ - __Pyx_TraceLine(116,0,__PYX_ERR(1, 116, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 116, __pyx_L5_except_error) + __Pyx_TraceLine(122,0,__PYX_ERR(1, 122, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 116, __pyx_L5_except_error) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 116, __pyx_L5_except_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 116, __pyx_L5_except_error) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_2) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -3772,161 +4042,85 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(1, 116, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(1, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":117 - * raise exceptions.ConnectionException(exc) + /* "reppy/robots.pyx":123 + * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: - * raise exceptions.MalformedUrl(exc) # <<<<<<<<<<<<<< + * wrap_exception(exceptions.MalformedUrl, exc) # <<<<<<<<<<<<<< * except TooManyRedirects as exc: - * raise exceptions.ExcessiveRedirects(exc) + * wrap_exception(exceptions.ExcessiveRedirects, exc) */ - __Pyx_TraceLine(117,0,__PYX_ERR(1, 117, __pyx_L5_except_error)) - __pyx_t_18 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 117, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_18); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 117, __pyx_L5_except_error) + __Pyx_TraceLine(123,0,__PYX_ERR(1, 123, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 123, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; - __pyx_t_18 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_1))) { - __pyx_t_18 = PyMethod_GET_SELF(__pyx_t_1); - if (likely(__pyx_t_18)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_1); - __Pyx_INCREF(__pyx_t_18); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_1, function); - } - } - if (!__pyx_t_18) { - __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 117, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_1)) { - PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 117, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; - __Pyx_GOTREF(__pyx_t_16); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { - PyObject *__pyx_temp[2] = {__pyx_t_18, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 117, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_18); __pyx_t_18 = 0; - __Pyx_GOTREF(__pyx_t_16); - } else - #endif - { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 117, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_18); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_18); __pyx_t_18 = NULL; - __Pyx_INCREF(__pyx_v_exc); - __Pyx_GIVEREF(__pyx_v_exc); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_v_exc); - __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_6, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 117, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } - } + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 123, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_Raise(__pyx_t_16, 0, 0, 0); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __PYX_ERR(1, 117, __pyx_L5_except_error) + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":118 + /* "reppy/robots.pyx":124 * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: - * raise exceptions.MalformedUrl(exc) + * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: # <<<<<<<<<<<<<< - * raise exceptions.ExcessiveRedirects(exc) + * wrap_exception(exceptions.ExcessiveRedirects, exc) * */ - __Pyx_TraceLine(118,0,__PYX_ERR(1, 118, __pyx_L5_except_error)) - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 118, __pyx_L5_except_error) + __Pyx_TraceLine(124,0,__PYX_ERR(1, 124, __pyx_L5_except_error)) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 124, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(1, 118, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(1, 124, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_7); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":119 - * raise exceptions.MalformedUrl(exc) + /* "reppy/robots.pyx":125 + * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: - * raise exceptions.ExcessiveRedirects(exc) # <<<<<<<<<<<<<< + * wrap_exception(exceptions.ExcessiveRedirects, exc) # <<<<<<<<<<<<<< * * def RobotsUrlMethod(cls, url): */ - __Pyx_TraceLine(119,0,__PYX_ERR(1, 119, __pyx_L5_except_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 119, __pyx_L5_except_error) + __Pyx_TraceLine(125,0,__PYX_ERR(1, 125, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 125, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 119, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 125, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { - __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); - if (likely(__pyx_t_1)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); - __Pyx_INCREF(__pyx_t_1); - __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_6, function); - } - } - if (!__pyx_t_1) { - __pyx_t_16 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 119, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_6)) { - PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 119, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_GOTREF(__pyx_t_16); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { - PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_v_exc}; - __pyx_t_16 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 119, __pyx_L5_except_error) - __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_GOTREF(__pyx_t_16); - } else - #endif - { - __pyx_t_18 = PyTuple_New(1+1); if (unlikely(!__pyx_t_18)) __PYX_ERR(1, 119, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_18); - __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_18, 0, __pyx_t_1); __pyx_t_1 = NULL; - __Pyx_INCREF(__pyx_v_exc); - __Pyx_GIVEREF(__pyx_v_exc); - PyTuple_SET_ITEM(__pyx_t_18, 0+1, __pyx_v_exc); - __pyx_t_16 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_18, NULL); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 119, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; - } - } - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_Raise(__pyx_t_16, 0, 0, 0); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __PYX_ERR(1, 119, __pyx_L5_except_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + goto __pyx_L4_exception_handled; } goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "reppy/robots.pyx":84 - * after_response_hook = kwargs.pop('after_response_hook', None) - * after_parse_hook = kwargs.pop('after_parse_hook', None) + /* "reppy/robots.pyx":90 + * after_response_hook(wrapped) + * raise wrapped * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True @@ -3944,6 +4138,12 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGIVEREF(__pyx_t_5); __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); goto __pyx_L0; + __pyx_L4_exception_handled:; + __Pyx_PyThreadState_assign + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); __pyx_L10_try_end:; } @@ -3965,25 +4165,25 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); __Pyx_XDECREF(__pyx_t_16); - __Pyx_XDECREF(__pyx_t_18); __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; - __Pyx_XDECREF(__pyx_v_after_response_hook); __Pyx_XDECREF(__pyx_v_after_parse_hook); + __Pyx_XDECREF(__pyx_v_wrap_exception); __Pyx_XDECREF(__pyx_v_res); __Pyx_XDECREF(__pyx_v_content); __Pyx_XDECREF(__pyx_v_expires); __Pyx_XDECREF(__pyx_v_robots); __Pyx_XDECREF(__pyx_v_exc); + __Pyx_DECREF(((PyObject *)__pyx_cur_scope)); __Pyx_XGIVEREF(__pyx_r); __Pyx_TraceReturn(__pyx_r, 0); __Pyx_RefNannyFinishContext(); return __pyx_r; } -/* "reppy/robots.pyx":121 - * raise exceptions.ExcessiveRedirects(exc) +/* "reppy/robots.pyx":127 + * wrap_exception(exceptions.ExcessiveRedirects, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' @@ -4020,11 +4220,11 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 121, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 127, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 121, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 127, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4037,7 +4237,7 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 121, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 127, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4058,40 +4258,40 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject std::string __pyx_t_2; std::string __pyx_t_3; PyObject *__pyx_t_4 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__10) + __Pyx_TraceFrameInit(__pyx_codeobj__12) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); - __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 121, 0, __PYX_ERR(1, 121, __pyx_L1_error)); + __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 127, 0, __PYX_ERR(1, 127, __pyx_L1_error)); - /* "reppy/robots.pyx":123 + /* "reppy/robots.pyx":129 * def RobotsUrlMethod(cls, url): * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) # <<<<<<<<<<<<<< * * cdef class Robots: */ - __Pyx_TraceLine(123,0,__PYX_ERR(1, 123, __pyx_L1_error)) + __Pyx_TraceLine(129,0,__PYX_ERR(1, 129, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 123, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 129, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 123, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 129, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 123, __pyx_L1_error) + __PYX_ERR(1, 129, __pyx_L1_error) } - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 123, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 129, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 123, __pyx_L1_error) + __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 129, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":121 - * raise exceptions.ExcessiveRedirects(exc) + /* "reppy/robots.pyx":127 + * wrap_exception(exceptions.ExcessiveRedirects, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' @@ -4111,7 +4311,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":141 +/* "reppy/robots.pyx":147 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -4150,7 +4350,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 141, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 147, __pyx_L3_error) } case 2: if (kw_args > 0) { @@ -4159,7 +4359,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 141, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 147, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4171,12 +4371,12 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } __pyx_v_url = values[0]; - __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 141, __pyx_L3_error) + __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 147, __pyx_L3_error) __pyx_v_expires = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 141, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 147, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4197,43 +4397,43 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob std::string __pyx_t_2; Rep::Robots *__pyx_t_3; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 141, 0, __PYX_ERR(1, 141, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 147, 0, __PYX_ERR(1, 147, __pyx_L1_error)); - /* "reppy/robots.pyx":142 + /* "reppy/robots.pyx":148 * * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) # <<<<<<<<<<<<<< * self.expires = expires * */ - __Pyx_TraceLine(142,0,__PYX_ERR(1, 142, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 142, __pyx_L1_error) + __Pyx_TraceLine(148,0,__PYX_ERR(1, 148, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 142, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 148, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = new Rep::Robots(__pyx_v_content, __pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 142, __pyx_L1_error) + __PYX_ERR(1, 148, __pyx_L1_error) } __pyx_v_self->robots = __pyx_t_3; - /* "reppy/robots.pyx":143 + /* "reppy/robots.pyx":149 * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) * self.expires = expires # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(143,0,__PYX_ERR(1, 143, __pyx_L1_error)) + __Pyx_TraceLine(149,0,__PYX_ERR(1, 149, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); __Pyx_GOTREF(__pyx_v_self->expires); __Pyx_DECREF(__pyx_v_self->expires); __pyx_v_self->expires = __pyx_v_expires; - /* "reppy/robots.pyx":141 + /* "reppy/robots.pyx":147 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -4254,7 +4454,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob return __pyx_r; } -/* "reppy/robots.pyx":145 +/* "reppy/robots.pyx":151 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4281,24 +4481,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[1], 145, 0, __PYX_ERR(1, 145, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[1], 151, 0, __PYX_ERR(1, 151, __pyx_L1_error)); - /* "reppy/robots.pyx":146 + /* "reppy/robots.pyx":152 * * def __str__(self): * return self.robots.str() # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __Pyx_TraceLine(146,0,__PYX_ERR(1, 146, __pyx_L1_error)) + __Pyx_TraceLine(152,0,__PYX_ERR(1, 152, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 146, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":145 + /* "reppy/robots.pyx":151 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4318,7 +4518,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":148 +/* "reppy/robots.pyx":154 * return self.robots.str() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4341,19 +4541,19 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - __Pyx_TraceCall("__dealloc__", __pyx_f[1], 148, 0, __PYX_ERR(1, 148, __pyx_L1_error)); + __Pyx_TraceCall("__dealloc__", __pyx_f[1], 154, 0, __PYX_ERR(1, 154, __pyx_L1_error)); - /* "reppy/robots.pyx":149 + /* "reppy/robots.pyx":155 * * def __dealloc__(self): * del self.robots # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(149,0,__PYX_ERR(1, 149, __pyx_L1_error)) + __Pyx_TraceLine(155,0,__PYX_ERR(1, 155, __pyx_L1_error)) delete __pyx_v_self->robots; - /* "reppy/robots.pyx":148 + /* "reppy/robots.pyx":154 * return self.robots.str() * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4370,7 +4570,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_RefNannyFinishContext(); } -/* "reppy/robots.pyx":152 +/* "reppy/robots.pyx":158 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4399,22 +4599,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 152, 0, __PYX_ERR(1, 152, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 158, 0, __PYX_ERR(1, 158, __pyx_L1_error)); - /* "reppy/robots.pyx":154 + /* "reppy/robots.pyx":160 * def sitemaps(self): * '''Get all the sitemaps in this robots.txt.''' * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< * * def allowed(self, path, name): */ - __Pyx_TraceLine(154,0,__PYX_ERR(1, 154, __pyx_L1_error)) + __Pyx_TraceLine(160,0,__PYX_ERR(1, 160, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -4422,14 +4622,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":152 + /* "reppy/robots.pyx":158 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4451,7 +4651,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ return __pyx_r; } -/* "reppy/robots.pyx":156 +/* "reppy/robots.pyx":162 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -4488,11 +4688,11 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 156, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 162, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 156, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 162, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4505,7 +4705,7 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 156, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 162, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4526,32 +4726,32 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp std::string __pyx_t_2; std::string __pyx_t_3; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[1], 156, 0, __PYX_ERR(1, 156, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[1], 162, 0, __PYX_ERR(1, 162, __pyx_L1_error)); - /* "reppy/robots.pyx":158 + /* "reppy/robots.pyx":164 * def allowed(self, path, name): * '''Is the provided path allowed for the provided agent?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) # <<<<<<<<<<<<<< * * def agent(self, name): */ - __Pyx_TraceLine(158,0,__PYX_ERR(1, 158, __pyx_L1_error)) + __Pyx_TraceLine(164,0,__PYX_ERR(1, 164, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 158, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 158, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 158, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 158, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 158, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":156 + /* "reppy/robots.pyx":162 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -4571,7 +4771,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":160 +/* "reppy/robots.pyx":166 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -4604,20 +4804,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ int __pyx_t_5; PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("agent", 0); - __Pyx_TraceCall("agent", __pyx_f[1], 160, 0, __PYX_ERR(1, 160, __pyx_L1_error)); + __Pyx_TraceCall("agent", __pyx_f[1], 166, 0, __PYX_ERR(1, 166, __pyx_L1_error)); - /* "reppy/robots.pyx":167 + /* "reppy/robots.pyx":173 * Agent object. * ''' * return Agent.from_robots(self, as_bytes(name)) # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(167,0,__PYX_ERR(1, 167, __pyx_L1_error)) + __Pyx_TraceLine(173,0,__PYX_ERR(1, 173, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 173, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 173, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; __pyx_t_5 = 0; @@ -4634,7 +4834,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 173, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -4643,14 +4843,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 173, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 173, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -4661,7 +4861,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 167, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 173, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -4670,7 +4870,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":160 + /* "reppy/robots.pyx":166 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -4694,7 +4894,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":170 +/* "reppy/robots.pyx":176 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4723,20 +4923,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 170, 0, __PYX_ERR(1, 170, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 176, 0, __PYX_ERR(1, 176, __pyx_L1_error)); - /* "reppy/robots.pyx":172 + /* "reppy/robots.pyx":178 * def expired(self): * '''True if the current time is past its expiration.''' * return time.time() > self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(172,0,__PYX_ERR(1, 172, __pyx_L1_error)) + __Pyx_TraceLine(178,0,__PYX_ERR(1, 178, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 172, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 178, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 172, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 178, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -4750,20 +4950,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } } if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 172, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 172, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 172, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 178, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":170 + /* "reppy/robots.pyx":176 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -4785,7 +4985,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":175 +/* "reppy/robots.pyx":181 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4811,22 +5011,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 175, 0, __PYX_ERR(1, 175, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 181, 0, __PYX_ERR(1, 181, __pyx_L1_error)); - /* "reppy/robots.pyx":177 + /* "reppy/robots.pyx":183 * def expires(self): * '''The expiration of this robots.txt.''' * return self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(177,0,__PYX_ERR(1, 177, __pyx_L1_error)) + __Pyx_TraceLine(183,0,__PYX_ERR(1, 183, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_self->expires); __pyx_r = __pyx_v_self->expires; goto __pyx_L0; - /* "reppy/robots.pyx":175 + /* "reppy/robots.pyx":181 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -4845,7 +5045,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":180 +/* "reppy/robots.pyx":186 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4877,21 +5077,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 180, 0, __PYX_ERR(1, 180, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[1], 186, 0, __PYX_ERR(1, 186, __pyx_L1_error)); - /* "reppy/robots.pyx":182 + /* "reppy/robots.pyx":188 * def ttl(self): * '''Remaining time for this response to be considered valid.''' * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(182,0,__PYX_ERR(1, 182, __pyx_L1_error)) + __Pyx_TraceLine(188,0,__PYX_ERR(1, 188, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 182, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 182, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -4905,24 +5105,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 182, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 182, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 188, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 182, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 182, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 182, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 182, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 182, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -4936,7 +5136,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":180 + /* "reppy/robots.pyx":186 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -4959,7 +5159,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 return __pyx_r; } -/* "reppy/robots.pyx":188 +/* "reppy/robots.pyx":194 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5000,7 +5200,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 188, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 194, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -5015,7 +5215,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 188, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 194, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5038,17 +5238,17 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 188, 0, __PYX_ERR(1, 188, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 194, 0, __PYX_ERR(1, 194, __pyx_L1_error)); - /* "reppy/robots.pyx":189 + /* "reppy/robots.pyx":195 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(189,0,__PYX_ERR(1, 189, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 189, __pyx_L1_error) + __Pyx_TraceLine(195,0,__PYX_ERR(1, 195, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 195, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -5065,7 +5265,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 189, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 195, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -5073,13 +5273,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 189, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 195, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 189, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 195, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -5096,14 +5296,14 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 189, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 195, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":188 + /* "reppy/robots.pyx":194 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5127,7 +5327,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":195 +/* "reppy/robots.pyx":201 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5167,7 +5367,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 195, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 201, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -5182,7 +5382,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 195, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 201, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5205,15 +5405,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 195, 0, __PYX_ERR(1, 195, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[1], 201, 0, __PYX_ERR(1, 201, __pyx_L1_error)); - /* "reppy/robots.pyx":196 + /* "reppy/robots.pyx":202 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(196,0,__PYX_ERR(1, 196, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 196, __pyx_L1_error) + __Pyx_TraceLine(202,0,__PYX_ERR(1, 202, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 202, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -5229,22 +5429,22 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__11, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 196, __pyx_L1_error) + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__13, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 202, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__11, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 196, __pyx_L1_error) + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__13, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 202, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 196, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 202, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -5255,20 +5455,20 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_4, __pyx_v_url); - __Pyx_INCREF(__pyx_kp_b__11); - __Pyx_GIVEREF(__pyx_kp_b__11); - PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b__11); + __Pyx_INCREF(__pyx_kp_b__13); + __Pyx_GIVEREF(__pyx_kp_b__13); + PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b__13); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 196, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 202, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":195 + /* "reppy/robots.pyx":201 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5707,7 +5907,7 @@ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyO * return f(value) */ __Pyx_TraceLine(65,0,__PYX_ERR(2, 65, __pyx_L1_error)) - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__13)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__15)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_wrap = __pyx_t_1; __pyx_t_1 = 0; @@ -6194,15 +6394,16 @@ static PyTypeObject __pyx_type_5reppy_6robots_AllowAll = { #endif }; -static struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[8]; -static int __pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; +static struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *__pyx_freelist_5reppy_6robots___pyx_scope_struct__FetchMethod[8]; +static int __pyx_freecount_5reppy_6robots___pyx_scope_struct__FetchMethod = 0; -static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { +static PyObject *__pyx_tp_new_5reppy_6robots___pyx_scope_struct__FetchMethod(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { PyObject *o; - if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { - o = (PyObject*)__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[--__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py]; - memset(o, 0, sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)); + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount_5reppy_6robots___pyx_scope_struct__FetchMethod > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod)))) { + o = (PyObject*)__pyx_freelist_5reppy_6robots___pyx_scope_struct__FetchMethod[--__pyx_freecount_5reppy_6robots___pyx_scope_struct__FetchMethod]; + memset(o, 0, sizeof(struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod)); (void) PyObject_INIT(o, t); + PyObject_GC_Track(o); } else { o = (*t->tp_alloc)(t, 0); if (unlikely(!o)) return 0; @@ -6210,20 +6411,48 @@ static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object__ return o; } -static void __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyObject *o) { - if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { - __pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py++] = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)o); +static void __pyx_tp_dealloc_5reppy_6robots___pyx_scope_struct__FetchMethod(PyObject *o) { + struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *p = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)o; + PyObject_GC_UnTrack(o); + Py_CLEAR(p->__pyx_v_after_response_hook); + Py_CLEAR(p->__pyx_v_url); + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount_5reppy_6robots___pyx_scope_struct__FetchMethod < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod)))) { + __pyx_freelist_5reppy_6robots___pyx_scope_struct__FetchMethod[__pyx_freecount_5reppy_6robots___pyx_scope_struct__FetchMethod++] = ((struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)o); } else { (*Py_TYPE(o)->tp_free)(o); } } -static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { +static int __pyx_tp_traverse_5reppy_6robots___pyx_scope_struct__FetchMethod(PyObject *o, visitproc v, void *a) { + int e; + struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *p = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)o; + if (p->__pyx_v_after_response_hook) { + e = (*v)(p->__pyx_v_after_response_hook, a); if (e) return e; + } + if (p->__pyx_v_url) { + e = (*v)(p->__pyx_v_url, a); if (e) return e; + } + return 0; +} + +static int __pyx_tp_clear_5reppy_6robots___pyx_scope_struct__FetchMethod(PyObject *o) { + PyObject* tmp; + struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *p = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)o; + tmp = ((PyObject*)p->__pyx_v_after_response_hook); + p->__pyx_v_after_response_hook = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + tmp = ((PyObject*)p->__pyx_v_url); + p->__pyx_v_url = Py_None; Py_INCREF(Py_None); + Py_XDECREF(tmp); + return 0; +} + +static PyTypeObject __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod = { PyVarObject_HEAD_INIT(0, 0) - "reppy.robots.__pyx_scope_struct____Pyx_CFunc_object____object___to_py", /*tp_name*/ - sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py), /*tp_basicsize*/ + "reppy.robots.__pyx_scope_struct__FetchMethod", /*tp_name*/ + sizeof(struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod), /*tp_basicsize*/ 0, /*tp_itemsize*/ - __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_dealloc*/ + __pyx_tp_dealloc_5reppy_6robots___pyx_scope_struct__FetchMethod, /*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -6243,10 +6472,10 @@ static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER|Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ + __pyx_tp_traverse_5reppy_6robots___pyx_scope_struct__FetchMethod, /*tp_traverse*/ + __pyx_tp_clear_5reppy_6robots___pyx_scope_struct__FetchMethod, /*tp_clear*/ 0, /*tp_richcompare*/ 0, /*tp_weaklistoffset*/ 0, /*tp_iter*/ @@ -6261,7 +6490,7 @@ static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { 0, /*tp_dictoffset*/ 0, /*tp_init*/ 0, /*tp_alloc*/ - __pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_new*/ + __pyx_tp_new_5reppy_6robots___pyx_scope_struct__FetchMethod, /*tp_new*/ 0, /*tp_free*/ 0, /*tp_is_gc*/ 0, /*tp_bases*/ @@ -6276,8 +6505,90 @@ static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { #endif }; -static PyMethodDef __pyx_methods[] = { - {0, 0, 0, 0} +static struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[8]; +static int __pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py = 0; + +static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, CYTHON_UNUSED PyObject *a, CYTHON_UNUSED PyObject *k) { + PyObject *o; + if (CYTHON_COMPILING_IN_CPYTHON && likely((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py > 0) & (t->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { + o = (PyObject*)__pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[--__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py]; + memset(o, 0, sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)); + (void) PyObject_INIT(o, t); + } else { + o = (*t->tp_alloc)(t, 0); + if (unlikely(!o)) return 0; + } + return o; +} + +static void __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyObject *o) { + if (CYTHON_COMPILING_IN_CPYTHON && ((__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py < 8) & (Py_TYPE(o)->tp_basicsize == sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py)))) { + __pyx_freelist___pyx_scope_struct____Pyx_CFunc_object____object___to_py[__pyx_freecount___pyx_scope_struct____Pyx_CFunc_object____object___to_py++] = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)o); + } else { + (*Py_TYPE(o)->tp_free)(o); + } +} + +static PyTypeObject __pyx_scope_struct____Pyx_CFunc_object____object___to_py = { + PyVarObject_HEAD_INIT(0, 0) + "reppy.robots.__pyx_scope_struct____Pyx_CFunc_object____object___to_py", /*tp_name*/ + sizeof(struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + __pyx_tp_dealloc___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + #if PY_MAJOR_VERSION < 3 + 0, /*tp_compare*/ + #endif + #if PY_MAJOR_VERSION >= 3 + 0, /*tp_as_async*/ + #endif + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_HAVE_VERSION_TAG|Py_TPFLAGS_CHECKTYPES|Py_TPFLAGS_HAVE_NEWBUFFER, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + 0, /*tp_methods*/ + 0, /*tp_members*/ + 0, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + __pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ + 0, /*tp_bases*/ + 0, /*tp_mro*/ + 0, /*tp_cache*/ + 0, /*tp_subclasses*/ + 0, /*tp_weaklist*/ + 0, /*tp_del*/ + 0, /*tp_version_tag*/ + #if PY_VERSION_HEX >= 0x030400a1 + 0, /*tp_finalize*/ + #endif +}; + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} }; #if PY_MAJOR_VERSION >= 3 @@ -6307,6 +6618,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_DEFAULT_TTL_POLICY, __pyx_k_DEFAULT_TTL_POLICY, sizeof(__pyx_k_DEFAULT_TTL_POLICY), 0, 0, 1, 1}, {&__pyx_n_s_ExcessiveRedirects, __pyx_k_ExcessiveRedirects, sizeof(__pyx_k_ExcessiveRedirects), 0, 0, 1, 1}, {&__pyx_n_s_FetchMethod, __pyx_k_FetchMethod, sizeof(__pyx_k_FetchMethod), 0, 0, 1, 1}, + {&__pyx_n_s_FetchMethod_locals_wrap_exceptio, __pyx_k_FetchMethod_locals_wrap_exceptio, sizeof(__pyx_k_FetchMethod_locals_wrap_exceptio), 0, 0, 1, 1}, {&__pyx_n_s_FromRobotsMethod, __pyx_k_FromRobotsMethod, sizeof(__pyx_k_FromRobotsMethod), 0, 0, 1, 1}, {&__pyx_kp_s_Got_i_for_s, __pyx_k_Got_i_for_s, sizeof(__pyx_k_Got_i_for_s), 0, 0, 1, 0}, {&__pyx_n_s_HeaderWithDefaultPolicy, __pyx_k_HeaderWithDefaultPolicy, sizeof(__pyx_k_HeaderWithDefaultPolicy), 0, 0, 1, 1}, @@ -6324,13 +6636,14 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_URLRequired, __pyx_k_URLRequired, sizeof(__pyx_k_URLRequired), 0, 0, 1, 1}, {&__pyx_kp_b_User_agent_Disallow, __pyx_k_User_agent_Disallow, sizeof(__pyx_k_User_agent_Disallow), 0, 0, 0, 0}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, - {&__pyx_n_s__11, __pyx_k__11, sizeof(__pyx_k__11), 0, 0, 1, 1}, - {&__pyx_kp_b__11, __pyx_k__11, sizeof(__pyx_k__11), 0, 0, 0, 0}, + {&__pyx_n_s__13, __pyx_k__13, sizeof(__pyx_k__13), 0, 0, 1, 1}, + {&__pyx_kp_b__13, __pyx_k__13, sizeof(__pyx_k__13), 0, 0, 0, 0}, {&__pyx_n_s_after_parse_hook, __pyx_k_after_parse_hook, sizeof(__pyx_k_after_parse_hook), 0, 0, 1, 1}, {&__pyx_n_s_after_response_hook, __pyx_k_after_response_hook, sizeof(__pyx_k_after_response_hook), 0, 0, 1, 1}, {&__pyx_n_s_agent, __pyx_k_agent, sizeof(__pyx_k_agent), 0, 0, 1, 1}, {&__pyx_n_s_amt, __pyx_k_amt, sizeof(__pyx_k_amt), 0, 0, 1, 1}, {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, + {&__pyx_n_s_cause, __pyx_k_cause, sizeof(__pyx_k_cause), 0, 0, 1, 1}, {&__pyx_n_s_cfunc_to_py, __pyx_k_cfunc_to_py, sizeof(__pyx_k_cfunc_to_py), 0, 0, 1, 1}, {&__pyx_n_s_closing, __pyx_k_closing, sizeof(__pyx_k_closing), 0, 0, 1, 1}, {&__pyx_n_s_cls, __pyx_k_cls, sizeof(__pyx_k_cls), 0, 0, 1, 1}, @@ -6341,6 +6654,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_default, __pyx_k_default, sizeof(__pyx_k_default), 0, 0, 1, 1}, {&__pyx_n_s_encode, __pyx_k_encode, sizeof(__pyx_k_encode), 0, 0, 1, 1}, {&__pyx_n_s_enter, __pyx_k_enter, sizeof(__pyx_k_enter), 0, 0, 1, 1}, + {&__pyx_n_s_etype, __pyx_k_etype, sizeof(__pyx_k_etype), 0, 0, 1, 1}, {&__pyx_n_s_exc, __pyx_k_exc, sizeof(__pyx_k_exc), 0, 0, 1, 1}, {&__pyx_n_s_exceptions, __pyx_k_exceptions, sizeof(__pyx_k_exceptions), 0, 0, 1, 1}, {&__pyx_n_s_exit, __pyx_k_exit, sizeof(__pyx_k_exit), 0, 0, 1, 1}, @@ -6383,11 +6697,13 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_k_vagrant_reppy_robots_pyx, sizeof(__pyx_k_vagrant_reppy_robots_pyx), 0, 0, 1, 0}, {&__pyx_n_s_value, __pyx_k_value, sizeof(__pyx_k_value), 0, 0, 1, 1}, {&__pyx_n_s_wrap, __pyx_k_wrap, sizeof(__pyx_k_wrap), 0, 0, 1, 1}, + {&__pyx_n_s_wrap_exception, __pyx_k_wrap_exception, sizeof(__pyx_k_wrap_exception), 0, 0, 1, 1}, + {&__pyx_n_s_wrapped, __pyx_k_wrapped, sizeof(__pyx_k_wrapped), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) - __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 154, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 160, __pyx_L1_error) __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) return 0; __pyx_L1_error:; @@ -6425,7 +6741,7 @@ static int __Pyx_InitCachedConstants(void) { * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< * after_parse_hook = kwargs.pop('after_parse_hook', None) - * try: + * def wrap_exception(etype, cause): */ __pyx_tuple__6 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 82, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__6); @@ -6435,26 +6751,38 @@ static int __Pyx_InitCachedConstants(void) { * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< - * try: - * # Limit the size of the request + * def wrap_exception(etype, cause): + * wrapped = etype(cause) */ __pyx_tuple__7 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 83, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__7); __Pyx_GIVEREF(__pyx_tuple__7); - /* "reppy/robots.pyx":87 + /* "reppy/robots.pyx":84 + * after_response_hook = kwargs.pop('after_response_hook', None) + * after_parse_hook = kwargs.pop('after_parse_hook', None) + * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< + * wrapped = etype(cause) + * wrapped.url = url + */ + __pyx_tuple__8 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(1, 84, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__8); + __Pyx_GIVEREF(__pyx_tuple__8); + __pyx_codeobj__9 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__8, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_wrap_exception, 84, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__9)) __PYX_ERR(1, 84, __pyx_L1_error) + + /* "reppy/robots.pyx":93 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __pyx_tuple__8 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(1, 87, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__8); - __Pyx_GIVEREF(__pyx_tuple__8); - __pyx_tuple__9 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(1, 87, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__9); - __Pyx_GIVEREF(__pyx_tuple__9); + __pyx_tuple__10 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(1, 93, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); + __pyx_tuple__11 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(1, 93, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__11); + __Pyx_GIVEREF(__pyx_tuple__11); /* "cfunc.to_py":65 * @cname("__Pyx_CFunc_object____object___to_py") @@ -6463,10 +6791,10 @@ static int __Pyx_InitCachedConstants(void) { * """wrap(value)""" * return f(value) */ - __pyx_tuple__12 = PyTuple_Pack(1, __pyx_n_s_value); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__12); - __Pyx_GIVEREF(__pyx_tuple__12); - __pyx_codeobj__13 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__12, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__13)) __PYX_ERR(2, 65, __pyx_L1_error) + __pyx_tuple__14 = PyTuple_Pack(1, __pyx_n_s_value); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(2, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__14); + __Pyx_GIVEREF(__pyx_tuple__14); + __pyx_codeobj__15 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__15)) __PYX_ERR(2, 65, __pyx_L1_error) /* "reppy/robots.pyx":33 * @@ -6475,10 +6803,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Construct an Agent from a CppAgent.''' * agent = Agent() */ - __pyx_tuple__14 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(1, 33, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__14); - __Pyx_GIVEREF(__pyx_tuple__14); - __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(1, 33, __pyx_L1_error) + __pyx_tuple__16 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 33, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__16); + __Pyx_GIVEREF(__pyx_tuple__16); + __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__16, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(1, 33, __pyx_L1_error) /* "reppy/robots.pyx":76 * @@ -6487,10 +6815,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __pyx_tuple__15 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 76, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__15); - __Pyx_GIVEREF(__pyx_tuple__15); - __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__15, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_tuple__17 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__17); + __Pyx_GIVEREF(__pyx_tuple__17); + __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__17, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 76, __pyx_L1_error) /* "reppy/robots.pyx":80 * return cls(url, as_bytes(content), expires) @@ -6499,22 +6827,22 @@ static int __Pyx_InitCachedConstants(void) { * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __pyx_tuple__16 = PyTuple_Pack(13, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 80, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__16); - __Pyx_GIVEREF(__pyx_tuple__16); - __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 13, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__16, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 80, __pyx_L1_error) + __pyx_tuple__18 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__18)) __PYX_ERR(1, 80, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__18); + __Pyx_GIVEREF(__pyx_tuple__18); + __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__18, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 80, __pyx_L1_error) - /* "reppy/robots.pyx":121 - * raise exceptions.ExcessiveRedirects(exc) + /* "reppy/robots.pyx":127 + * wrap_exception(exceptions.ExcessiveRedirects, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __pyx_tuple__17 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 121, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__17); - __Pyx_GIVEREF(__pyx_tuple__17); - __pyx_codeobj__10 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__17, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 121, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__10)) __PYX_ERR(1, 121, __pyx_L1_error) + __pyx_tuple__19 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__19)) __PYX_ERR(1, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__19); + __Pyx_GIVEREF(__pyx_tuple__19); + __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__19, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 127, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(1, 127, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -6629,20 +6957,23 @@ PyMODINIT_FUNC PyInit_robots(void) __pyx_type_5reppy_6robots_Agent.tp_print = 0; if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 43, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; - if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 125, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 131, __pyx_L1_error) __pyx_type_5reppy_6robots_Robots.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 125, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 131, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 185, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 191, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 185, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 191, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 192, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 198, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 192, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 198, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; + if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(1, 80, __pyx_L1_error) + __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_print = 0; + __pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod = &__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod; if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; __pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = &__pyx_scope_struct____Pyx_CFunc_object____object___to_py; @@ -6830,7 +7161,7 @@ PyMODINIT_FUNC PyInit_robots(void) __Pyx_INCREF(__pyx_n_s_exceptions); __Pyx_GIVEREF(__pyx_n_s_exceptions); PyList_SET_ITEM(__pyx_t_2, 2, __pyx_n_s_exceptions); - __pyx_t_1 = __Pyx_Import(__pyx_n_s__11, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 19, __pyx_L1_error) + __pyx_t_1 = __Pyx_Import(__pyx_n_s__13, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_util); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) @@ -6903,89 +7234,89 @@ PyMODINIT_FUNC PyInit_robots(void) if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(1, 80, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":121 - * raise exceptions.ExcessiveRedirects(exc) + /* "reppy/robots.pyx":127 + * wrap_exception(exceptions.ExcessiveRedirects, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __Pyx_TraceLine(121,0,__PYX_ERR(1, 121, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 121, __pyx_L1_error) + __Pyx_TraceLine(127,0,__PYX_ERR(1, 127, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 127, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 121, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 127, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":130 + /* "reppy/robots.pyx":136 * # The default TTL policy is to cache for 3600 seconds or what's provided in the * # headers, and a minimum of 600 seconds * DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) # <<<<<<<<<<<<<< * * # Class methods */ - __Pyx_TraceLine(130,0,__PYX_ERR(1, 130, __pyx_L1_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 130, __pyx_L1_error) + __Pyx_TraceLine(136,0,__PYX_ERR(1, 136, __pyx_L1_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 130, __pyx_L1_error) + __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 130, __pyx_L1_error) - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 130, __pyx_L1_error) - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 130, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 136, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 136, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 130, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 136, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":133 + /* "reppy/robots.pyx":139 * * # Class methods * parse = classmethod(ParseMethod) # <<<<<<<<<<<<<< * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) */ - __Pyx_TraceLine(133,0,__PYX_ERR(1, 133, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 133, __pyx_L1_error) + __Pyx_TraceLine(139,0,__PYX_ERR(1, 139, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 133, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 133, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 139, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":134 + /* "reppy/robots.pyx":140 * # Class methods * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) # <<<<<<<<<<<<<< * robots_url = classmethod(RobotsUrlMethod) * */ - __Pyx_TraceLine(134,0,__PYX_ERR(1, 134, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 134, __pyx_L1_error) + __Pyx_TraceLine(140,0,__PYX_ERR(1, 140, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 134, __pyx_L1_error) + __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 134, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 140, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":135 + /* "reppy/robots.pyx":141 * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) # <<<<<<<<<<<<<< * * # Data members */ - __Pyx_TraceLine(135,0,__PYX_ERR(1, 135, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 135, __pyx_L1_error) + __Pyx_TraceLine(141,0,__PYX_ERR(1, 141, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 135, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 135, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 141, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); @@ -7569,29 +7900,13 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObjec } #endif -/* PyObjectCallNoArg */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, NULL, 0); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { - return __Pyx_PyObject_CallMethO(func, NULL); - } - } - return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); +/* None */ + static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); } -#endif /* PyErrFetchRestore */ - #if CYTHON_FAST_THREAD_STATE + #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; tmp_type = tstate->curexc_type; @@ -7615,7 +7930,7 @@ static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject #endif /* RaiseException */ - #if PY_MAJOR_VERSION < 3 + #if PY_MAJOR_VERSION < 3 static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, CYTHON_UNUSED PyObject *cause) { __Pyx_PyThreadState_declare @@ -7777,269 +8092,47 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject } #endif -/* PyIntBinop */ - #if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { - if (op1 == op2) { - Py_RETURN_TRUE; - } - #if PY_MAJOR_VERSION < 3 - if (likely(PyInt_CheckExact(op1))) { - const long b = intval; - long a = PyInt_AS_LONG(op1); - if (a == b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; +/* FetchCommonType */ + static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* fake_module; + PyTypeObject* cached_type = NULL; + fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); + if (!fake_module) return NULL; + Py_INCREF(fake_module); + cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + if (cached_type) { + if (!PyType_Check((PyObject*)cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", + type->tp_name); + goto bad; } - } - #endif - #if CYTHON_USE_PYLONG_INTERNALS - if (likely(PyLong_CheckExact(op1))) { - const long b = intval; - long a; - const digit* digits = ((PyLongObject*)op1)->ob_digit; - const Py_ssize_t size = Py_SIZE(op1); - if (likely(__Pyx_sst_abs(size) <= 1)) { - a = likely(size) ? digits[0] : 0; - if (size == -1) a = -a; - } else { - switch (size) { - case -2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case -3: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 3: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case -4: - if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 4: - if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 - default: return PyLong_Type.tp_richcompare(op1, op2, Py_EQ); - #else - default: Py_RETURN_FALSE; - #endif - } + if (cached_type->tp_basicsize != type->tp_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + type->tp_name); + goto bad; } - if (a == b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } - } - #endif - if (PyFloat_CheckExact(op1)) { - const long b = intval; - double a = PyFloat_AS_DOUBLE(op1); - if ((double)a == (double)b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } + } else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; } - return PyObject_RichCompare(op1, op2, Py_EQ); -} -#endif - -/* SaveResetException */ - #if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - *type = tstate->exc_type; - *value = tstate->exc_value; - *tb = tstate->exc_traceback; - Py_XINCREF(*type); - Py_XINCREF(*value); - Py_XINCREF(*tb); -} -static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = type; - tstate->exc_value = value; - tstate->exc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} -#endif - -/* GetException */ - #if CYTHON_FAST_THREAD_STATE -static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { -#else -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { -#endif - PyObject *local_type, *local_value, *local_tb; -#if CYTHON_FAST_THREAD_STATE - PyObject *tmp_type, *tmp_value, *tmp_tb; - local_type = tstate->curexc_type; - local_value = tstate->curexc_value; - local_tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; -#else - PyErr_Fetch(&local_type, &local_value, &local_tb); -#endif - PyErr_NormalizeException(&local_type, &local_value, &local_tb); -#if CYTHON_FAST_THREAD_STATE - if (unlikely(tstate->curexc_type)) -#else - if (unlikely(PyErr_Occurred())) -#endif - goto bad; - #if PY_MAJOR_VERSION >= 3 - if (local_tb) { - if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) - goto bad; - } - #endif - Py_XINCREF(local_tb); - Py_XINCREF(local_type); - Py_XINCREF(local_value); - *type = local_type; - *value = local_value; - *tb = local_tb; -#if CYTHON_FAST_THREAD_STATE - tmp_type = tstate->exc_type; - tmp_value = tstate->exc_value; - tmp_tb = tstate->exc_traceback; - tstate->exc_type = local_type; - tstate->exc_value = local_value; - tstate->exc_traceback = local_tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -#else - PyErr_SetExcInfo(local_type, local_value, local_tb); -#endif - return 0; -bad: - *type = 0; - *value = 0; - *tb = 0; - Py_XDECREF(local_type); - Py_XDECREF(local_value); - Py_XDECREF(local_tb); - return -1; -} - -/* PyErrExceptionMatches */ - #if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err) { - PyObject *exc_type = tstate->curexc_type; - if (exc_type == err) return 1; - if (unlikely(!exc_type)) return 0; - return PyErr_GivenExceptionMatches(exc_type, err); -} -#endif - -/* WriteUnraisableException */ - static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, - CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, - int full_traceback, CYTHON_UNUSED int nogil) { - PyObject *old_exc, *old_val, *old_tb; - PyObject *ctx; - __Pyx_PyThreadState_declare -#ifdef WITH_THREAD - PyGILState_STATE state; - if (nogil) - state = PyGILState_Ensure(); -#ifdef _MSC_VER - else state = (PyGILState_STATE)-1; -#endif -#endif - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); - if (full_traceback) { - Py_XINCREF(old_exc); - Py_XINCREF(old_val); - Py_XINCREF(old_tb); - __Pyx_ErrRestore(old_exc, old_val, old_tb); - PyErr_PrintEx(1); - } - #if PY_MAJOR_VERSION < 3 - ctx = PyString_FromString(name); - #else - ctx = PyUnicode_FromString(name); - #endif - __Pyx_ErrRestore(old_exc, old_val, old_tb); - if (!ctx) { - PyErr_WriteUnraisable(Py_None); - } else { - PyErr_WriteUnraisable(ctx); - Py_DECREF(ctx); - } -#ifdef WITH_THREAD - if (nogil) - PyGILState_Release(state); -#endif -} - -/* FetchCommonType */ - static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); - if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); - goto bad; - } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; - } -done: - Py_DECREF(fake_module); - return cached_type; -bad: - Py_XDECREF(cached_type); - cached_type = NULL; - goto done; +done: + Py_DECREF(fake_module); + return cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; } /* CythonFunction */ - static PyObject * + static PyObject * __Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) { if (unlikely(op->func_doc == NULL)) { @@ -8621,6 +8714,249 @@ static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, Py Py_INCREF(dict); } +/* PyObjectCallNoArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, NULL, 0); + } +#endif +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { +#else + if (likely(PyCFunction_Check(func))) { +#endif + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); + } + } + return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); +} +#endif + +/* PyIntBinop */ + #if !CYTHON_COMPILING_IN_PYPY +static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { + if (op1 == op2) { + Py_RETURN_TRUE; + } + #if PY_MAJOR_VERSION < 3 + if (likely(PyInt_CheckExact(op1))) { + const long b = intval; + long a = PyInt_AS_LONG(op1); + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(PyLong_CheckExact(op1))) { + const long b = intval; + long a; + const digit* digits = ((PyLongObject*)op1)->ob_digit; + const Py_ssize_t size = Py_SIZE(op1); + if (likely(__Pyx_sst_abs(size) <= 1)) { + a = likely(size) ? digits[0] : 0; + if (size == -1) a = -a; + } else { + switch (size) { + case -2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 2: + if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { + a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 3: + if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { + a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case -4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + case 4: + if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { + a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); + break; + } + #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 + default: return PyLong_Type.tp_richcompare(op1, op2, Py_EQ); + #else + default: Py_RETURN_FALSE; + #endif + } + } + if (a == b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + #endif + if (PyFloat_CheckExact(op1)) { + const long b = intval; + double a = PyFloat_AS_DOUBLE(op1); + if ((double)a == (double)b) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + } + return PyObject_RichCompare(op1, op2, Py_EQ); +} +#endif + +/* SaveResetException */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +#endif + +/* GetException */ + #if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { +#endif + PyObject *local_type, *local_value, *local_tb; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + #if PY_MAJOR_VERSION >= 3 + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } + #endif + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +} + +/* PyErrExceptionMatches */ + #if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err) { + PyObject *exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; + return PyErr_GivenExceptionMatches(exc_type, err); +} +#endif + +/* WriteUnraisableException */ + static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, + CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, + int full_traceback, CYTHON_UNUSED int nogil) { + PyObject *old_exc, *old_val, *old_tb; + PyObject *ctx; + __Pyx_PyThreadState_declare +#ifdef WITH_THREAD + PyGILState_STATE state; + if (nogil) + state = PyGILState_Ensure(); +#ifdef _MSC_VER + else state = (PyGILState_STATE)-1; +#endif +#endif + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); + if (full_traceback) { + Py_XINCREF(old_exc); + Py_XINCREF(old_val); + Py_XINCREF(old_tb); + __Pyx_ErrRestore(old_exc, old_val, old_tb); + PyErr_PrintEx(1); + } + #if PY_MAJOR_VERSION < 3 + ctx = PyString_FromString(name); + #else + ctx = PyUnicode_FromString(name); + #endif + __Pyx_ErrRestore(old_exc, old_val, old_tb); + if (!ctx) { + PyErr_WriteUnraisable(Py_None); + } else { + PyErr_WriteUnraisable(ctx); + Py_DECREF(ctx); + } +#ifdef WITH_THREAD + if (nogil) + PyGILState_Release(state); +#endif +} + /* Import */ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { PyObject *empty_list = 0; diff --git a/reppy/robots.pyx b/reppy/robots.pyx index e7f5389..f90db3c 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -81,6 +81,12 @@ def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): '''Get the robots.txt at the provided URL.''' after_response_hook = kwargs.pop('after_response_hook', None) after_parse_hook = kwargs.pop('after_parse_hook', None) + def wrap_exception(etype, cause): + wrapped = etype(cause) + wrapped.url = url + if after_response_hook is not None: + after_response_hook(wrapped) + raise wrapped try: # Limit the size of the request kwargs['stream'] = True @@ -110,13 +116,13 @@ def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): raise exceptions.BadStatusCode( 'Got %i for %s' % (res.status_code, url), res.status_code) except SSLError as exc: - raise exceptions.SSLException(exc) + wrap_exception(exceptions.SSLException, exc) except ConnectionError as exc: - raise exceptions.ConnectionException(exc) + wrap_exception(exceptions.ConnectionException, exc) except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: - raise exceptions.MalformedUrl(exc) + wrap_exception(exceptions.MalformedUrl, exc) except TooManyRedirects as exc: - raise exceptions.ExcessiveRedirects(exc) + wrap_exception(exceptions.ExcessiveRedirects, exc) def RobotsUrlMethod(cls, url): '''Get the robots.txt URL that corresponds to the provided one.''' diff --git a/setup.py b/setup.py index 5a99fa0..bd8eb49 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.8', + version='0.4.9', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like diff --git a/tests/test_robots.py b/tests/test_robots.py index 91d1374..cb5990b 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -368,6 +368,20 @@ def hook(response): 'http://example.com/robots.txt', after_response_hook=hook) self.assertTrue(state["called"]) + def test_after_response_hook_on_error(self): + '''Calls after_response_hook when error occurs during fetch''' + state = {"called": False} + expected_url = 'http://localhost:8080/robots.txt' + + def hook(response): + state["called"] = True + self.assertIsInstance( + response, robots.exceptions.ConnectionException) + self.assertEquals(response.url, expected_url) + with self.assertRaises(robots.exceptions.ConnectionException): + robots.Robots.fetch(expected_url, after_response_hook=hook) + self.assertTrue(state["called"]) + def test_after_parse_hook(self): '''Calls after_parse_hook after parsing robots.txt''' state = {"called": False} From 5c2d2ee0e4a93754cd4f7aa15d0ebc9a3cf539b7 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 17 Nov 2017 15:51:44 -0800 Subject: [PATCH 080/113] Lower log level for TTL extraction to warn. These aren't fatal errors and servers can give back all sorts of wacky strings. The default values should handle these cases fine, so some people may not want the noisy error in their logs. Lowering the severity makes that easier and the exception level is equivalent to ERROR which is not something that should be silenced. Fixes #76. --- reppy/ttl.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/reppy/ttl.py b/reppy/ttl.py index c8ac49f..bc8ae88 100644 --- a/reppy/ttl.py +++ b/reppy/ttl.py @@ -40,7 +40,8 @@ def ttl(self, response): try: return max(self.minimum, int(value.strip())) except ValueError: - logger.exception('Could not parse %s=%s', name, value) + logger.warn( + 'Could not parse %s=%s', name, value, exc_info=1) # Check the Expires header expires = response.headers.get('expires') @@ -51,7 +52,8 @@ def ttl(self, response): try: date = parse_date(date) except ValueError: - logger.exception('Could not parse date string %s', date) + logger.warn( + 'Could not parse date string %s', date, exc_info=1) date = time.time() else: date = time.time() @@ -59,6 +61,7 @@ def ttl(self, response): try: return max(self.minimum, parse_date(expires) - date) except ValueError: - logger.exception('Could not parse date string %s', expires) + logger.warn( + 'Could not parse date string %s', expires, exc_info=1) return self.default From 86eea284e68882699b1cac2a1264f34c1907ce9c Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Fri, 17 Nov 2017 15:57:10 -0800 Subject: [PATCH 081/113] Add Crawl-Delay to string representation of Agent. Pulls in changes from seomoz/rep-cpp#31 Also, add some tests demonstrating functionality. --- reppy/rep-cpp | 2 +- tests/test_agent.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index ee70c28..8abadd8 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit ee70c280c574a8cf6c896c5ae333adf9dbe2be6d +Subproject commit 8abadd8005233ce53c2ca9c14ecf0d3a8f31b3a9 diff --git a/tests/test_agent.py b/tests/test_agent.py index b048390..a125c81 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -117,3 +117,21 @@ def test_params_only(self): ''', 'agent') self.assertFalse(agent.allowed('/;')) self.assertTrue(agent.allowed('/')) + + def test_str(self): + '''str() shows directives.''' + agent = self.parse(''' + User-agent: agent + Disallow: / + ''', 'agent') + self.assertEquals(str(agent), '[Directive(Disallow: /)]') + + def test_str_crawl_delay(self): + '''str() shows crawl-delay.''' + agent = self.parse(''' + User-agent: agent + Crawl-Delay: 1 + Disallow: / + ''', 'agent') + self.assertEquals( + str(agent), 'Crawl-Delay: 1 [Directive(Disallow: /)]') From 0247f7bf54facb3a6628fcb7f80b673c73e076a8 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 20 Nov 2017 09:50:57 -0800 Subject: [PATCH 082/113] Fix __str__ on Python 3. --- reppy/robots.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reppy/robots.pyx b/reppy/robots.pyx index f90db3c..54edd13 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -48,7 +48,7 @@ cdef class Agent: from_robots = classmethod(FromRobotsMethod) def __str__(self): - return self.agent.str() + return self.agent.str().decode('utf8') @property def delay(self): @@ -149,7 +149,7 @@ cdef class Robots: self.expires = expires def __str__(self): - return self.robots.str() + return self.robots.str().decode('utf8') def __dealloc__(self): del self.robots From 29cebd4f79c84ff4079a68b35f0b96349e2951e1 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 20 Nov 2017 10:48:44 -0800 Subject: [PATCH 083/113] Try upgrading Cython. --- dev-requirements.txt | 2 +- reppy/robots.cpp | 3368 ++++++++++++++++++++++++++++-------------- 2 files changed, 2264 insertions(+), 1106 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 2a40a3e..cdb77a3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ cachetools==2.0.0 colorama==0.3.7 coverage==4.0.3 -Cython==0.25.1 +Cython==0.27.3 funcsigs==1.0.2 mock==2.0.0 nose==1.3.7 diff --git a/reppy/robots.cpp b/reppy/robots.cpp index ef3c3d5..0c65874 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -1,13 +1,14 @@ -/* Generated by Cython 0.25.1 */ +/* Generated by Cython 0.27.3 */ #define PY_SSIZE_T_CLEAN #include "Python.h" #ifndef Py_PYTHON_H #error Python headers needed to compile C extensions, please install development version of Python. -#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) - #error Cython requires Python 2.6+ or Python 3.2+. +#elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000) + #error Cython requires Python 2.6+ or Python 3.3+. #else -#define CYTHON_ABI "0_25_1" +#define CYTHON_ABI "0_27_3" +#define CYTHON_FUTURE_DIVISION 0 #include #ifndef offsetof #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) @@ -29,8 +30,9 @@ #ifndef DL_EXPORT #define DL_EXPORT(t) t #endif +#define __PYX_COMMA , #ifndef HAVE_LONG_LONG - #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) + #if PY_VERSION_HEX >= 0x02070000 #define HAVE_LONG_LONG #endif #endif @@ -46,8 +48,14 @@ #define CYTHON_COMPILING_IN_CPYTHON 0 #undef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 0 - #undef CYTHON_USE_ASYNC_SLOTS - #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #if PY_VERSION_HEX < 0x03050000 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif #undef CYTHON_USE_PYLIST_INTERNALS #define CYTHON_USE_PYLIST_INTERNALS 0 #undef CYTHON_USE_UNICODE_INTERNALS @@ -66,6 +74,10 @@ #define CYTHON_FAST_THREAD_STATE 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 #elif defined(PYSTON_VERSION) #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 1 @@ -73,6 +85,8 @@ #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 #undef CYTHON_USE_ASYNC_SLOTS #define CYTHON_USE_ASYNC_SLOTS 0 #undef CYTHON_USE_PYLIST_INTERNALS @@ -97,6 +111,10 @@ #define CYTHON_FAST_THREAD_STATE 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 #else #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 0 @@ -104,6 +122,12 @@ #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #elif !defined(CYTHON_USE_PYTYPE_LOOKUP) + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif #if PY_MAJOR_VERSION < 3 #undef CYTHON_USE_ASYNC_SLOTS #define CYTHON_USE_ASYNC_SLOTS 0 @@ -143,6 +167,12 @@ #ifndef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 1 #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT (0 && PY_VERSION_HEX >= 0x03050000) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PY_VERSION_HEX >= 0x030400a1) + #endif #endif #if !defined(CYTHON_FAST_PYCCALL) #define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) @@ -181,19 +211,44 @@ #ifndef Py_TPFLAGS_HAVE_FINALIZE #define Py_TPFLAGS_HAVE_FINALIZE 0 #endif -#ifndef METH_FASTCALL - #define METH_FASTCALL 0x80 - typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); +#if PY_VERSION_HEX < 0x030700A0 || !defined(METH_FASTCALL) + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames); #else #define __Pyx_PyCFunctionFast _PyCFunctionFast + #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords #endif #if CYTHON_FAST_PYCCALL #define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))) + ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS))))) #else #define __Pyx_PyFastCFunction_Check(func) 0 #endif +#if !CYTHON_FAST_THREAD_STATE || PY_VERSION_HEX < 0x02070000 + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x03060000 + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#elif PY_VERSION_HEX >= 0x03000000 + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_Current +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#if PY_MAJOR_VERSION >= 3 || CYTHON_FUTURE_DIVISION + #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#else + #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) + #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) +#endif #if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ @@ -277,7 +332,6 @@ #ifndef PySet_CheckExact #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) #endif -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) #define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) #if PY_MAJOR_VERSION >= 3 #define PyIntObject PyLongObject @@ -317,20 +371,28 @@ #else #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) #endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif #if CYTHON_USE_ASYNC_SLOTS #if PY_VERSION_HEX >= 0x030500B1 #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) #else + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif +#else + #define __Pyx_PyType_AsAsync(obj) NULL +#endif +#ifndef __Pyx_PyAsyncMethodsStruct typedef struct { unaryfunc am_await; unaryfunc am_aiter; unaryfunc am_anext; } __Pyx_PyAsyncMethodsStruct; - #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) - #endif -#else - #define __Pyx_PyType_AsAsync(obj) NULL #endif #ifndef CYTHON_RESTRICT #if defined(__GNUC__) @@ -343,13 +405,81 @@ #define CYTHON_RESTRICT #endif #endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_MAYBE_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_MAYBE_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif #define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int32 uint32_t; + #endif + #endif +#else + #include +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) && __cplusplus >= 201103L + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #elif __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__ ) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif #ifndef __cplusplus #error "Cython files generated with the C++ option must be compiled with a C++ compiler." #endif #ifndef CYTHON_INLINE - #define CYTHON_INLINE inline + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif #endif template void __Pyx_call_destructor(T& x) { @@ -361,9 +491,10 @@ class __Pyx_FakeReference { __Pyx_FakeReference() : ptr(NULL) { } __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } T *operator->() { return ptr; } + T *operator&() { return ptr; } operator T&() { return *ptr; } - template bool operator ==(U other) { return *ptr == other; }; - template bool operator !=(U other) { return *ptr != other; }; + template bool operator ==(U other) { return *ptr == other; } + template bool operator !=(U other) { return *ptr != other; } private: T *ptr; }; @@ -393,14 +524,6 @@ static CYTHON_INLINE float __PYX_NAN() { __pyx_filename = __pyx_f[f_index]; __pyx_lineno = lineno; __pyx_clineno = __LINE__; goto Ln_error; \ } -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) -#else - #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) - #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) -#endif - #ifndef __PYX_EXTERN_C #ifdef __cplusplus #define __PYX_EXTERN_C extern "C" @@ -425,30 +548,10 @@ static CYTHON_INLINE float __PYX_NAN() { #include #endif /* _OPENMP */ -#ifdef PYREX_WITHOUT_ASSERTIONS +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) #define CYTHON_WITHOUT_ASSERTIONS #endif -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif -#endif typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; @@ -476,8 +579,8 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc #define __Pyx_sst_abs(value) abs(value) #elif SIZEOF_LONG >= SIZEOF_SIZE_T #define __Pyx_sst_abs(value) labs(value) -#elif defined (_MSC_VER) && defined (_M_X64) - #define __Pyx_sst_abs(value) _abs64(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define __Pyx_sst_abs(value) llabs(value) #elif defined (__GNUC__) @@ -485,8 +588,8 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc #else #define __Pyx_sst_abs(value) ((value<0) ? -value : value) #endif -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject*); -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); #define __Pyx_PyByteArray_FromString(s) PyByteArray_FromStringAndSize((const char*)s, strlen((const char*)s)) #define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) #define __Pyx_PyBytes_FromString PyBytes_FromString @@ -499,23 +602,27 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #define __Pyx_PyStr_FromString __Pyx_PyUnicode_FromString #define __Pyx_PyStr_FromStringAndSize __Pyx_PyUnicode_FromStringAndSize #endif -#define __Pyx_PyObject_AsSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) +#define __Pyx_PyObject_AsWritableString(s) ((char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) #define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) #define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) #define __Pyx_PyStr_FromCString(s) __Pyx_PyStr_FromString((const char*)s) #define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) -#if PY_MAJOR_VERSION < 3 -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) -{ +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { const Py_UNICODE *u_end = u; while (*u_end++) ; return (size_t)(u_end - u - 1); } -#else -#define __Pyx_Py_UNICODE_strlen Py_UNICODE_strlen -#endif #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode #define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode @@ -524,6 +631,8 @@ static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) #define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); #if CYTHON_ASSUME_SAFE_MACROS @@ -622,10 +731,12 @@ static int __Pyx_init_sys_getdefaultencoding_params(void) { #define likely(x) (x) #define unlikely(x) (x) #endif /* __GNUC__ */ +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } -static PyObject *__pyx_m; +static PyObject *__pyx_m = NULL; static PyObject *__pyx_d; static PyObject *__pyx_b; +static PyObject *__pyx_cython_runtime; static PyObject *__pyx_empty_tuple; static PyObject *__pyx_empty_bytes; static PyObject *__pyx_empty_unicode; @@ -637,8 +748,8 @@ static const char *__pyx_filename; static const char *__pyx_f[] = { "reppy/robots.pxd", - "reppy/robots.pyx", "stringsource", + "reppy/robots.pyx", }; /*--- Type declarations ---*/ @@ -857,10 +968,10 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); if (CYTHON_TRACE_NOGIL) {\ PyThreadState *tstate;\ PyGILState_STATE state = PyGILState_Ensure();\ - tstate = PyThreadState_GET();\ + tstate = __Pyx_PyThreadState_Current;\ if (unlikely(tstate->use_tracing) && !tstate->tracing &&\ (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {\ - __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&__pyx_frame_code, &__pyx_frame, funcname, srcfile, firstlineno);\ + __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&__pyx_frame_code, &__pyx_frame, tstate, funcname, srcfile, firstlineno);\ }\ PyGILState_Release(state);\ if (unlikely(__Pyx_use_tracing < 0)) goto_error;\ @@ -869,7 +980,7 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); PyThreadState* tstate = PyThreadState_GET();\ if (unlikely(tstate->use_tracing) && !tstate->tracing &&\ (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {\ - __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&__pyx_frame_code, &__pyx_frame, funcname, srcfile, firstlineno);\ + __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&__pyx_frame_code, &__pyx_frame, tstate, funcname, srcfile, firstlineno);\ if (unlikely(__Pyx_use_tracing < 0)) goto_error;\ }\ } @@ -878,14 +989,14 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); { PyThreadState* tstate = PyThreadState_GET();\ if (unlikely(tstate->use_tracing) && !tstate->tracing &&\ (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {\ - __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&__pyx_frame_code, &__pyx_frame, funcname, srcfile, firstlineno);\ + __Pyx_use_tracing = __Pyx_TraceSetupAndCall(&__pyx_frame_code, &__pyx_frame, tstate, funcname, srcfile, firstlineno);\ if (unlikely(__Pyx_use_tracing < 0)) goto_error;\ }\ } #endif #define __Pyx_TraceException()\ if (likely(!__Pyx_use_tracing)); else {\ - PyThreadState* tstate = PyThreadState_GET();\ + PyThreadState* tstate = __Pyx_PyThreadState_Current;\ if (tstate->use_tracing &&\ (tstate->c_profilefunc || (CYTHON_TRACE && tstate->c_tracefunc))) {\ tstate->tracing++;\ @@ -924,14 +1035,14 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); if (CYTHON_TRACE_NOGIL) {\ PyThreadState *tstate;\ PyGILState_STATE state = PyGILState_Ensure();\ - tstate = PyThreadState_GET();\ + tstate = __Pyx_PyThreadState_Current;\ if (tstate->use_tracing) {\ __Pyx_call_return_trace_func(tstate, __pyx_frame, (PyObject*)result);\ }\ PyGILState_Release(state);\ }\ } else {\ - PyThreadState* tstate = PyThreadState_GET();\ + PyThreadState* tstate = __Pyx_PyThreadState_Current;\ if (tstate->use_tracing) {\ __Pyx_call_return_trace_func(tstate, __pyx_frame, (PyObject*)result);\ }\ @@ -940,18 +1051,18 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); #else #define __Pyx_TraceReturn(result, nogil)\ if (likely(!__Pyx_use_tracing)); else {\ - PyThreadState* tstate = PyThreadState_GET();\ + PyThreadState* tstate = __Pyx_PyThreadState_Current;\ if (tstate->use_tracing) {\ __Pyx_call_return_trace_func(tstate, __pyx_frame, (PyObject*)result);\ }\ } #endif static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const char *srcfile, int firstlineno); - static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, const char *funcname, const char *srcfile, int firstlineno); + static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, PyThreadState* tstate, const char *funcname, const char *srcfile, int firstlineno); #else #define __Pyx_TraceDeclarations #define __Pyx_TraceFrameInit(codeobj) - #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error) if (1); else goto_error; + #define __Pyx_TraceCall(funcname, srcfile, firstlineno, nogil, goto_error) if ((1)); else goto_error; #define __Pyx_TraceException() #define __Pyx_TraceReturn(result, nogil) #endif @@ -983,16 +1094,16 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); int ret = 0;\ PyThreadState *tstate;\ PyGILState_STATE state = PyGILState_Ensure();\ - tstate = PyThreadState_GET();\ - if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) {\ + tstate = __Pyx_PyThreadState_Current;\ + if (unlikely(tstate->use_tracing && tstate->c_tracefunc && __pyx_frame->f_trace)) {\ ret = __Pyx_call_line_trace_func(tstate, __pyx_frame, lineno);\ }\ PyGILState_Release(state);\ if (unlikely(ret)) goto_error;\ }\ } else {\ - PyThreadState* tstate = PyThreadState_GET();\ - if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) {\ + PyThreadState* tstate = __Pyx_PyThreadState_Current;\ + if (unlikely(tstate->use_tracing && tstate->c_tracefunc && __pyx_frame->f_trace)) {\ int ret = __Pyx_call_line_trace_func(tstate, __pyx_frame, lineno);\ if (unlikely(ret)) goto_error;\ }\ @@ -1001,15 +1112,15 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); #else #define __Pyx_TraceLine(lineno, nogil, goto_error)\ if (likely(!__Pyx_use_tracing)); else {\ - PyThreadState* tstate = PyThreadState_GET();\ - if (unlikely(tstate->use_tracing && tstate->c_tracefunc)) {\ + PyThreadState* tstate = __Pyx_PyThreadState_Current;\ + if (unlikely(tstate->use_tracing && tstate->c_tracefunc && __pyx_frame->f_trace)) {\ int ret = __Pyx_call_line_trace_func(tstate, __pyx_frame, lineno);\ if (unlikely(ret)) goto_error;\ }\ } #endif #else - #define __Pyx_TraceLine(lineno, nogil, goto_error) if (1); else goto_error; + #define __Pyx_TraceLine(lineno, nogil, goto_error) if ((1)); else goto_error; #endif /* PyObjectCall.proto */ @@ -1035,8 +1146,81 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ const char* function_name); /* ArgTypeTest.proto */ -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact); +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* IncludeCppStringH.proto */ +#include + +/* decode_c_string_utf16.proto */ +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = 0; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16LE(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = -1; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16BE(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = 1; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} + +/* decode_c_bytes.proto */ +static CYTHON_INLINE PyObject* __Pyx_decode_c_bytes( + const char* cstring, Py_ssize_t length, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)); + +/* decode_cpp_string.proto */ +static CYTHON_INLINE PyObject* __Pyx_decode_cpp_string( + std::string cppstring, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { + return __Pyx_decode_c_bytes( + cppstring.data(), cppstring.size(), start, stop, encoding, errors, decode_func); +} + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#define __Pyx_PyErr_Occurred() __pyx_tstate->curexc_type +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); /* PyFunctionFastCall.proto */ #if CYTHON_FAST_PYCALL @@ -1085,33 +1269,6 @@ static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr #define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v) #endif -/* PyThreadStateGet.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; -#define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); -#else -#define __Pyx_PyThreadState_declare -#define __Pyx_PyThreadState_assign -#endif - -/* PyErrFetchRestore.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); -#else -#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) -#endif - -/* RaiseException.proto */ -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); - /* FetchCommonType.proto */ static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); @@ -1170,7 +1327,7 @@ static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, static int __pyx_CyFunction_init(void); /* PyObjectLookupSpecial.proto */ -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { PyObject *res; PyTypeObject *tp = Py_TYPE(obj); @@ -1262,6 +1419,9 @@ static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { /* IncludeStringH.proto */ #include +/* SetupReduce.proto */ +static int __Pyx_setup_reduce(PyObject* type_obj); + /* Import.proto */ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); @@ -1272,6 +1432,13 @@ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); #include "descrobject.h" static PyObject* __Pyx_Method_ClassMethod(PyObject *method); +/* CLineInTraceback.proto */ +#ifdef CYTHON_CLINE_IN_TRACEBACK +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#else +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#endif + /* CodeObjectCache.proto */ typedef struct { PyCodeObject* code_object; @@ -1306,6 +1473,18 @@ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); /* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +#define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) +#endif + /* CheckBinaryVersion.proto */ static int __Pyx_check_binary_version(void); @@ -1339,14 +1518,16 @@ static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_st static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*)(PyObject *)); /*proto*/ static PyObject *__pyx_convert_vector_to_py_std_3a__3a_string(const std::vector &); /*proto*/ #define __Pyx_MODULE_NAME "reppy.robots" +extern int __pyx_module_is_main_reppy__robots; int __pyx_module_is_main_reppy__robots = 0; /* Implementation of 'reppy.robots' */ static PyObject *__pyx_builtin_ValueError; +static PyObject *__pyx_builtin_TypeError; static PyObject *__pyx_builtin_map; static PyObject *__pyx_builtin_range; static const char __pyx_k_PY3[] = "PY3"; -static const char __pyx_k__13[] = ""; +static const char __pyx_k__19[] = ""; static const char __pyx_k_amt[] = "amt"; static const char __pyx_k_cls[] = "cls"; static const char __pyx_k_exc[] = "exc"; @@ -1383,6 +1564,8 @@ static const char __pyx_k_encode[] = "encode"; static const char __pyx_k_import[] = "__import__"; static const char __pyx_k_kwargs[] = "kwargs"; static const char __pyx_k_logger[] = "logger"; +static const char __pyx_k_name_2[] = "__name__"; +static const char __pyx_k_reduce[] = "__reduce__"; static const char __pyx_k_robots[] = "robots"; static const char __pyx_k_stream[] = "stream"; static const char __pyx_k_closing[] = "closing"; @@ -1392,8 +1575,12 @@ static const char __pyx_k_expires[] = "expires"; static const char __pyx_k_minimum[] = "minimum"; static const char __pyx_k_wrapped[] = "wrapped"; static const char __pyx_k_SSLError[] = "SSLError"; +static const char __pyx_k_getstate[] = "__getstate__"; static const char __pyx_k_max_size[] = "max_size"; static const char __pyx_k_requests[] = "requests"; +static const char __pyx_k_setstate[] = "__setstate__"; +static const char __pyx_k_TypeError[] = "TypeError"; +static const char __pyx_k_reduce_ex[] = "__reduce_ex__"; static const char __pyx_k_InvalidURL[] = "InvalidURL"; static const char __pyx_k_ValueError[] = "ValueError"; static const char __pyx_k_contextlib[] = "contextlib"; @@ -1414,25 +1601,30 @@ static const char __pyx_k_stringsource[] = "stringsource"; static const char __pyx_k_BadStatusCode[] = "BadStatusCode"; static const char __pyx_k_InvalidSchema[] = "InvalidSchema"; static const char __pyx_k_MissingSchema[] = "MissingSchema"; +static const char __pyx_k_reduce_cython[] = "__reduce_cython__"; static const char __pyx_k_ContentTooLong[] = "ContentTooLong"; static const char __pyx_k_decode_content[] = "decode_content"; static const char __pyx_k_wrap_exception[] = "wrap_exception"; static const char __pyx_k_ConnectionError[] = "ConnectionError"; static const char __pyx_k_RobotsUrlMethod[] = "RobotsUrlMethod"; +static const char __pyx_k_setstate_cython[] = "__setstate_cython__"; static const char __pyx_k_FromRobotsMethod[] = "FromRobotsMethod"; static const char __pyx_k_TooManyRedirects[] = "TooManyRedirects"; static const char __pyx_k_after_parse_hook[] = "after_parse_hook"; +static const char __pyx_k_reppy_robots_pyx[] = "reppy/robots.pyx"; static const char __pyx_k_DEFAULT_TTL_POLICY[] = "DEFAULT_TTL_POLICY"; static const char __pyx_k_ExcessiveRedirects[] = "ExcessiveRedirects"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; static const char __pyx_k_ConnectionException[] = "ConnectionException"; static const char __pyx_k_User_agent_Disallow[] = "User-agent: *\nDisallow: /"; static const char __pyx_k_after_response_hook[] = "after_response_hook"; static const char __pyx_k_requests_exceptions[] = "requests.exceptions"; static const char __pyx_k_HeaderWithDefaultPolicy[] = "HeaderWithDefaultPolicy"; -static const char __pyx_k_vagrant_reppy_robots_pyx[] = "/vagrant/reppy/robots.pyx"; static const char __pyx_k_Content_larger_than_s_bytes[] = "Content larger than %s bytes"; static const char __pyx_k_Pyx_CFunc_object____object___t[] = "__Pyx_CFunc_object____object___to_py..wrap"; +static const char __pyx_k_self_robots_cannot_be_converted[] = "self.robots cannot be converted to a Python object for pickling"; static const char __pyx_k_FetchMethod_locals_wrap_exceptio[] = "FetchMethod..wrap_exception"; +static const char __pyx_k_self_agent_cannot_be_converted_t[] = "self.agent cannot be converted to a Python object for pickling"; static PyObject *__pyx_n_s_BadStatusCode; static PyObject *__pyx_n_s_ConnectionError; static PyObject *__pyx_n_s_ConnectionException; @@ -1456,11 +1648,12 @@ static PyObject *__pyx_n_s_RobotsUrlMethod; static PyObject *__pyx_n_s_SSLError; static PyObject *__pyx_n_s_SSLException; static PyObject *__pyx_n_s_TooManyRedirects; +static PyObject *__pyx_n_s_TypeError; static PyObject *__pyx_n_s_URLRequired; static PyObject *__pyx_kp_b_User_agent_Disallow; static PyObject *__pyx_n_s_ValueError; -static PyObject *__pyx_n_s__13; -static PyObject *__pyx_kp_b__13; +static PyObject *__pyx_n_s__19; +static PyObject *__pyx_kp_b__19; static PyObject *__pyx_n_s_after_parse_hook; static PyObject *__pyx_n_s_after_response_hook; static PyObject *__pyx_n_s_agent; @@ -1468,6 +1661,7 @@ static PyObject *__pyx_n_s_amt; static PyObject *__pyx_n_s_args; static PyObject *__pyx_n_s_cause; static PyObject *__pyx_n_s_cfunc_to_py; +static PyObject *__pyx_n_s_cline_in_traceback; static PyObject *__pyx_n_s_closing; static PyObject *__pyx_n_s_cls; static PyObject *__pyx_n_s_content; @@ -1485,6 +1679,7 @@ static PyObject *__pyx_n_s_expires; static PyObject *__pyx_n_s_fetch; static PyObject *__pyx_n_s_from_robots; static PyObject *__pyx_n_s_get; +static PyObject *__pyx_n_s_getstate; static PyObject *__pyx_n_s_import; static PyObject *__pyx_n_s_init; static PyObject *__pyx_n_s_kwargs; @@ -1494,18 +1689,27 @@ static PyObject *__pyx_n_s_map; static PyObject *__pyx_n_s_max_size; static PyObject *__pyx_n_s_minimum; static PyObject *__pyx_n_s_name; +static PyObject *__pyx_n_s_name_2; static PyObject *__pyx_n_s_parse; static PyObject *__pyx_n_s_path; static PyObject *__pyx_n_s_pop; static PyObject *__pyx_n_s_range; static PyObject *__pyx_n_s_raw; static PyObject *__pyx_n_s_read; +static PyObject *__pyx_n_s_reduce; +static PyObject *__pyx_n_s_reduce_cython; +static PyObject *__pyx_n_s_reduce_ex; static PyObject *__pyx_n_s_reppy_robots; +static PyObject *__pyx_kp_s_reppy_robots_pyx; static PyObject *__pyx_n_s_requests; static PyObject *__pyx_n_s_requests_exceptions; static PyObject *__pyx_n_s_res; static PyObject *__pyx_n_s_robots; static PyObject *__pyx_n_s_robots_url; +static PyObject *__pyx_kp_s_self_agent_cannot_be_converted_t; +static PyObject *__pyx_kp_s_self_robots_cannot_be_converted; +static PyObject *__pyx_n_s_setstate; +static PyObject *__pyx_n_s_setstate_cython; static PyObject *__pyx_n_s_six; static PyObject *__pyx_n_s_status_code; static PyObject *__pyx_n_s_stream; @@ -1517,7 +1721,6 @@ static PyObject *__pyx_n_s_ttl_policy; static PyObject *__pyx_n_s_url; static PyObject *__pyx_kp_s_utf_8; static PyObject *__pyx_n_s_util; -static PyObject *__pyx_kp_s_vagrant_reppy_robots_pyx; static PyObject *__pyx_n_s_value; static PyObject *__pyx_n_s_wrap; static PyObject *__pyx_n_s_wrap_exception; @@ -1528,6 +1731,8 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_8__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_content, PyObject *__pyx_v_expires); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject *__pyx_self, PyObject *__pyx_v_etype, PyObject *__pyx_v_cause); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs); /* proto */ @@ -1541,8 +1746,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_10__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_6Robots_12__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6robots_AllowNone *__pyx_v_self, PyObject *__pyx_v_url, PyObject *__pyx_v_expires); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_9AllowNone_2__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_AllowNone *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_9AllowNone_4__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_AllowNone *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6robots_AllowAll *__pyx_v_self, PyObject *__pyx_v_url, PyObject *__pyx_v_expires); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_8AllowAll_2__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_AllowAll *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_8AllowAll_4__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_AllowAll *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_wrap(PyObject *__pyx_self, PyObject *__pyx_v_value); /* proto */ static PyObject *__pyx_tp_new_5reppy_6robots_Agent(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5reppy_6robots_Robots(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ @@ -1561,22 +1772,30 @@ static PyObject *__pyx_int_3600; static PyObject *__pyx_int_1048576; static PyObject *__pyx_tuple_; static PyObject *__pyx_tuple__2; -static PyObject *__pyx_tuple__6; -static PyObject *__pyx_tuple__7; +static PyObject *__pyx_tuple__4; +static PyObject *__pyx_tuple__5; static PyObject *__pyx_tuple__8; +static PyObject *__pyx_tuple__9; static PyObject *__pyx_tuple__10; -static PyObject *__pyx_tuple__11; -static PyObject *__pyx_tuple__14; +static PyObject *__pyx_tuple__12; +static PyObject *__pyx_tuple__13; +static PyObject *__pyx_tuple__15; static PyObject *__pyx_tuple__16; static PyObject *__pyx_tuple__17; static PyObject *__pyx_tuple__18; -static PyObject *__pyx_tuple__19; +static PyObject *__pyx_tuple__20; +static PyObject *__pyx_tuple__21; +static PyObject *__pyx_tuple__22; +static PyObject *__pyx_tuple__24; +static PyObject *__pyx_tuple__25; +static PyObject *__pyx_tuple__26; +static PyObject *__pyx_tuple__27; static PyObject *__pyx_codeobj__3; -static PyObject *__pyx_codeobj__4; -static PyObject *__pyx_codeobj__5; -static PyObject *__pyx_codeobj__9; -static PyObject *__pyx_codeobj__12; -static PyObject *__pyx_codeobj__15; +static PyObject *__pyx_codeobj__6; +static PyObject *__pyx_codeobj__7; +static PyObject *__pyx_codeobj__11; +static PyObject *__pyx_codeobj__14; +static PyObject *__pyx_codeobj__23; /* "reppy/robots.pyx":21 * from . import util, logger, exceptions @@ -1595,7 +1814,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; __Pyx_RefNannySetupContext("as_bytes", 0); - __Pyx_TraceCall("as_bytes", __pyx_f[1], 21, 0, __PYX_ERR(1, 21, __pyx_L1_error)); + __Pyx_TraceCall("as_bytes", __pyx_f[2], 21, 0, __PYX_ERR(2, 21, __pyx_L1_error)); /* "reppy/robots.pyx":22 * @@ -1604,7 +1823,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { * return value * return value.encode('utf-8') */ - __Pyx_TraceLine(22,0,__PYX_ERR(1, 22, __pyx_L1_error)) + __Pyx_TraceLine(22,0,__PYX_ERR(2, 22, __pyx_L1_error)) __pyx_t_1 = PyBytes_Check(__pyx_v_value); __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { @@ -1616,7 +1835,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { * return value.encode('utf-8') * */ - __Pyx_TraceLine(23,0,__PYX_ERR(1, 23, __pyx_L1_error)) + __Pyx_TraceLine(23,0,__PYX_ERR(2, 23, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_value); __pyx_r = __pyx_v_value; @@ -1638,11 +1857,11 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { * * cdef as_string(value): */ - __Pyx_TraceLine(24,0,__PYX_ERR(1, 24, __pyx_L1_error)) + __Pyx_TraceLine(24,0,__PYX_ERR(2, 24, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 24, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 24, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_4; @@ -1687,7 +1906,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { int __pyx_t_3; int __pyx_t_4; __Pyx_RefNannySetupContext("as_string", 0); - __Pyx_TraceCall("as_string", __pyx_f[1], 26, 0, __PYX_ERR(1, 26, __pyx_L1_error)); + __Pyx_TraceCall("as_string", __pyx_f[2], 26, 0, __PYX_ERR(2, 26, __pyx_L1_error)); /* "reppy/robots.pyx":27 * @@ -1696,13 +1915,13 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { * if isinstance(value, bytes): * return value.decode('utf-8') */ - __Pyx_TraceLine(27,0,__PYX_ERR(1, 27, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_six); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 27, __pyx_L1_error) + __Pyx_TraceLine(27,0,__PYX_ERR(2, 27, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_six); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 27, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_PY3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 27, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_PY3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 27, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(1, 27, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(2, 27, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_3) { @@ -1713,7 +1932,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { * return value.decode('utf-8') * return value */ - __Pyx_TraceLine(28,0,__PYX_ERR(1, 28, __pyx_L1_error)) + __Pyx_TraceLine(28,0,__PYX_ERR(2, 28, __pyx_L1_error)) __pyx_t_3 = PyBytes_Check(__pyx_v_value); __pyx_t_4 = (__pyx_t_3 != 0); if (__pyx_t_4) { @@ -1725,11 +1944,11 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { * return value * */ - __Pyx_TraceLine(29,0,__PYX_ERR(1, 29, __pyx_L1_error)) + __Pyx_TraceLine(29,0,__PYX_ERR(2, 29, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_decode); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 29, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_decode); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 29, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 29, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 29, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; @@ -1761,7 +1980,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { * * */ - __Pyx_TraceLine(30,0,__PYX_ERR(1, 30, __pyx_L1_error)) + __Pyx_TraceLine(30,0,__PYX_ERR(2, 30, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_value); __pyx_r = __pyx_v_value; @@ -1815,8 +2034,11 @@ static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } @@ -1825,19 +2047,21 @@ static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_robots)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 1); __PYX_ERR(1, 33, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 1); __PYX_ERR(2, 33, __pyx_L3_error) } + CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 2); __PYX_ERR(1, 33, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 2); __PYX_ERR(2, 33, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "FromRobotsMethod") < 0)) __PYX_ERR(1, 33, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "FromRobotsMethod") < 0)) __PYX_ERR(2, 33, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -1848,17 +2072,17 @@ static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, } __pyx_v_cls = values[0]; __pyx_v_robots = ((struct __pyx_obj_5reppy_6robots_Robots *)values[1]); - __pyx_v_name = __pyx_convert_string_from_py_std__in_string(values[2]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 33, __pyx_L3_error) + __pyx_v_name = __pyx_convert_string_from_py_std__in_string(values[2]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 33, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 33, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 33, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.FromRobotsMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_robots), __pyx_ptype_5reppy_6robots_Robots, 1, "robots", 0))) __PYX_ERR(1, 33, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_robots), __pyx_ptype_5reppy_6robots_Robots, 1, "robots", 0))) __PYX_ERR(2, 33, __pyx_L1_error) __pyx_r = __pyx_pf_5reppy_6robots_FromRobotsMethod(__pyx_self, __pyx_v_cls, __pyx_v_robots, __pyx_v_name); /* function exit code */ @@ -1878,7 +2102,7 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject PyObject *__pyx_t_1 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__3) __Pyx_RefNannySetupContext("FromRobotsMethod", 0); - __Pyx_TraceCall("FromRobotsMethod", __pyx_f[1], 33, 0, __PYX_ERR(1, 33, __pyx_L1_error)); + __Pyx_TraceCall("FromRobotsMethod", __pyx_f[2], 33, 0, __PYX_ERR(2, 33, __pyx_L1_error)); /* "reppy/robots.pyx":35 * def FromRobotsMethod(cls, Robots robots, const string& name): @@ -1887,8 +2111,8 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject * # This is somewhat inefficient due to the copying, but it is * # required to be copied because we often toss the containing */ - __Pyx_TraceLine(35,0,__PYX_ERR(1, 35, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 35, __pyx_L1_error) + __Pyx_TraceLine(35,0,__PYX_ERR(2, 35, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 35, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_agent = ((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_t_1); __pyx_t_1 = 0; @@ -1900,7 +2124,7 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject * return agent * */ - __Pyx_TraceLine(40,0,__PYX_ERR(1, 40, __pyx_L1_error)) + __Pyx_TraceLine(40,0,__PYX_ERR(2, 40, __pyx_L1_error)) __pyx_v_agent->agent = __pyx_v_robots->robots->agent(__pyx_v_name); /* "reppy/robots.pyx":41 @@ -1910,7 +2134,7 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject * * cdef class Agent: */ - __Pyx_TraceLine(41,0,__PYX_ERR(1, 41, __pyx_L1_error)) + __Pyx_TraceLine(41,0,__PYX_ERR(2, 41, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_agent)); __pyx_r = ((PyObject *)__pyx_v_agent); @@ -1941,7 +2165,7 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject * from_robots = classmethod(FromRobotsMethod) * * def __str__(self): # <<<<<<<<<<<<<< - * return self.agent.str() + * return self.agent.str().decode('utf8') * */ @@ -1964,18 +2188,18 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[1], 50, 0, __PYX_ERR(1, 50, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[2], 50, 0, __PYX_ERR(2, 50, __pyx_L1_error)); /* "reppy/robots.pyx":51 * * def __str__(self): - * return self.agent.str() # <<<<<<<<<<<<<< + * return self.agent.str().decode('utf8') # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(51,0,__PYX_ERR(1, 51, __pyx_L1_error)) + __Pyx_TraceLine(51,0,__PYX_ERR(2, 51, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->agent.str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 51, __pyx_L1_error) + __pyx_t_1 = __Pyx_decode_cpp_string(__pyx_v_self->agent.str(), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 51, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -1985,7 +2209,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ * from_robots = classmethod(FromRobotsMethod) * * def __str__(self): # <<<<<<<<<<<<<< - * return self.agent.str() + * return self.agent.str().decode('utf8') * */ @@ -2030,7 +2254,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ int __pyx_t_1; PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 54, 0, __PYX_ERR(1, 54, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 54, 0, __PYX_ERR(2, 54, __pyx_L1_error)); /* "reppy/robots.pyx":56 * def delay(self): @@ -2039,7 +2263,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ * if value > 0: * return value */ - __Pyx_TraceLine(56,0,__PYX_ERR(1, 56, __pyx_L1_error)) + __Pyx_TraceLine(56,0,__PYX_ERR(2, 56, __pyx_L1_error)) __pyx_v_value = __pyx_v_self->agent.delay(); /* "reppy/robots.pyx":57 @@ -2049,7 +2273,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ * return value * return None */ - __Pyx_TraceLine(57,0,__PYX_ERR(1, 57, __pyx_L1_error)) + __Pyx_TraceLine(57,0,__PYX_ERR(2, 57, __pyx_L1_error)) __pyx_t_1 = ((__pyx_v_value > 0.0) != 0); if (__pyx_t_1) { @@ -2060,9 +2284,9 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ * return None * */ - __Pyx_TraceLine(58,0,__PYX_ERR(1, 58, __pyx_L1_error)) + __Pyx_TraceLine(58,0,__PYX_ERR(2, 58, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 58, __pyx_L1_error) + __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 58, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; @@ -2084,7 +2308,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ * * def allow(self, path): */ - __Pyx_TraceLine(59,0,__PYX_ERR(1, 59, __pyx_L1_error)) + __Pyx_TraceLine(59,0,__PYX_ERR(2, 59, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; @@ -2139,7 +2363,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allow", 0); - __Pyx_TraceCall("allow", __pyx_f[1], 61, 0, __PYX_ERR(1, 61, __pyx_L1_error)); + __Pyx_TraceCall("allow", __pyx_f[2], 61, 0, __PYX_ERR(2, 61, __pyx_L1_error)); /* "reppy/robots.pyx":63 * def allow(self, path): @@ -2148,10 +2372,10 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 * return self * */ - __Pyx_TraceLine(63,0,__PYX_ERR(1, 63, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 63, __pyx_L1_error) + __Pyx_TraceLine(63,0,__PYX_ERR(2, 63, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 63, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 63, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 63, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.allow(__pyx_t_2); @@ -2162,7 +2386,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 * * def disallow(self, path): */ - __Pyx_TraceLine(64,0,__PYX_ERR(1, 64, __pyx_L1_error)) + __Pyx_TraceLine(64,0,__PYX_ERR(2, 64, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); @@ -2217,7 +2441,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("disallow", 0); - __Pyx_TraceCall("disallow", __pyx_f[1], 66, 0, __PYX_ERR(1, 66, __pyx_L1_error)); + __Pyx_TraceCall("disallow", __pyx_f[2], 66, 0, __PYX_ERR(2, 66, __pyx_L1_error)); /* "reppy/robots.pyx":68 * def disallow(self, path): @@ -2226,10 +2450,10 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp * return self * */ - __Pyx_TraceLine(68,0,__PYX_ERR(1, 68, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 68, __pyx_L1_error) + __Pyx_TraceLine(68,0,__PYX_ERR(2, 68, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 68, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 68, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 68, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.disallow(__pyx_t_2); @@ -2240,7 +2464,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp * * def allowed(self, path): */ - __Pyx_TraceLine(69,0,__PYX_ERR(1, 69, __pyx_L1_error)) + __Pyx_TraceLine(69,0,__PYX_ERR(2, 69, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); @@ -2295,7 +2519,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[1], 71, 0, __PYX_ERR(1, 71, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[2], 71, 0, __PYX_ERR(2, 71, __pyx_L1_error)); /* "reppy/robots.pyx":73 * def allowed(self, path): @@ -2304,13 +2528,13 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy * * */ - __Pyx_TraceLine(73,0,__PYX_ERR(1, 73, __pyx_L1_error)) + __Pyx_TraceLine(73,0,__PYX_ERR(2, 73, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 73, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 73, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 73, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 73, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 73, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 73, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -2336,6 +2560,121 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy return __pyx_r; } +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_9__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_9__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_8__reduce_cython__(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_5Agent_8__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + __Pyx_TraceCall("__reduce_cython__", __pyx_f[1], 1, 0, __PYX_ERR(1, 1, __pyx_L1_error)); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self.agent cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + */ + __Pyx_TraceLine(2,0,__PYX_ERR(1, 2, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Agent.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_11__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_11__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_10__setstate_cython__(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + __Pyx_TraceCall("__setstate_cython__", __pyx_f[1], 3, 0, __PYX_ERR(1, 3, __pyx_L1_error)); + + /* "(tree fragment)":4 + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.agent cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Agent.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + /* "reppy/robots.pyx":76 * * @@ -2365,9 +2704,13 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } @@ -2376,16 +2719,19 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(1, 76, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(2, 76, __pyx_L3_error) } + CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(1, 76, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(2, 76, __pyx_L3_error) } + CYTHON_FALLTHROUGH; case 3: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); @@ -2393,11 +2739,12 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(1, 76, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(2, 76, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[0] = PyTuple_GET_ITEM(__pyx_args, 0); @@ -2412,7 +2759,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 76, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 76, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.ParseMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2435,9 +2782,9 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_4 = NULL; int __pyx_t_5; PyObject *__pyx_t_6 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__4) + __Pyx_TraceFrameInit(__pyx_codeobj__6) __Pyx_RefNannySetupContext("ParseMethod", 0); - __Pyx_TraceCall("ParseMethod", __pyx_f[1], 76, 0, __PYX_ERR(1, 76, __pyx_L1_error)); + __Pyx_TraceCall("ParseMethod", __pyx_f[2], 76, 0, __PYX_ERR(2, 76, __pyx_L1_error)); /* "reppy/robots.pyx":78 * def ParseMethod(cls, url, content, expires=None): @@ -2446,9 +2793,9 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): */ - __Pyx_TraceLine(78,0,__PYX_ERR(1, 78, __pyx_L1_error)) + __Pyx_TraceLine(78,0,__PYX_ERR(2, 78, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 78, __pyx_L1_error) + __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 78, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_v_cls); __pyx_t_3 = __pyx_v_cls; __pyx_t_4 = NULL; @@ -2466,7 +2813,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 78, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 78, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -2475,14 +2822,14 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 78, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 78, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 78, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 78, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -2496,7 +2843,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_5, __pyx_v_expires); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 78, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 78, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -2575,9 +2922,13 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb switch (pos_args) { default: case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; } kw_args = PyDict_Size(__pyx_kwds); @@ -2585,16 +2936,19 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(1, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(2, 80, __pyx_L3_error) } + CYTHON_FALLTHROUGH; case 2: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_ttl_policy); if (value) { values[2] = value; kw_args--; } } + CYTHON_FALLTHROUGH; case 3: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_max_size); @@ -2603,13 +2957,15 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } if (unlikely(kw_args > 0)) { const Py_ssize_t used_pos_args = (pos_args < 4) ? pos_args : 4; - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(1, 80, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(2, 80, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { default: case 4: values[3] = PyTuple_GET_ITEM(__pyx_args, 3); + CYTHON_FALLTHROUGH; case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; @@ -2625,7 +2981,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 80, __pyx_L3_error) __pyx_L3_error:; __Pyx_DECREF(__pyx_v_args); __pyx_v_args = 0; __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; @@ -2667,7 +3023,9 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } @@ -2676,14 +3034,15 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_etype)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cause)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(1, 84, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(2, 84, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "wrap_exception") < 0)) __PYX_ERR(1, 84, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "wrap_exception") < 0)) __PYX_ERR(2, 84, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -2696,7 +3055,7 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 84, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 84, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.FetchMethod.wrap_exception", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2725,7 +3084,7 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_RefNannySetupContext("wrap_exception", 0); __pyx_outer_scope = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *) __Pyx_CyFunction_GetClosure(__pyx_self); __pyx_cur_scope = __pyx_outer_scope; - __Pyx_TraceCall("wrap_exception", __pyx_f[1], 84, 0, __PYX_ERR(1, 84, __pyx_L1_error)); + __Pyx_TraceCall("wrap_exception", __pyx_f[2], 84, 0, __PYX_ERR(2, 84, __pyx_L1_error)); /* "reppy/robots.pyx":85 * after_parse_hook = kwargs.pop('after_parse_hook', None) @@ -2734,7 +3093,7 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * * wrapped.url = url * if after_response_hook is not None: */ - __Pyx_TraceLine(85,0,__PYX_ERR(1, 85, __pyx_L1_error)) + __Pyx_TraceLine(85,0,__PYX_ERR(2, 85, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_etype); __pyx_t_2 = __pyx_v_etype; __pyx_t_3 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { @@ -2747,13 +3106,13 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * } } if (!__pyx_t_3) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 85, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 85, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -2761,19 +3120,19 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 85, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 85, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __pyx_t_3 = NULL; __Pyx_INCREF(__pyx_v_cause); __Pyx_GIVEREF(__pyx_v_cause); PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_cause); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 85, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } @@ -2789,9 +3148,9 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * * if after_response_hook is not None: * after_response_hook(wrapped) */ - __Pyx_TraceLine(86,0,__PYX_ERR(1, 86, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_url)) { __Pyx_RaiseClosureNameError("url"); __PYX_ERR(1, 86, __pyx_L1_error) } - if (__Pyx_PyObject_SetAttrStr(__pyx_v_wrapped, __pyx_n_s_url, __pyx_cur_scope->__pyx_v_url) < 0) __PYX_ERR(1, 86, __pyx_L1_error) + __Pyx_TraceLine(86,0,__PYX_ERR(2, 86, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_url)) { __Pyx_RaiseClosureNameError("url"); __PYX_ERR(2, 86, __pyx_L1_error) } + if (__Pyx_PyObject_SetAttrStr(__pyx_v_wrapped, __pyx_n_s_url, __pyx_cur_scope->__pyx_v_url) < 0) __PYX_ERR(2, 86, __pyx_L1_error) /* "reppy/robots.pyx":87 * wrapped = etype(cause) @@ -2800,8 +3159,8 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * * after_response_hook(wrapped) * raise wrapped */ - __Pyx_TraceLine(87,0,__PYX_ERR(1, 87, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(1, 87, __pyx_L1_error) } + __Pyx_TraceLine(87,0,__PYX_ERR(2, 87, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 87, __pyx_L1_error) } __pyx_t_5 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); __pyx_t_6 = (__pyx_t_5 != 0); if (__pyx_t_6) { @@ -2813,8 +3172,8 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * * raise wrapped * try: */ - __Pyx_TraceLine(88,0,__PYX_ERR(1, 88, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(1, 88, __pyx_L1_error) } + __Pyx_TraceLine(88,0,__PYX_ERR(2, 88, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 88, __pyx_L1_error) } __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); __pyx_t_2 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_4 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { @@ -2827,13 +3186,13 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * } } if (!__pyx_t_4) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -2841,19 +3200,19 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 88, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); __pyx_t_4 = NULL; __Pyx_INCREF(__pyx_v_wrapped); __Pyx_GIVEREF(__pyx_v_wrapped); PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_v_wrapped); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } @@ -2877,9 +3236,9 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * * try: * # Limit the size of the request */ - __Pyx_TraceLine(89,0,__PYX_ERR(1, 89, __pyx_L1_error)) + __Pyx_TraceLine(89,0,__PYX_ERR(2, 89, __pyx_L1_error)) __Pyx_Raise(__pyx_v_wrapped, 0, 0, 0); - __PYX_ERR(1, 89, __pyx_L1_error) + __PYX_ERR(2, 89, __pyx_L1_error) /* "reppy/robots.pyx":84 * after_response_hook = kwargs.pop('after_response_hook', None) @@ -2941,17 +3300,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ int __pyx_t_15; PyObject *__pyx_t_16 = NULL; PyObject *__pyx_t_17 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__5) + __Pyx_TraceFrameInit(__pyx_codeobj__7) __Pyx_RefNannySetupContext("FetchMethod", 0); __pyx_cur_scope = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)__pyx_tp_new_5reppy_6robots___pyx_scope_struct__FetchMethod(__pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(1, 80, __pyx_L1_error) + __PYX_ERR(2, 80, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } - __Pyx_TraceCall("FetchMethod", __pyx_f[1], 80, 0, __PYX_ERR(1, 80, __pyx_L1_error)); + __Pyx_TraceCall("FetchMethod", __pyx_f[2], 80, 0, __PYX_ERR(2, 80, __pyx_L1_error)); __pyx_cur_scope->__pyx_v_url = __pyx_v_url; __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); @@ -2963,10 +3322,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): */ - __Pyx_TraceLine(82,0,__PYX_ERR(1, 82, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 82, __pyx_L1_error) + __Pyx_TraceLine(82,0,__PYX_ERR(2, 82, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 82, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 82, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 82, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_t_2); @@ -2980,10 +3339,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * def wrap_exception(etype, cause): * wrapped = etype(cause) */ - __Pyx_TraceLine(83,0,__PYX_ERR(1, 83, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 83, __pyx_L1_error) + __Pyx_TraceLine(83,0,__PYX_ERR(2, 83, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 83, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 83, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 83, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_after_parse_hook = __pyx_t_1; @@ -2996,8 +3355,8 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrapped = etype(cause) * wrapped.url = url */ - __Pyx_TraceLine(84,0,__PYX_ERR(1, 84, __pyx_L1_error)) - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__9)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 84, __pyx_L1_error) + __Pyx_TraceLine(84,0,__PYX_ERR(2, 84, __pyx_L1_error)) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 84, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_wrap_exception = __pyx_t_1; __pyx_t_1 = 0; @@ -3009,7 +3368,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_TraceLine(90,0,__PYX_ERR(1, 90, __pyx_L3_error)) + __Pyx_TraceLine(90,0,__PYX_ERR(2, 90, __pyx_L3_error)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign @@ -3026,8 +3385,8 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) */ - __Pyx_TraceLine(92,0,__PYX_ERR(1, 92, __pyx_L3_error)) - if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(1, 92, __pyx_L3_error) + __Pyx_TraceLine(92,0,__PYX_ERR(2, 92, __pyx_L3_error)) + if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(2, 92, __pyx_L3_error) /* "reppy/robots.pyx":93 * # Limit the size of the request @@ -3036,24 +3395,24 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __Pyx_TraceLine(93,0,__PYX_ERR(1, 93, __pyx_L3_error)) + __Pyx_TraceLine(93,0,__PYX_ERR(2, 93, __pyx_L3_error)) /*with:*/ { - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_cur_scope->__pyx_v_url); - __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -3068,14 +3427,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -3084,28 +3443,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { - __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); __pyx_t_6 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(1, 93, __pyx_L3_error) + __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 93, __pyx_L11_error) + __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 93, __pyx_L9_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { @@ -3118,10 +3477,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (__pyx_t_6) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 93, __pyx_L11_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 93, __pyx_L9_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 93, __pyx_L11_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 93, __pyx_L9_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3147,17 +3506,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): */ - __Pyx_TraceLine(94,0,__PYX_ERR(1, 94, __pyx_L17_error)) - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 94, __pyx_L17_error) + __Pyx_TraceLine(94,0,__PYX_ERR(2, 94, __pyx_L13_error)) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 94, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = PyDict_New(); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 94, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(1, 94, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 94, __pyx_L17_error) - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 94, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(2, 94, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 94, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 94, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3171,21 +3530,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) */ - __Pyx_TraceLine(96,0,__PYX_ERR(1, 96, __pyx_L17_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 96, __pyx_L17_error) + __Pyx_TraceLine(96,0,__PYX_ERR(2, 96, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 96, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 96, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = PyDict_New(); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 96, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(1, 96, __pyx_L17_error) - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(1, 96, __pyx_L17_error) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 96, __pyx_L17_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(2, 96, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 96, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 96, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 96, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 96, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_13) { @@ -3196,10 +3555,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * 'Content larger than %s bytes' % max_size) * */ - __Pyx_TraceLine(97,0,__PYX_ERR(1, 97, __pyx_L17_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 97, __pyx_L17_error) + __Pyx_TraceLine(97,0,__PYX_ERR(2, 97, __pyx_L13_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 97, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -3210,8 +3569,8 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * * if after_response_hook is not None: */ - __Pyx_TraceLine(98,0,__PYX_ERR(1, 98, __pyx_L17_error)) - __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 98, __pyx_L17_error) + __Pyx_TraceLine(98,0,__PYX_ERR(2, 98, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -3224,14 +3583,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 97, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 97, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -3240,20 +3599,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 97, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 97, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_2); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 97, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3261,7 +3620,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(1, 97, __pyx_L17_error) + __PYX_ERR(2, 97, __pyx_L13_error) /* "reppy/robots.pyx":96 * content = res.raw.read(amt=max_size, decode_content=True) @@ -3279,7 +3638,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * after_response_hook(res) * */ - __Pyx_TraceLine(100,0,__PYX_ERR(1, 100, __pyx_L17_error)) + __Pyx_TraceLine(100,0,__PYX_ERR(2, 100, __pyx_L13_error)) __pyx_t_13 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { @@ -3291,7 +3650,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * * # Get the TTL policy's ruling on the ttl */ - __Pyx_TraceLine(101,0,__PYX_ERR(1, 101, __pyx_L17_error)) + __Pyx_TraceLine(101,0,__PYX_ERR(2, 101, __pyx_L13_error)) __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); __pyx_t_7 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_8 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -3304,13 +3663,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3318,19 +3677,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 101, __pyx_L17_error) + __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_2, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 101, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } @@ -3354,21 +3713,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * * if res.status_code == 200: */ - __Pyx_TraceLine(104,0,__PYX_ERR(1, 104, __pyx_L17_error)) - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 104, __pyx_L17_error) + __Pyx_TraceLine(104,0,__PYX_ERR(2, 104, __pyx_L13_error)) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 104, __pyx_L13_error) if (!__pyx_t_14) { } else { __Pyx_INCREF(__pyx_v_ttl_policy); __pyx_t_7 = __pyx_v_ttl_policy; - goto __pyx_L27_bool_binop_done; + goto __pyx_L21_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 104, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_2); __pyx_t_7 = __pyx_t_2; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_L27_bool_binop_done:; - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 104, __pyx_L17_error) + __pyx_L21_bool_binop_done:; + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = NULL; @@ -3382,13 +3741,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 104, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 104, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3396,19 +3755,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 104, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 104, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 104, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3424,13 +3783,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: */ - __Pyx_TraceLine(106,0,__PYX_ERR(1, 106, __pyx_L17_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 106, __pyx_L17_error) + __Pyx_TraceLine(106,0,__PYX_ERR(2, 106, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 106, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 106, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 106, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 106, __pyx_L17_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 106, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_14) { @@ -3441,8 +3800,8 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * if after_parse_hook is not None: * after_parse_hook(robots) */ - __Pyx_TraceLine(107,0,__PYX_ERR(1, 107, __pyx_L17_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 107, __pyx_L17_error) + __Pyx_TraceLine(107,0,__PYX_ERR(2, 107, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 107, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_8 = NULL; __pyx_t_15 = 0; @@ -3459,7 +3818,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 107, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else @@ -3467,13 +3826,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 107, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 107, __pyx_L17_error) + __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 107, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); if (__pyx_t_8) { __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; @@ -3487,7 +3846,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_15, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 107, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 107, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } @@ -3502,7 +3861,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * after_parse_hook(robots) * return robots */ - __Pyx_TraceLine(108,0,__PYX_ERR(1, 108, __pyx_L17_error)) + __Pyx_TraceLine(108,0,__PYX_ERR(2, 108, __pyx_L13_error)) __pyx_t_14 = (__pyx_v_after_parse_hook != Py_None); __pyx_t_13 = (__pyx_t_14 != 0); if (__pyx_t_13) { @@ -3514,7 +3873,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * return robots * elif res.status_code in (401, 403): */ - __Pyx_TraceLine(109,0,__PYX_ERR(1, 109, __pyx_L17_error)) + __Pyx_TraceLine(109,0,__PYX_ERR(2, 109, __pyx_L13_error)) __Pyx_INCREF(__pyx_v_after_parse_hook); __pyx_t_1 = __pyx_v_after_parse_hook; __pyx_t_7 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_1))) { @@ -3527,13 +3886,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 109, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 109, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else @@ -3541,19 +3900,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 109, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 109, __pyx_L17_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 109, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_robots); __Pyx_GIVEREF(__pyx_v_robots); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_robots); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 109, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 109, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3577,11 +3936,11 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * elif res.status_code in (401, 403): * return AllowNone(url, expires) */ - __Pyx_TraceLine(110,0,__PYX_ERR(1, 110, __pyx_L17_error)) + __Pyx_TraceLine(110,0,__PYX_ERR(2, 110, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_robots); __pyx_r = __pyx_v_robots; - goto __pyx_L21_try_return; + goto __pyx_L17_try_return; /* "reppy/robots.pyx":106 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) @@ -3599,24 +3958,24 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: */ - __Pyx_TraceLine(111,0,__PYX_ERR(1, 111, __pyx_L17_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 111, __pyx_L17_error) + __Pyx_TraceLine(111,0,__PYX_ERR(2, 111, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 111, __pyx_L17_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (!__pyx_t_14) { } else { __pyx_t_13 = __pyx_t_14; - goto __pyx_L31_bool_binop_done; + goto __pyx_L25_bool_binop_done; } - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 111, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(1, 111, __pyx_L17_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_13 = __pyx_t_14; - __pyx_L31_bool_binop_done:; + __pyx_L25_bool_binop_done:; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { @@ -3628,9 +3987,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) */ - __Pyx_TraceLine(112,0,__PYX_ERR(1, 112, __pyx_L17_error)) + __Pyx_TraceLine(112,0,__PYX_ERR(2, 112, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 112, __pyx_L17_error) + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 112, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); @@ -3638,12 +3997,12 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 112, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 112, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; - goto __pyx_L21_try_return; + goto __pyx_L17_try_return; /* "reppy/robots.pyx":111 * after_parse_hook(robots) @@ -3661,26 +4020,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * return AllowAll(url, expires) * else: */ - __Pyx_TraceLine(113,0,__PYX_ERR(1, 113, __pyx_L17_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 113, __pyx_L17_error) + __Pyx_TraceLine(113,0,__PYX_ERR(2, 113, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 113, __pyx_L17_error) + __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 113, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_13) { } else { __pyx_t_14 = __pyx_t_13; - goto __pyx_L33_bool_binop_done; + goto __pyx_L27_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 113, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 113, __pyx_L17_error) + __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(1, 113, __pyx_L17_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_14 = __pyx_t_13; - __pyx_L33_bool_binop_done:; + __pyx_L27_bool_binop_done:; if (__pyx_t_14) { /* "reppy/robots.pyx":114 @@ -3690,9 +4049,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * else: * raise exceptions.BadStatusCode( */ - __Pyx_TraceLine(114,0,__PYX_ERR(1, 114, __pyx_L17_error)) + __Pyx_TraceLine(114,0,__PYX_ERR(2, 114, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 114, __pyx_L17_error) + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); @@ -3700,12 +4059,12 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 114, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; - goto __pyx_L21_try_return; + goto __pyx_L17_try_return; /* "reppy/robots.pyx":113 * elif res.status_code in (401, 403): @@ -3723,11 +4082,11 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: */ - __Pyx_TraceLine(116,0,__PYX_ERR(1, 116, __pyx_L17_error)) + __Pyx_TraceLine(116,0,__PYX_ERR(2, 116, __pyx_L13_error)) /*else*/ { - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 116, __pyx_L17_error) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 116, __pyx_L17_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -3738,10 +4097,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) */ - __Pyx_TraceLine(117,0,__PYX_ERR(1, 117, __pyx_L17_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 117, __pyx_L17_error) + __Pyx_TraceLine(117,0,__PYX_ERR(2, 117, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 117, __pyx_L17_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); @@ -3749,10 +4108,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_cur_scope->__pyx_v_url); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 117, __pyx_L17_error) + __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 117, __pyx_L17_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; __pyx_t_15 = 0; @@ -3769,7 +4128,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 116, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -3779,7 +4138,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 116, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -3787,7 +4146,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } else #endif { - __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 116, __pyx_L17_error) + __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_6) { __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; @@ -3798,14 +4157,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_15, __pyx_t_7); __pyx_t_1 = 0; __pyx_t_7 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 116, __pyx_L17_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(1, 116, __pyx_L17_error) + __PYX_ERR(2, 116, __pyx_L13_error) } /* "reppy/robots.pyx":93 @@ -3816,8 +4175,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * # Try to read an additional byte, to see if the response is too big */ } - __pyx_L17_error:; - __Pyx_PyThreadState_assign + __pyx_L13_error:; __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3826,20 +4184,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; /*except:*/ { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(1, 93, __pyx_L19_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 93, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); - __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 93, __pyx_L19_except_error) + __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 93, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_7, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_17)) __PYX_ERR(1, 93, __pyx_L19_except_error) + if (unlikely(!__pyx_t_17)) __PYX_ERR(2, 93, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_17); __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_17); __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; - if (__pyx_t_14 < 0) __PYX_ERR(1, 93, __pyx_L19_except_error) + if (__pyx_t_14 < 0) __PYX_ERR(2, 93, __pyx_L15_except_error) __pyx_t_13 = ((!(__pyx_t_14 != 0)) != 0); if (__pyx_t_13) { __Pyx_GIVEREF(__pyx_t_2); @@ -3847,29 +4205,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGIVEREF(__pyx_t_16); __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_8, __pyx_t_16); __pyx_t_2 = 0; __pyx_t_8 = 0; __pyx_t_16 = 0; - __PYX_ERR(1, 93, __pyx_L19_except_error) + __PYX_ERR(2, 93, __pyx_L15_except_error) } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - goto __pyx_L18_exception_handled; + goto __pyx_L14_exception_handled; } - __pyx_L19_except_error:; - __Pyx_PyThreadState_assign + __pyx_L15_except_error:; __Pyx_XGIVEREF(__pyx_t_10); __Pyx_XGIVEREF(__pyx_t_11); __Pyx_XGIVEREF(__pyx_t_12); __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); goto __pyx_L3_error; - __pyx_L21_try_return:; - __Pyx_PyThreadState_assign + __pyx_L17_try_return:; __Pyx_XGIVEREF(__pyx_t_10); __Pyx_XGIVEREF(__pyx_t_11); __Pyx_XGIVEREF(__pyx_t_12); __Pyx_ExceptionReset(__pyx_t_10, __pyx_t_11, __pyx_t_12); - goto __pyx_L14_return; - __pyx_L18_exception_handled:; - __Pyx_PyThreadState_assign + goto __pyx_L10_return; + __pyx_L14_exception_handled:; __Pyx_XGIVEREF(__pyx_t_10); __Pyx_XGIVEREF(__pyx_t_11); __Pyx_XGIVEREF(__pyx_t_12); @@ -3879,21 +4234,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ /*finally:*/ { /*normal exit:*/{ if (__pyx_t_9) { - __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__10, NULL); + __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__12, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_12)) __PYX_ERR(1, 93, __pyx_L3_error) + if (unlikely(!__pyx_t_12)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; } - goto __pyx_L16; + goto __pyx_L12; } - __pyx_L14_return: { + __pyx_L10_return: { __pyx_t_12 = __pyx_r; __pyx_r = 0; if (__pyx_t_9) { - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__11, NULL); + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__13, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_11)) __PYX_ERR(1, 93, __pyx_L3_error) + if (unlikely(!__pyx_t_11)) __PYX_ERR(2, 93, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } @@ -3901,13 +4256,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_12 = 0; goto __pyx_L7_try_return; } - __pyx_L16:; + __pyx_L12:; } - goto __pyx_L38; - __pyx_L11_error:; + goto __pyx_L32; + __pyx_L9_error:; __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; goto __pyx_L3_error; - __pyx_L38:; + __pyx_L32:; } /* "reppy/robots.pyx":90 @@ -3921,9 +4276,8 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; - goto __pyx_L10_try_end; + goto __pyx_L8_try_end; __pyx_L3_error:; - __Pyx_PyThreadState_assign __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3938,14 +4292,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: */ - __Pyx_TraceLine(118,0,__PYX_ERR(1, 118, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 118, __pyx_L5_except_error) + __Pyx_TraceLine(118,0,__PYX_ERR(2, 118, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 118, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(1, 118, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(2, 118, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); @@ -3959,13 +4313,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) */ - __Pyx_TraceLine(119,0,__PYX_ERR(1, 119, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 119, __pyx_L5_except_error) + __Pyx_TraceLine(119,0,__PYX_ERR(2, 119, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 119, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 119, __pyx_L5_except_error) + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3982,14 +4336,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: */ - __Pyx_TraceLine(120,0,__PYX_ERR(1, 120, __pyx_L5_except_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 120, __pyx_L5_except_error) + __Pyx_TraceLine(120,0,__PYX_ERR(2, 120, __pyx_L5_except_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(1, 120, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 120, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); @@ -4003,13 +4357,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) */ - __Pyx_TraceLine(121,0,__PYX_ERR(1, 121, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 121, __pyx_L5_except_error) + __Pyx_TraceLine(121,0,__PYX_ERR(2, 121, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 121, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 121, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 121, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 121, __pyx_L5_except_error) + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 121, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -4026,14 +4380,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: */ - __Pyx_TraceLine(122,0,__PYX_ERR(1, 122, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 122, __pyx_L5_except_error) + __Pyx_TraceLine(122,0,__PYX_ERR(2, 122, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 122, __pyx_L5_except_error) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 122, __pyx_L5_except_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(1, 122, __pyx_L5_except_error) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_2) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4042,7 +4396,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(1, 122, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); @@ -4056,13 +4410,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * except TooManyRedirects as exc: * wrap_exception(exceptions.ExcessiveRedirects, exc) */ - __Pyx_TraceLine(123,0,__PYX_ERR(1, 123, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 123, __pyx_L5_except_error) + __Pyx_TraceLine(123,0,__PYX_ERR(2, 123, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 123, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 123, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4079,14 +4433,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrap_exception(exceptions.ExcessiveRedirects, exc) * */ - __Pyx_TraceLine(124,0,__PYX_ERR(1, 124, __pyx_L5_except_error)) - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(1, 124, __pyx_L5_except_error) + __Pyx_TraceLine(124,0,__PYX_ERR(2, 124, __pyx_L5_except_error)) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 124, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(1, 124, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(2, 124, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_7); @@ -4100,13 +4454,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * * def RobotsUrlMethod(cls, url): */ - __Pyx_TraceLine(125,0,__PYX_ERR(1, 125, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 125, __pyx_L5_except_error) + __Pyx_TraceLine(125,0,__PYX_ERR(2, 125, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 125, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(1, 125, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4125,26 +4479,23 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_PyThreadState_assign __Pyx_XGIVEREF(__pyx_t_3); __Pyx_XGIVEREF(__pyx_t_4); __Pyx_XGIVEREF(__pyx_t_5); __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); goto __pyx_L1_error; __pyx_L7_try_return:; - __Pyx_PyThreadState_assign __Pyx_XGIVEREF(__pyx_t_3); __Pyx_XGIVEREF(__pyx_t_4); __Pyx_XGIVEREF(__pyx_t_5); __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); goto __pyx_L0; __pyx_L4_exception_handled:; - __Pyx_PyThreadState_assign __Pyx_XGIVEREF(__pyx_t_3); __Pyx_XGIVEREF(__pyx_t_4); __Pyx_XGIVEREF(__pyx_t_5); __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); - __pyx_L10_try_end:; + __pyx_L8_try_end:; } /* "reppy/robots.pyx":80 @@ -4208,7 +4559,9 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } @@ -4217,14 +4570,15 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(1, 127, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(2, 127, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(1, 127, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(2, 127, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4237,7 +4591,7 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 127, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 127, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4258,9 +4612,9 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject std::string __pyx_t_2; std::string __pyx_t_3; PyObject *__pyx_t_4 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__12) + __Pyx_TraceFrameInit(__pyx_codeobj__14) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); - __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[1], 127, 0, __PYX_ERR(1, 127, __pyx_L1_error)); + __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[2], 127, 0, __PYX_ERR(2, 127, __pyx_L1_error)); /* "reppy/robots.pyx":129 * def RobotsUrlMethod(cls, url): @@ -4269,21 +4623,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject * * cdef class Robots: */ - __Pyx_TraceLine(129,0,__PYX_ERR(1, 129, __pyx_L1_error)) + __Pyx_TraceLine(129,0,__PYX_ERR(2, 129, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 129, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 129, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 129, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 129, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 129, __pyx_L1_error) + __PYX_ERR(2, 129, __pyx_L1_error) } - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 129, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 129, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 129, __pyx_L1_error) + __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 129, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_4; @@ -4337,8 +4691,11 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } @@ -4347,11 +4704,13 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(1, 147, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(2, 147, __pyx_L3_error) } + CYTHON_FALLTHROUGH; case 2: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); @@ -4359,11 +4718,12 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 147, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 147, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 3: values[2] = PyTuple_GET_ITEM(__pyx_args, 2); + CYTHON_FALLTHROUGH; case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; @@ -4371,12 +4731,12 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } __pyx_v_url = values[0]; - __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 147, __pyx_L3_error) + __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 147, __pyx_L3_error) __pyx_v_expires = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 147, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 147, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4397,7 +4757,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob std::string __pyx_t_2; Rep::Robots *__pyx_t_3; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 147, 0, __PYX_ERR(1, 147, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 147, 0, __PYX_ERR(2, 147, __pyx_L1_error)); /* "reppy/robots.pyx":148 * @@ -4406,16 +4766,16 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob * self.expires = expires * */ - __Pyx_TraceLine(148,0,__PYX_ERR(1, 148, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 148, __pyx_L1_error) + __Pyx_TraceLine(148,0,__PYX_ERR(2, 148, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 148, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 148, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = new Rep::Robots(__pyx_v_content, __pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(1, 148, __pyx_L1_error) + __PYX_ERR(2, 148, __pyx_L1_error) } __pyx_v_self->robots = __pyx_t_3; @@ -4426,7 +4786,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob * * def __str__(self): */ - __Pyx_TraceLine(149,0,__PYX_ERR(1, 149, __pyx_L1_error)) + __Pyx_TraceLine(149,0,__PYX_ERR(2, 149, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); __Pyx_GOTREF(__pyx_v_self->expires); @@ -4458,7 +4818,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< - * return self.robots.str() + * return self.robots.str().decode('utf8') * */ @@ -4481,18 +4841,18 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[1], 151, 0, __PYX_ERR(1, 151, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[2], 151, 0, __PYX_ERR(2, 151, __pyx_L1_error)); /* "reppy/robots.pyx":152 * * def __str__(self): - * return self.robots.str() # <<<<<<<<<<<<<< + * return self.robots.str().decode('utf8') # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __Pyx_TraceLine(152,0,__PYX_ERR(1, 152, __pyx_L1_error)) + __Pyx_TraceLine(152,0,__PYX_ERR(2, 152, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 152, __pyx_L1_error) + __pyx_t_1 = __Pyx_decode_cpp_string(__pyx_v_self->robots->str(), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 152, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -4502,7 +4862,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< - * return self.robots.str() + * return self.robots.str().decode('utf8') * */ @@ -4519,7 +4879,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp } /* "reppy/robots.pyx":154 - * return self.robots.str() + * return self.robots.str().decode('utf8') * * def __dealloc__(self): # <<<<<<<<<<<<<< * del self.robots @@ -4541,7 +4901,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - __Pyx_TraceCall("__dealloc__", __pyx_f[1], 154, 0, __PYX_ERR(1, 154, __pyx_L1_error)); + __Pyx_TraceCall("__dealloc__", __pyx_f[2], 154, 0, __PYX_ERR(2, 154, __pyx_L1_error)); /* "reppy/robots.pyx":155 * @@ -4550,11 +4910,11 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy * * @property */ - __Pyx_TraceLine(155,0,__PYX_ERR(1, 155, __pyx_L1_error)) + __Pyx_TraceLine(155,0,__PYX_ERR(2, 155, __pyx_L1_error)) delete __pyx_v_self->robots; /* "reppy/robots.pyx":154 - * return self.robots.str() + * return self.robots.str().decode('utf8') * * def __dealloc__(self): # <<<<<<<<<<<<<< * del self.robots @@ -4564,7 +4924,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy /* function exit code */ goto __pyx_L0; __pyx_L1_error:; - __Pyx_WriteUnraisable("reppy.robots.Robots.__dealloc__", __pyx_clineno, __pyx_lineno, __pyx_filename, 0, 0); + __Pyx_WriteUnraisable("reppy.robots.Robots.__dealloc__", __pyx_clineno, __pyx_lineno, __pyx_filename, 1, 0); __pyx_L0:; __Pyx_TraceReturn(Py_None, 0); __Pyx_RefNannyFinishContext(); @@ -4599,7 +4959,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 158, 0, __PYX_ERR(1, 158, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 158, 0, __PYX_ERR(2, 158, __pyx_L1_error)); /* "reppy/robots.pyx":160 * def sitemaps(self): @@ -4608,13 +4968,13 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ * * def allowed(self, path, name): */ - __Pyx_TraceLine(160,0,__PYX_ERR(1, 160, __pyx_L1_error)) + __Pyx_TraceLine(160,0,__PYX_ERR(2, 160, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 160, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 160, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 160, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -4622,7 +4982,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 160, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; @@ -4676,7 +5036,9 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } @@ -4685,14 +5047,15 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(1, 162, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(2, 162, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(1, 162, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(2, 162, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4705,7 +5068,7 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 162, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 162, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4726,7 +5089,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp std::string __pyx_t_2; std::string __pyx_t_3; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[1], 162, 0, __PYX_ERR(1, 162, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[2], 162, 0, __PYX_ERR(2, 162, __pyx_L1_error)); /* "reppy/robots.pyx":164 * def allowed(self, path, name): @@ -4735,17 +5098,17 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp * * def agent(self, name): */ - __Pyx_TraceLine(164,0,__PYX_ERR(1, 164, __pyx_L1_error)) + __Pyx_TraceLine(164,0,__PYX_ERR(2, 164, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 164, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 164, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 164, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -4804,7 +5167,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ int __pyx_t_5; PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("agent", 0); - __Pyx_TraceCall("agent", __pyx_f[1], 166, 0, __PYX_ERR(1, 166, __pyx_L1_error)); + __Pyx_TraceCall("agent", __pyx_f[2], 166, 0, __PYX_ERR(2, 166, __pyx_L1_error)); /* "reppy/robots.pyx":173 * Agent object. @@ -4813,11 +5176,11 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ * * @property */ - __Pyx_TraceLine(173,0,__PYX_ERR(1, 173, __pyx_L1_error)) + __Pyx_TraceLine(173,0,__PYX_ERR(2, 173, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 173, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 173, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 173, __pyx_L1_error) + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 173, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; __pyx_t_5 = 0; @@ -4834,7 +5197,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 173, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 173, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -4843,14 +5206,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 173, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 173, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(1, 173, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 173, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -4861,7 +5224,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 173, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 173, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -4923,7 +5286,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 176, 0, __PYX_ERR(1, 176, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 176, 0, __PYX_ERR(2, 176, __pyx_L1_error)); /* "reppy/robots.pyx":178 * def expired(self): @@ -4932,11 +5295,11 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o * * @property */ - __Pyx_TraceLine(178,0,__PYX_ERR(1, 178, __pyx_L1_error)) + __Pyx_TraceLine(178,0,__PYX_ERR(2, 178, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 178, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 178, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -4950,14 +5313,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } } if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 178, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 178, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 178, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; @@ -5011,7 +5374,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 181, 0, __PYX_ERR(1, 181, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 181, 0, __PYX_ERR(2, 181, __pyx_L1_error)); /* "reppy/robots.pyx":183 * def expires(self): @@ -5020,7 +5383,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o * * @property */ - __Pyx_TraceLine(183,0,__PYX_ERR(1, 183, __pyx_L1_error)) + __Pyx_TraceLine(183,0,__PYX_ERR(2, 183, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_self->expires); __pyx_r = __pyx_v_self->expires; @@ -5077,7 +5440,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[1], 186, 0, __PYX_ERR(1, 186, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 186, 0, __PYX_ERR(2, 186, __pyx_L1_error)); /* "reppy/robots.pyx":188 * def ttl(self): @@ -5086,12 +5449,12 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 * * */ - __Pyx_TraceLine(188,0,__PYX_ERR(1, 188, __pyx_L1_error)) + __Pyx_TraceLine(188,0,__PYX_ERR(2, 188, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -5105,24 +5468,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 188, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 188, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 188, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(2, 188, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 188, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 188, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -5159,6 +5522,121 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 return __pyx_r; } +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_11__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_11__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_10__reduce_cython__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6Robots_10__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + __Pyx_TraceCall("__reduce_cython__", __pyx_f[1], 1, 0, __PYX_ERR(1, 1, __pyx_L1_error)); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + __Pyx_TraceLine(2,0,__PYX_ERR(1, 2, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__15, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Robots.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_13__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_6Robots_13__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_6Robots_12__setstate_cython__(((struct __pyx_obj_5reppy_6robots_Robots *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_6Robots_12__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + __Pyx_TraceCall("__setstate_cython__", __pyx_f[1], 3, 0, __PYX_ERR(1, 3, __pyx_L1_error)); + + /* "(tree fragment)":4 + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__16, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.Robots.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + /* "reppy/robots.pyx":194 * '''No requests are allowed.''' * @@ -5184,7 +5662,9 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } @@ -5193,6 +5673,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); @@ -5200,11 +5681,12 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 194, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 194, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; @@ -5215,7 +5697,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 194, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 194, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5238,7 +5720,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 194, 0, __PYX_ERR(1, 194, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 194, 0, __PYX_ERR(2, 194, __pyx_L1_error)); /* "reppy/robots.pyx":195 * @@ -5247,8 +5729,8 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 * * */ - __Pyx_TraceLine(195,0,__PYX_ERR(1, 195, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 195, __pyx_L1_error) + __Pyx_TraceLine(195,0,__PYX_ERR(2, 195, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 195, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -5265,7 +5747,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 195, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 195, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -5273,13 +5755,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 195, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 195, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 195, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 195, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -5296,7 +5778,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 195, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 195, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } @@ -5327,6 +5809,121 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 return __pyx_r; } +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_9AllowNone_3__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_9AllowNone_3__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_9AllowNone_2__reduce_cython__(((struct __pyx_obj_5reppy_6robots_AllowNone *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_9AllowNone_2__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_AllowNone *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + __Pyx_TraceCall("__reduce_cython__", __pyx_f[1], 1, 0, __PYX_ERR(1, 1, __pyx_L1_error)); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + __Pyx_TraceLine(2,0,__PYX_ERR(1, 2, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__17, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.AllowNone.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_9AllowNone_5__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_9AllowNone_5__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_9AllowNone_4__setstate_cython__(((struct __pyx_obj_5reppy_6robots_AllowNone *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_9AllowNone_4__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_AllowNone *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + __Pyx_TraceCall("__setstate_cython__", __pyx_f[1], 3, 0, __PYX_ERR(1, 3, __pyx_L1_error)); + + /* "(tree fragment)":4 + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__18, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.AllowNone.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + /* "reppy/robots.pyx":201 * '''All requests are allowed.''' * @@ -5351,7 +5948,9 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P const Py_ssize_t pos_args = PyTuple_GET_SIZE(__pyx_args); switch (pos_args) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); + CYTHON_FALLTHROUGH; case 0: break; default: goto __pyx_L5_argtuple_error; } @@ -5360,6 +5959,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P case 0: if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; + CYTHON_FALLTHROUGH; case 1: if (kw_args > 0) { PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); @@ -5367,11 +5967,12 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(1, 201, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 201, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { case 2: values[1] = PyTuple_GET_ITEM(__pyx_args, 1); + CYTHON_FALLTHROUGH; case 1: values[0] = PyTuple_GET_ITEM(__pyx_args, 0); break; default: goto __pyx_L5_argtuple_error; @@ -5382,7 +5983,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(1, 201, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 201, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5405,15 +6006,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[1], 201, 0, __PYX_ERR(1, 201, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 201, 0, __PYX_ERR(2, 201, __pyx_L1_error)); /* "reppy/robots.pyx":202 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(202,0,__PYX_ERR(1, 202, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 202, __pyx_L1_error) + __Pyx_TraceLine(202,0,__PYX_ERR(2, 202, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 202, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -5429,22 +6030,22 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__13, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 202, __pyx_L1_error) + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 202, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__13, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 202, __pyx_L1_error) + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 202, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 202, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 202, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -5455,13 +6056,13 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_4, __pyx_v_url); - __Pyx_INCREF(__pyx_kp_b__13); - __Pyx_GIVEREF(__pyx_kp_b__13); - PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b__13); + __Pyx_INCREF(__pyx_kp_b__19); + __Pyx_GIVEREF(__pyx_kp_b__19); + PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b__19); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 202, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 202, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } @@ -5491,43 +6092,158 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r return __pyx_r; } +/* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_8AllowAll_3__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_8AllowAll_3__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_8AllowAll_2__reduce_cython__(((struct __pyx_obj_5reppy_6robots_AllowAll *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_8AllowAll_2__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_AllowAll *__pyx_v_self) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__reduce_cython__", 0); + __Pyx_TraceCall("__reduce_cython__", __pyx_f[1], 1, 0, __PYX_ERR(1, 1, __pyx_L1_error)); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + __Pyx_TraceLine(2,0,__PYX_ERR(1, 2, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__20, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 2, __pyx_L1_error) + + /* "(tree fragment)":1 + * def __reduce_cython__(self): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.AllowAll.__reduce_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + +/* Python wrapper */ +static PyObject *__pyx_pw_5reppy_6robots_8AllowAll_5__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_8AllowAll_5__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_8AllowAll_4__setstate_cython__(((struct __pyx_obj_5reppy_6robots_AllowAll *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_5reppy_6robots_8AllowAll_4__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_AllowAll *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { + PyObject *__pyx_r = NULL; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + __Pyx_RefNannySetupContext("__setstate_cython__", 0); + __Pyx_TraceCall("__setstate_cython__", __pyx_f[1], 3, 0, __PYX_ERR(1, 3, __pyx_L1_error)); + + /* "(tree fragment)":4 + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__21, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_Raise(__pyx_t_1, 0, 0, 0); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __PYX_ERR(1, 4, __pyx_L1_error) + + /* "(tree fragment)":3 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): # <<<<<<<<<<<<<< + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("reppy.robots.AllowAll.__setstate_cython__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_TraceReturn(__pyx_r, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + /* "string.from_py":13 * * @cname("__pyx_convert_string_from_py_std__in_string") * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) */ static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v_o) { Py_ssize_t __pyx_v_length; - char *__pyx_v_data; + char const *__pyx_v_data; std::string __pyx_r; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations - char *__pyx_t_1; + char const *__pyx_t_1; __Pyx_RefNannySetupContext("__pyx_convert_string_from_py_std__in_string", 0); - __Pyx_TraceCall("__pyx_convert_string_from_py_std__in_string", __pyx_f[2], 13, 0, __PYX_ERR(2, 13, __pyx_L1_error)); + __Pyx_TraceCall("__pyx_convert_string_from_py_std__in_string", __pyx_f[1], 13, 0, __PYX_ERR(1, 13, __pyx_L1_error)); /* "string.from_py":15 * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< + * cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) # <<<<<<<<<<<<<< * return string(data, length) * */ - __Pyx_TraceLine(15,0,__PYX_ERR(2, 15, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == NULL)) __PYX_ERR(2, 15, __pyx_L1_error) + __Pyx_TraceLine(15,0,__PYX_ERR(1, 15, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_AsStringAndSize(__pyx_v_o, (&__pyx_v_length)); if (unlikely(__pyx_t_1 == ((char const *)NULL))) __PYX_ERR(1, 15, __pyx_L1_error) __pyx_v_data = __pyx_t_1; /* "string.from_py":16 * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) * return string(data, length) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(16,0,__PYX_ERR(2, 16, __pyx_L1_error)) + __Pyx_TraceLine(16,0,__PYX_ERR(1, 16, __pyx_L1_error)) __pyx_r = std::string(__pyx_v_data, __pyx_v_length); goto __pyx_L0; @@ -5536,12 +6252,13 @@ static std::string __pyx_convert_string_from_py_std__in_string(PyObject *__pyx_v * @cname("__pyx_convert_string_from_py_std__in_string") * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< * cdef Py_ssize_t length - * cdef char* data = __Pyx_PyObject_AsStringAndSize(o, &length) + * cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) */ /* function exit code */ __pyx_L1_error:; __Pyx_AddTraceback("string.from_py.__pyx_convert_string_from_py_std__in_string", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); __pyx_L0:; __Pyx_TraceReturn(Py_None, 0); __Pyx_RefNannyFinishContext(); @@ -5562,18 +6279,18 @@ static CYTHON_INLINE PyObject *__pyx_convert_PyObject_string_to_py_std__in_strin __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__pyx_convert_PyObject_string_to_py_std__in_string", 0); - __Pyx_TraceCall("__pyx_convert_PyObject_string_to_py_std__in_string", __pyx_f[2], 31, 0, __PYX_ERR(2, 31, __pyx_L1_error)); + __Pyx_TraceCall("__pyx_convert_PyObject_string_to_py_std__in_string", __pyx_f[1], 31, 0, __PYX_ERR(1, 31, __pyx_L1_error)); /* "string.to_py":32 * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< * cdef extern from *: - * cdef object __Pyx_PyUnicode_FromStringAndSize(char*, size_t) + * cdef object __Pyx_PyUnicode_FromStringAndSize(const char*, size_t) */ - __Pyx_TraceLine(32,0,__PYX_ERR(2, 32, __pyx_L1_error)) + __Pyx_TraceLine(32,0,__PYX_ERR(1, 32, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyObject_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 32, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 32, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5613,18 +6330,18 @@ static CYTHON_INLINE PyObject *__pyx_convert_PyUnicode_string_to_py_std__in_stri __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__pyx_convert_PyUnicode_string_to_py_std__in_string", 0); - __Pyx_TraceCall("__pyx_convert_PyUnicode_string_to_py_std__in_string", __pyx_f[2], 37, 0, __PYX_ERR(2, 37, __pyx_L1_error)); + __Pyx_TraceCall("__pyx_convert_PyUnicode_string_to_py_std__in_string", __pyx_f[1], 37, 0, __PYX_ERR(1, 37, __pyx_L1_error)); /* "string.to_py":38 * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< * cdef extern from *: - * cdef object __Pyx_PyStr_FromStringAndSize(char*, size_t) + * cdef object __Pyx_PyStr_FromStringAndSize(const char*, size_t) */ - __Pyx_TraceLine(38,0,__PYX_ERR(2, 38, __pyx_L1_error)) + __Pyx_TraceLine(38,0,__PYX_ERR(1, 38, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyUnicode_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 38, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyUnicode_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 38, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5664,18 +6381,18 @@ static CYTHON_INLINE PyObject *__pyx_convert_PyStr_string_to_py_std__in_string(s __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__pyx_convert_PyStr_string_to_py_std__in_string", 0); - __Pyx_TraceCall("__pyx_convert_PyStr_string_to_py_std__in_string", __pyx_f[2], 43, 0, __PYX_ERR(2, 43, __pyx_L1_error)); + __Pyx_TraceCall("__pyx_convert_PyStr_string_to_py_std__in_string", __pyx_f[1], 43, 0, __PYX_ERR(1, 43, __pyx_L1_error)); /* "string.to_py":44 * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< * cdef extern from *: - * cdef object __Pyx_PyBytes_FromStringAndSize(char*, size_t) + * cdef object __Pyx_PyBytes_FromStringAndSize(const char*, size_t) */ - __Pyx_TraceLine(44,0,__PYX_ERR(2, 44, __pyx_L1_error)) + __Pyx_TraceLine(44,0,__PYX_ERR(1, 44, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyStr_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 44, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyStr_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 44, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5715,18 +6432,18 @@ static CYTHON_INLINE PyObject *__pyx_convert_PyBytes_string_to_py_std__in_string __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__pyx_convert_PyBytes_string_to_py_std__in_string", 0); - __Pyx_TraceCall("__pyx_convert_PyBytes_string_to_py_std__in_string", __pyx_f[2], 49, 0, __PYX_ERR(2, 49, __pyx_L1_error)); + __Pyx_TraceCall("__pyx_convert_PyBytes_string_to_py_std__in_string", __pyx_f[1], 49, 0, __PYX_ERR(1, 49, __pyx_L1_error)); /* "string.to_py":50 * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< * cdef extern from *: - * cdef object __Pyx_PyByteArray_FromStringAndSize(char*, size_t) + * cdef object __Pyx_PyByteArray_FromStringAndSize(const char*, size_t) */ - __Pyx_TraceLine(50,0,__PYX_ERR(2, 50, __pyx_L1_error)) + __Pyx_TraceLine(50,0,__PYX_ERR(1, 50, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 50, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBytes_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 50, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5766,7 +6483,7 @@ static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_st __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__pyx_convert_PyByteArray_string_to_py_std__in_string", 0); - __Pyx_TraceCall("__pyx_convert_PyByteArray_string_to_py_std__in_string", __pyx_f[2], 55, 0, __PYX_ERR(2, 55, __pyx_L1_error)); + __Pyx_TraceCall("__pyx_convert_PyByteArray_string_to_py_std__in_string", __pyx_f[1], 55, 0, __PYX_ERR(1, 55, __pyx_L1_error)); /* "string.to_py":56 * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") @@ -5774,9 +6491,9 @@ static CYTHON_INLINE PyObject *__pyx_convert_PyByteArray_string_to_py_std__in_st * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) # <<<<<<<<<<<<<< * */ - __Pyx_TraceLine(56,0,__PYX_ERR(2, 56, __pyx_L1_error)) + __Pyx_TraceLine(56,0,__PYX_ERR(1, 56, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_PyByteArray_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 56, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyByteArray_FromStringAndSize(__pyx_v_s.data(), __pyx_v_s.size()); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 56, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5835,7 +6552,7 @@ static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_ __Pyx_RefNannySetupContext("wrap", 0); __pyx_outer_scope = (struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *) __Pyx_CyFunction_GetClosure(__pyx_self); __pyx_cur_scope = __pyx_outer_scope; - __Pyx_TraceCall("wrap", __pyx_f[2], 65, 0, __PYX_ERR(2, 65, __pyx_L1_error)); + __Pyx_TraceCall("wrap", __pyx_f[1], 65, 0, __PYX_ERR(1, 65, __pyx_L1_error)); /* "cfunc.to_py":67 * def wrap(object value): @@ -5844,9 +6561,9 @@ static PyObject *__pyx_pf_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_ * return wrap * */ - __Pyx_TraceLine(67,0,__PYX_ERR(2, 67, __pyx_L1_error)) + __Pyx_TraceLine(67,0,__PYX_ERR(1, 67, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_cur_scope->__pyx_v_f(__pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 67, __pyx_L1_error) + __pyx_t_1 = __pyx_cur_scope->__pyx_v_f(__pyx_v_value); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 67, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -5892,11 +6609,11 @@ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyO if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(2, 64, __pyx_L1_error) + __PYX_ERR(1, 64, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } - __Pyx_TraceCall("__Pyx_CFunc_object____object___to_py", __pyx_f[2], 64, 0, __PYX_ERR(2, 64, __pyx_L1_error)); + __Pyx_TraceCall("__Pyx_CFunc_object____object___to_py", __pyx_f[1], 64, 0, __PYX_ERR(1, 64, __pyx_L1_error)); __pyx_cur_scope->__pyx_v_f = __pyx_v_f; /* "cfunc.to_py":65 @@ -5906,8 +6623,8 @@ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyO * """wrap(value)""" * return f(value) */ - __Pyx_TraceLine(65,0,__PYX_ERR(2, 65, __pyx_L1_error)) - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__15)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 65, __pyx_L1_error) + __Pyx_TraceLine(65,0,__PYX_ERR(1, 65, __pyx_L1_error)) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__23)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 65, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_wrap = __pyx_t_1; __pyx_t_1 = 0; @@ -5919,7 +6636,7 @@ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyO * * */ - __Pyx_TraceLine(68,0,__PYX_ERR(2, 68, __pyx_L1_error)) + __Pyx_TraceLine(68,0,__PYX_ERR(1, 68, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_wrap); __pyx_r = __pyx_v_wrap; @@ -5947,11 +6664,11 @@ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyO return __pyx_r; } -/* "vector.to_py":67 +/* "vector.to_py":60 * * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): # <<<<<<<<<<<<<< - * return [X_to_py(v[i]) for i in range(v.size())] + * return [v[i] for i in range(v.size())] * */ @@ -5965,36 +6682,36 @@ static PyObject *__pyx_convert_vector_to_py_std_3a__3a_string(const std::vector< size_t __pyx_t_3; PyObject *__pyx_t_4 = NULL; __Pyx_RefNannySetupContext("__pyx_convert_vector_to_py_std_3a__3a_string", 0); - __Pyx_TraceCall("__pyx_convert_vector_to_py_std_3a__3a_string", __pyx_f[2], 67, 0, __PYX_ERR(2, 67, __pyx_L1_error)); + __Pyx_TraceCall("__pyx_convert_vector_to_py_std_3a__3a_string", __pyx_f[1], 60, 0, __PYX_ERR(1, 60, __pyx_L1_error)); - /* "vector.to_py":68 + /* "vector.to_py":61 * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): - * return [X_to_py(v[i]) for i in range(v.size())] # <<<<<<<<<<<<<< + * return [v[i] for i in range(v.size())] # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(68,0,__PYX_ERR(2, 68, __pyx_L1_error)) + __Pyx_TraceLine(61,0,__PYX_ERR(1, 61, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 68, __pyx_L1_error) + __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 61, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_v_v.size(); for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { __pyx_v_i = __pyx_t_3; - __pyx_t_4 = __pyx_convert_PyObject_string_to_py_std__in_string((__pyx_v_v[__pyx_v_i])); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 68, __pyx_L1_error) + __pyx_t_4 = __pyx_convert_PyBytes_string_to_py_std__in_string((__pyx_v_v[__pyx_v_i])); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 61, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) __PYX_ERR(2, 68, __pyx_L1_error) + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) __PYX_ERR(1, 61, __pyx_L1_error) __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "vector.to_py":67 + /* "vector.to_py":60 * * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): # <<<<<<<<<<<<<< - * return [X_to_py(v[i]) for i in range(v.size())] + * return [v[i] for i in range(v.size())] * */ @@ -6027,8 +6744,8 @@ static PyObject *__pyx_tp_new_5reppy_6robots_Agent(PyTypeObject *t, CYTHON_UNUSE static void __pyx_tp_dealloc_5reppy_6robots_Agent(PyObject *o) { struct __pyx_obj_5reppy_6robots_Agent *p = (struct __pyx_obj_5reppy_6robots_Agent *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && (!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))) { if (PyObject_CallFinalizerFromDealloc(o)) return; } #endif @@ -6044,6 +6761,8 @@ static PyMethodDef __pyx_methods_5reppy_6robots_Agent[] = { {"allow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_3allow, METH_O, __pyx_doc_5reppy_6robots_5Agent_2allow}, {"disallow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_5disallow, METH_O, __pyx_doc_5reppy_6robots_5Agent_4disallow}, {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_7allowed, METH_O, __pyx_doc_5reppy_6robots_5Agent_6allowed}, + {"__reduce_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_9__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_11__setstate_cython__, METH_O, 0}, {0, 0, 0, 0} }; @@ -6126,8 +6845,8 @@ static PyObject *__pyx_tp_new_5reppy_6robots_Robots(PyTypeObject *t, CYTHON_UNUS static void __pyx_tp_dealloc_5reppy_6robots_Robots(PyObject *o) { struct __pyx_obj_5reppy_6robots_Robots *p = (struct __pyx_obj_5reppy_6robots_Robots *)o; - #if PY_VERSION_HEX >= 0x030400a1 - if (unlikely(Py_TYPE(o)->tp_finalize) && !_PyGC_FINALIZED(o)) { + #if CYTHON_USE_TP_FINALIZE + if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE) && Py_TYPE(o)->tp_finalize) && !_PyGC_FINALIZED(o)) { if (PyObject_CallFinalizerFromDealloc(o)) return; } #endif @@ -6181,6 +6900,8 @@ static PyObject *__pyx_getprop_5reppy_6robots_6Robots_ttl(PyObject *o, CYTHON_UN static PyMethodDef __pyx_methods_5reppy_6robots_Robots[] = { {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_7allowed, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_6Robots_6allowed}, {"agent", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_9agent, METH_O, __pyx_doc_5reppy_6robots_6Robots_8agent}, + {"__reduce_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_11__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_13__setstate_cython__, METH_O, 0}, {0, 0, 0, 0} }; @@ -6257,6 +6978,8 @@ static PyObject *__pyx_tp_new_5reppy_6robots_AllowNone(PyTypeObject *t, PyObject } static PyMethodDef __pyx_methods_5reppy_6robots_AllowNone[] = { + {"__reduce_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_9AllowNone_3__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_9AllowNone_5__setstate_cython__, METH_O, 0}, {0, 0, 0, 0} }; @@ -6329,6 +7052,8 @@ static PyObject *__pyx_tp_new_5reppy_6robots_AllowAll(PyTypeObject *t, PyObject } static PyMethodDef __pyx_methods_5reppy_6robots_AllowAll[] = { + {"__reduce_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_8AllowAll_3__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_8AllowAll_5__setstate_cython__, METH_O, 0}, {0, 0, 0, 0} }; @@ -6592,17 +7317,31 @@ static PyMethodDef __pyx_methods[] = { }; #if PY_MAJOR_VERSION >= 3 +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_robots(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_robots}, + {0, NULL} +}; +#endif + static struct PyModuleDef __pyx_moduledef = { - #if PY_VERSION_HEX < 0x03020000 - { PyObject_HEAD_INIT(NULL) NULL, 0, NULL }, - #else PyModuleDef_HEAD_INIT, - #endif "robots", 0, /* m_doc */ + #if CYTHON_PEP489_MULTI_PHASE_INIT + 0, /* m_size */ + #else -1, /* m_size */ + #endif __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else NULL, /* m_reload */ + #endif NULL, /* m_traverse */ NULL, /* m_clear */ NULL /* m_free */ @@ -6633,11 +7372,12 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_SSLError, __pyx_k_SSLError, sizeof(__pyx_k_SSLError), 0, 0, 1, 1}, {&__pyx_n_s_SSLException, __pyx_k_SSLException, sizeof(__pyx_k_SSLException), 0, 0, 1, 1}, {&__pyx_n_s_TooManyRedirects, __pyx_k_TooManyRedirects, sizeof(__pyx_k_TooManyRedirects), 0, 0, 1, 1}, + {&__pyx_n_s_TypeError, __pyx_k_TypeError, sizeof(__pyx_k_TypeError), 0, 0, 1, 1}, {&__pyx_n_s_URLRequired, __pyx_k_URLRequired, sizeof(__pyx_k_URLRequired), 0, 0, 1, 1}, {&__pyx_kp_b_User_agent_Disallow, __pyx_k_User_agent_Disallow, sizeof(__pyx_k_User_agent_Disallow), 0, 0, 0, 0}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, - {&__pyx_n_s__13, __pyx_k__13, sizeof(__pyx_k__13), 0, 0, 1, 1}, - {&__pyx_kp_b__13, __pyx_k__13, sizeof(__pyx_k__13), 0, 0, 0, 0}, + {&__pyx_n_s__19, __pyx_k__19, sizeof(__pyx_k__19), 0, 0, 1, 1}, + {&__pyx_kp_b__19, __pyx_k__19, sizeof(__pyx_k__19), 0, 0, 0, 0}, {&__pyx_n_s_after_parse_hook, __pyx_k_after_parse_hook, sizeof(__pyx_k_after_parse_hook), 0, 0, 1, 1}, {&__pyx_n_s_after_response_hook, __pyx_k_after_response_hook, sizeof(__pyx_k_after_response_hook), 0, 0, 1, 1}, {&__pyx_n_s_agent, __pyx_k_agent, sizeof(__pyx_k_agent), 0, 0, 1, 1}, @@ -6645,6 +7385,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_args, __pyx_k_args, sizeof(__pyx_k_args), 0, 0, 1, 1}, {&__pyx_n_s_cause, __pyx_k_cause, sizeof(__pyx_k_cause), 0, 0, 1, 1}, {&__pyx_n_s_cfunc_to_py, __pyx_k_cfunc_to_py, sizeof(__pyx_k_cfunc_to_py), 0, 0, 1, 1}, + {&__pyx_n_s_cline_in_traceback, __pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 0, 1, 1}, {&__pyx_n_s_closing, __pyx_k_closing, sizeof(__pyx_k_closing), 0, 0, 1, 1}, {&__pyx_n_s_cls, __pyx_k_cls, sizeof(__pyx_k_cls), 0, 0, 1, 1}, {&__pyx_n_s_content, __pyx_k_content, sizeof(__pyx_k_content), 0, 0, 1, 1}, @@ -6662,6 +7403,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_fetch, __pyx_k_fetch, sizeof(__pyx_k_fetch), 0, 0, 1, 1}, {&__pyx_n_s_from_robots, __pyx_k_from_robots, sizeof(__pyx_k_from_robots), 0, 0, 1, 1}, {&__pyx_n_s_get, __pyx_k_get, sizeof(__pyx_k_get), 0, 0, 1, 1}, + {&__pyx_n_s_getstate, __pyx_k_getstate, sizeof(__pyx_k_getstate), 0, 0, 1, 1}, {&__pyx_n_s_import, __pyx_k_import, sizeof(__pyx_k_import), 0, 0, 1, 1}, {&__pyx_n_s_init, __pyx_k_init, sizeof(__pyx_k_init), 0, 0, 1, 1}, {&__pyx_n_s_kwargs, __pyx_k_kwargs, sizeof(__pyx_k_kwargs), 0, 0, 1, 1}, @@ -6671,18 +7413,27 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_max_size, __pyx_k_max_size, sizeof(__pyx_k_max_size), 0, 0, 1, 1}, {&__pyx_n_s_minimum, __pyx_k_minimum, sizeof(__pyx_k_minimum), 0, 0, 1, 1}, {&__pyx_n_s_name, __pyx_k_name, sizeof(__pyx_k_name), 0, 0, 1, 1}, + {&__pyx_n_s_name_2, __pyx_k_name_2, sizeof(__pyx_k_name_2), 0, 0, 1, 1}, {&__pyx_n_s_parse, __pyx_k_parse, sizeof(__pyx_k_parse), 0, 0, 1, 1}, {&__pyx_n_s_path, __pyx_k_path, sizeof(__pyx_k_path), 0, 0, 1, 1}, {&__pyx_n_s_pop, __pyx_k_pop, sizeof(__pyx_k_pop), 0, 0, 1, 1}, {&__pyx_n_s_range, __pyx_k_range, sizeof(__pyx_k_range), 0, 0, 1, 1}, {&__pyx_n_s_raw, __pyx_k_raw, sizeof(__pyx_k_raw), 0, 0, 1, 1}, {&__pyx_n_s_read, __pyx_k_read, sizeof(__pyx_k_read), 0, 0, 1, 1}, + {&__pyx_n_s_reduce, __pyx_k_reduce, sizeof(__pyx_k_reduce), 0, 0, 1, 1}, + {&__pyx_n_s_reduce_cython, __pyx_k_reduce_cython, sizeof(__pyx_k_reduce_cython), 0, 0, 1, 1}, + {&__pyx_n_s_reduce_ex, __pyx_k_reduce_ex, sizeof(__pyx_k_reduce_ex), 0, 0, 1, 1}, {&__pyx_n_s_reppy_robots, __pyx_k_reppy_robots, sizeof(__pyx_k_reppy_robots), 0, 0, 1, 1}, + {&__pyx_kp_s_reppy_robots_pyx, __pyx_k_reppy_robots_pyx, sizeof(__pyx_k_reppy_robots_pyx), 0, 0, 1, 0}, {&__pyx_n_s_requests, __pyx_k_requests, sizeof(__pyx_k_requests), 0, 0, 1, 1}, {&__pyx_n_s_requests_exceptions, __pyx_k_requests_exceptions, sizeof(__pyx_k_requests_exceptions), 0, 0, 1, 1}, {&__pyx_n_s_res, __pyx_k_res, sizeof(__pyx_k_res), 0, 0, 1, 1}, {&__pyx_n_s_robots, __pyx_k_robots, sizeof(__pyx_k_robots), 0, 0, 1, 1}, {&__pyx_n_s_robots_url, __pyx_k_robots_url, sizeof(__pyx_k_robots_url), 0, 0, 1, 1}, + {&__pyx_kp_s_self_agent_cannot_be_converted_t, __pyx_k_self_agent_cannot_be_converted_t, sizeof(__pyx_k_self_agent_cannot_be_converted_t), 0, 0, 1, 0}, + {&__pyx_kp_s_self_robots_cannot_be_converted, __pyx_k_self_robots_cannot_be_converted, sizeof(__pyx_k_self_robots_cannot_be_converted), 0, 0, 1, 0}, + {&__pyx_n_s_setstate, __pyx_k_setstate, sizeof(__pyx_k_setstate), 0, 0, 1, 1}, + {&__pyx_n_s_setstate_cython, __pyx_k_setstate_cython, sizeof(__pyx_k_setstate_cython), 0, 0, 1, 1}, {&__pyx_n_s_six, __pyx_k_six, sizeof(__pyx_k_six), 0, 0, 1, 1}, {&__pyx_n_s_status_code, __pyx_k_status_code, sizeof(__pyx_k_status_code), 0, 0, 1, 1}, {&__pyx_n_s_stream, __pyx_k_stream, sizeof(__pyx_k_stream), 0, 0, 1, 1}, @@ -6694,7 +7445,6 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_url, __pyx_k_url, sizeof(__pyx_k_url), 0, 0, 1, 1}, {&__pyx_kp_s_utf_8, __pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 0, 1, 0}, {&__pyx_n_s_util, __pyx_k_util, sizeof(__pyx_k_util), 0, 0, 1, 1}, - {&__pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_k_vagrant_reppy_robots_pyx, sizeof(__pyx_k_vagrant_reppy_robots_pyx), 0, 0, 1, 0}, {&__pyx_n_s_value, __pyx_k_value, sizeof(__pyx_k_value), 0, 0, 1, 1}, {&__pyx_n_s_wrap, __pyx_k_wrap, sizeof(__pyx_k_wrap), 0, 0, 1, 1}, {&__pyx_n_s_wrap_exception, __pyx_k_wrap_exception, sizeof(__pyx_k_wrap_exception), 0, 0, 1, 1}, @@ -6703,8 +7453,9 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { }; static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) - __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(1, 160, __pyx_L1_error) - __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(2, 68, __pyx_L1_error) + __pyx_builtin_TypeError = __Pyx_GetBuiltinName(__pyx_n_s_TypeError); if (!__pyx_builtin_TypeError) __PYX_ERR(1, 2, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(2, 160, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(1, 61, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -6721,7 +7472,7 @@ static int __Pyx_InitCachedConstants(void) { * * cdef as_string(value): */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple_)) __PYX_ERR(1, 24, __pyx_L1_error) + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple_)) __PYX_ERR(2, 24, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_); @@ -6732,10 +7483,29 @@ static int __Pyx_InitCachedConstants(void) { * return value * */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 29, __pyx_L1_error) + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(2, 29, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self.agent cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + */ + __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_self_agent_cannot_be_converted_t); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__4); + __Pyx_GIVEREF(__pyx_tuple__4); + + /* "(tree fragment)":4 + * raise TypeError("self.agent cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.agent cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_self_agent_cannot_be_converted_t); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__5); + __Pyx_GIVEREF(__pyx_tuple__5); + /* "reppy/robots.pyx":82 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' @@ -6743,9 +7513,9 @@ static int __Pyx_InitCachedConstants(void) { * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): */ - __pyx_tuple__6 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(1, 82, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__6); - __Pyx_GIVEREF(__pyx_tuple__6); + __pyx_tuple__8 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(2, 82, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__8); + __Pyx_GIVEREF(__pyx_tuple__8); /* "reppy/robots.pyx":83 * '''Get the robots.txt at the provided URL.''' @@ -6754,9 +7524,9 @@ static int __Pyx_InitCachedConstants(void) { * def wrap_exception(etype, cause): * wrapped = etype(cause) */ - __pyx_tuple__7 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__7)) __PYX_ERR(1, 83, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__7); - __Pyx_GIVEREF(__pyx_tuple__7); + __pyx_tuple__9 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(2, 83, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__9); + __Pyx_GIVEREF(__pyx_tuple__9); /* "reppy/robots.pyx":84 * after_response_hook = kwargs.pop('after_response_hook', None) @@ -6765,10 +7535,10 @@ static int __Pyx_InitCachedConstants(void) { * wrapped = etype(cause) * wrapped.url = url */ - __pyx_tuple__8 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(1, 84, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__8); - __Pyx_GIVEREF(__pyx_tuple__8); - __pyx_codeobj__9 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__8, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_wrap_exception, 84, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__9)) __PYX_ERR(1, 84, __pyx_L1_error) + __pyx_tuple__10 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 84, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); + __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_wrap_exception, 84, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 84, __pyx_L1_error) /* "reppy/robots.pyx":93 * # Limit the size of the request @@ -6777,12 +7547,69 @@ static int __Pyx_InitCachedConstants(void) { * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __pyx_tuple__10 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(1, 93, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); - __pyx_tuple__11 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(1, 93, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__11); - __Pyx_GIVEREF(__pyx_tuple__11); + __pyx_tuple__12 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 93, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__12); + __Pyx_GIVEREF(__pyx_tuple__12); + __pyx_tuple__13 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(2, 93, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__13); + __Pyx_GIVEREF(__pyx_tuple__13); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + __pyx_tuple__15 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__15); + __Pyx_GIVEREF(__pyx_tuple__15); + + /* "(tree fragment)":4 + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __pyx_tuple__16 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__16); + __Pyx_GIVEREF(__pyx_tuple__16); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + __pyx_tuple__17 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__17); + __Pyx_GIVEREF(__pyx_tuple__17); + + /* "(tree fragment)":4 + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __pyx_tuple__18 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__18)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__18); + __Pyx_GIVEREF(__pyx_tuple__18); + + /* "(tree fragment)":2 + * def __reduce_cython__(self): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + */ + __pyx_tuple__20 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__20)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__20); + __Pyx_GIVEREF(__pyx_tuple__20); + + /* "(tree fragment)":4 + * raise TypeError("self.robots cannot be converted to a Python object for pickling") + * def __setstate_cython__(self, __pyx_state): + * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< + */ + __pyx_tuple__21 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__21)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__21); + __Pyx_GIVEREF(__pyx_tuple__21); /* "cfunc.to_py":65 * @cname("__Pyx_CFunc_object____object___to_py") @@ -6791,10 +7618,10 @@ static int __Pyx_InitCachedConstants(void) { * """wrap(value)""" * return f(value) */ - __pyx_tuple__14 = PyTuple_Pack(1, __pyx_n_s_value); if (unlikely(!__pyx_tuple__14)) __PYX_ERR(2, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__14); - __Pyx_GIVEREF(__pyx_tuple__14); - __pyx_codeobj__15 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__14, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__15)) __PYX_ERR(2, 65, __pyx_L1_error) + __pyx_tuple__22 = PyTuple_Pack(1, __pyx_n_s_value); if (unlikely(!__pyx_tuple__22)) __PYX_ERR(1, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__22); + __Pyx_GIVEREF(__pyx_tuple__22); + __pyx_codeobj__23 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__22, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__23)) __PYX_ERR(1, 65, __pyx_L1_error) /* "reppy/robots.pyx":33 * @@ -6803,10 +7630,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Construct an Agent from a CppAgent.''' * agent = Agent() */ - __pyx_tuple__16 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 33, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__16); - __Pyx_GIVEREF(__pyx_tuple__16); - __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__16, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(1, 33, __pyx_L1_error) + __pyx_tuple__24 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(2, 33, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__24); + __Pyx_GIVEREF(__pyx_tuple__24); + __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__24, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(2, 33, __pyx_L1_error) /* "reppy/robots.pyx":76 * @@ -6815,10 +7642,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __pyx_tuple__17 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 76, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__17); - __Pyx_GIVEREF(__pyx_tuple__17); - __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__17, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_ParseMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(1, 76, __pyx_L1_error) + __pyx_tuple__25 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(2, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__25); + __Pyx_GIVEREF(__pyx_tuple__25); + __pyx_codeobj__6 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_ParseMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__6)) __PYX_ERR(2, 76, __pyx_L1_error) /* "reppy/robots.pyx":80 * return cls(url, as_bytes(content), expires) @@ -6827,10 +7654,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __pyx_tuple__18 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__18)) __PYX_ERR(1, 80, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__18); - __Pyx_GIVEREF(__pyx_tuple__18); - __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__18, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_FetchMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(1, 80, __pyx_L1_error) + __pyx_tuple__26 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(2, 80, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__26); + __Pyx_GIVEREF(__pyx_tuple__26); + __pyx_codeobj__7 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__26, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FetchMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__7)) __PYX_ERR(2, 80, __pyx_L1_error) /* "reppy/robots.pyx":127 * wrap_exception(exceptions.ExcessiveRedirects, exc) @@ -6839,10 +7666,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __pyx_tuple__19 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__19)) __PYX_ERR(1, 127, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__19); - __Pyx_GIVEREF(__pyx_tuple__19); - __pyx_codeobj__12 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, 0, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__19, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_vagrant_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 127, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__12)) __PYX_ERR(1, 127, __pyx_L1_error) + __pyx_tuple__27 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(2, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__27); + __Pyx_GIVEREF(__pyx_tuple__27); + __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__27, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 127, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) __PYX_ERR(2, 127, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -6851,16 +7678,16 @@ static int __Pyx_InitCachedConstants(void) { } static int __Pyx_InitGlobals(void) { - if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(1, 1, __pyx_L1_error); - __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_int_200 = PyInt_FromLong(200); if (unlikely(!__pyx_int_200)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_int_400 = PyInt_FromLong(400); if (unlikely(!__pyx_int_400)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_int_401 = PyInt_FromLong(401); if (unlikely(!__pyx_int_401)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_int_403 = PyInt_FromLong(403); if (unlikely(!__pyx_int_403)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_int_500 = PyInt_FromLong(500); if (unlikely(!__pyx_int_500)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_int_600 = PyInt_FromLong(600); if (unlikely(!__pyx_int_600)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_int_3600 = PyInt_FromLong(3600); if (unlikely(!__pyx_int_3600)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_int_1048576 = PyInt_FromLong(1048576L); if (unlikely(!__pyx_int_1048576)) __PYX_ERR(1, 1, __pyx_L1_error) + if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(2, 1, __pyx_L1_error); + __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_int_200 = PyInt_FromLong(200); if (unlikely(!__pyx_int_200)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_int_400 = PyInt_FromLong(400); if (unlikely(!__pyx_int_400)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_int_401 = PyInt_FromLong(401); if (unlikely(!__pyx_int_401)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_int_403 = PyInt_FromLong(403); if (unlikely(!__pyx_int_403)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_int_500 = PyInt_FromLong(500); if (unlikely(!__pyx_int_500)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_int_600 = PyInt_FromLong(600); if (unlikely(!__pyx_int_600)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_int_3600 = PyInt_FromLong(3600); if (unlikely(!__pyx_int_3600)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_int_1048576 = PyInt_FromLong(1048576L); if (unlikely(!__pyx_int_1048576)) __PYX_ERR(2, 1, __pyx_L1_error) return 0; __pyx_L1_error:; return -1; @@ -6872,13 +7699,57 @@ PyMODINIT_FUNC initrobots(void) #else PyMODINIT_FUNC PyInit_robots(void); /*proto*/ PyMODINIT_FUNC PyInit_robots(void) -#endif +#if CYTHON_PEP489_MULTI_PHASE_INIT { - __Pyx_TraceDeclarations - PyObject *__pyx_t_1 = NULL; - PyObject *__pyx_t_2 = NULL; - PyObject *__pyx_t_3 = NULL; - __Pyx_RefNannyDeclarations + return PyModuleDef_Init(&__pyx_moduledef); +} +static int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name) { + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + result = PyDict_SetItemString(moddict, to_name, value); + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static PyObject* __pyx_pymod_create(PyObject *spec, CYTHON_UNUSED PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__") < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) +#endif +#endif +{ + __Pyx_TraceDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m && __pyx_m == __pyx_pyinit_module) return 0; + #endif #if CYTHON_REFNANNY __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); if (!__Pyx_RefNanny) { @@ -6889,24 +7760,27 @@ PyMODINIT_FUNC PyInit_robots(void) } #endif __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_robots(void)", 0); - if (__Pyx_check_binary_version() < 0) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(1, 1, __pyx_L1_error) + if (__Pyx_check_binary_version() < 0) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(2, 1, __pyx_L1_error) #ifdef __Pyx_CyFunction_USED - if (__pyx_CyFunction_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__pyx_CyFunction_init() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #endif #ifdef __Pyx_FusedFunction_USED - if (__pyx_FusedFunction_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__pyx_FusedFunction_init() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #endif #ifdef __Pyx_Coroutine_USED - if (__pyx_Coroutine_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__pyx_Coroutine_init() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #endif #ifdef __Pyx_Generator_USED - if (__pyx_Generator_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__pyx_Generator_init() < 0) __PYX_ERR(2, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #endif #ifdef __Pyx_StopAsyncIteration_USED - if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__pyx_StopAsyncIteration_init() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #endif /*--- Library function declarations ---*/ /*--- Threads initialization code ---*/ @@ -6916,65 +7790,75 @@ PyMODINIT_FUNC PyInit_robots(void) #endif #endif /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_m = __pyx_pyinit_module; + Py_INCREF(__pyx_m); + #else #if PY_MAJOR_VERSION < 3 __pyx_m = Py_InitModule4("robots", __pyx_methods, 0, 0, PYTHON_API_VERSION); Py_XINCREF(__pyx_m); #else __pyx_m = PyModule_Create(&__pyx_moduledef); #endif - if (unlikely(!__pyx_m)) __PYX_ERR(1, 1, __pyx_L1_error) - __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(1, 1, __pyx_L1_error) + if (unlikely(!__pyx_m)) __PYX_ERR(2, 1, __pyx_L1_error) + #endif + __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(2, 1, __pyx_L1_error) Py_INCREF(__pyx_d); - __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(1, 1, __pyx_L1_error) + __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(2, 1, __pyx_L1_error) + __pyx_cython_runtime = PyImport_AddModule((char *) "cython_runtime"); if (unlikely(!__pyx_cython_runtime)) __PYX_ERR(2, 1, __pyx_L1_error) #if CYTHON_COMPILING_IN_PYPY Py_INCREF(__pyx_b); #endif - if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(1, 1, __pyx_L1_error); + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(2, 1, __pyx_L1_error); /*--- Initialize various global constants etc. ---*/ - if (__Pyx_InitGlobals() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__Pyx_InitGlobals() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #endif if (__pyx_module_is_main_reppy__robots) { - if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(2, 1, __pyx_L1_error) } #if PY_MAJOR_VERSION >= 3 { - PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(1, 1, __pyx_L1_error) + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(2, 1, __pyx_L1_error) if (!PyDict_GetItemString(modules, "reppy.robots")) { - if (unlikely(PyDict_SetItemString(modules, "reppy.robots", __pyx_m) < 0)) __PYX_ERR(1, 1, __pyx_L1_error) + if (unlikely(PyDict_SetItemString(modules, "reppy.robots", __pyx_m) < 0)) __PYX_ERR(2, 1, __pyx_L1_error) } } #endif /*--- Builtin init code ---*/ - if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(2, 1, __pyx_L1_error) /*--- Constants init code ---*/ - if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(2, 1, __pyx_L1_error) /*--- Global init code ---*/ /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 43, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 43, __pyx_L1_error) __pyx_type_5reppy_6robots_Agent.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(1, 43, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 43, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 43, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; - if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 131, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 131, __pyx_L1_error) __pyx_type_5reppy_6robots_Robots.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(1, 131, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 131, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 131, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 191, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 191, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(1, 191, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 191, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 191, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 198, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 198, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(1, 198, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 198, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 198, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; - if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(1, 80, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(2, 80, __pyx_L1_error) __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_print = 0; __pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod = &__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod; - if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(2, 64, __pyx_L1_error) + if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(1, 64, __pyx_L1_error) __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; __pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = &__pyx_scope_struct____Pyx_CFunc_object____object___to_py; /*--- Type import code ---*/ @@ -6982,9 +7866,9 @@ PyMODINIT_FUNC PyInit_robots(void) /*--- Function import code ---*/ /*--- Execution code ---*/ #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) - if (__Pyx_patch_abc() < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (__Pyx_patch_abc() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #endif - __Pyx_TraceCall("PyMODINIT_FUNC PyInit_robots(void)", __pyx_f[1], 1, 0, __PYX_ERR(1, 1, __pyx_L1_error)); + __Pyx_TraceCall("PyMODINIT_FUNC PyInit_robots(void)", __pyx_f[2], 1, 0, __PYX_ERR(2, 1, __pyx_L1_error)); /* "reppy/robots.pyx":4 * # distutils: define_macros=CYTHON_TRACE=1 @@ -6993,18 +7877,18 @@ PyMODINIT_FUNC PyInit_robots(void) * import time * */ - __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_TraceLine(4,0,__PYX_ERR(2, 4, __pyx_L1_error)) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 4, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_n_s_closing); __Pyx_GIVEREF(__pyx_n_s_closing); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_closing); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_contextlib, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_contextlib, __pyx_t_1, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 4, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_closing); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_closing); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 4, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_closing, __pyx_t_1) < 0) __PYX_ERR(1, 4, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_closing, __pyx_t_1) < 0) __PYX_ERR(2, 4, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -7015,10 +7899,10 @@ PyMODINIT_FUNC PyInit_robots(void) * * import requests */ - __Pyx_TraceLine(5,0,__PYX_ERR(1, 5, __pyx_L1_error)) - __pyx_t_2 = __Pyx_Import(__pyx_n_s_time, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 5, __pyx_L1_error) + __Pyx_TraceLine(5,0,__PYX_ERR(2, 5, __pyx_L1_error)) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_time, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 5, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_time, __pyx_t_2) < 0) __PYX_ERR(1, 5, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_time, __pyx_t_2) < 0) __PYX_ERR(2, 5, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* "reppy/robots.pyx":7 @@ -7028,10 +7912,10 @@ PyMODINIT_FUNC PyInit_robots(void) * from requests.exceptions import ( * SSLError, */ - __Pyx_TraceLine(7,0,__PYX_ERR(1, 7, __pyx_L1_error)) - __pyx_t_2 = __Pyx_Import(__pyx_n_s_requests, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 7, __pyx_L1_error) + __Pyx_TraceLine(7,0,__PYX_ERR(2, 7, __pyx_L1_error)) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_requests, 0, -1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 7, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_requests, __pyx_t_2) < 0) __PYX_ERR(1, 7, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_requests, __pyx_t_2) < 0) __PYX_ERR(2, 7, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* "reppy/robots.pyx":9 @@ -7041,8 +7925,8 @@ PyMODINIT_FUNC PyInit_robots(void) * ConnectionError, * URLRequired, */ - __Pyx_TraceLine(9,0,__PYX_ERR(1, 9, __pyx_L1_error)) - __pyx_t_2 = PyList_New(7); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 9, __pyx_L1_error) + __Pyx_TraceLine(9,0,__PYX_ERR(2, 9, __pyx_L1_error)) + __pyx_t_2 = PyList_New(7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 9, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_SSLError); __Pyx_GIVEREF(__pyx_n_s_SSLError); @@ -7073,37 +7957,37 @@ PyMODINIT_FUNC PyInit_robots(void) * SSLError, * ConnectionError, */ - __Pyx_TraceLine(8,0,__PYX_ERR(1, 8, __pyx_L1_error)) - __pyx_t_1 = __Pyx_Import(__pyx_n_s_requests_exceptions, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 8, __pyx_L1_error) + __Pyx_TraceLine(8,0,__PYX_ERR(2, 8, __pyx_L1_error)) + __pyx_t_1 = __Pyx_Import(__pyx_n_s_requests_exceptions, __pyx_t_2, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_SSLError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_SSLError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_SSLError, __pyx_t_2) < 0) __PYX_ERR(1, 9, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_SSLError, __pyx_t_2) < 0) __PYX_ERR(2, 9, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ConnectionError, __pyx_t_2) < 0) __PYX_ERR(1, 10, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ConnectionError, __pyx_t_2) < 0) __PYX_ERR(2, 10, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_URLRequired); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_URLRequired); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_URLRequired, __pyx_t_2) < 0) __PYX_ERR(1, 11, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_URLRequired, __pyx_t_2) < 0) __PYX_ERR(2, 11, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_MissingSchema, __pyx_t_2) < 0) __PYX_ERR(1, 12, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_MissingSchema, __pyx_t_2) < 0) __PYX_ERR(2, 12, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_InvalidSchema, __pyx_t_2) < 0) __PYX_ERR(1, 13, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_InvalidSchema, __pyx_t_2) < 0) __PYX_ERR(2, 13, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_InvalidURL, __pyx_t_2) < 0) __PYX_ERR(1, 14, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_InvalidURL, __pyx_t_2) < 0) __PYX_ERR(2, 14, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 8, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 8, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_TooManyRedirects, __pyx_t_2) < 0) __PYX_ERR(1, 15, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_TooManyRedirects, __pyx_t_2) < 0) __PYX_ERR(2, 15, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -7114,10 +7998,10 @@ PyMODINIT_FUNC PyInit_robots(void) * * from .ttl import HeaderWithDefaultPolicy */ - __Pyx_TraceLine(16,0,__PYX_ERR(1, 16, __pyx_L1_error)) - __pyx_t_1 = __Pyx_Import(__pyx_n_s_six, 0, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 16, __pyx_L1_error) + __Pyx_TraceLine(16,0,__PYX_ERR(2, 16, __pyx_L1_error)) + __pyx_t_1 = __Pyx_Import(__pyx_n_s_six, 0, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 16, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_six, __pyx_t_1) < 0) __PYX_ERR(1, 16, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_six, __pyx_t_1) < 0) __PYX_ERR(2, 16, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "reppy/robots.pyx":18 @@ -7127,18 +8011,18 @@ PyMODINIT_FUNC PyInit_robots(void) * from . import util, logger, exceptions * */ - __Pyx_TraceLine(18,0,__PYX_ERR(1, 18, __pyx_L1_error)) - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 18, __pyx_L1_error) + __Pyx_TraceLine(18,0,__PYX_ERR(2, 18, __pyx_L1_error)) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 18, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_n_s_HeaderWithDefaultPolicy); __Pyx_GIVEREF(__pyx_n_s_HeaderWithDefaultPolicy); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_HeaderWithDefaultPolicy); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_ttl, __pyx_t_1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 18, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_ttl, __pyx_t_1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 18, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 18, __pyx_L1_error) + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 18, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_HeaderWithDefaultPolicy, __pyx_t_1) < 0) __PYX_ERR(1, 18, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_HeaderWithDefaultPolicy, __pyx_t_1) < 0) __PYX_ERR(2, 18, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -7149,8 +8033,8 @@ PyMODINIT_FUNC PyInit_robots(void) * * cdef as_bytes(value): */ - __Pyx_TraceLine(19,0,__PYX_ERR(1, 19, __pyx_L1_error)) - __pyx_t_2 = PyList_New(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) + __Pyx_TraceLine(19,0,__PYX_ERR(2, 19, __pyx_L1_error)) + __pyx_t_2 = PyList_New(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_util); __Pyx_GIVEREF(__pyx_n_s_util); @@ -7161,20 +8045,20 @@ PyMODINIT_FUNC PyInit_robots(void) __Pyx_INCREF(__pyx_n_s_exceptions); __Pyx_GIVEREF(__pyx_n_s_exceptions); PyList_SET_ITEM(__pyx_t_2, 2, __pyx_n_s_exceptions); - __pyx_t_1 = __Pyx_Import(__pyx_n_s__13, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 19, __pyx_L1_error) + __pyx_t_1 = __Pyx_Import(__pyx_n_s__19, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_util); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_util); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_util, __pyx_t_2) < 0) __PYX_ERR(1, 19, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_util, __pyx_t_2) < 0) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_logger); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_logger); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_logger, __pyx_t_2) < 0) __PYX_ERR(1, 19, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_logger, __pyx_t_2) < 0) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 19, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_exceptions, __pyx_t_2) < 0) __PYX_ERR(1, 19, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_exceptions, __pyx_t_2) < 0) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -7185,10 +8069,10 @@ PyMODINIT_FUNC PyInit_robots(void) * '''Construct an Agent from a CppAgent.''' * agent = Agent() */ - __Pyx_TraceLine(33,0,__PYX_ERR(1, 33, __pyx_L1_error)) - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_1FromRobotsMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 33, __pyx_L1_error) + __Pyx_TraceLine(33,0,__PYX_ERR(2, 33, __pyx_L1_error)) + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_1FromRobotsMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 33, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FromRobotsMethod, __pyx_t_1) < 0) __PYX_ERR(1, 33, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FromRobotsMethod, __pyx_t_1) < 0) __PYX_ERR(2, 33, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "reppy/robots.pyx":48 @@ -7198,13 +8082,13 @@ PyMODINIT_FUNC PyInit_robots(void) * * def __str__(self): */ - __Pyx_TraceLine(48,0,__PYX_ERR(1, 48, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 48, __pyx_L1_error) + __Pyx_TraceLine(48,0,__PYX_ERR(2, 48, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 48, __pyx_L1_error) + __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 48, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Agent->tp_dict, __pyx_n_s_from_robots, __pyx_t_2) < 0) __PYX_ERR(1, 48, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Agent->tp_dict, __pyx_n_s_from_robots, __pyx_t_2) < 0) __PYX_ERR(2, 48, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Agent); @@ -7215,10 +8099,10 @@ PyMODINIT_FUNC PyInit_robots(void) * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __Pyx_TraceLine(76,0,__PYX_ERR(1, 76, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 76, __pyx_L1_error) + __Pyx_TraceLine(76,0,__PYX_ERR(2, 76, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 76, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(1, 76, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(2, 76, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* "reppy/robots.pyx":80 @@ -7228,10 +8112,10 @@ PyMODINIT_FUNC PyInit_robots(void) * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __Pyx_TraceLine(80,0,__PYX_ERR(1, 80, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 80, __pyx_L1_error) + __Pyx_TraceLine(80,0,__PYX_ERR(2, 80, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(1, 80, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(2, 80, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* "reppy/robots.pyx":127 @@ -7241,10 +8125,10 @@ PyMODINIT_FUNC PyInit_robots(void) * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __Pyx_TraceLine(127,0,__PYX_ERR(1, 127, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 127, __pyx_L1_error) + __Pyx_TraceLine(127,0,__PYX_ERR(2, 127, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 127, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(1, 127, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(2, 127, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; /* "reppy/robots.pyx":136 @@ -7254,18 +8138,18 @@ PyMODINIT_FUNC PyInit_robots(void) * * # Class methods */ - __Pyx_TraceLine(136,0,__PYX_ERR(1, 136, __pyx_L1_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(1, 136, __pyx_L1_error) + __Pyx_TraceLine(136,0,__PYX_ERR(2, 136, __pyx_L1_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 136, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(1, 136, __pyx_L1_error) - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(1, 136, __pyx_L1_error) - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 136, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(2, 136, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(2, 136, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(1, 136, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(2, 136, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); @@ -7276,13 +8160,13 @@ PyMODINIT_FUNC PyInit_robots(void) * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) */ - __Pyx_TraceLine(139,0,__PYX_ERR(1, 139, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 139, __pyx_L1_error) + __Pyx_TraceLine(139,0,__PYX_ERR(2, 139, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 139, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(1, 139, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(2, 139, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); @@ -7293,13 +8177,13 @@ PyMODINIT_FUNC PyInit_robots(void) * robots_url = classmethod(RobotsUrlMethod) * */ - __Pyx_TraceLine(140,0,__PYX_ERR(1, 140, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 140, __pyx_L1_error) + __Pyx_TraceLine(140,0,__PYX_ERR(2, 140, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 140, __pyx_L1_error) + __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 140, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(1, 140, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(2, 140, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); @@ -7310,13 +8194,13 @@ PyMODINIT_FUNC PyInit_robots(void) * * # Data members */ - __Pyx_TraceLine(141,0,__PYX_ERR(1, 141, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 141, __pyx_L1_error) + __Pyx_TraceLine(141,0,__PYX_ERR(2, 141, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 141, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 141, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(1, 141, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(2, 141, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); @@ -7325,17 +8209,17 @@ PyMODINIT_FUNC PyInit_robots(void) * # distutils: define_macros=CYTHON_TRACE=1 * */ - __Pyx_TraceLine(1,0,__PYX_ERR(1, 1, __pyx_L1_error)) - __pyx_t_1 = PyDict_New(); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 1, __pyx_L1_error) + __Pyx_TraceLine(1,0,__PYX_ERR(2, 1, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 1, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(1, 1, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(2, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "vector.to_py":67 + /* "vector.to_py":60 * * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): # <<<<<<<<<<<<<< - * return [X_to_py(v[i]) for i in range(v.size())] + * return [v[i] for i in range(v.size())] * */ __Pyx_TraceReturn(Py_None, 0); @@ -7349,7 +8233,7 @@ PyMODINIT_FUNC PyInit_robots(void) __Pyx_XDECREF(__pyx_t_3); if (__pyx_m) { if (__pyx_d) { - __Pyx_AddTraceback("init reppy.robots", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_AddTraceback("init reppy.robots", 0, __pyx_lineno, __pyx_filename); } Py_DECREF(__pyx_m); __pyx_m = 0; } else if (!PyErr_Occurred()) { @@ -7357,10 +8241,12 @@ PyMODINIT_FUNC PyInit_robots(void) } __pyx_L0:; __Pyx_RefNannyFinishContext(); - #if PY_MAJOR_VERSION < 3 - return; - #else + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #elif PY_MAJOR_VERSION >= 3 return __pyx_m; + #else + return; #endif } @@ -7400,12 +8286,12 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { #if CYTHON_PROFILE static int __Pyx_TraceSetupAndCall(PyCodeObject** code, PyFrameObject** frame, + PyThreadState* tstate, const char *funcname, const char *srcfile, int firstlineno) { PyObject *type, *value, *traceback; int retval; - PyThreadState* tstate = PyThreadState_GET(); if (*frame == NULL || !CYTHON_PROFILE_REUSE_FRAME) { if (*code == NULL) { *code = __Pyx_createFrameCodeObject(funcname, srcfile, firstlineno); @@ -7470,7 +8356,7 @@ static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const cha #endif 0, 0, - 0, + CO_OPTIMIZED | CO_NEWLOCALS, __pyx_empty_bytes, /*PyObject *code,*/ __pyx_empty_tuple, /*PyObject *consts,*/ __pyx_empty_tuple, /*PyObject *names,*/ @@ -7670,243 +8556,55 @@ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { } /* ArgTypeTest */ - static void __Pyx_RaiseArgumentTypeInvalid(const char* name, PyObject *obj, PyTypeObject *type) { - PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); -} -static CYTHON_INLINE int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed, - const char *name, int exact) + static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) { if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; } - if (none_allowed && obj == Py_None) return 1; else if (exact) { - if (likely(Py_TYPE(obj) == type)) return 1; #if PY_MAJOR_VERSION == 2 - else if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; + if ((type == &PyBaseString_Type) && likely(__Pyx_PyBaseString_CheckExact(obj))) return 1; #endif } else { - if (likely(PyObject_TypeCheck(obj, type))) return 1; + if (likely(__Pyx_TypeCheck(obj, type))) return 1; } - __Pyx_RaiseArgumentTypeInvalid(name, obj, type); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", + name, type->tp_name, Py_TYPE(obj)->tp_name); return 0; } -/* PyFunctionFastCall */ - #if CYTHON_FAST_PYCALL -#include "frameobject.h" -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = PyThreadState_GET(); - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; - } - fastlocals = f->f_localsplus; - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; - } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return result; -} -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; -#endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; - PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; - } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && -#endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; - } - } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; - } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; - } - nk = i / 2; - } - else { - kwtuple = NULL; - k = NULL; - } - closure = PyFunction_GET_CLOSURE(func); -#if PY_MAJOR_VERSION >= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); -#endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); - } - else { - d = NULL; - nd = 0; - } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); -#else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); -#endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; -} -#endif // CPython < 3.6 -#endif // CYTHON_FAST_PYCALL - -/* PyCFunctionFastCall */ - #if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - PyObject *result; - int flags; - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST)); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); -} -#endif // CYTHON_FAST_PYCCALL - -/* PyObjectCallMethO */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); - } - return result; -} -#endif - -/* PyObjectCallOneArg */ - #if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif +/* decode_c_bytes */ + static CYTHON_INLINE PyObject* __Pyx_decode_c_bytes( + const char* cstring, Py_ssize_t length, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { + if (unlikely((start < 0) | (stop < 0))) { + if (start < 0) { + start += length; + if (start < 0) + start = 0; } + if (stop < 0) + stop += length; + } + if (stop > length) + stop = length; + length = stop - start; + if (unlikely(length <= 0)) + return PyUnicode_FromUnicode(NULL, 0); + cstring += start; + if (decode_func) { + return decode_func(cstring, length, errors); + } else { + return PyUnicode_Decode(cstring, length, encoding, errors); } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -#endif - -/* None */ - static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { - PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); } /* PyErrFetchRestore */ - #if CYTHON_FAST_THREAD_STATE + #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; tmp_type = tstate->curexc_type; @@ -7930,7 +8628,7 @@ static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject #endif /* RaiseException */ - #if PY_MAJOR_VERSION < 3 + #if PY_MAJOR_VERSION < 3 static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, CYTHON_UNUSED PyObject *cause) { __Pyx_PyThreadState_declare @@ -8045,55 +8743,259 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject "raise: exception class must be a subclass of BaseException"); goto bad; } -#if PY_VERSION_HEX >= 0x03030000 - if (cause) { + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if CYTHON_COMPILING_IN_PYPY + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#else + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} +#endif + +/* PyFunctionFastCall */ + #if CYTHON_FAST_PYCALL +#include "frameobject.h" +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = f->f_localsplus; + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); #else - if (cause && cause != Py_None) { + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); #endif - PyObject *fixed_cause; - if (cause == Py_None) { - fixed_cause = NULL; - } else if (PyExceptionClass_Check(cause)) { - fixed_cause = PyObject_CallObject(cause, NULL); - if (fixed_cause == NULL) - goto bad; - } else if (PyExceptionInstance_Check(cause)) { - fixed_cause = cause; - Py_INCREF(fixed_cause); - } else { - PyErr_SetString(PyExc_TypeError, - "exception causes must derive from " - "BaseException"); - goto bad; - } - PyException_SetCause(value, fixed_cause); + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif +#endif + +/* PyCFunctionFastCall */ + #if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + int flags = PyCFunction_GET_FLAGS(func); + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS))); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { + return (*((__Pyx_PyCFunctionFastWithKeywords)meth)) (self, args, nargs, NULL); + } else { + return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs); } - PyErr_SetObject(type, value); - if (tb) { -#if CYTHON_COMPILING_IN_PYPY - PyObject *tmp_type, *tmp_value, *tmp_tb; - PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); - Py_INCREF(tb); - PyErr_Restore(tmp_type, tmp_value, tb); - Py_XDECREF(tmp_tb); -#else - PyThreadState *tstate = PyThreadState_GET(); - PyObject* tmp_tb = tstate->curexc_traceback; - if (tb != tmp_tb) { - Py_INCREF(tb); - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_tb); - } +} #endif + +/* PyObjectCallMethO */ + #if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); } -bad: - Py_XDECREF(owned_instance); - return; + return result; +} +#endif + +/* PyObjectCallOneArg */ + #if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif + if (likely(PyCFunction_Check(func))) { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif + } + } + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; } #endif +/* None */ + static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); +} + /* FetchCommonType */ - static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { PyObject* fake_module; PyTypeObject* cached_type = NULL; fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); @@ -8132,7 +9034,7 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject } /* CythonFunction */ - static PyObject * + static PyObject * __Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) { if (unlikely(op->func_doc == NULL)) { @@ -8414,7 +9316,7 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { {0, 0, 0, 0, 0} }; static PyMemberDef __pyx_CyFunction_members[] = { - {(char *) "__module__", T_OBJECT, offsetof(__pyx_CyFunctionObject, func.m_module), PY_WRITE_RESTRICTED, 0}, + {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), PY_WRITE_RESTRICTED, 0}, {0, 0, 0, 0, 0} }; static PyObject * @@ -8492,14 +9394,18 @@ __Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) } return 0; } -static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) { - PyObject_GC_UnTrack(m); if (__Pyx_CyFunction_weakreflist(m) != NULL) PyObject_ClearWeakRefs((PyObject *) m); __Pyx_CyFunction_clear(m); PyObject_GC_Del(m); } +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) { Py_VISIT(m->func_closure); @@ -8574,10 +9480,16 @@ static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, Py if (likely(kw == NULL || PyDict_Size(kw) == 0)) { size = PyTuple_GET_SIZE(arg); if (likely(size == 1)) { - PyObject *result, *arg0 = PySequence_ITEM(arg, 0); - if (unlikely(!arg0)) return NULL; + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) Py_DECREF(arg0); + #endif return result; } PyErr_Format(PyExc_TypeError, @@ -8684,7 +9596,7 @@ static PyTypeObject __pyx_CyFunctionType_type = { }; static int __pyx_CyFunction_init(void) { __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); - if (__pyx_CyFunctionType == NULL) { + if (unlikely(__pyx_CyFunctionType == NULL)) { return -1; } return 0; @@ -8692,7 +9604,7 @@ static int __pyx_CyFunction_init(void) { static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *func, size_t size, int pyobjects) { __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; m->defaults = PyObject_Malloc(size); - if (!m->defaults) + if (unlikely(!m->defaults)) return PyErr_NoMemory(); memset(m->defaults, 0, size); m->defaults_pyobjects = pyobjects; @@ -8715,7 +9627,7 @@ static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, Py } /* PyObjectCallNoArg */ - #if CYTHON_COMPILING_IN_CPYTHON + #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { #if CYTHON_FAST_PYCALL if (PyFunction_Check(func)) { @@ -8723,7 +9635,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { } #endif #ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { + if (likely(PyCFunction_Check(func) || __Pyx_TypeCheck(func, __pyx_CyFunctionType))) { #else if (likely(PyCFunction_Check(func))) { #endif @@ -8736,7 +9648,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { #endif /* PyIntBinop */ - #if !CYTHON_COMPILING_IN_PYPY + #if !CYTHON_COMPILING_IN_PYPY static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { if (op1 == op2) { Py_RETURN_TRUE; @@ -8821,23 +9733,38 @@ static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED #endif /* SaveResetException */ - #if CYTHON_FAST_THREAD_STATE + #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if PY_VERSION_HEX >= 0x030700A2 + *type = tstate->exc_state.exc_type; + *value = tstate->exc_state.exc_value; + *tb = tstate->exc_state.exc_traceback; + #else *type = tstate->exc_type; *value = tstate->exc_value; *tb = tstate->exc_traceback; + #endif Py_XINCREF(*type); Py_XINCREF(*value); Py_XINCREF(*tb); } static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030700A2 + tmp_type = tstate->exc_state.exc_type; + tmp_value = tstate->exc_state.exc_value; + tmp_tb = tstate->exc_state.exc_traceback; + tstate->exc_state.exc_type = type; + tstate->exc_state.exc_value = value; + tstate->exc_state.exc_traceback = tb; + #else tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; tstate->exc_type = type; tstate->exc_value = value; tstate->exc_traceback = tb; + #endif Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); @@ -8845,7 +9772,7 @@ static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject #endif /* GetException */ - #if CYTHON_FAST_THREAD_STATE + #if CYTHON_FAST_THREAD_STATE static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { #else static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { @@ -8882,12 +9809,21 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) *value = local_value; *tb = local_tb; #if CYTHON_FAST_THREAD_STATE + #if PY_VERSION_HEX >= 0x030700A2 + tmp_type = tstate->exc_state.exc_type; + tmp_value = tstate->exc_state.exc_value; + tmp_tb = tstate->exc_state.exc_traceback; + tstate->exc_state.exc_type = local_type; + tstate->exc_state.exc_value = local_value; + tstate->exc_state.exc_traceback = local_tb; + #else tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; tstate->exc_type = local_type; tstate->exc_value = local_value; tstate->exc_traceback = local_tb; + #endif Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); @@ -8906,17 +9842,32 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) } /* PyErrExceptionMatches */ - #if CYTHON_FAST_THREAD_STATE + #if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); +#if PY_MAJOR_VERSION >= 3 + for (i=0; icurexc_type; if (exc_type == err) return 1; if (unlikely(!exc_type)) return 0; - return PyErr_GivenExceptionMatches(exc_type, err); + if (unlikely(PyTuple_Check(err))) + return __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + return __Pyx_PyErr_GivenExceptionMatches(exc_type, err); } #endif /* WriteUnraisableException */ - static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, + static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, int full_traceback, CYTHON_UNUSED int nogil) { PyObject *old_exc, *old_val, *old_tb; @@ -8957,14 +9908,90 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta #endif } +/* SetupReduce */ + static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { + int ret; + PyObject *name_attr; + name_attr = __Pyx_PyObject_GetAttrStr(meth, __pyx_n_s_name_2); + if (likely(name_attr)) { + ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); + } else { + ret = -1; + } + if (unlikely(ret < 0)) { + PyErr_Clear(); + ret = 0; + } + Py_XDECREF(name_attr); + return ret; +} +static int __Pyx_setup_reduce(PyObject* type_obj) { + int ret = 0; + PyObject *object_reduce = NULL; + PyObject *object_reduce_ex = NULL; + PyObject *reduce = NULL; + PyObject *reduce_ex = NULL; + PyObject *reduce_cython = NULL; + PyObject *setstate = NULL; + PyObject *setstate_cython = NULL; +#if CYTHON_USE_PYTYPE_LOOKUP + if (_PyType_Lookup((PyTypeObject*)type_obj, __pyx_n_s_getstate)) goto GOOD; +#else + if (PyObject_HasAttr(type_obj, __pyx_n_s_getstate)) goto GOOD; +#endif +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce_ex = _PyType_Lookup(&PyBaseObject_Type, __pyx_n_s_reduce_ex); if (!object_reduce_ex) goto BAD; +#else + object_reduce_ex = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_n_s_reduce_ex); if (!object_reduce_ex) goto BAD; +#endif + reduce_ex = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce_ex); if (unlikely(!reduce_ex)) goto BAD; + if (reduce_ex == object_reduce_ex) { +#if CYTHON_USE_PYTYPE_LOOKUP + object_reduce = _PyType_Lookup(&PyBaseObject_Type, __pyx_n_s_reduce); if (!object_reduce) goto BAD; +#else + object_reduce = __Pyx_PyObject_GetAttrStr((PyObject*)&PyBaseObject_Type, __pyx_n_s_reduce); if (!object_reduce) goto BAD; +#endif + reduce = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce); if (unlikely(!reduce)) goto BAD; + if (reduce == object_reduce || __Pyx_setup_reduce_is_named(reduce, __pyx_n_s_reduce_cython)) { + reduce_cython = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_reduce_cython); if (unlikely(!reduce_cython)) goto BAD; + ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_reduce, reduce_cython); if (unlikely(ret < 0)) goto BAD; + ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_reduce_cython); if (unlikely(ret < 0)) goto BAD; + setstate = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_setstate); + if (!setstate) PyErr_Clear(); + if (!setstate || __Pyx_setup_reduce_is_named(setstate, __pyx_n_s_setstate_cython)) { + setstate_cython = __Pyx_PyObject_GetAttrStr(type_obj, __pyx_n_s_setstate_cython); if (unlikely(!setstate_cython)) goto BAD; + ret = PyDict_SetItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_setstate, setstate_cython); if (unlikely(ret < 0)) goto BAD; + ret = PyDict_DelItem(((PyTypeObject*)type_obj)->tp_dict, __pyx_n_s_setstate_cython); if (unlikely(ret < 0)) goto BAD; + } + PyType_Modified((PyTypeObject*)type_obj); + } + } + goto GOOD; +BAD: + if (!PyErr_Occurred()) + PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", ((PyTypeObject*)type_obj)->tp_name); + ret = -1; +GOOD: +#if !CYTHON_USE_PYTYPE_LOOKUP + Py_XDECREF(object_reduce); + Py_XDECREF(object_reduce_ex); +#endif + Py_XDECREF(reduce); + Py_XDECREF(reduce_ex); + Py_XDECREF(reduce_cython); + Py_XDECREF(setstate); + Py_XDECREF(setstate_cython); + return ret; +} + /* Import */ - static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { PyObject *empty_list = 0; PyObject *module = 0; PyObject *global_dict = 0; PyObject *empty_dict = 0; PyObject *list; - #if PY_VERSION_HEX < 0x03030000 + #if PY_MAJOR_VERSION < 3 PyObject *py_import; py_import = __Pyx_PyObject_GetAttrStr(__pyx_b, __pyx_n_s_import); if (!py_import) @@ -8988,17 +10015,8 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta #if PY_MAJOR_VERSION >= 3 if (level == -1) { if (strchr(__Pyx_MODULE_NAME, '.')) { - #if PY_VERSION_HEX < 0x03030000 - PyObject *py_level = PyInt_FromLong(1); - if (!py_level) - goto bad; - module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); - Py_DECREF(py_level); - #else module = PyImport_ImportModuleLevelObject( name, global_dict, empty_dict, list, 1); - #endif if (!module) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) goto bad; @@ -9009,7 +10027,7 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta } #endif if (!module) { - #if PY_VERSION_HEX < 0x03030000 + #if PY_MAJOR_VERSION < 3 PyObject *py_level = PyInt_FromLong(level); if (!py_level) goto bad; @@ -9023,7 +10041,7 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta } } bad: - #if PY_VERSION_HEX < 0x03030000 + #if PY_MAJOR_VERSION < 3 Py_XDECREF(py_import); #endif Py_XDECREF(empty_list); @@ -9032,7 +10050,7 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta } /* ImportFrom */ - static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Format(PyExc_ImportError, @@ -9046,13 +10064,13 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta } /* ClassMethod */ - static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { -#if CYTHON_COMPILING_IN_PYPY + static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM <= 0x05080000 if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) { return PyClassMethod_New(method); } #else -#if CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_PYSTON || CYTHON_COMPILING_IN_PYPY if (PyMethodDescr_Check(method)) { #else static PyTypeObject *methoddescr_type = NULL; @@ -9062,7 +10080,7 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta methoddescr_type = Py_TYPE(meth); Py_DECREF(meth); } - if (PyObject_TypeCheck(method, methoddescr_type)) { + if (__Pyx_TypeCheck(method, methoddescr_type)) { #endif PyMethodDescrObject *descr = (PyMethodDescrObject *)method; #if PY_VERSION_HEX < 0x03020000 @@ -9080,7 +10098,7 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta return PyClassMethod_New(method); } #ifdef __Pyx_CyFunction_USED - else if (PyObject_TypeCheck(method, __pyx_CyFunctionType)) { + else if (__Pyx_TypeCheck(method, __pyx_CyFunctionType)) { return PyClassMethod_New(method); } #endif @@ -9090,8 +10108,45 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta return NULL; } +/* CLineInTraceback */ + #ifndef CYTHON_CLINE_IN_TRACEBACK +static int __Pyx_CLineForTraceback(CYTHON_UNUSED PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + use_cline = PyDict_GetItem(*cython_runtime_dict, __pyx_n_s_cline_in_traceback); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStr(__pyx_cython_runtime, __pyx_n_s_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + PyObject_SetAttr(__pyx_cython_runtime, __pyx_n_s_cline_in_traceback, Py_False); + } + else if (PyObject_Not(use_cline) != 0) { + c_line = 0; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + /* CodeObjectCache */ - static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { return count; @@ -9171,7 +10226,7 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { } /* AddTraceback */ - #include "compile.h" + #include "compile.h" #include "frameobject.h" #include "traceback.h" static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( @@ -9230,18 +10285,22 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename) { PyCodeObject *py_code = 0; PyFrameObject *py_frame = 0; - py_code = __pyx_find_code_object(c_line ? c_line : py_line); + PyThreadState *tstate = __Pyx_PyThreadState_Current; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); if (!py_code) { py_code = __Pyx_CreateCodeObjectForTraceback( funcname, c_line, py_line, filename); if (!py_code) goto bad; - __pyx_insert_code_object(c_line ? c_line : py_line, py_code); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); } py_frame = PyFrame_New( - PyThreadState_GET(), /*PyThreadState *tstate,*/ - py_code, /*PyCodeObject *code,*/ - __pyx_d, /*PyObject *globals,*/ - 0 /*PyObject *locals*/ + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ ); if (!py_frame) goto bad; __Pyx_PyFrame_SetLineNumber(py_frame, py_line); @@ -9252,7 +10311,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { const long neg_one = (long) -1, const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { @@ -9283,7 +10342,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntFromPyVerify */ - #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) #define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) @@ -9305,7 +10364,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntFromPy */ - static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 @@ -9494,7 +10553,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntFromPy */ - static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { const long neg_one = (long) -1, const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 @@ -9683,7 +10742,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntFromPy */ - static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { const int neg_one = (int) -1, const_zero = (int) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 @@ -9871,8 +10930,80 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, return (int) -1; } +/* FastTypeChecks */ + #if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = a->tp_base; + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +#if PY_MAJOR_VERSION == 2 +static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { + PyObject *exception, *value, *tb; + int res; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&exception, &value, &tb); + res = exc_type1 ? PyObject_IsSubclass(err, exc_type1) : 0; + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + if (!res) { + res = PyObject_IsSubclass(err, exc_type2); + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + } + __Pyx_ErrRestore(exception, value, tb); + return res; +} +#else +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + int res = exc_type1 ? __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type1) : 0; + if (!res) { + res = __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } + return res; +} +#endif +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject* exc_type) { + if (likely(err == exc_type)) return 1; + if (likely(PyExceptionClass_Check(err))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, NULL, exc_type); + } + return PyErr_GivenExceptionMatches(err, exc_type); +} +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *exc_type1, PyObject *exc_type2) { + if (likely(err == exc_type1 || err == exc_type2)) return 1; + if (likely(PyExceptionClass_Check(err))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, exc_type1, exc_type2); + } + return (PyErr_GivenExceptionMatches(err, exc_type1) || PyErr_GivenExceptionMatches(err, exc_type2)); +} +#endif + /* CheckBinaryVersion */ - static int __Pyx_check_binary_version(void) { + static int __Pyx_check_binary_version(void) { char ctversion[4], rtversion[4]; PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); @@ -9888,7 +11019,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* InitStrings */ - static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { + static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { #if PY_MAJOR_VERSION < 3 if (t->is_unicode) { @@ -9913,6 +11044,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, #endif if (!*t->p) return -1; + if (PyObject_Hash(*t->p) == -1) + PyErr_Clear(); ++t; } return 0; @@ -9921,50 +11054,57 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { return __Pyx_PyUnicode_FromStringAndSize(c_str, (Py_ssize_t)strlen(c_str)); } -static CYTHON_INLINE char* __Pyx_PyObject_AsString(PyObject* o) { +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { Py_ssize_t ignore; return __Pyx_PyObject_AsStringAndSize(o, &ignore); } -static CYTHON_INLINE char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { -#if CYTHON_COMPILING_IN_CPYTHON && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT) - if ( -#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - __Pyx_sys_getdefaultencoding_not_ascii && -#endif - PyUnicode_Check(o)) { -#if PY_VERSION_HEX < 0x03030000 - char* defenc_c; - PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); - if (!defenc) return NULL; - defenc_c = PyBytes_AS_STRING(defenc); +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT +#if !CYTHON_PEP393_ENABLED +static const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + char* defenc_c; + PyObject* defenc = _PyUnicode_AsDefaultEncodedString(o, NULL); + if (!defenc) return NULL; + defenc_c = PyBytes_AS_STRING(defenc); #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - { - char* end = defenc_c + PyBytes_GET_SIZE(defenc); - char* c; - for (c = defenc_c; c < end; c++) { - if ((unsigned char) (*c) >= 128) { - PyUnicode_AsASCIIString(o); - return NULL; - } + { + char* end = defenc_c + PyBytes_GET_SIZE(defenc); + char* c; + for (c = defenc_c; c < end; c++) { + if ((unsigned char) (*c) >= 128) { + PyUnicode_AsASCIIString(o); + return NULL; } } + } #endif - *length = PyBytes_GET_SIZE(defenc); - return defenc_c; + *length = PyBytes_GET_SIZE(defenc); + return defenc_c; +} #else - if (__Pyx_PyUnicode_READY(o) == -1) return NULL; +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII - if (PyUnicode_IS_ASCII(o)) { - *length = PyUnicode_GET_LENGTH(o); - return PyUnicode_AsUTF8(o); - } else { - PyUnicode_AsASCIIString(o); - return NULL; - } + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } #else - return PyUnicode_AsUTF8AndSize(o, length); + return PyUnicode_AsUTF8AndSize(o, length); #endif +} +#endif +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT + if ( +#if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + __Pyx_sys_getdefaultencoding_not_ascii && #endif + PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); } else #endif #if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) @@ -9988,6 +11128,26 @@ static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { if (is_true | (x == Py_False) | (x == Py_None)) return is_true; else return PyObject_IsTrue(x); } +static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const char* type_name) { +#if PY_MAJOR_VERSION >= 3 + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type %.200s). " + "The ability to return an instance of a strict subclass of int " + "is deprecated, and may be removed in a future version of Python.", + Py_TYPE(result)->tp_name)) { + Py_DECREF(result); + return NULL; + } + return result; + } +#endif + PyErr_Format(PyExc_TypeError, + "__%.4s__ returned non-%.4s (type %.200s)", + type_name, type_name, Py_TYPE(result)->tp_name); + Py_DECREF(result); + return NULL; +} static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { #if CYTHON_USE_TYPE_SLOTS PyNumberMethods *m; @@ -9995,9 +11155,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { const char *name = NULL; PyObject *res = NULL; #if PY_MAJOR_VERSION < 3 - if (PyInt_Check(x) || PyLong_Check(x)) + if (likely(PyInt_Check(x) || PyLong_Check(x))) #else - if (PyLong_Check(x)) + if (likely(PyLong_Check(x))) #endif return __Pyx_NewRef(x); #if CYTHON_USE_TYPE_SLOTS @@ -10005,32 +11165,30 @@ static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { #if PY_MAJOR_VERSION < 3 if (m && m->nb_int) { name = "int"; - res = PyNumber_Int(x); + res = m->nb_int(x); } else if (m && m->nb_long) { name = "long"; - res = PyNumber_Long(x); + res = m->nb_long(x); } #else - if (m && m->nb_int) { + if (likely(m && m->nb_int)) { name = "int"; - res = PyNumber_Long(x); + res = m->nb_int(x); } #endif #else - res = PyNumber_Int(x); + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Int(x); + } #endif - if (res) { + if (likely(res)) { #if PY_MAJOR_VERSION < 3 - if (!PyInt_Check(res) && !PyLong_Check(res)) { + if (unlikely(!PyInt_Check(res) && !PyLong_Check(res))) { #else - if (!PyLong_Check(res)) { + if (unlikely(!PyLong_CheckExact(res))) { #endif - PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - name, name, Py_TYPE(res)->tp_name); - Py_DECREF(res); - return NULL; + return __Pyx_PyNumber_IntOrLongWrongResultType(res, name); } } else if (!PyErr_Occurred()) { From fcae85ad04eac2506d1020a9c6b2bd6026310af7 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Sat, 12 May 2018 08:27:36 -0700 Subject: [PATCH 084/113] Update local Python version to 2.7.15. --- .python-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.python-version b/.python-version index 4712731..f24054f 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -2.7.12 +2.7.15 From af22c24f989ebf9aed887e1adcb0f0b90348cc16 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Wed, 30 May 2018 18:13:56 -0700 Subject: [PATCH 085/113] Drop unused imports. --- tests/test_cache/test_cache.py | 3 --- tests/util.py | 1 - 2 files changed, 4 deletions(-) diff --git a/tests/test_cache/test_cache.py b/tests/test_cache/test_cache.py index 3a27854..476ca09 100644 --- a/tests/test_cache/test_cache.py +++ b/tests/test_cache/test_cache.py @@ -2,11 +2,8 @@ '''Tests about our caching utilities.''' -import contextlib -import os import unittest -import requests_mock import mock from reppy import cache diff --git a/tests/util.py b/tests/util.py index 6ab980f..6104e2b 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,7 +1,6 @@ '''Testing utilities.''' import contextlib -import json import os import six From c1dd31464fdecdecef1ec2655c55115ab6d4fec7 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Tue, 10 Jul 2018 12:08:00 -0700 Subject: [PATCH 086/113] Python 3.7 is now released. So, test for 3.7. --- .travis.yml | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 80cedab..17aad42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,20 @@ language: python -dist: trusty - -python: - - "2.7" - - "3.3" - - "3.4" - - "3.5" - - "3.6" - - "3.7-dev" +sudo: false matrix: - allow_failures: - - python: "3.7-dev" + include: + - python: "2.7" + dist: xenial + - python: "3.3" + dist: trusty + - python: "3.4" + dist: trusty + - python: "3.5" + dist: xenial + - python: "3.6" + dist: xenial + - python: "3.7" + dist: xenial install: pip install -r dev-requirements.txt From ac3d9071a6b2624d01373d308f3619a07648b76e Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Tue, 10 Jul 2018 12:14:43 -0700 Subject: [PATCH 087/113] Use `sudo: true` for 3.7 on xenial. Not sure why this fixes the build issue, but it was recommended in travis-ci/travis-ci#9069. Apparently without `sudo` it appears to try to download https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/14.04/x86_64/python-3.7.tar.bz2 which doesn't exist, instead of https://s3.amazonaws.com/travis-python-archives/binaries/ubuntu/16.04/x86_64/python-3.7.tar.bz2 which does exist. --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 17aad42..44ed91d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,20 +1,25 @@ language: python -sudo: false matrix: include: - python: "2.7" dist: xenial + sudo: false - python: "3.3" dist: trusty + sudo: false - python: "3.4" dist: trusty + sudo: false - python: "3.5" dist: xenial + sudo: false - python: "3.6" dist: xenial + sudo: false - python: "3.7" dist: xenial + sudo: true install: pip install -r dev-requirements.txt From a95a35bba88f706947ce88143d9ff667a21f11c5 Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Mon, 23 Jul 2018 15:08:18 -0700 Subject: [PATCH 088/113] Bump to latest rep-cpp. --- reppy/rep-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index 8abadd8..a2f304e 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit 8abadd8005233ce53c2ca9c14ecf0d3a8f31b3a9 +Subproject commit a2f304e3731287be1e6b45c53815460db175c448 From 312953f7a9faf19316c67c6e6672d9392d973c8a Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Tue, 24 Jul 2018 09:46:35 -0700 Subject: [PATCH 089/113] Add latest Python classifiers. Also, bump to 0.4.10pre1 to see if this will work for a `pip install` for 3.7. --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index bd8eb49..56bb4df 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.9', + version='0.4.10rc1', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like @@ -96,9 +96,12 @@ 'Intended Audience :: Developers', 'Topic :: Internet :: WWW/HTTP', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5' + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7' ], **kwargs ) From 2c58e66eff17fd03909372de159f8889ca715b1b Mon Sep 17 00:00:00 2001 From: Brandon Forehand Date: Thu, 26 Jul 2018 13:55:16 -0700 Subject: [PATCH 090/113] Release 0.4.10 I've confirmed via pypi that this works for Python 3.7.0. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 56bb4df..f0007f6 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.10rc1', + version='0.4.10', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From d12a7d53e704b5ac9c16278d37c0d5f64ab8f4aa Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Mon, 17 Sep 2018 17:45:12 -0700 Subject: [PATCH 091/113] ReppyException type to wrap requests's ReadTimeout --- reppy/exceptions.py | 4 ++++ reppy/robots.pyx | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/reppy/exceptions.py b/reppy/exceptions.py index d3913b2..30ded93 100644 --- a/reppy/exceptions.py +++ b/reppy/exceptions.py @@ -52,6 +52,10 @@ class ExcessiveRedirects(ReppyException): '''A TooManyRedirects error.''' pass +class ReadTimeout(ReppyException): + '''A ReadTimeout error from the HTTP library.''' + pass + class BadStatusCode(ReppyException): '''An exception for 5xx status codes.''' pass diff --git a/reppy/robots.pyx b/reppy/robots.pyx index 54edd13..e5297d9 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -12,7 +12,8 @@ from requests.exceptions import ( MissingSchema, InvalidSchema, InvalidURL, - TooManyRedirects) + TooManyRedirects, + ReadTimeout) import six from .ttl import HeaderWithDefaultPolicy @@ -123,6 +124,8 @@ def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): wrap_exception(exceptions.MalformedUrl, exc) except TooManyRedirects as exc: wrap_exception(exceptions.ExcessiveRedirects, exc) + except ReadTimeout as exc: + wrap_exception(exceptions.ReadTimeout, exc) def RobotsUrlMethod(cls, url): '''Get the robots.txt URL that corresponds to the provided one.''' From ab593ce2ea9c1371a4d668c4fe7ca3d4c0ca0bcf Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Tue, 18 Sep 2018 14:36:16 -0700 Subject: [PATCH 092/113] regenerate robots.cpp with Cython --- reppy/robots.cpp | 1226 ++++++++++++++++++++++++---------------------- 1 file changed, 640 insertions(+), 586 deletions(-) diff --git a/reppy/robots.cpp b/reppy/robots.cpp index 0c65874..f7858d9 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -760,7 +760,7 @@ struct __pyx_obj_5reppy_6robots_AllowAll; struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod; struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; -/* "reppy/robots.pyx":43 +/* "reppy/robots.pyx":44 * return agent * * cdef class Agent: # <<<<<<<<<<<<<< @@ -773,7 +773,7 @@ struct __pyx_obj_5reppy_6robots_Agent { }; -/* "reppy/robots.pyx":131 +/* "reppy/robots.pyx":134 * return as_string(CppRobots.robotsUrl(as_bytes(url))) * * cdef class Robots: # <<<<<<<<<<<<<< @@ -787,7 +787,7 @@ struct __pyx_obj_5reppy_6robots_Robots { }; -/* "reppy/robots.pyx":191 +/* "reppy/robots.pyx":194 * * * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< @@ -799,7 +799,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone { }; -/* "reppy/robots.pyx":198 +/* "reppy/robots.pyx":201 * * * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< @@ -811,7 +811,7 @@ struct __pyx_obj_5reppy_6robots_AllowAll { }; -/* "reppy/robots.pyx":80 +/* "reppy/robots.pyx":81 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -1590,6 +1590,7 @@ static const char __pyx_k_ttl_policy[] = "ttl_policy"; static const char __pyx_k_FetchMethod[] = "FetchMethod"; static const char __pyx_k_Got_i_for_s[] = "Got %i for %s"; static const char __pyx_k_ParseMethod[] = "ParseMethod"; +static const char __pyx_k_ReadTimeout[] = "ReadTimeout"; static const char __pyx_k_URLRequired[] = "URLRequired"; static const char __pyx_k_cfunc_to_py[] = "cfunc.to_py"; static const char __pyx_k_from_robots[] = "from_robots"; @@ -1644,6 +1645,7 @@ static PyObject *__pyx_n_s_MissingSchema; static PyObject *__pyx_n_s_PY3; static PyObject *__pyx_n_s_ParseMethod; static PyObject *__pyx_n_s_Pyx_CFunc_object____object___t; +static PyObject *__pyx_n_s_ReadTimeout; static PyObject *__pyx_n_s_RobotsUrlMethod; static PyObject *__pyx_n_s_SSLError; static PyObject *__pyx_n_s_SSLException; @@ -1797,7 +1799,7 @@ static PyObject *__pyx_codeobj__11; static PyObject *__pyx_codeobj__14; static PyObject *__pyx_codeobj__23; -/* "reppy/robots.pyx":21 +/* "reppy/robots.pyx":22 * from . import util, logger, exceptions * * cdef as_bytes(value): # <<<<<<<<<<<<<< @@ -1814,34 +1816,34 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; __Pyx_RefNannySetupContext("as_bytes", 0); - __Pyx_TraceCall("as_bytes", __pyx_f[2], 21, 0, __PYX_ERR(2, 21, __pyx_L1_error)); + __Pyx_TraceCall("as_bytes", __pyx_f[2], 22, 0, __PYX_ERR(2, 22, __pyx_L1_error)); - /* "reppy/robots.pyx":22 + /* "reppy/robots.pyx":23 * * cdef as_bytes(value): * if isinstance(value, bytes): # <<<<<<<<<<<<<< * return value * return value.encode('utf-8') */ - __Pyx_TraceLine(22,0,__PYX_ERR(2, 22, __pyx_L1_error)) + __Pyx_TraceLine(23,0,__PYX_ERR(2, 23, __pyx_L1_error)) __pyx_t_1 = PyBytes_Check(__pyx_v_value); __pyx_t_2 = (__pyx_t_1 != 0); if (__pyx_t_2) { - /* "reppy/robots.pyx":23 + /* "reppy/robots.pyx":24 * cdef as_bytes(value): * if isinstance(value, bytes): * return value # <<<<<<<<<<<<<< * return value.encode('utf-8') * */ - __Pyx_TraceLine(23,0,__PYX_ERR(2, 23, __pyx_L1_error)) + __Pyx_TraceLine(24,0,__PYX_ERR(2, 24, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_value); __pyx_r = __pyx_v_value; goto __pyx_L0; - /* "reppy/robots.pyx":22 + /* "reppy/robots.pyx":23 * * cdef as_bytes(value): * if isinstance(value, bytes): # <<<<<<<<<<<<<< @@ -1850,25 +1852,25 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { */ } - /* "reppy/robots.pyx":24 + /* "reppy/robots.pyx":25 * if isinstance(value, bytes): * return value * return value.encode('utf-8') # <<<<<<<<<<<<<< * * cdef as_string(value): */ - __Pyx_TraceLine(24,0,__PYX_ERR(2, 24, __pyx_L1_error)) + __Pyx_TraceLine(25,0,__PYX_ERR(2, 25, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 24, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 24, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":21 + /* "reppy/robots.pyx":22 * from . import util, logger, exceptions * * cdef as_bytes(value): # <<<<<<<<<<<<<< @@ -1889,7 +1891,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { return __pyx_r; } -/* "reppy/robots.pyx":26 +/* "reppy/robots.pyx":27 * return value.encode('utf-8') * * cdef as_string(value): # <<<<<<<<<<<<<< @@ -1906,56 +1908,56 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { int __pyx_t_3; int __pyx_t_4; __Pyx_RefNannySetupContext("as_string", 0); - __Pyx_TraceCall("as_string", __pyx_f[2], 26, 0, __PYX_ERR(2, 26, __pyx_L1_error)); + __Pyx_TraceCall("as_string", __pyx_f[2], 27, 0, __PYX_ERR(2, 27, __pyx_L1_error)); - /* "reppy/robots.pyx":27 + /* "reppy/robots.pyx":28 * * cdef as_string(value): * if six.PY3: # <<<<<<<<<<<<<< * if isinstance(value, bytes): * return value.decode('utf-8') */ - __Pyx_TraceLine(27,0,__PYX_ERR(2, 27, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_six); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 27, __pyx_L1_error) + __Pyx_TraceLine(28,0,__PYX_ERR(2, 28, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_six); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 28, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_PY3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 27, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_PY3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 28, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(2, 27, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(2, 28, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_3) { - /* "reppy/robots.pyx":28 + /* "reppy/robots.pyx":29 * cdef as_string(value): * if six.PY3: * if isinstance(value, bytes): # <<<<<<<<<<<<<< * return value.decode('utf-8') * return value */ - __Pyx_TraceLine(28,0,__PYX_ERR(2, 28, __pyx_L1_error)) + __Pyx_TraceLine(29,0,__PYX_ERR(2, 29, __pyx_L1_error)) __pyx_t_3 = PyBytes_Check(__pyx_v_value); __pyx_t_4 = (__pyx_t_3 != 0); if (__pyx_t_4) { - /* "reppy/robots.pyx":29 + /* "reppy/robots.pyx":30 * if six.PY3: * if isinstance(value, bytes): * return value.decode('utf-8') # <<<<<<<<<<<<<< * return value * */ - __Pyx_TraceLine(29,0,__PYX_ERR(2, 29, __pyx_L1_error)) + __Pyx_TraceLine(30,0,__PYX_ERR(2, 30, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_decode); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 29, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_decode); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 30, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 29, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 30, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":28 + /* "reppy/robots.pyx":29 * cdef as_string(value): * if six.PY3: * if isinstance(value, bytes): # <<<<<<<<<<<<<< @@ -1964,7 +1966,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { */ } - /* "reppy/robots.pyx":27 + /* "reppy/robots.pyx":28 * * cdef as_string(value): * if six.PY3: # <<<<<<<<<<<<<< @@ -1973,20 +1975,20 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { */ } - /* "reppy/robots.pyx":30 + /* "reppy/robots.pyx":31 * if isinstance(value, bytes): * return value.decode('utf-8') * return value # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(30,0,__PYX_ERR(2, 30, __pyx_L1_error)) + __Pyx_TraceLine(31,0,__PYX_ERR(2, 31, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_value); __pyx_r = __pyx_v_value; goto __pyx_L0; - /* "reppy/robots.pyx":26 + /* "reppy/robots.pyx":27 * return value.encode('utf-8') * * cdef as_string(value): # <<<<<<<<<<<<<< @@ -2007,7 +2009,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { return __pyx_r; } -/* "reppy/robots.pyx":33 +/* "reppy/robots.pyx":34 * * * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< @@ -2051,17 +2053,17 @@ static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_robots)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 1); __PYX_ERR(2, 33, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 1); __PYX_ERR(2, 34, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 2); __PYX_ERR(2, 33, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 2); __PYX_ERR(2, 34, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "FromRobotsMethod") < 0)) __PYX_ERR(2, 33, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "FromRobotsMethod") < 0)) __PYX_ERR(2, 34, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -2072,17 +2074,17 @@ static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, } __pyx_v_cls = values[0]; __pyx_v_robots = ((struct __pyx_obj_5reppy_6robots_Robots *)values[1]); - __pyx_v_name = __pyx_convert_string_from_py_std__in_string(values[2]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 33, __pyx_L3_error) + __pyx_v_name = __pyx_convert_string_from_py_std__in_string(values[2]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 34, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 33, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 34, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.FromRobotsMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_robots), __pyx_ptype_5reppy_6robots_Robots, 1, "robots", 0))) __PYX_ERR(2, 33, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_robots), __pyx_ptype_5reppy_6robots_Robots, 1, "robots", 0))) __PYX_ERR(2, 34, __pyx_L1_error) __pyx_r = __pyx_pf_5reppy_6robots_FromRobotsMethod(__pyx_self, __pyx_v_cls, __pyx_v_robots, __pyx_v_name); /* function exit code */ @@ -2102,45 +2104,45 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject PyObject *__pyx_t_1 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__3) __Pyx_RefNannySetupContext("FromRobotsMethod", 0); - __Pyx_TraceCall("FromRobotsMethod", __pyx_f[2], 33, 0, __PYX_ERR(2, 33, __pyx_L1_error)); + __Pyx_TraceCall("FromRobotsMethod", __pyx_f[2], 34, 0, __PYX_ERR(2, 34, __pyx_L1_error)); - /* "reppy/robots.pyx":35 + /* "reppy/robots.pyx":36 * def FromRobotsMethod(cls, Robots robots, const string& name): * '''Construct an Agent from a CppAgent.''' * agent = Agent() # <<<<<<<<<<<<<< * # This is somewhat inefficient due to the copying, but it is * # required to be copied because we often toss the containing */ - __Pyx_TraceLine(35,0,__PYX_ERR(2, 35, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 35, __pyx_L1_error) + __Pyx_TraceLine(36,0,__PYX_ERR(2, 36, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 36, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_agent = ((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":40 + /* "reppy/robots.pyx":41 * # Robots object as a temporary thus we'd leave the underlying * # Agent object dangling without a full copy. * agent.agent = robots.robots.agent(name) # <<<<<<<<<<<<<< * return agent * */ - __Pyx_TraceLine(40,0,__PYX_ERR(2, 40, __pyx_L1_error)) + __Pyx_TraceLine(41,0,__PYX_ERR(2, 41, __pyx_L1_error)) __pyx_v_agent->agent = __pyx_v_robots->robots->agent(__pyx_v_name); - /* "reppy/robots.pyx":41 + /* "reppy/robots.pyx":42 * # Agent object dangling without a full copy. * agent.agent = robots.robots.agent(name) * return agent # <<<<<<<<<<<<<< * * cdef class Agent: */ - __Pyx_TraceLine(41,0,__PYX_ERR(2, 41, __pyx_L1_error)) + __Pyx_TraceLine(42,0,__PYX_ERR(2, 42, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_agent)); __pyx_r = ((PyObject *)__pyx_v_agent); goto __pyx_L0; - /* "reppy/robots.pyx":33 + /* "reppy/robots.pyx":34 * * * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< @@ -2161,7 +2163,7 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":50 +/* "reppy/robots.pyx":51 * from_robots = classmethod(FromRobotsMethod) * * def __str__(self): # <<<<<<<<<<<<<< @@ -2188,24 +2190,24 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[2], 50, 0, __PYX_ERR(2, 50, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[2], 51, 0, __PYX_ERR(2, 51, __pyx_L1_error)); - /* "reppy/robots.pyx":51 + /* "reppy/robots.pyx":52 * * def __str__(self): * return self.agent.str().decode('utf8') # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(51,0,__PYX_ERR(2, 51, __pyx_L1_error)) + __Pyx_TraceLine(52,0,__PYX_ERR(2, 52, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_decode_cpp_string(__pyx_v_self->agent.str(), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 51, __pyx_L1_error) + __pyx_t_1 = __Pyx_decode_cpp_string(__pyx_v_self->agent.str(), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":50 + /* "reppy/robots.pyx":51 * from_robots = classmethod(FromRobotsMethod) * * def __str__(self): # <<<<<<<<<<<<<< @@ -2225,7 +2227,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":54 +/* "reppy/robots.pyx":55 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -2254,45 +2256,45 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ int __pyx_t_1; PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 54, 0, __PYX_ERR(2, 54, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 55, 0, __PYX_ERR(2, 55, __pyx_L1_error)); - /* "reppy/robots.pyx":56 + /* "reppy/robots.pyx":57 * def delay(self): * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() # <<<<<<<<<<<<<< * if value > 0: * return value */ - __Pyx_TraceLine(56,0,__PYX_ERR(2, 56, __pyx_L1_error)) + __Pyx_TraceLine(57,0,__PYX_ERR(2, 57, __pyx_L1_error)) __pyx_v_value = __pyx_v_self->agent.delay(); - /* "reppy/robots.pyx":57 + /* "reppy/robots.pyx":58 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< * return value * return None */ - __Pyx_TraceLine(57,0,__PYX_ERR(2, 57, __pyx_L1_error)) + __Pyx_TraceLine(58,0,__PYX_ERR(2, 58, __pyx_L1_error)) __pyx_t_1 = ((__pyx_v_value > 0.0) != 0); if (__pyx_t_1) { - /* "reppy/robots.pyx":58 + /* "reppy/robots.pyx":59 * cdef float value = self.agent.delay() * if value > 0: * return value # <<<<<<<<<<<<<< * return None * */ - __Pyx_TraceLine(58,0,__PYX_ERR(2, 58, __pyx_L1_error)) + __Pyx_TraceLine(59,0,__PYX_ERR(2, 59, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 58, __pyx_L1_error) + __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 59, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":57 + /* "reppy/robots.pyx":58 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< @@ -2301,20 +2303,20 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ */ } - /* "reppy/robots.pyx":59 + /* "reppy/robots.pyx":60 * if value > 0: * return value * return None # <<<<<<<<<<<<<< * * def allow(self, path): */ - __Pyx_TraceLine(59,0,__PYX_ERR(2, 59, __pyx_L1_error)) + __Pyx_TraceLine(60,0,__PYX_ERR(2, 60, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; goto __pyx_L0; - /* "reppy/robots.pyx":54 + /* "reppy/robots.pyx":55 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -2334,7 +2336,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ return __pyx_r; } -/* "reppy/robots.pyx":61 +/* "reppy/robots.pyx":62 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -2363,36 +2365,36 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allow", 0); - __Pyx_TraceCall("allow", __pyx_f[2], 61, 0, __PYX_ERR(2, 61, __pyx_L1_error)); + __Pyx_TraceCall("allow", __pyx_f[2], 62, 0, __PYX_ERR(2, 62, __pyx_L1_error)); - /* "reppy/robots.pyx":63 + /* "reppy/robots.pyx":64 * def allow(self, path): * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(63,0,__PYX_ERR(2, 63, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 63, __pyx_L1_error) + __Pyx_TraceLine(64,0,__PYX_ERR(2, 64, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 64, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 63, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 64, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.allow(__pyx_t_2); - /* "reppy/robots.pyx":64 + /* "reppy/robots.pyx":65 * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def disallow(self, path): */ - __Pyx_TraceLine(64,0,__PYX_ERR(2, 64, __pyx_L1_error)) + __Pyx_TraceLine(65,0,__PYX_ERR(2, 65, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":61 + /* "reppy/robots.pyx":62 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -2412,7 +2414,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":66 +/* "reppy/robots.pyx":67 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2441,36 +2443,36 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("disallow", 0); - __Pyx_TraceCall("disallow", __pyx_f[2], 66, 0, __PYX_ERR(2, 66, __pyx_L1_error)); + __Pyx_TraceCall("disallow", __pyx_f[2], 67, 0, __PYX_ERR(2, 67, __pyx_L1_error)); - /* "reppy/robots.pyx":68 + /* "reppy/robots.pyx":69 * def disallow(self, path): * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(68,0,__PYX_ERR(2, 68, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 68, __pyx_L1_error) + __Pyx_TraceLine(69,0,__PYX_ERR(2, 69, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 69, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 68, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 69, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.disallow(__pyx_t_2); - /* "reppy/robots.pyx":69 + /* "reppy/robots.pyx":70 * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def allowed(self, path): */ - __Pyx_TraceLine(69,0,__PYX_ERR(2, 69, __pyx_L1_error)) + __Pyx_TraceLine(70,0,__PYX_ERR(2, 70, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":66 + /* "reppy/robots.pyx":67 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2490,7 +2492,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":71 +/* "reppy/robots.pyx":72 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2519,28 +2521,28 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[2], 71, 0, __PYX_ERR(2, 71, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[2], 72, 0, __PYX_ERR(2, 72, __pyx_L1_error)); - /* "reppy/robots.pyx":73 + /* "reppy/robots.pyx":74 * def allowed(self, path): * '''Is the provided URL allowed?''' * return self.agent.allowed(as_bytes(path)) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(73,0,__PYX_ERR(2, 73, __pyx_L1_error)) + __Pyx_TraceLine(74,0,__PYX_ERR(2, 74, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 73, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 74, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 73, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 74, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 73, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 74, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":71 + /* "reppy/robots.pyx":72 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2675,7 +2677,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__setstate_cython__(CYTHON_UNU return __pyx_r; } -/* "reppy/robots.pyx":76 +/* "reppy/robots.pyx":77 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2723,13 +2725,13 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(2, 76, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(2, 77, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(2, 76, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(2, 77, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 3: @@ -2739,7 +2741,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(2, 76, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(2, 77, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2759,7 +2761,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 76, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 77, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.ParseMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2784,18 +2786,18 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_6 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__6) __Pyx_RefNannySetupContext("ParseMethod", 0); - __Pyx_TraceCall("ParseMethod", __pyx_f[2], 76, 0, __PYX_ERR(2, 76, __pyx_L1_error)); + __Pyx_TraceCall("ParseMethod", __pyx_f[2], 77, 0, __PYX_ERR(2, 77, __pyx_L1_error)); - /* "reppy/robots.pyx":78 + /* "reppy/robots.pyx":79 * def ParseMethod(cls, url, content, expires=None): * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) # <<<<<<<<<<<<<< * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): */ - __Pyx_TraceLine(78,0,__PYX_ERR(2, 78, __pyx_L1_error)) + __Pyx_TraceLine(79,0,__PYX_ERR(2, 79, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 78, __pyx_L1_error) + __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 79, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_v_cls); __pyx_t_3 = __pyx_v_cls; __pyx_t_4 = NULL; @@ -2813,7 +2815,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 78, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 79, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -2822,14 +2824,14 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 78, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 79, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 78, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 79, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -2843,7 +2845,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_5, __pyx_v_expires); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 78, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 79, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -2852,7 +2854,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":76 + /* "reppy/robots.pyx":77 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2876,7 +2878,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":80 +/* "reppy/robots.pyx":81 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -2940,7 +2942,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(2, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(2, 81, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: @@ -2957,7 +2959,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } if (unlikely(kw_args > 0)) { const Py_ssize_t used_pos_args = (pos_args < 4) ? pos_args : 4; - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(2, 80, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(2, 81, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2981,7 +2983,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 81, __pyx_L3_error) __pyx_L3_error:; __Pyx_DECREF(__pyx_v_args); __pyx_v_args = 0; __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; @@ -2998,7 +3000,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb return __pyx_r; } -/* "reppy/robots.pyx":84 +/* "reppy/robots.pyx":85 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< @@ -3038,11 +3040,11 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cause)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(2, 84, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(2, 85, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "wrap_exception") < 0)) __PYX_ERR(2, 84, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "wrap_exception") < 0)) __PYX_ERR(2, 85, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -3055,7 +3057,7 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 84, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 85, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.FetchMethod.wrap_exception", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3084,16 +3086,16 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_RefNannySetupContext("wrap_exception", 0); __pyx_outer_scope = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *) __Pyx_CyFunction_GetClosure(__pyx_self); __pyx_cur_scope = __pyx_outer_scope; - __Pyx_TraceCall("wrap_exception", __pyx_f[2], 84, 0, __PYX_ERR(2, 84, __pyx_L1_error)); + __Pyx_TraceCall("wrap_exception", __pyx_f[2], 85, 0, __PYX_ERR(2, 85, __pyx_L1_error)); - /* "reppy/robots.pyx":85 + /* "reppy/robots.pyx":86 * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): * wrapped = etype(cause) # <<<<<<<<<<<<<< * wrapped.url = url * if after_response_hook is not None: */ - __Pyx_TraceLine(85,0,__PYX_ERR(2, 85, __pyx_L1_error)) + __Pyx_TraceLine(86,0,__PYX_ERR(2, 86, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_etype); __pyx_t_2 = __pyx_v_etype; __pyx_t_3 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { @@ -3106,13 +3108,13 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * } } if (!__pyx_t_3) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3120,19 +3122,19 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 85, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 86, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __pyx_t_3 = NULL; __Pyx_INCREF(__pyx_v_cause); __Pyx_GIVEREF(__pyx_v_cause); PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_cause); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } @@ -3141,39 +3143,39 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __pyx_v_wrapped = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":86 + /* "reppy/robots.pyx":87 * def wrap_exception(etype, cause): * wrapped = etype(cause) * wrapped.url = url # <<<<<<<<<<<<<< * if after_response_hook is not None: * after_response_hook(wrapped) */ - __Pyx_TraceLine(86,0,__PYX_ERR(2, 86, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_url)) { __Pyx_RaiseClosureNameError("url"); __PYX_ERR(2, 86, __pyx_L1_error) } - if (__Pyx_PyObject_SetAttrStr(__pyx_v_wrapped, __pyx_n_s_url, __pyx_cur_scope->__pyx_v_url) < 0) __PYX_ERR(2, 86, __pyx_L1_error) + __Pyx_TraceLine(87,0,__PYX_ERR(2, 87, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_url)) { __Pyx_RaiseClosureNameError("url"); __PYX_ERR(2, 87, __pyx_L1_error) } + if (__Pyx_PyObject_SetAttrStr(__pyx_v_wrapped, __pyx_n_s_url, __pyx_cur_scope->__pyx_v_url) < 0) __PYX_ERR(2, 87, __pyx_L1_error) - /* "reppy/robots.pyx":87 + /* "reppy/robots.pyx":88 * wrapped = etype(cause) * wrapped.url = url * if after_response_hook is not None: # <<<<<<<<<<<<<< * after_response_hook(wrapped) * raise wrapped */ - __Pyx_TraceLine(87,0,__PYX_ERR(2, 87, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 87, __pyx_L1_error) } + __Pyx_TraceLine(88,0,__PYX_ERR(2, 88, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 88, __pyx_L1_error) } __pyx_t_5 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); __pyx_t_6 = (__pyx_t_5 != 0); if (__pyx_t_6) { - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":89 * wrapped.url = url * if after_response_hook is not None: * after_response_hook(wrapped) # <<<<<<<<<<<<<< * raise wrapped * try: */ - __Pyx_TraceLine(88,0,__PYX_ERR(2, 88, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 88, __pyx_L1_error) } + __Pyx_TraceLine(89,0,__PYX_ERR(2, 89, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 89, __pyx_L1_error) } __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); __pyx_t_2 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_4 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { @@ -3186,13 +3188,13 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * } } if (!__pyx_t_4) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3200,19 +3202,19 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 88, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); __pyx_t_4 = NULL; __Pyx_INCREF(__pyx_v_wrapped); __Pyx_GIVEREF(__pyx_v_wrapped); PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_v_wrapped); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } @@ -3220,7 +3222,7 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":87 + /* "reppy/robots.pyx":88 * wrapped = etype(cause) * wrapped.url = url * if after_response_hook is not None: # <<<<<<<<<<<<<< @@ -3229,18 +3231,18 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * */ } - /* "reppy/robots.pyx":89 + /* "reppy/robots.pyx":90 * if after_response_hook is not None: * after_response_hook(wrapped) * raise wrapped # <<<<<<<<<<<<<< * try: * # Limit the size of the request */ - __Pyx_TraceLine(89,0,__PYX_ERR(2, 89, __pyx_L1_error)) + __Pyx_TraceLine(90,0,__PYX_ERR(2, 90, __pyx_L1_error)) __Pyx_Raise(__pyx_v_wrapped, 0, 0, 0); - __PYX_ERR(2, 89, __pyx_L1_error) + __PYX_ERR(2, 90, __pyx_L1_error) - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":85 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< @@ -3263,7 +3265,7 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * return __pyx_r; } -/* "reppy/robots.pyx":80 +/* "reppy/robots.pyx":81 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -3306,69 +3308,69 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(2, 80, __pyx_L1_error) + __PYX_ERR(2, 81, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } - __Pyx_TraceCall("FetchMethod", __pyx_f[2], 80, 0, __PYX_ERR(2, 80, __pyx_L1_error)); + __Pyx_TraceCall("FetchMethod", __pyx_f[2], 81, 0, __PYX_ERR(2, 81, __pyx_L1_error)); __pyx_cur_scope->__pyx_v_url = __pyx_v_url; __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); - /* "reppy/robots.pyx":82 + /* "reppy/robots.pyx":83 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): */ - __Pyx_TraceLine(82,0,__PYX_ERR(2, 82, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 82, __pyx_L1_error) + __Pyx_TraceLine(83,0,__PYX_ERR(2, 83, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 83, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 82, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 83, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_t_2); __pyx_cur_scope->__pyx_v_after_response_hook = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":83 + /* "reppy/robots.pyx":84 * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< * def wrap_exception(etype, cause): * wrapped = etype(cause) */ - __Pyx_TraceLine(83,0,__PYX_ERR(2, 83, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 83, __pyx_L1_error) + __Pyx_TraceLine(84,0,__PYX_ERR(2, 84, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 84, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 83, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 84, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_after_parse_hook = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":85 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< * wrapped = etype(cause) * wrapped.url = url */ - __Pyx_TraceLine(84,0,__PYX_ERR(2, 84, __pyx_L1_error)) - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 84, __pyx_L1_error) + __Pyx_TraceLine(85,0,__PYX_ERR(2, 85, __pyx_L1_error)) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_wrap_exception = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":91 * after_response_hook(wrapped) * raise wrapped * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_TraceLine(90,0,__PYX_ERR(2, 90, __pyx_L3_error)) + __Pyx_TraceLine(91,0,__PYX_ERR(2, 91, __pyx_L3_error)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign @@ -3378,41 +3380,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGOTREF(__pyx_t_5); /*try:*/ { - /* "reppy/robots.pyx":92 + /* "reppy/robots.pyx":93 * try: * # Limit the size of the request * kwargs['stream'] = True # <<<<<<<<<<<<<< * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) */ - __Pyx_TraceLine(92,0,__PYX_ERR(2, 92, __pyx_L3_error)) - if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(2, 92, __pyx_L3_error) + __Pyx_TraceLine(93,0,__PYX_ERR(2, 93, __pyx_L3_error)) + if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(2, 93, __pyx_L3_error) - /* "reppy/robots.pyx":93 + /* "reppy/robots.pyx":94 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __Pyx_TraceLine(93,0,__PYX_ERR(2, 93, __pyx_L3_error)) + __Pyx_TraceLine(94,0,__PYX_ERR(2, 94, __pyx_L3_error)) /*with:*/ { - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_cur_scope->__pyx_v_url); - __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -3427,14 +3429,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -3443,28 +3445,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { - __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); __pyx_t_6 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(2, 93, __pyx_L3_error) + __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 93, __pyx_L9_error) + __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L9_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { @@ -3477,10 +3479,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (__pyx_t_6) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 93, __pyx_L9_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 94, __pyx_L9_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 93, __pyx_L9_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 94, __pyx_L9_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3499,78 +3501,78 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_res = __pyx_t_7; __pyx_t_7 = 0; - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":95 * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) # <<<<<<<<<<<<<< * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): */ - __Pyx_TraceLine(94,0,__PYX_ERR(2, 94, __pyx_L13_error)) - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L13_error) + __Pyx_TraceLine(95,0,__PYX_ERR(2, 95, __pyx_L13_error)) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 95, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 95, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(2, 94, __pyx_L13_error) - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 94, __pyx_L13_error) - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 94, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(2, 95, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 95, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 95, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_v_content = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":96 + /* "reppy/robots.pyx":97 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) */ - __Pyx_TraceLine(96,0,__PYX_ERR(2, 96, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 96, __pyx_L13_error) + __Pyx_TraceLine(97,0,__PYX_ERR(2, 97, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 96, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 96, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(2, 96, __pyx_L13_error) - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 96, __pyx_L13_error) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 96, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(2, 97, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 96, __pyx_L13_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 97, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_13) { - /* "reppy/robots.pyx":97 + /* "reppy/robots.pyx":98 * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( # <<<<<<<<<<<<<< * 'Content larger than %s bytes' % max_size) * */ - __Pyx_TraceLine(97,0,__PYX_ERR(2, 97, __pyx_L13_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L13_error) + __Pyx_TraceLine(98,0,__PYX_ERR(2, 98, __pyx_L13_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":98 + /* "reppy/robots.pyx":99 * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) # <<<<<<<<<<<<<< * * if after_response_hook is not None: */ - __Pyx_TraceLine(98,0,__PYX_ERR(2, 98, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 98, __pyx_L13_error) + __Pyx_TraceLine(99,0,__PYX_ERR(2, 99, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 99, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -3583,14 +3585,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -3599,20 +3601,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_2); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3620,9 +3622,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(2, 97, __pyx_L13_error) + __PYX_ERR(2, 98, __pyx_L13_error) - /* "reppy/robots.pyx":96 + /* "reppy/robots.pyx":97 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< @@ -3631,26 +3633,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":100 + /* "reppy/robots.pyx":101 * 'Content larger than %s bytes' % max_size) * * if after_response_hook is not None: # <<<<<<<<<<<<<< * after_response_hook(res) * */ - __Pyx_TraceLine(100,0,__PYX_ERR(2, 100, __pyx_L13_error)) + __Pyx_TraceLine(101,0,__PYX_ERR(2, 101, __pyx_L13_error)) __pyx_t_13 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { - /* "reppy/robots.pyx":101 + /* "reppy/robots.pyx":102 * * if after_response_hook is not None: * after_response_hook(res) # <<<<<<<<<<<<<< * * # Get the TTL policy's ruling on the ttl */ - __Pyx_TraceLine(101,0,__PYX_ERR(2, 101, __pyx_L13_error)) + __Pyx_TraceLine(102,0,__PYX_ERR(2, 102, __pyx_L13_error)) __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); __pyx_t_7 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_8 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -3663,13 +3665,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 102, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 102, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3677,19 +3679,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 102, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 102, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_2, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 102, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } @@ -3697,7 +3699,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":100 + /* "reppy/robots.pyx":101 * 'Content larger than %s bytes' % max_size) * * if after_response_hook is not None: # <<<<<<<<<<<<<< @@ -3706,28 +3708,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":104 + /* "reppy/robots.pyx":105 * * # Get the TTL policy's ruling on the ttl * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) # <<<<<<<<<<<<<< * * if res.status_code == 200: */ - __Pyx_TraceLine(104,0,__PYX_ERR(2, 104, __pyx_L13_error)) - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 104, __pyx_L13_error) + __Pyx_TraceLine(105,0,__PYX_ERR(2, 105, __pyx_L13_error)) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 105, __pyx_L13_error) if (!__pyx_t_14) { } else { __Pyx_INCREF(__pyx_v_ttl_policy); __pyx_t_7 = __pyx_v_ttl_policy; goto __pyx_L21_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 104, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_2); __pyx_t_7 = __pyx_t_2; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_L21_bool_binop_done:; - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 104, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = NULL; @@ -3741,13 +3743,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3755,19 +3757,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 104, __pyx_L13_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3776,32 +3778,32 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_expires = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":106 + /* "reppy/robots.pyx":107 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: */ - __Pyx_TraceLine(106,0,__PYX_ERR(2, 106, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 106, __pyx_L13_error) + __Pyx_TraceLine(107,0,__PYX_ERR(2, 107, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 107, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 106, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 107, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 106, __pyx_L13_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 107, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_14) { - /* "reppy/robots.pyx":107 + /* "reppy/robots.pyx":108 * * if res.status_code == 200: * robots = cls.parse(url, content, expires) # <<<<<<<<<<<<<< * if after_parse_hook is not None: * after_parse_hook(robots) */ - __Pyx_TraceLine(107,0,__PYX_ERR(2, 107, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 107, __pyx_L13_error) + __Pyx_TraceLine(108,0,__PYX_ERR(2, 108, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_8 = NULL; __pyx_t_15 = 0; @@ -3818,7 +3820,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 107, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else @@ -3826,13 +3828,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 107, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 107, __pyx_L13_error) + __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); if (__pyx_t_8) { __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; @@ -3846,7 +3848,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_15, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 107, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } @@ -3854,26 +3856,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_robots = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":108 + /* "reppy/robots.pyx":109 * if res.status_code == 200: * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: # <<<<<<<<<<<<<< * after_parse_hook(robots) * return robots */ - __Pyx_TraceLine(108,0,__PYX_ERR(2, 108, __pyx_L13_error)) + __Pyx_TraceLine(109,0,__PYX_ERR(2, 109, __pyx_L13_error)) __pyx_t_14 = (__pyx_v_after_parse_hook != Py_None); __pyx_t_13 = (__pyx_t_14 != 0); if (__pyx_t_13) { - /* "reppy/robots.pyx":109 + /* "reppy/robots.pyx":110 * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: * after_parse_hook(robots) # <<<<<<<<<<<<<< * return robots * elif res.status_code in (401, 403): */ - __Pyx_TraceLine(109,0,__PYX_ERR(2, 109, __pyx_L13_error)) + __Pyx_TraceLine(110,0,__PYX_ERR(2, 110, __pyx_L13_error)) __Pyx_INCREF(__pyx_v_after_parse_hook); __pyx_t_1 = __pyx_v_after_parse_hook; __pyx_t_7 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_1))) { @@ -3886,13 +3888,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 109, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 109, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else @@ -3900,19 +3902,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 109, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 109, __pyx_L13_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 110, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_robots); __Pyx_GIVEREF(__pyx_v_robots); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_robots); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 109, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3920,7 +3922,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":108 + /* "reppy/robots.pyx":109 * if res.status_code == 200: * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: # <<<<<<<<<<<<<< @@ -3929,20 +3931,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":110 + /* "reppy/robots.pyx":111 * if after_parse_hook is not None: * after_parse_hook(robots) * return robots # <<<<<<<<<<<<<< * elif res.status_code in (401, 403): * return AllowNone(url, expires) */ - __Pyx_TraceLine(110,0,__PYX_ERR(2, 110, __pyx_L13_error)) + __Pyx_TraceLine(111,0,__PYX_ERR(2, 111, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_robots); __pyx_r = __pyx_v_robots; goto __pyx_L17_try_return; - /* "reppy/robots.pyx":106 + /* "reppy/robots.pyx":107 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< @@ -3951,28 +3953,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":111 + /* "reppy/robots.pyx":112 * after_parse_hook(robots) * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: */ - __Pyx_TraceLine(111,0,__PYX_ERR(2, 111, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) + __Pyx_TraceLine(112,0,__PYX_ERR(2, 112, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 112, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 112, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 111, __pyx_L13_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 112, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (!__pyx_t_14) { } else { __pyx_t_13 = __pyx_t_14; goto __pyx_L25_bool_binop_done; } - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 112, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 111, __pyx_L13_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 112, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_13 = __pyx_t_14; __pyx_L25_bool_binop_done:; @@ -3980,16 +3982,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { - /* "reppy/robots.pyx":112 + /* "reppy/robots.pyx":113 * return robots * elif res.status_code in (401, 403): * return AllowNone(url, expires) # <<<<<<<<<<<<<< * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) */ - __Pyx_TraceLine(112,0,__PYX_ERR(2, 112, __pyx_L13_error)) + __Pyx_TraceLine(113,0,__PYX_ERR(2, 113, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 112, __pyx_L13_error) + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); @@ -3997,14 +3999,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 112, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L17_try_return; - /* "reppy/robots.pyx":111 + /* "reppy/robots.pyx":112 * after_parse_hook(robots) * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< @@ -4013,45 +4015,45 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":113 + /* "reppy/robots.pyx":114 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< * return AllowAll(url, expires) * else: */ - __Pyx_TraceLine(113,0,__PYX_ERR(2, 113, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 113, __pyx_L13_error) + __Pyx_TraceLine(114,0,__PYX_ERR(2, 114, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_13) { } else { __pyx_t_14 = __pyx_t_13; goto __pyx_L27_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_14 = __pyx_t_13; __pyx_L27_bool_binop_done:; if (__pyx_t_14) { - /* "reppy/robots.pyx":114 + /* "reppy/robots.pyx":115 * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) # <<<<<<<<<<<<<< * else: * raise exceptions.BadStatusCode( */ - __Pyx_TraceLine(114,0,__PYX_ERR(2, 114, __pyx_L13_error)) + __Pyx_TraceLine(115,0,__PYX_ERR(2, 115, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 114, __pyx_L13_error) + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 115, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); @@ -4059,14 +4061,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 115, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L17_try_return; - /* "reppy/robots.pyx":113 + /* "reppy/robots.pyx":114 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< @@ -4075,32 +4077,32 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":116 + /* "reppy/robots.pyx":117 * return AllowAll(url, expires) * else: * raise exceptions.BadStatusCode( # <<<<<<<<<<<<<< * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: */ - __Pyx_TraceLine(116,0,__PYX_ERR(2, 116, __pyx_L13_error)) + __Pyx_TraceLine(117,0,__PYX_ERR(2, 117, __pyx_L13_error)) /*else*/ { - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 116, __pyx_L13_error) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 116, __pyx_L13_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":117 + /* "reppy/robots.pyx":118 * else: * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) # <<<<<<<<<<<<<< * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) */ - __Pyx_TraceLine(117,0,__PYX_ERR(2, 117, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) + __Pyx_TraceLine(118,0,__PYX_ERR(2, 118, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); @@ -4108,10 +4110,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_cur_scope->__pyx_v_url); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; __pyx_t_15 = 0; @@ -4128,7 +4130,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -4138,7 +4140,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -4146,7 +4148,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } else #endif { - __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 116, __pyx_L13_error) + __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_6) { __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; @@ -4157,17 +4159,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_15, __pyx_t_7); __pyx_t_1 = 0; __pyx_t_7 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(2, 116, __pyx_L13_error) + __PYX_ERR(2, 117, __pyx_L13_error) } - /* "reppy/robots.pyx":93 + /* "reppy/robots.pyx":94 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< @@ -4184,20 +4186,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; /*except:*/ { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 93, __pyx_L15_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 94, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); - __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 93, __pyx_L15_except_error) + __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_7, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_17)) __PYX_ERR(2, 93, __pyx_L15_except_error) + if (unlikely(!__pyx_t_17)) __PYX_ERR(2, 94, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_17); __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_17); __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; - if (__pyx_t_14 < 0) __PYX_ERR(2, 93, __pyx_L15_except_error) + if (__pyx_t_14 < 0) __PYX_ERR(2, 94, __pyx_L15_except_error) __pyx_t_13 = ((!(__pyx_t_14 != 0)) != 0); if (__pyx_t_13) { __Pyx_GIVEREF(__pyx_t_2); @@ -4205,7 +4207,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGIVEREF(__pyx_t_16); __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_8, __pyx_t_16); __pyx_t_2 = 0; __pyx_t_8 = 0; __pyx_t_16 = 0; - __PYX_ERR(2, 93, __pyx_L15_except_error) + __PYX_ERR(2, 94, __pyx_L15_except_error) } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -4236,7 +4238,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__12, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_12)) __PYX_ERR(2, 93, __pyx_L3_error) + if (unlikely(!__pyx_t_12)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; } @@ -4248,7 +4250,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__13, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_11)) __PYX_ERR(2, 93, __pyx_L3_error) + if (unlikely(!__pyx_t_11)) __PYX_ERR(2, 94, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } @@ -4265,7 +4267,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L32:; } - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":91 * after_response_hook(wrapped) * raise wrapped * try: # <<<<<<<<<<<<<< @@ -4285,41 +4287,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; - /* "reppy/robots.pyx":118 + /* "reppy/robots.pyx":119 * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: */ - __Pyx_TraceLine(118,0,__PYX_ERR(2, 118, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 118, __pyx_L5_except_error) + __Pyx_TraceLine(119,0,__PYX_ERR(2, 119, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(2, 118, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(2, 119, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":119 + /* "reppy/robots.pyx":120 * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) # <<<<<<<<<<<<<< * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) */ - __Pyx_TraceLine(119,0,__PYX_ERR(2, 119, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 119, __pyx_L5_except_error) + __Pyx_TraceLine(120,0,__PYX_ERR(2, 120, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 120, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 119, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 120, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 119, __pyx_L5_except_error) + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 120, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -4329,41 +4331,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":120 + /* "reppy/robots.pyx":121 * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: */ - __Pyx_TraceLine(120,0,__PYX_ERR(2, 120, __pyx_L5_except_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L5_except_error) + __Pyx_TraceLine(121,0,__PYX_ERR(2, 121, __pyx_L5_except_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 121, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 120, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 121, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":121 + /* "reppy/robots.pyx":122 * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) # <<<<<<<<<<<<<< * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) */ - __Pyx_TraceLine(121,0,__PYX_ERR(2, 121, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 121, __pyx_L5_except_error) + __Pyx_TraceLine(122,0,__PYX_ERR(2, 122, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 121, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 121, __pyx_L5_except_error) + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -4373,21 +4375,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":122 + /* "reppy/robots.pyx":123 * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: */ - __Pyx_TraceLine(122,0,__PYX_ERR(2, 122, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 122, __pyx_L5_except_error) + __Pyx_TraceLine(123,0,__PYX_ERR(2, 123, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 122, __pyx_L5_except_error) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 122, __pyx_L5_except_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 122, __pyx_L5_except_error) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_2) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4396,27 +4398,27 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 122, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":123 + /* "reppy/robots.pyx":124 * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) # <<<<<<<<<<<<<< * except TooManyRedirects as exc: * wrap_exception(exceptions.ExcessiveRedirects, exc) */ - __Pyx_TraceLine(123,0,__PYX_ERR(2, 123, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __Pyx_TraceLine(124,0,__PYX_ERR(2, 124, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 124, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 124, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 124, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4426,41 +4428,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":124 + /* "reppy/robots.pyx":125 * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.ExcessiveRedirects, exc) - * + * except ReadTimeout as exc: */ - __Pyx_TraceLine(124,0,__PYX_ERR(2, 124, __pyx_L5_except_error)) - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 124, __pyx_L5_except_error) + __Pyx_TraceLine(125,0,__PYX_ERR(2, 125, __pyx_L5_except_error)) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(2, 124, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_7); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":125 + /* "reppy/robots.pyx":126 * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: * wrap_exception(exceptions.ExcessiveRedirects, exc) # <<<<<<<<<<<<<< - * - * def RobotsUrlMethod(cls, url): + * except ReadTimeout as exc: + * wrap_exception(exceptions.ReadTimeout, exc) */ - __Pyx_TraceLine(125,0,__PYX_ERR(2, 125, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 125, __pyx_L5_except_error) + __Pyx_TraceLine(126,0,__PYX_ERR(2, 126, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 125, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 125, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4469,10 +4471,54 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; goto __pyx_L4_exception_handled; } + + /* "reppy/robots.pyx":127 + * except TooManyRedirects as exc: + * wrap_exception(exceptions.ExcessiveRedirects, exc) + * except ReadTimeout as exc: # <<<<<<<<<<<<<< + * wrap_exception(exceptions.ReadTimeout, exc) + * + */ + __Pyx_TraceLine(127,0,__PYX_ERR(2, 127, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 127, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_7); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + if (__pyx_t_15) { + __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 127, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_GOTREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_t_8); + __Pyx_INCREF(__pyx_t_2); + __pyx_v_exc = __pyx_t_2; + + /* "reppy/robots.pyx":128 + * wrap_exception(exceptions.ExcessiveRedirects, exc) + * except ReadTimeout as exc: + * wrap_exception(exceptions.ReadTimeout, exc) # <<<<<<<<<<<<<< + * + * def RobotsUrlMethod(cls, url): + */ + __Pyx_TraceLine(128,0,__PYX_ERR(2, 128, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_16); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + goto __pyx_L4_exception_handled; + } goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":91 * after_response_hook(wrapped) * raise wrapped * try: # <<<<<<<<<<<<<< @@ -4498,7 +4544,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L8_try_end:; } - /* "reppy/robots.pyx":80 + /* "reppy/robots.pyx":81 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -4533,8 +4579,8 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":127 - * wrap_exception(exceptions.ExcessiveRedirects, exc) +/* "reppy/robots.pyx":130 + * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' @@ -4574,11 +4620,11 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(2, 127, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(2, 130, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(2, 127, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(2, 130, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4591,7 +4637,7 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 127, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 130, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4614,38 +4660,38 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject PyObject *__pyx_t_4 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__14) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); - __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[2], 127, 0, __PYX_ERR(2, 127, __pyx_L1_error)); + __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[2], 130, 0, __PYX_ERR(2, 130, __pyx_L1_error)); - /* "reppy/robots.pyx":129 + /* "reppy/robots.pyx":132 * def RobotsUrlMethod(cls, url): * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) # <<<<<<<<<<<<<< * * cdef class Robots: */ - __Pyx_TraceLine(129,0,__PYX_ERR(2, 129, __pyx_L1_error)) + __Pyx_TraceLine(132,0,__PYX_ERR(2, 132, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 129, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 132, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 129, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 132, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(2, 129, __pyx_L1_error) + __PYX_ERR(2, 132, __pyx_L1_error) } - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 129, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 132, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 129, __pyx_L1_error) + __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 132, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":127 - * wrap_exception(exceptions.ExcessiveRedirects, exc) + /* "reppy/robots.pyx":130 + * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' @@ -4665,7 +4711,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":147 +/* "reppy/robots.pyx":150 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -4708,7 +4754,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(2, 147, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(2, 150, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: @@ -4718,7 +4764,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 147, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 150, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4731,12 +4777,12 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } __pyx_v_url = values[0]; - __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 147, __pyx_L3_error) + __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 150, __pyx_L3_error) __pyx_v_expires = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 147, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 150, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4757,43 +4803,43 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob std::string __pyx_t_2; Rep::Robots *__pyx_t_3; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[2], 147, 0, __PYX_ERR(2, 147, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 150, 0, __PYX_ERR(2, 150, __pyx_L1_error)); - /* "reppy/robots.pyx":148 + /* "reppy/robots.pyx":151 * * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) # <<<<<<<<<<<<<< * self.expires = expires * */ - __Pyx_TraceLine(148,0,__PYX_ERR(2, 148, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 148, __pyx_L1_error) + __Pyx_TraceLine(151,0,__PYX_ERR(2, 151, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 151, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 148, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 151, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = new Rep::Robots(__pyx_v_content, __pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(2, 148, __pyx_L1_error) + __PYX_ERR(2, 151, __pyx_L1_error) } __pyx_v_self->robots = __pyx_t_3; - /* "reppy/robots.pyx":149 + /* "reppy/robots.pyx":152 * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) * self.expires = expires # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(149,0,__PYX_ERR(2, 149, __pyx_L1_error)) + __Pyx_TraceLine(152,0,__PYX_ERR(2, 152, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); __Pyx_GOTREF(__pyx_v_self->expires); __Pyx_DECREF(__pyx_v_self->expires); __pyx_v_self->expires = __pyx_v_expires; - /* "reppy/robots.pyx":147 + /* "reppy/robots.pyx":150 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -4814,7 +4860,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob return __pyx_r; } -/* "reppy/robots.pyx":151 +/* "reppy/robots.pyx":154 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4841,24 +4887,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[2], 151, 0, __PYX_ERR(2, 151, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[2], 154, 0, __PYX_ERR(2, 154, __pyx_L1_error)); - /* "reppy/robots.pyx":152 + /* "reppy/robots.pyx":155 * * def __str__(self): * return self.robots.str().decode('utf8') # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __Pyx_TraceLine(152,0,__PYX_ERR(2, 152, __pyx_L1_error)) + __Pyx_TraceLine(155,0,__PYX_ERR(2, 155, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_decode_cpp_string(__pyx_v_self->robots->str(), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 152, __pyx_L1_error) + __pyx_t_1 = __Pyx_decode_cpp_string(__pyx_v_self->robots->str(), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 155, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":151 + /* "reppy/robots.pyx":154 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4878,7 +4924,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":154 +/* "reppy/robots.pyx":157 * return self.robots.str().decode('utf8') * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4901,19 +4947,19 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - __Pyx_TraceCall("__dealloc__", __pyx_f[2], 154, 0, __PYX_ERR(2, 154, __pyx_L1_error)); + __Pyx_TraceCall("__dealloc__", __pyx_f[2], 157, 0, __PYX_ERR(2, 157, __pyx_L1_error)); - /* "reppy/robots.pyx":155 + /* "reppy/robots.pyx":158 * * def __dealloc__(self): * del self.robots # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(155,0,__PYX_ERR(2, 155, __pyx_L1_error)) + __Pyx_TraceLine(158,0,__PYX_ERR(2, 158, __pyx_L1_error)) delete __pyx_v_self->robots; - /* "reppy/robots.pyx":154 + /* "reppy/robots.pyx":157 * return self.robots.str().decode('utf8') * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4930,7 +4976,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_RefNannyFinishContext(); } -/* "reppy/robots.pyx":158 +/* "reppy/robots.pyx":161 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4959,22 +5005,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 158, 0, __PYX_ERR(2, 158, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 161, 0, __PYX_ERR(2, 161, __pyx_L1_error)); - /* "reppy/robots.pyx":160 + /* "reppy/robots.pyx":163 * def sitemaps(self): * '''Get all the sitemaps in this robots.txt.''' * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< * * def allowed(self, path, name): */ - __Pyx_TraceLine(160,0,__PYX_ERR(2, 160, __pyx_L1_error)) + __Pyx_TraceLine(163,0,__PYX_ERR(2, 163, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 160, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 163, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 160, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 163, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 160, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 163, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -4982,14 +5028,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 160, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 163, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":158 + /* "reppy/robots.pyx":161 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -5011,7 +5057,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ return __pyx_r; } -/* "reppy/robots.pyx":162 +/* "reppy/robots.pyx":165 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -5051,11 +5097,11 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(2, 162, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(2, 165, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(2, 162, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(2, 165, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -5068,7 +5114,7 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 162, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 165, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5089,32 +5135,32 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp std::string __pyx_t_2; std::string __pyx_t_3; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[2], 162, 0, __PYX_ERR(2, 162, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[2], 165, 0, __PYX_ERR(2, 165, __pyx_L1_error)); - /* "reppy/robots.pyx":164 + /* "reppy/robots.pyx":167 * def allowed(self, path, name): * '''Is the provided path allowed for the provided agent?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) # <<<<<<<<<<<<<< * * def agent(self, name): */ - __Pyx_TraceLine(164,0,__PYX_ERR(2, 164, __pyx_L1_error)) + __Pyx_TraceLine(167,0,__PYX_ERR(2, 167, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 164, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 164, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 167, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 164, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 164, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 167, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 164, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 167, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":162 + /* "reppy/robots.pyx":165 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -5134,7 +5180,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":166 +/* "reppy/robots.pyx":169 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -5167,20 +5213,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ int __pyx_t_5; PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("agent", 0); - __Pyx_TraceCall("agent", __pyx_f[2], 166, 0, __PYX_ERR(2, 166, __pyx_L1_error)); + __Pyx_TraceCall("agent", __pyx_f[2], 169, 0, __PYX_ERR(2, 169, __pyx_L1_error)); - /* "reppy/robots.pyx":173 + /* "reppy/robots.pyx":176 * Agent object. * ''' * return Agent.from_robots(self, as_bytes(name)) # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(173,0,__PYX_ERR(2, 173, __pyx_L1_error)) + __Pyx_TraceLine(176,0,__PYX_ERR(2, 176, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 173, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 176, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 173, __pyx_L1_error) + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 176, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; __pyx_t_5 = 0; @@ -5197,7 +5243,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 173, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 176, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -5206,14 +5252,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 173, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 176, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 173, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 176, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -5224,7 +5270,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 173, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 176, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -5233,7 +5279,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":166 + /* "reppy/robots.pyx":169 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -5257,7 +5303,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":176 +/* "reppy/robots.pyx":179 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -5286,20 +5332,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 176, 0, __PYX_ERR(2, 176, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 179, 0, __PYX_ERR(2, 179, __pyx_L1_error)); - /* "reppy/robots.pyx":178 + /* "reppy/robots.pyx":181 * def expired(self): * '''True if the current time is past its expiration.''' * return time.time() > self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(178,0,__PYX_ERR(2, 178, __pyx_L1_error)) + __Pyx_TraceLine(181,0,__PYX_ERR(2, 181, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 178, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 178, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -5313,20 +5359,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } } if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 178, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 178, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 178, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":176 + /* "reppy/robots.pyx":179 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -5348,7 +5394,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":181 +/* "reppy/robots.pyx":184 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -5374,22 +5420,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 181, 0, __PYX_ERR(2, 181, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 184, 0, __PYX_ERR(2, 184, __pyx_L1_error)); - /* "reppy/robots.pyx":183 + /* "reppy/robots.pyx":186 * def expires(self): * '''The expiration of this robots.txt.''' * return self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(183,0,__PYX_ERR(2, 183, __pyx_L1_error)) + __Pyx_TraceLine(186,0,__PYX_ERR(2, 186, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_self->expires); __pyx_r = __pyx_v_self->expires; goto __pyx_L0; - /* "reppy/robots.pyx":181 + /* "reppy/robots.pyx":184 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -5408,7 +5454,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":186 +/* "reppy/robots.pyx":189 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -5440,21 +5486,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 186, 0, __PYX_ERR(2, 186, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 189, 0, __PYX_ERR(2, 189, __pyx_L1_error)); - /* "reppy/robots.pyx":188 + /* "reppy/robots.pyx":191 * def ttl(self): * '''Remaining time for this response to be considered valid.''' * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(188,0,__PYX_ERR(2, 188, __pyx_L1_error)) + __Pyx_TraceLine(191,0,__PYX_ERR(2, 191, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 188, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 191, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 188, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 191, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -5468,24 +5514,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 188, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 191, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 188, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 191, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 188, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 191, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 188, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 191, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 188, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 191, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(2, 188, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(2, 191, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 188, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 191, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -5499,7 +5545,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":186 + /* "reppy/robots.pyx":189 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -5637,7 +5683,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_12__setstate_cython__(CYTHON_UN return __pyx_r; } -/* "reppy/robots.pyx":194 +/* "reppy/robots.pyx":197 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5681,7 +5727,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 194, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 197, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -5697,7 +5743,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 194, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 197, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5720,17 +5766,17 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[2], 194, 0, __PYX_ERR(2, 194, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 197, 0, __PYX_ERR(2, 197, __pyx_L1_error)); - /* "reppy/robots.pyx":195 + /* "reppy/robots.pyx":198 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(195,0,__PYX_ERR(2, 195, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 195, __pyx_L1_error) + __Pyx_TraceLine(198,0,__PYX_ERR(2, 198, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 198, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -5747,7 +5793,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 195, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 198, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -5755,13 +5801,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 195, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 198, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 195, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 198, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -5778,14 +5824,14 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 195, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 198, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":194 + /* "reppy/robots.pyx":197 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5924,7 +5970,7 @@ static PyObject *__pyx_pf_5reppy_6robots_9AllowNone_4__setstate_cython__(CYTHON_ return __pyx_r; } -/* "reppy/robots.pyx":201 +/* "reppy/robots.pyx":204 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5967,7 +6013,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 201, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 204, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -5983,7 +6029,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 201, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 204, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -6006,15 +6052,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[2], 201, 0, __PYX_ERR(2, 201, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 204, 0, __PYX_ERR(2, 204, __pyx_L1_error)); - /* "reppy/robots.pyx":202 + /* "reppy/robots.pyx":205 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(202,0,__PYX_ERR(2, 202, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 202, __pyx_L1_error) + __Pyx_TraceLine(205,0,__PYX_ERR(2, 205, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 205, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -6031,7 +6077,7 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 202, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 205, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -6039,13 +6085,13 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 202, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 205, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 202, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 205, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -6062,14 +6108,14 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 202, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 205, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":201 + /* "reppy/robots.pyx":204 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -7368,6 +7414,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_PY3, __pyx_k_PY3, sizeof(__pyx_k_PY3), 0, 0, 1, 1}, {&__pyx_n_s_ParseMethod, __pyx_k_ParseMethod, sizeof(__pyx_k_ParseMethod), 0, 0, 1, 1}, {&__pyx_n_s_Pyx_CFunc_object____object___t, __pyx_k_Pyx_CFunc_object____object___t, sizeof(__pyx_k_Pyx_CFunc_object____object___t), 0, 0, 1, 1}, + {&__pyx_n_s_ReadTimeout, __pyx_k_ReadTimeout, sizeof(__pyx_k_ReadTimeout), 0, 0, 1, 1}, {&__pyx_n_s_RobotsUrlMethod, __pyx_k_RobotsUrlMethod, sizeof(__pyx_k_RobotsUrlMethod), 0, 0, 1, 1}, {&__pyx_n_s_SSLError, __pyx_k_SSLError, sizeof(__pyx_k_SSLError), 0, 0, 1, 1}, {&__pyx_n_s_SSLException, __pyx_k_SSLException, sizeof(__pyx_k_SSLException), 0, 0, 1, 1}, @@ -7454,7 +7501,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) __pyx_builtin_TypeError = __Pyx_GetBuiltinName(__pyx_n_s_TypeError); if (!__pyx_builtin_TypeError) __PYX_ERR(1, 2, __pyx_L1_error) - __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(2, 160, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(2, 163, __pyx_L1_error) __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(1, 61, __pyx_L1_error) return 0; __pyx_L1_error:; @@ -7465,25 +7512,25 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "reppy/robots.pyx":24 + /* "reppy/robots.pyx":25 * if isinstance(value, bytes): * return value * return value.encode('utf-8') # <<<<<<<<<<<<<< * * cdef as_string(value): */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple_)) __PYX_ERR(2, 24, __pyx_L1_error) + __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple_)) __PYX_ERR(2, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_); - /* "reppy/robots.pyx":29 + /* "reppy/robots.pyx":30 * if six.PY3: * if isinstance(value, bytes): * return value.decode('utf-8') # <<<<<<<<<<<<<< * return value * */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(2, 29, __pyx_L1_error) + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(2, 30, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); @@ -7506,51 +7553,51 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__5); __Pyx_GIVEREF(__pyx_tuple__5); - /* "reppy/robots.pyx":82 + /* "reppy/robots.pyx":83 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): */ - __pyx_tuple__8 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(2, 82, __pyx_L1_error) + __pyx_tuple__8 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(2, 83, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__8); __Pyx_GIVEREF(__pyx_tuple__8); - /* "reppy/robots.pyx":83 + /* "reppy/robots.pyx":84 * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< * def wrap_exception(etype, cause): * wrapped = etype(cause) */ - __pyx_tuple__9 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(2, 83, __pyx_L1_error) + __pyx_tuple__9 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(2, 84, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__9); __Pyx_GIVEREF(__pyx_tuple__9); - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":85 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< * wrapped = etype(cause) * wrapped.url = url */ - __pyx_tuple__10 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 84, __pyx_L1_error) + __pyx_tuple__10 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__10); __Pyx_GIVEREF(__pyx_tuple__10); - __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_wrap_exception, 84, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 84, __pyx_L1_error) + __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_wrap_exception, 85, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 85, __pyx_L1_error) - /* "reppy/robots.pyx":93 + /* "reppy/robots.pyx":94 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __pyx_tuple__12 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 93, __pyx_L1_error) + __pyx_tuple__12 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 94, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__12); __Pyx_GIVEREF(__pyx_tuple__12); - __pyx_tuple__13 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(2, 93, __pyx_L1_error) + __pyx_tuple__13 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(2, 94, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__13); __Pyx_GIVEREF(__pyx_tuple__13); @@ -7623,53 +7670,53 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(__pyx_tuple__22); __pyx_codeobj__23 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__22, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__23)) __PYX_ERR(1, 65, __pyx_L1_error) - /* "reppy/robots.pyx":33 + /* "reppy/robots.pyx":34 * * * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< * '''Construct an Agent from a CppAgent.''' * agent = Agent() */ - __pyx_tuple__24 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(2, 33, __pyx_L1_error) + __pyx_tuple__24 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(2, 34, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__24); __Pyx_GIVEREF(__pyx_tuple__24); - __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__24, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 33, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(2, 33, __pyx_L1_error) + __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__24, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 34, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(2, 34, __pyx_L1_error) - /* "reppy/robots.pyx":76 + /* "reppy/robots.pyx":77 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __pyx_tuple__25 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(2, 76, __pyx_L1_error) + __pyx_tuple__25 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(2, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__25); __Pyx_GIVEREF(__pyx_tuple__25); - __pyx_codeobj__6 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_ParseMethod, 76, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__6)) __PYX_ERR(2, 76, __pyx_L1_error) + __pyx_codeobj__6 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_ParseMethod, 77, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__6)) __PYX_ERR(2, 77, __pyx_L1_error) - /* "reppy/robots.pyx":80 + /* "reppy/robots.pyx":81 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __pyx_tuple__26 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(2, 80, __pyx_L1_error) + __pyx_tuple__26 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(2, 81, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__26); __Pyx_GIVEREF(__pyx_tuple__26); - __pyx_codeobj__7 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__26, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FetchMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__7)) __PYX_ERR(2, 80, __pyx_L1_error) + __pyx_codeobj__7 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__26, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FetchMethod, 81, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__7)) __PYX_ERR(2, 81, __pyx_L1_error) - /* "reppy/robots.pyx":127 - * wrap_exception(exceptions.ExcessiveRedirects, exc) + /* "reppy/robots.pyx":130 + * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __pyx_tuple__27 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(2, 127, __pyx_L1_error) + __pyx_tuple__27 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(2, 130, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__27); __Pyx_GIVEREF(__pyx_tuple__27); - __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__27, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 127, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) __PYX_ERR(2, 127, __pyx_L1_error) + __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__27, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 130, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) __PYX_ERR(2, 130, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -7833,29 +7880,29 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 43, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 44, __pyx_L1_error) __pyx_type_5reppy_6robots_Agent.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 43, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 43, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 44, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 44, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; - if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 131, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 134, __pyx_L1_error) __pyx_type_5reppy_6robots_Robots.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 131, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 131, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 134, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 134, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 191, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 194, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 191, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 191, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 194, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 194, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 198, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 201, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 198, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 198, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 201, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 201, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; - if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(2, 80, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(2, 81, __pyx_L1_error) __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_print = 0; __pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod = &__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod; if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(1, 64, __pyx_L1_error) @@ -7926,7 +7973,7 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) * URLRequired, */ __Pyx_TraceLine(9,0,__PYX_ERR(2, 9, __pyx_L1_error)) - __pyx_t_2 = PyList_New(7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 9, __pyx_L1_error) + __pyx_t_2 = PyList_New(8); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 9, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_SSLError); __Pyx_GIVEREF(__pyx_n_s_SSLError); @@ -7949,6 +7996,9 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) __Pyx_INCREF(__pyx_n_s_TooManyRedirects); __Pyx_GIVEREF(__pyx_n_s_TooManyRedirects); PyList_SET_ITEM(__pyx_t_2, 6, __pyx_n_s_TooManyRedirects); + __Pyx_INCREF(__pyx_n_s_ReadTimeout); + __Pyx_GIVEREF(__pyx_n_s_ReadTimeout); + PyList_SET_ITEM(__pyx_t_2, 7, __pyx_n_s_ReadTimeout); /* "reppy/robots.pyx":8 * @@ -7989,52 +8039,56 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) __Pyx_GOTREF(__pyx_t_2); if (PyDict_SetItem(__pyx_d, __pyx_n_s_TooManyRedirects, __pyx_t_2) < 0) __PYX_ERR(2, 15, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ReadTimeout, __pyx_t_2) < 0) __PYX_ERR(2, 16, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":16 - * InvalidURL, - * TooManyRedirects) + /* "reppy/robots.pyx":17 + * TooManyRedirects, + * ReadTimeout) * import six # <<<<<<<<<<<<<< * * from .ttl import HeaderWithDefaultPolicy */ - __Pyx_TraceLine(16,0,__PYX_ERR(2, 16, __pyx_L1_error)) - __pyx_t_1 = __Pyx_Import(__pyx_n_s_six, 0, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 16, __pyx_L1_error) + __Pyx_TraceLine(17,0,__PYX_ERR(2, 17, __pyx_L1_error)) + __pyx_t_1 = __Pyx_Import(__pyx_n_s_six, 0, -1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 17, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_six, __pyx_t_1) < 0) __PYX_ERR(2, 16, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_six, __pyx_t_1) < 0) __PYX_ERR(2, 17, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":18 + /* "reppy/robots.pyx":19 * import six * * from .ttl import HeaderWithDefaultPolicy # <<<<<<<<<<<<<< * from . import util, logger, exceptions * */ - __Pyx_TraceLine(18,0,__PYX_ERR(2, 18, __pyx_L1_error)) - __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 18, __pyx_L1_error) + __Pyx_TraceLine(19,0,__PYX_ERR(2, 19, __pyx_L1_error)) + __pyx_t_1 = PyList_New(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_n_s_HeaderWithDefaultPolicy); __Pyx_GIVEREF(__pyx_n_s_HeaderWithDefaultPolicy); PyList_SET_ITEM(__pyx_t_1, 0, __pyx_n_s_HeaderWithDefaultPolicy); - __pyx_t_2 = __Pyx_Import(__pyx_n_s_ttl, __pyx_t_1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 18, __pyx_L1_error) + __pyx_t_2 = __Pyx_Import(__pyx_n_s_ttl, __pyx_t_1, 1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 18, __pyx_L1_error) + __pyx_t_1 = __Pyx_ImportFrom(__pyx_t_2, __pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_HeaderWithDefaultPolicy, __pyx_t_1) < 0) __PYX_ERR(2, 18, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_HeaderWithDefaultPolicy, __pyx_t_1) < 0) __PYX_ERR(2, 19, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":19 + /* "reppy/robots.pyx":20 * * from .ttl import HeaderWithDefaultPolicy * from . import util, logger, exceptions # <<<<<<<<<<<<<< * * cdef as_bytes(value): */ - __Pyx_TraceLine(19,0,__PYX_ERR(2, 19, __pyx_L1_error)) - __pyx_t_2 = PyList_New(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 19, __pyx_L1_error) + __Pyx_TraceLine(20,0,__PYX_ERR(2, 20, __pyx_L1_error)) + __pyx_t_2 = PyList_New(3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_n_s_util); __Pyx_GIVEREF(__pyx_n_s_util); @@ -8045,162 +8099,162 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) __Pyx_INCREF(__pyx_n_s_exceptions); __Pyx_GIVEREF(__pyx_n_s_exceptions); PyList_SET_ITEM(__pyx_t_2, 2, __pyx_n_s_exceptions); - __pyx_t_1 = __Pyx_Import(__pyx_n_s__19, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 19, __pyx_L1_error) + __pyx_t_1 = __Pyx_Import(__pyx_n_s__19, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_util); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 19, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_util); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_util, __pyx_t_2) < 0) __PYX_ERR(2, 19, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_util, __pyx_t_2) < 0) __PYX_ERR(2, 20, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_logger); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 19, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_logger); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_logger, __pyx_t_2) < 0) __PYX_ERR(2, 19, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_logger, __pyx_t_2) < 0) __PYX_ERR(2, 20, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 19, __pyx_L1_error) + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_exceptions, __pyx_t_2) < 0) __PYX_ERR(2, 19, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_exceptions, __pyx_t_2) < 0) __PYX_ERR(2, 20, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":33 + /* "reppy/robots.pyx":34 * * * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< * '''Construct an Agent from a CppAgent.''' * agent = Agent() */ - __Pyx_TraceLine(33,0,__PYX_ERR(2, 33, __pyx_L1_error)) - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_1FromRobotsMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 33, __pyx_L1_error) + __Pyx_TraceLine(34,0,__PYX_ERR(2, 34, __pyx_L1_error)) + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_1FromRobotsMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 34, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FromRobotsMethod, __pyx_t_1) < 0) __PYX_ERR(2, 33, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FromRobotsMethod, __pyx_t_1) < 0) __PYX_ERR(2, 34, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":48 + /* "reppy/robots.pyx":49 * cdef CppAgent agent * * from_robots = classmethod(FromRobotsMethod) # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(48,0,__PYX_ERR(2, 48, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 48, __pyx_L1_error) + __Pyx_TraceLine(49,0,__PYX_ERR(2, 49, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 48, __pyx_L1_error) + __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 49, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Agent->tp_dict, __pyx_n_s_from_robots, __pyx_t_2) < 0) __PYX_ERR(2, 48, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Agent->tp_dict, __pyx_n_s_from_robots, __pyx_t_2) < 0) __PYX_ERR(2, 49, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Agent); - /* "reppy/robots.pyx":76 + /* "reppy/robots.pyx":77 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __Pyx_TraceLine(76,0,__PYX_ERR(2, 76, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 76, __pyx_L1_error) + __Pyx_TraceLine(77,0,__PYX_ERR(2, 77, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(2, 76, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(2, 77, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":80 + /* "reppy/robots.pyx":81 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __Pyx_TraceLine(80,0,__PYX_ERR(2, 80, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 80, __pyx_L1_error) + __Pyx_TraceLine(81,0,__PYX_ERR(2, 81, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 81, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(2, 80, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(2, 81, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":127 - * wrap_exception(exceptions.ExcessiveRedirects, exc) + /* "reppy/robots.pyx":130 + * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __Pyx_TraceLine(127,0,__PYX_ERR(2, 127, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 127, __pyx_L1_error) + __Pyx_TraceLine(130,0,__PYX_ERR(2, 130, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 130, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(2, 127, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(2, 130, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":136 + /* "reppy/robots.pyx":139 * # The default TTL policy is to cache for 3600 seconds or what's provided in the * # headers, and a minimum of 600 seconds * DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) # <<<<<<<<<<<<<< * * # Class methods */ - __Pyx_TraceLine(136,0,__PYX_ERR(2, 136, __pyx_L1_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 136, __pyx_L1_error) + __Pyx_TraceLine(139,0,__PYX_ERR(2, 139, __pyx_L1_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 136, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(2, 136, __pyx_L1_error) - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(2, 136, __pyx_L1_error) - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 136, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(2, 139, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(2, 139, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 139, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(2, 136, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(2, 139, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":139 + /* "reppy/robots.pyx":142 * * # Class methods * parse = classmethod(ParseMethod) # <<<<<<<<<<<<<< * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) */ - __Pyx_TraceLine(139,0,__PYX_ERR(2, 139, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 139, __pyx_L1_error) + __Pyx_TraceLine(142,0,__PYX_ERR(2, 142, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 142, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 139, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 142, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(2, 139, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(2, 142, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":140 + /* "reppy/robots.pyx":143 * # Class methods * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) # <<<<<<<<<<<<<< * robots_url = classmethod(RobotsUrlMethod) * */ - __Pyx_TraceLine(140,0,__PYX_ERR(2, 140, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 140, __pyx_L1_error) + __Pyx_TraceLine(143,0,__PYX_ERR(2, 143, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 143, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 140, __pyx_L1_error) + __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 143, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(2, 140, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(2, 143, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":141 + /* "reppy/robots.pyx":144 * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) # <<<<<<<<<<<<<< * * # Data members */ - __Pyx_TraceLine(141,0,__PYX_ERR(2, 141, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 141, __pyx_L1_error) + __Pyx_TraceLine(144,0,__PYX_ERR(2, 144, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 141, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 144, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(2, 141, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(2, 144, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); From 7dcec7e1cbbe7645c40b35c6ab91b0712940e4e9 Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Wed, 19 Sep 2018 09:41:57 -0700 Subject: [PATCH 093/113] Bump version and update maintainer --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index f0007f6..911bcdf 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.10', + version='0.4.11', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like @@ -69,8 +69,8 @@ - Configurable user agent for fetching robots.txt - Automatic refetching based on expiration ''', - maintainer='Brandon Forehand', - maintainer_email='brandon@moz.com', + maintainer='Lindsey Reno', + maintainer_email='lindsey@moz.com', url='http://github.com/seomoz/reppy', license='MIT', platforms='Posix; MacOS X', From cdfb058d424d1b5f21c108b8cb24eba76a94264d Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Mon, 8 Oct 2018 12:44:24 -0700 Subject: [PATCH 094/113] log reppy errors would have saved me lots of hassle --- reppy/cache/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reppy/cache/__init__.py b/reppy/cache/__init__.py index 2f45297..f15f0ec 100644 --- a/reppy/cache/__init__.py +++ b/reppy/cache/__init__.py @@ -8,6 +8,7 @@ from .policy import DefaultObjectPolicy, ReraiseExceptionPolicy from ..robots import Robots, AllowNone, Agent +from .. import logger class ExpiringObject(object): @@ -61,6 +62,7 @@ def factory(self, url): try: return self.fetch(url) except BaseException as exc: + logger.exception('Reppy error on %s' % url) return self.cache_policy.exception(url, exc) def fetch(self, url): From 09a76710c2b83b2898f126301ef595b6dbc3dabe Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Mon, 8 Oct 2018 16:39:22 -0700 Subject: [PATCH 095/113] Add test for str() UnicodeDecodeError problem --- tests/test_robots.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_robots.py b/tests/test_robots.py index cb5990b..ae81649 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -270,6 +270,23 @@ def test_utf8_bom(self): self.assertTrue(robot.allowed('http://example.com/path', 'agent')) self.assertFalse(robot.allowed('http://example.com/path', 'other')) + def test_str_function(self): + ''' + If there is valid UTF-8, str() should return a representation of the + directives. + + This came out of a UnicodeDecodeError happening in Python 2, when we + were unduly decoding the bytes (via UTF-8) to unicode, then implictly + converting back to bytes via UTF-8. + ''' + robot = robots.Robots.parse('http://example.com/robots.txt', + codecs.BOM_UTF8 + b''' + User-Agent: \xc3\xa4gent + Allow: /swedish-chef + ''') + s = str(robot) + self.assertTrue('ägent' in s) + def test_utf16_bom(self): '''If there's a utf-16 BOM, we should parse it as such''' robot = robots.Robots.parse('http://example.com/robots.txt', From f055d73ef02ae54adccaab4cf5afd39452347ae2 Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Mon, 8 Oct 2018 15:57:15 -0700 Subject: [PATCH 096/113] Fix UnicodeDecode/Encode errors for Python 2 For Python 2, the `Robots.__str__` function was returning a unicode object instead of a `str` (bytes) object. This had two problems: * For invalid UTF-8, the `decode()` method was raising a UnicodeDecodeError. Really, we should just be returning the raw bytes so we should not be worrying about encodings here. However, in Python 3 this problem will still exist. I'm not sure what the right answer is here, if we want to use `.decode('utf-8', 'replace')`? In this case -- `Robots.__str__` is just debug information -- we might want to just do that. * Even for valid UTF-8, if there are non-ASCII characters, Python 2 will convert the unicode object back to bytes via the `str()` function, using the ASCII encoding. This will result in a UnicodeEncodeError. This problem was Python 2 specific. For Python 2, both these have been fixed in this commit. --- reppy/robots.pyx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/reppy/robots.pyx b/reppy/robots.pyx index e5297d9..65e4c02 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -24,6 +24,9 @@ cdef as_bytes(value): return value return value.encode('utf-8') +# For contexts which require a 'str' type, convert bytes to unicode if needed +# (i.e., Python 3). Note: could raise UnicodeDecodeError in Python 3 if input +# is invalid UTF-8 cdef as_string(value): if six.PY3: if isinstance(value, bytes): @@ -49,7 +52,7 @@ cdef class Agent: from_robots = classmethod(FromRobotsMethod) def __str__(self): - return self.agent.str().decode('utf8') + return as_string(self.agent.str()) @property def delay(self): @@ -152,7 +155,9 @@ cdef class Robots: self.expires = expires def __str__(self): - return self.robots.str().decode('utf8') + # Note: this could raise a UnicodeDecodeError in Python 3 if the + # robots.txt had invalid UTF-8 + return as_string(self.robots.str()) def __dealloc__(self): del self.robots From 0de7f72d2bea4960fc35702ad5d82b93f58cdd13 Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Mon, 8 Oct 2018 16:22:53 -0700 Subject: [PATCH 097/113] update robots.cpp --- reppy/robots.cpp | 1219 ++++++++++++++++++++++------------------------ 1 file changed, 585 insertions(+), 634 deletions(-) diff --git a/reppy/robots.cpp b/reppy/robots.cpp index f7858d9..8f29741 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -760,7 +760,7 @@ struct __pyx_obj_5reppy_6robots_AllowAll; struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod; struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py; -/* "reppy/robots.pyx":44 +/* "reppy/robots.pyx":47 * return agent * * cdef class Agent: # <<<<<<<<<<<<<< @@ -773,7 +773,7 @@ struct __pyx_obj_5reppy_6robots_Agent { }; -/* "reppy/robots.pyx":134 +/* "reppy/robots.pyx":137 * return as_string(CppRobots.robotsUrl(as_bytes(url))) * * cdef class Robots: # <<<<<<<<<<<<<< @@ -787,7 +787,7 @@ struct __pyx_obj_5reppy_6robots_Robots { }; -/* "reppy/robots.pyx":194 +/* "reppy/robots.pyx":199 * * * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< @@ -799,7 +799,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone { }; -/* "reppy/robots.pyx":201 +/* "reppy/robots.pyx":206 * * * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< @@ -811,7 +811,7 @@ struct __pyx_obj_5reppy_6robots_AllowAll { }; -/* "reppy/robots.pyx":81 +/* "reppy/robots.pyx":84 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -1151,38 +1151,6 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ __Pyx__ArgTypeTest(obj, type, name, exact)) static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); -/* IncludeCppStringH.proto */ -#include - -/* decode_c_string_utf16.proto */ -static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, const char *errors) { - int byteorder = 0; - return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); -} -static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16LE(const char *s, Py_ssize_t size, const char *errors) { - int byteorder = -1; - return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); -} -static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16BE(const char *s, Py_ssize_t size, const char *errors) { - int byteorder = 1; - return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); -} - -/* decode_c_bytes.proto */ -static CYTHON_INLINE PyObject* __Pyx_decode_c_bytes( - const char* cstring, Py_ssize_t length, Py_ssize_t start, Py_ssize_t stop, - const char* encoding, const char* errors, - PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)); - -/* decode_cpp_string.proto */ -static CYTHON_INLINE PyObject* __Pyx_decode_cpp_string( - std::string cppstring, Py_ssize_t start, Py_ssize_t stop, - const char* encoding, const char* errors, - PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { - return __Pyx_decode_c_bytes( - cppstring.data(), cppstring.size(), start, stop, encoding, errors, decode_func); -} - /* PyThreadStateGet.proto */ #if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; @@ -1857,7 +1825,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { * return value * return value.encode('utf-8') # <<<<<<<<<<<<<< * - * cdef as_string(value): + * # For contexts which require a 'str' type, convert bytes to unicode if needed */ __Pyx_TraceLine(25,0,__PYX_ERR(2, 25, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); @@ -1891,9 +1859,9 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { return __pyx_r; } -/* "reppy/robots.pyx":27 - * return value.encode('utf-8') - * +/* "reppy/robots.pyx":30 + * # (i.e., Python 3). Note: could raise UnicodeDecodeError in Python 3 if input + * # is invalid UTF-8 * cdef as_string(value): # <<<<<<<<<<<<<< * if six.PY3: * if isinstance(value, bytes): @@ -1908,56 +1876,56 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { int __pyx_t_3; int __pyx_t_4; __Pyx_RefNannySetupContext("as_string", 0); - __Pyx_TraceCall("as_string", __pyx_f[2], 27, 0, __PYX_ERR(2, 27, __pyx_L1_error)); + __Pyx_TraceCall("as_string", __pyx_f[2], 30, 0, __PYX_ERR(2, 30, __pyx_L1_error)); - /* "reppy/robots.pyx":28 - * + /* "reppy/robots.pyx":31 + * # is invalid UTF-8 * cdef as_string(value): * if six.PY3: # <<<<<<<<<<<<<< * if isinstance(value, bytes): * return value.decode('utf-8') */ - __Pyx_TraceLine(28,0,__PYX_ERR(2, 28, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_six); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 28, __pyx_L1_error) + __Pyx_TraceLine(31,0,__PYX_ERR(2, 31, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_six); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 31, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_PY3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 28, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_PY3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 31, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(2, 28, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) __PYX_ERR(2, 31, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_3) { - /* "reppy/robots.pyx":29 + /* "reppy/robots.pyx":32 * cdef as_string(value): * if six.PY3: * if isinstance(value, bytes): # <<<<<<<<<<<<<< * return value.decode('utf-8') * return value */ - __Pyx_TraceLine(29,0,__PYX_ERR(2, 29, __pyx_L1_error)) + __Pyx_TraceLine(32,0,__PYX_ERR(2, 32, __pyx_L1_error)) __pyx_t_3 = PyBytes_Check(__pyx_v_value); __pyx_t_4 = (__pyx_t_3 != 0); if (__pyx_t_4) { - /* "reppy/robots.pyx":30 + /* "reppy/robots.pyx":33 * if six.PY3: * if isinstance(value, bytes): * return value.decode('utf-8') # <<<<<<<<<<<<<< * return value * */ - __Pyx_TraceLine(30,0,__PYX_ERR(2, 30, __pyx_L1_error)) + __Pyx_TraceLine(33,0,__PYX_ERR(2, 33, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_decode); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 30, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_decode); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 33, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 30, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 33, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":29 + /* "reppy/robots.pyx":32 * cdef as_string(value): * if six.PY3: * if isinstance(value, bytes): # <<<<<<<<<<<<<< @@ -1966,8 +1934,8 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { */ } - /* "reppy/robots.pyx":28 - * + /* "reppy/robots.pyx":31 + * # is invalid UTF-8 * cdef as_string(value): * if six.PY3: # <<<<<<<<<<<<<< * if isinstance(value, bytes): @@ -1975,22 +1943,22 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { */ } - /* "reppy/robots.pyx":31 + /* "reppy/robots.pyx":34 * if isinstance(value, bytes): * return value.decode('utf-8') * return value # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(31,0,__PYX_ERR(2, 31, __pyx_L1_error)) + __Pyx_TraceLine(34,0,__PYX_ERR(2, 34, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_value); __pyx_r = __pyx_v_value; goto __pyx_L0; - /* "reppy/robots.pyx":27 - * return value.encode('utf-8') - * + /* "reppy/robots.pyx":30 + * # (i.e., Python 3). Note: could raise UnicodeDecodeError in Python 3 if input + * # is invalid UTF-8 * cdef as_string(value): # <<<<<<<<<<<<<< * if six.PY3: * if isinstance(value, bytes): @@ -2009,7 +1977,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { return __pyx_r; } -/* "reppy/robots.pyx":34 +/* "reppy/robots.pyx":37 * * * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< @@ -2053,17 +2021,17 @@ static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_robots)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 1); __PYX_ERR(2, 34, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 1); __PYX_ERR(2, 37, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 2); __PYX_ERR(2, 34, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 2); __PYX_ERR(2, 37, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "FromRobotsMethod") < 0)) __PYX_ERR(2, 34, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "FromRobotsMethod") < 0)) __PYX_ERR(2, 37, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 3) { goto __pyx_L5_argtuple_error; @@ -2074,17 +2042,17 @@ static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, } __pyx_v_cls = values[0]; __pyx_v_robots = ((struct __pyx_obj_5reppy_6robots_Robots *)values[1]); - __pyx_v_name = __pyx_convert_string_from_py_std__in_string(values[2]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 34, __pyx_L3_error) + __pyx_v_name = __pyx_convert_string_from_py_std__in_string(values[2]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 37, __pyx_L3_error) } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 34, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 37, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.FromRobotsMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); return NULL; __pyx_L4_argument_unpacking_done:; - if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_robots), __pyx_ptype_5reppy_6robots_Robots, 1, "robots", 0))) __PYX_ERR(2, 34, __pyx_L1_error) + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_robots), __pyx_ptype_5reppy_6robots_Robots, 1, "robots", 0))) __PYX_ERR(2, 37, __pyx_L1_error) __pyx_r = __pyx_pf_5reppy_6robots_FromRobotsMethod(__pyx_self, __pyx_v_cls, __pyx_v_robots, __pyx_v_name); /* function exit code */ @@ -2104,45 +2072,45 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject PyObject *__pyx_t_1 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__3) __Pyx_RefNannySetupContext("FromRobotsMethod", 0); - __Pyx_TraceCall("FromRobotsMethod", __pyx_f[2], 34, 0, __PYX_ERR(2, 34, __pyx_L1_error)); + __Pyx_TraceCall("FromRobotsMethod", __pyx_f[2], 37, 0, __PYX_ERR(2, 37, __pyx_L1_error)); - /* "reppy/robots.pyx":36 + /* "reppy/robots.pyx":39 * def FromRobotsMethod(cls, Robots robots, const string& name): * '''Construct an Agent from a CppAgent.''' * agent = Agent() # <<<<<<<<<<<<<< * # This is somewhat inefficient due to the copying, but it is * # required to be copied because we often toss the containing */ - __Pyx_TraceLine(36,0,__PYX_ERR(2, 36, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 36, __pyx_L1_error) + __Pyx_TraceLine(39,0,__PYX_ERR(2, 39, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 39, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_agent = ((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":41 + /* "reppy/robots.pyx":44 * # Robots object as a temporary thus we'd leave the underlying * # Agent object dangling without a full copy. * agent.agent = robots.robots.agent(name) # <<<<<<<<<<<<<< * return agent * */ - __Pyx_TraceLine(41,0,__PYX_ERR(2, 41, __pyx_L1_error)) + __Pyx_TraceLine(44,0,__PYX_ERR(2, 44, __pyx_L1_error)) __pyx_v_agent->agent = __pyx_v_robots->robots->agent(__pyx_v_name); - /* "reppy/robots.pyx":42 + /* "reppy/robots.pyx":45 * # Agent object dangling without a full copy. * agent.agent = robots.robots.agent(name) * return agent # <<<<<<<<<<<<<< * * cdef class Agent: */ - __Pyx_TraceLine(42,0,__PYX_ERR(2, 42, __pyx_L1_error)) + __Pyx_TraceLine(45,0,__PYX_ERR(2, 45, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_agent)); __pyx_r = ((PyObject *)__pyx_v_agent); goto __pyx_L0; - /* "reppy/robots.pyx":34 + /* "reppy/robots.pyx":37 * * * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< @@ -2163,11 +2131,11 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":51 +/* "reppy/robots.pyx":54 * from_robots = classmethod(FromRobotsMethod) * * def __str__(self): # <<<<<<<<<<<<<< - * return self.agent.str().decode('utf8') + * return as_string(self.agent.str()) * */ @@ -2189,35 +2157,40 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[2], 51, 0, __PYX_ERR(2, 51, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[2], 54, 0, __PYX_ERR(2, 54, __pyx_L1_error)); - /* "reppy/robots.pyx":52 + /* "reppy/robots.pyx":55 * * def __str__(self): - * return self.agent.str().decode('utf8') # <<<<<<<<<<<<<< + * return as_string(self.agent.str()) # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(52,0,__PYX_ERR(2, 52, __pyx_L1_error)) + __Pyx_TraceLine(55,0,__PYX_ERR(2, 55, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_decode_cpp_string(__pyx_v_self->agent.str(), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 52, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->agent.str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 55, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; + __pyx_t_2 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 55, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":51 + /* "reppy/robots.pyx":54 * from_robots = classmethod(FromRobotsMethod) * * def __str__(self): # <<<<<<<<<<<<<< - * return self.agent.str().decode('utf8') + * return as_string(self.agent.str()) * */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); __Pyx_AddTraceback("reppy.robots.Agent.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; @@ -2227,7 +2200,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":55 +/* "reppy/robots.pyx":58 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -2256,45 +2229,45 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ int __pyx_t_1; PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 55, 0, __PYX_ERR(2, 55, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 58, 0, __PYX_ERR(2, 58, __pyx_L1_error)); - /* "reppy/robots.pyx":57 + /* "reppy/robots.pyx":60 * def delay(self): * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() # <<<<<<<<<<<<<< * if value > 0: * return value */ - __Pyx_TraceLine(57,0,__PYX_ERR(2, 57, __pyx_L1_error)) + __Pyx_TraceLine(60,0,__PYX_ERR(2, 60, __pyx_L1_error)) __pyx_v_value = __pyx_v_self->agent.delay(); - /* "reppy/robots.pyx":58 + /* "reppy/robots.pyx":61 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< * return value * return None */ - __Pyx_TraceLine(58,0,__PYX_ERR(2, 58, __pyx_L1_error)) + __Pyx_TraceLine(61,0,__PYX_ERR(2, 61, __pyx_L1_error)) __pyx_t_1 = ((__pyx_v_value > 0.0) != 0); if (__pyx_t_1) { - /* "reppy/robots.pyx":59 + /* "reppy/robots.pyx":62 * cdef float value = self.agent.delay() * if value > 0: * return value # <<<<<<<<<<<<<< * return None * */ - __Pyx_TraceLine(59,0,__PYX_ERR(2, 59, __pyx_L1_error)) + __Pyx_TraceLine(62,0,__PYX_ERR(2, 62, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 59, __pyx_L1_error) + __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 62, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":58 + /* "reppy/robots.pyx":61 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< @@ -2303,20 +2276,20 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ */ } - /* "reppy/robots.pyx":60 + /* "reppy/robots.pyx":63 * if value > 0: * return value * return None # <<<<<<<<<<<<<< * * def allow(self, path): */ - __Pyx_TraceLine(60,0,__PYX_ERR(2, 60, __pyx_L1_error)) + __Pyx_TraceLine(63,0,__PYX_ERR(2, 63, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; goto __pyx_L0; - /* "reppy/robots.pyx":55 + /* "reppy/robots.pyx":58 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -2336,7 +2309,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ return __pyx_r; } -/* "reppy/robots.pyx":62 +/* "reppy/robots.pyx":65 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -2365,36 +2338,36 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allow", 0); - __Pyx_TraceCall("allow", __pyx_f[2], 62, 0, __PYX_ERR(2, 62, __pyx_L1_error)); + __Pyx_TraceCall("allow", __pyx_f[2], 65, 0, __PYX_ERR(2, 65, __pyx_L1_error)); - /* "reppy/robots.pyx":64 + /* "reppy/robots.pyx":67 * def allow(self, path): * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(64,0,__PYX_ERR(2, 64, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 64, __pyx_L1_error) + __Pyx_TraceLine(67,0,__PYX_ERR(2, 67, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 67, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 64, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 67, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.allow(__pyx_t_2); - /* "reppy/robots.pyx":65 + /* "reppy/robots.pyx":68 * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def disallow(self, path): */ - __Pyx_TraceLine(65,0,__PYX_ERR(2, 65, __pyx_L1_error)) + __Pyx_TraceLine(68,0,__PYX_ERR(2, 68, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":62 + /* "reppy/robots.pyx":65 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -2414,7 +2387,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":67 +/* "reppy/robots.pyx":70 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2443,36 +2416,36 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("disallow", 0); - __Pyx_TraceCall("disallow", __pyx_f[2], 67, 0, __PYX_ERR(2, 67, __pyx_L1_error)); + __Pyx_TraceCall("disallow", __pyx_f[2], 70, 0, __PYX_ERR(2, 70, __pyx_L1_error)); - /* "reppy/robots.pyx":69 + /* "reppy/robots.pyx":72 * def disallow(self, path): * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(69,0,__PYX_ERR(2, 69, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 69, __pyx_L1_error) + __Pyx_TraceLine(72,0,__PYX_ERR(2, 72, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 72, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 69, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 72, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.disallow(__pyx_t_2); - /* "reppy/robots.pyx":70 + /* "reppy/robots.pyx":73 * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def allowed(self, path): */ - __Pyx_TraceLine(70,0,__PYX_ERR(2, 70, __pyx_L1_error)) + __Pyx_TraceLine(73,0,__PYX_ERR(2, 73, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":67 + /* "reppy/robots.pyx":70 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2492,7 +2465,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":72 +/* "reppy/robots.pyx":75 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2521,28 +2494,28 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[2], 72, 0, __PYX_ERR(2, 72, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[2], 75, 0, __PYX_ERR(2, 75, __pyx_L1_error)); - /* "reppy/robots.pyx":74 + /* "reppy/robots.pyx":77 * def allowed(self, path): * '''Is the provided URL allowed?''' * return self.agent.allowed(as_bytes(path)) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(74,0,__PYX_ERR(2, 74, __pyx_L1_error)) + __Pyx_TraceLine(77,0,__PYX_ERR(2, 77, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 74, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 74, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 77, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 74, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 77, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":72 + /* "reppy/robots.pyx":75 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2677,7 +2650,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__setstate_cython__(CYTHON_UNU return __pyx_r; } -/* "reppy/robots.pyx":77 +/* "reppy/robots.pyx":80 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2725,13 +2698,13 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(2, 77, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(2, 80, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(2, 77, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(2, 80, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 3: @@ -2741,7 +2714,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(2, 77, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(2, 80, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2761,7 +2734,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 77, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 80, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.ParseMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2786,18 +2759,18 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_6 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__6) __Pyx_RefNannySetupContext("ParseMethod", 0); - __Pyx_TraceCall("ParseMethod", __pyx_f[2], 77, 0, __PYX_ERR(2, 77, __pyx_L1_error)); + __Pyx_TraceCall("ParseMethod", __pyx_f[2], 80, 0, __PYX_ERR(2, 80, __pyx_L1_error)); - /* "reppy/robots.pyx":79 + /* "reppy/robots.pyx":82 * def ParseMethod(cls, url, content, expires=None): * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) # <<<<<<<<<<<<<< * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): */ - __Pyx_TraceLine(79,0,__PYX_ERR(2, 79, __pyx_L1_error)) + __Pyx_TraceLine(82,0,__PYX_ERR(2, 82, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 79, __pyx_L1_error) + __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 82, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_v_cls); __pyx_t_3 = __pyx_v_cls; __pyx_t_4 = NULL; @@ -2815,7 +2788,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 79, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 82, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -2824,14 +2797,14 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 79, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 82, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 79, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 82, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -2845,7 +2818,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_5, __pyx_v_expires); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 79, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 82, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -2854,7 +2827,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":77 + /* "reppy/robots.pyx":80 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2878,7 +2851,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":81 +/* "reppy/robots.pyx":84 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -2942,7 +2915,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(2, 81, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(2, 84, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: @@ -2959,7 +2932,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } if (unlikely(kw_args > 0)) { const Py_ssize_t used_pos_args = (pos_args < 4) ? pos_args : 4; - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(2, 81, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(2, 84, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2983,7 +2956,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 81, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 84, __pyx_L3_error) __pyx_L3_error:; __Pyx_DECREF(__pyx_v_args); __pyx_v_args = 0; __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; @@ -3000,7 +2973,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb return __pyx_r; } -/* "reppy/robots.pyx":85 +/* "reppy/robots.pyx":88 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< @@ -3040,11 +3013,11 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cause)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(2, 85, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(2, 88, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "wrap_exception") < 0)) __PYX_ERR(2, 85, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "wrap_exception") < 0)) __PYX_ERR(2, 88, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -3057,7 +3030,7 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 85, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 88, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.FetchMethod.wrap_exception", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3086,16 +3059,16 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_RefNannySetupContext("wrap_exception", 0); __pyx_outer_scope = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *) __Pyx_CyFunction_GetClosure(__pyx_self); __pyx_cur_scope = __pyx_outer_scope; - __Pyx_TraceCall("wrap_exception", __pyx_f[2], 85, 0, __PYX_ERR(2, 85, __pyx_L1_error)); + __Pyx_TraceCall("wrap_exception", __pyx_f[2], 88, 0, __PYX_ERR(2, 88, __pyx_L1_error)); - /* "reppy/robots.pyx":86 + /* "reppy/robots.pyx":89 * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): * wrapped = etype(cause) # <<<<<<<<<<<<<< * wrapped.url = url * if after_response_hook is not None: */ - __Pyx_TraceLine(86,0,__PYX_ERR(2, 86, __pyx_L1_error)) + __Pyx_TraceLine(89,0,__PYX_ERR(2, 89, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_etype); __pyx_t_2 = __pyx_v_etype; __pyx_t_3 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { @@ -3108,13 +3081,13 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * } } if (!__pyx_t_3) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3122,19 +3095,19 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 86, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __pyx_t_3 = NULL; __Pyx_INCREF(__pyx_v_cause); __Pyx_GIVEREF(__pyx_v_cause); PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_cause); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } @@ -3143,39 +3116,39 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __pyx_v_wrapped = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":87 + /* "reppy/robots.pyx":90 * def wrap_exception(etype, cause): * wrapped = etype(cause) * wrapped.url = url # <<<<<<<<<<<<<< * if after_response_hook is not None: * after_response_hook(wrapped) */ - __Pyx_TraceLine(87,0,__PYX_ERR(2, 87, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_url)) { __Pyx_RaiseClosureNameError("url"); __PYX_ERR(2, 87, __pyx_L1_error) } - if (__Pyx_PyObject_SetAttrStr(__pyx_v_wrapped, __pyx_n_s_url, __pyx_cur_scope->__pyx_v_url) < 0) __PYX_ERR(2, 87, __pyx_L1_error) + __Pyx_TraceLine(90,0,__PYX_ERR(2, 90, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_url)) { __Pyx_RaiseClosureNameError("url"); __PYX_ERR(2, 90, __pyx_L1_error) } + if (__Pyx_PyObject_SetAttrStr(__pyx_v_wrapped, __pyx_n_s_url, __pyx_cur_scope->__pyx_v_url) < 0) __PYX_ERR(2, 90, __pyx_L1_error) - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":91 * wrapped = etype(cause) * wrapped.url = url * if after_response_hook is not None: # <<<<<<<<<<<<<< * after_response_hook(wrapped) * raise wrapped */ - __Pyx_TraceLine(88,0,__PYX_ERR(2, 88, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 88, __pyx_L1_error) } + __Pyx_TraceLine(91,0,__PYX_ERR(2, 91, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 91, __pyx_L1_error) } __pyx_t_5 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); __pyx_t_6 = (__pyx_t_5 != 0); if (__pyx_t_6) { - /* "reppy/robots.pyx":89 + /* "reppy/robots.pyx":92 * wrapped.url = url * if after_response_hook is not None: * after_response_hook(wrapped) # <<<<<<<<<<<<<< * raise wrapped * try: */ - __Pyx_TraceLine(89,0,__PYX_ERR(2, 89, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 89, __pyx_L1_error) } + __Pyx_TraceLine(92,0,__PYX_ERR(2, 92, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 92, __pyx_L1_error) } __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); __pyx_t_2 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_4 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { @@ -3188,13 +3161,13 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * } } if (!__pyx_t_4) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3202,19 +3175,19 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); __pyx_t_4 = NULL; __Pyx_INCREF(__pyx_v_wrapped); __Pyx_GIVEREF(__pyx_v_wrapped); PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_v_wrapped); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } @@ -3222,7 +3195,7 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":91 * wrapped = etype(cause) * wrapped.url = url * if after_response_hook is not None: # <<<<<<<<<<<<<< @@ -3231,18 +3204,18 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * */ } - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":93 * if after_response_hook is not None: * after_response_hook(wrapped) * raise wrapped # <<<<<<<<<<<<<< * try: * # Limit the size of the request */ - __Pyx_TraceLine(90,0,__PYX_ERR(2, 90, __pyx_L1_error)) + __Pyx_TraceLine(93,0,__PYX_ERR(2, 93, __pyx_L1_error)) __Pyx_Raise(__pyx_v_wrapped, 0, 0, 0); - __PYX_ERR(2, 90, __pyx_L1_error) + __PYX_ERR(2, 93, __pyx_L1_error) - /* "reppy/robots.pyx":85 + /* "reppy/robots.pyx":88 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< @@ -3265,7 +3238,7 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * return __pyx_r; } -/* "reppy/robots.pyx":81 +/* "reppy/robots.pyx":84 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -3308,69 +3281,69 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(2, 81, __pyx_L1_error) + __PYX_ERR(2, 84, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } - __Pyx_TraceCall("FetchMethod", __pyx_f[2], 81, 0, __PYX_ERR(2, 81, __pyx_L1_error)); + __Pyx_TraceCall("FetchMethod", __pyx_f[2], 84, 0, __PYX_ERR(2, 84, __pyx_L1_error)); __pyx_cur_scope->__pyx_v_url = __pyx_v_url; __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); - /* "reppy/robots.pyx":83 + /* "reppy/robots.pyx":86 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): */ - __Pyx_TraceLine(83,0,__PYX_ERR(2, 83, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 83, __pyx_L1_error) + __Pyx_TraceLine(86,0,__PYX_ERR(2, 86, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 83, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 86, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_t_2); __pyx_cur_scope->__pyx_v_after_response_hook = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":87 * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< * def wrap_exception(etype, cause): * wrapped = etype(cause) */ - __Pyx_TraceLine(84,0,__PYX_ERR(2, 84, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 84, __pyx_L1_error) + __Pyx_TraceLine(87,0,__PYX_ERR(2, 87, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 87, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 84, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 87, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_after_parse_hook = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":85 + /* "reppy/robots.pyx":88 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< * wrapped = etype(cause) * wrapped.url = url */ - __Pyx_TraceLine(85,0,__PYX_ERR(2, 85, __pyx_L1_error)) - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) + __Pyx_TraceLine(88,0,__PYX_ERR(2, 88, __pyx_L1_error)) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_wrap_exception = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":91 + /* "reppy/robots.pyx":94 * after_response_hook(wrapped) * raise wrapped * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_TraceLine(91,0,__PYX_ERR(2, 91, __pyx_L3_error)) + __Pyx_TraceLine(94,0,__PYX_ERR(2, 94, __pyx_L3_error)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign @@ -3380,41 +3353,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGOTREF(__pyx_t_5); /*try:*/ { - /* "reppy/robots.pyx":93 + /* "reppy/robots.pyx":96 * try: * # Limit the size of the request * kwargs['stream'] = True # <<<<<<<<<<<<<< * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) */ - __Pyx_TraceLine(93,0,__PYX_ERR(2, 93, __pyx_L3_error)) - if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(2, 93, __pyx_L3_error) + __Pyx_TraceLine(96,0,__PYX_ERR(2, 96, __pyx_L3_error)) + if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(2, 96, __pyx_L3_error) - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":97 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __Pyx_TraceLine(94,0,__PYX_ERR(2, 94, __pyx_L3_error)) + __Pyx_TraceLine(97,0,__PYX_ERR(2, 97, __pyx_L3_error)) /*with:*/ { - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_cur_scope->__pyx_v_url); - __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -3429,14 +3402,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -3445,28 +3418,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { - __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); __pyx_t_6 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(2, 94, __pyx_L3_error) + __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L9_error) + __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L9_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { @@ -3479,10 +3452,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (__pyx_t_6) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 94, __pyx_L9_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L9_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 94, __pyx_L9_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L9_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3501,78 +3474,78 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_res = __pyx_t_7; __pyx_t_7 = 0; - /* "reppy/robots.pyx":95 + /* "reppy/robots.pyx":98 * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) # <<<<<<<<<<<<<< * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): */ - __Pyx_TraceLine(95,0,__PYX_ERR(2, 95, __pyx_L13_error)) - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 95, __pyx_L13_error) + __Pyx_TraceLine(98,0,__PYX_ERR(2, 98, __pyx_L13_error)) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 95, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(2, 95, __pyx_L13_error) - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 95, __pyx_L13_error) - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 95, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(2, 98, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 98, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 98, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_v_content = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":97 + /* "reppy/robots.pyx":100 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) */ - __Pyx_TraceLine(97,0,__PYX_ERR(2, 97, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L13_error) + __Pyx_TraceLine(100,0,__PYX_ERR(2, 100, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(2, 97, __pyx_L13_error) - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 97, __pyx_L13_error) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(2, 100, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 100, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 97, __pyx_L13_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 100, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_13) { - /* "reppy/robots.pyx":98 + /* "reppy/robots.pyx":101 * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( # <<<<<<<<<<<<<< * 'Content larger than %s bytes' % max_size) * */ - __Pyx_TraceLine(98,0,__PYX_ERR(2, 98, __pyx_L13_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 98, __pyx_L13_error) + __Pyx_TraceLine(101,0,__PYX_ERR(2, 101, __pyx_L13_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 98, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":99 + /* "reppy/robots.pyx":102 * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) # <<<<<<<<<<<<<< * * if after_response_hook is not None: */ - __Pyx_TraceLine(99,0,__PYX_ERR(2, 99, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 99, __pyx_L13_error) + __Pyx_TraceLine(102,0,__PYX_ERR(2, 102, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 102, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -3585,14 +3558,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -3601,20 +3574,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 98, __pyx_L13_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_2); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3622,9 +3595,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(2, 98, __pyx_L13_error) + __PYX_ERR(2, 101, __pyx_L13_error) - /* "reppy/robots.pyx":97 + /* "reppy/robots.pyx":100 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< @@ -3633,26 +3606,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":101 + /* "reppy/robots.pyx":104 * 'Content larger than %s bytes' % max_size) * * if after_response_hook is not None: # <<<<<<<<<<<<<< * after_response_hook(res) * */ - __Pyx_TraceLine(101,0,__PYX_ERR(2, 101, __pyx_L13_error)) + __Pyx_TraceLine(104,0,__PYX_ERR(2, 104, __pyx_L13_error)) __pyx_t_13 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { - /* "reppy/robots.pyx":102 + /* "reppy/robots.pyx":105 * * if after_response_hook is not None: * after_response_hook(res) # <<<<<<<<<<<<<< * * # Get the TTL policy's ruling on the ttl */ - __Pyx_TraceLine(102,0,__PYX_ERR(2, 102, __pyx_L13_error)) + __Pyx_TraceLine(105,0,__PYX_ERR(2, 105, __pyx_L13_error)) __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); __pyx_t_7 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_8 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -3665,13 +3638,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 102, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 102, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3679,19 +3652,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 102, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 102, __pyx_L13_error) + __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_2, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 102, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } @@ -3699,7 +3672,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":101 + /* "reppy/robots.pyx":104 * 'Content larger than %s bytes' % max_size) * * if after_response_hook is not None: # <<<<<<<<<<<<<< @@ -3708,28 +3681,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":105 + /* "reppy/robots.pyx":108 * * # Get the TTL policy's ruling on the ttl * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) # <<<<<<<<<<<<<< * * if res.status_code == 200: */ - __Pyx_TraceLine(105,0,__PYX_ERR(2, 105, __pyx_L13_error)) - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 105, __pyx_L13_error) + __Pyx_TraceLine(108,0,__PYX_ERR(2, 108, __pyx_L13_error)) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 108, __pyx_L13_error) if (!__pyx_t_14) { } else { __Pyx_INCREF(__pyx_v_ttl_policy); __pyx_t_7 = __pyx_v_ttl_policy; goto __pyx_L21_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_2); __pyx_t_7 = __pyx_t_2; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_L21_bool_binop_done:; - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = NULL; @@ -3743,13 +3716,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3757,19 +3730,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3778,32 +3751,32 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_expires = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":107 + /* "reppy/robots.pyx":110 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: */ - __Pyx_TraceLine(107,0,__PYX_ERR(2, 107, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 107, __pyx_L13_error) + __Pyx_TraceLine(110,0,__PYX_ERR(2, 110, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 110, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 107, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 107, __pyx_L13_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 110, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_14) { - /* "reppy/robots.pyx":108 + /* "reppy/robots.pyx":111 * * if res.status_code == 200: * robots = cls.parse(url, content, expires) # <<<<<<<<<<<<<< * if after_parse_hook is not None: * after_parse_hook(robots) */ - __Pyx_TraceLine(108,0,__PYX_ERR(2, 108, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) + __Pyx_TraceLine(111,0,__PYX_ERR(2, 111, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_8 = NULL; __pyx_t_15 = 0; @@ -3820,7 +3793,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else @@ -3828,13 +3801,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); if (__pyx_t_8) { __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; @@ -3848,7 +3821,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_15, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } @@ -3856,26 +3829,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_robots = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":109 + /* "reppy/robots.pyx":112 * if res.status_code == 200: * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: # <<<<<<<<<<<<<< * after_parse_hook(robots) * return robots */ - __Pyx_TraceLine(109,0,__PYX_ERR(2, 109, __pyx_L13_error)) + __Pyx_TraceLine(112,0,__PYX_ERR(2, 112, __pyx_L13_error)) __pyx_t_14 = (__pyx_v_after_parse_hook != Py_None); __pyx_t_13 = (__pyx_t_14 != 0); if (__pyx_t_13) { - /* "reppy/robots.pyx":110 + /* "reppy/robots.pyx":113 * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: * after_parse_hook(robots) # <<<<<<<<<<<<<< * return robots * elif res.status_code in (401, 403): */ - __Pyx_TraceLine(110,0,__PYX_ERR(2, 110, __pyx_L13_error)) + __Pyx_TraceLine(113,0,__PYX_ERR(2, 113, __pyx_L13_error)) __Pyx_INCREF(__pyx_v_after_parse_hook); __pyx_t_1 = __pyx_v_after_parse_hook; __pyx_t_7 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_1))) { @@ -3888,13 +3861,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else @@ -3902,19 +3875,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 110, __pyx_L13_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_robots); __Pyx_GIVEREF(__pyx_v_robots); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_robots); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3922,7 +3895,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":109 + /* "reppy/robots.pyx":112 * if res.status_code == 200: * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: # <<<<<<<<<<<<<< @@ -3931,20 +3904,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":111 + /* "reppy/robots.pyx":114 * if after_parse_hook is not None: * after_parse_hook(robots) * return robots # <<<<<<<<<<<<<< * elif res.status_code in (401, 403): * return AllowNone(url, expires) */ - __Pyx_TraceLine(111,0,__PYX_ERR(2, 111, __pyx_L13_error)) + __Pyx_TraceLine(114,0,__PYX_ERR(2, 114, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_robots); __pyx_r = __pyx_v_robots; goto __pyx_L17_try_return; - /* "reppy/robots.pyx":107 + /* "reppy/robots.pyx":110 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< @@ -3953,28 +3926,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":112 + /* "reppy/robots.pyx":115 * after_parse_hook(robots) * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: */ - __Pyx_TraceLine(112,0,__PYX_ERR(2, 112, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 112, __pyx_L13_error) + __Pyx_TraceLine(115,0,__PYX_ERR(2, 115, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 115, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 112, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 115, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 112, __pyx_L13_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 115, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (!__pyx_t_14) { } else { __pyx_t_13 = __pyx_t_14; goto __pyx_L25_bool_binop_done; } - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 112, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 115, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 112, __pyx_L13_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 115, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_13 = __pyx_t_14; __pyx_L25_bool_binop_done:; @@ -3982,16 +3955,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { - /* "reppy/robots.pyx":113 + /* "reppy/robots.pyx":116 * return robots * elif res.status_code in (401, 403): * return AllowNone(url, expires) # <<<<<<<<<<<<<< * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) */ - __Pyx_TraceLine(113,0,__PYX_ERR(2, 113, __pyx_L13_error)) + __Pyx_TraceLine(116,0,__PYX_ERR(2, 116, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); @@ -3999,14 +3972,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L17_try_return; - /* "reppy/robots.pyx":112 + /* "reppy/robots.pyx":115 * after_parse_hook(robots) * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< @@ -4015,45 +3988,45 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":114 + /* "reppy/robots.pyx":117 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< * return AllowAll(url, expires) * else: */ - __Pyx_TraceLine(114,0,__PYX_ERR(2, 114, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 114, __pyx_L13_error) + __Pyx_TraceLine(117,0,__PYX_ERR(2, 117, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) + __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 114, __pyx_L13_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_13) { } else { __pyx_t_14 = __pyx_t_13; goto __pyx_L27_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 114, __pyx_L13_error) + __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 114, __pyx_L13_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 117, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_14 = __pyx_t_13; __pyx_L27_bool_binop_done:; if (__pyx_t_14) { - /* "reppy/robots.pyx":115 + /* "reppy/robots.pyx":118 * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) # <<<<<<<<<<<<<< * else: * raise exceptions.BadStatusCode( */ - __Pyx_TraceLine(115,0,__PYX_ERR(2, 115, __pyx_L13_error)) + __Pyx_TraceLine(118,0,__PYX_ERR(2, 118, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 115, __pyx_L13_error) + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); @@ -4061,14 +4034,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 115, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L17_try_return; - /* "reppy/robots.pyx":114 + /* "reppy/robots.pyx":117 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< @@ -4077,32 +4050,32 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":117 + /* "reppy/robots.pyx":120 * return AllowAll(url, expires) * else: * raise exceptions.BadStatusCode( # <<<<<<<<<<<<<< * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: */ - __Pyx_TraceLine(117,0,__PYX_ERR(2, 117, __pyx_L13_error)) + __Pyx_TraceLine(120,0,__PYX_ERR(2, 120, __pyx_L13_error)) /*else*/ { - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":118 + /* "reppy/robots.pyx":121 * else: * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) # <<<<<<<<<<<<<< * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) */ - __Pyx_TraceLine(118,0,__PYX_ERR(2, 118, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) + __Pyx_TraceLine(121,0,__PYX_ERR(2, 121, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 121, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 118, __pyx_L13_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 121, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); @@ -4110,10 +4083,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_cur_scope->__pyx_v_url); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 121, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 118, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 121, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; __pyx_t_15 = 0; @@ -4130,7 +4103,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -4140,7 +4113,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -4148,7 +4121,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } else #endif { - __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_6) { __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; @@ -4159,17 +4132,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_15, __pyx_t_7); __pyx_t_1 = 0; __pyx_t_7 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(2, 117, __pyx_L13_error) + __PYX_ERR(2, 120, __pyx_L13_error) } - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":97 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< @@ -4186,20 +4159,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; /*except:*/ { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 94, __pyx_L15_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 97, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); - __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 94, __pyx_L15_except_error) + __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_7, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_17)) __PYX_ERR(2, 94, __pyx_L15_except_error) + if (unlikely(!__pyx_t_17)) __PYX_ERR(2, 97, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_17); __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_17); __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; - if (__pyx_t_14 < 0) __PYX_ERR(2, 94, __pyx_L15_except_error) + if (__pyx_t_14 < 0) __PYX_ERR(2, 97, __pyx_L15_except_error) __pyx_t_13 = ((!(__pyx_t_14 != 0)) != 0); if (__pyx_t_13) { __Pyx_GIVEREF(__pyx_t_2); @@ -4207,7 +4180,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGIVEREF(__pyx_t_16); __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_8, __pyx_t_16); __pyx_t_2 = 0; __pyx_t_8 = 0; __pyx_t_16 = 0; - __PYX_ERR(2, 94, __pyx_L15_except_error) + __PYX_ERR(2, 97, __pyx_L15_except_error) } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -4238,7 +4211,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__12, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_12)) __PYX_ERR(2, 94, __pyx_L3_error) + if (unlikely(!__pyx_t_12)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; } @@ -4250,7 +4223,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__13, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_11)) __PYX_ERR(2, 94, __pyx_L3_error) + if (unlikely(!__pyx_t_11)) __PYX_ERR(2, 97, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } @@ -4267,7 +4240,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L32:; } - /* "reppy/robots.pyx":91 + /* "reppy/robots.pyx":94 * after_response_hook(wrapped) * raise wrapped * try: # <<<<<<<<<<<<<< @@ -4287,41 +4260,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; - /* "reppy/robots.pyx":119 + /* "reppy/robots.pyx":122 * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: */ - __Pyx_TraceLine(119,0,__PYX_ERR(2, 119, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 119, __pyx_L5_except_error) + __Pyx_TraceLine(122,0,__PYX_ERR(2, 122, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(2, 119, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(2, 122, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":120 + /* "reppy/robots.pyx":123 * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) # <<<<<<<<<<<<<< * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) */ - __Pyx_TraceLine(120,0,__PYX_ERR(2, 120, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 120, __pyx_L5_except_error) + __Pyx_TraceLine(123,0,__PYX_ERR(2, 123, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 120, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 120, __pyx_L5_except_error) + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 123, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -4331,41 +4304,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":121 + /* "reppy/robots.pyx":124 * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: */ - __Pyx_TraceLine(121,0,__PYX_ERR(2, 121, __pyx_L5_except_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 121, __pyx_L5_except_error) + __Pyx_TraceLine(124,0,__PYX_ERR(2, 124, __pyx_L5_except_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 124, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 121, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 124, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":122 + /* "reppy/robots.pyx":125 * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) # <<<<<<<<<<<<<< * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) */ - __Pyx_TraceLine(122,0,__PYX_ERR(2, 122, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 122, __pyx_L5_except_error) + __Pyx_TraceLine(125,0,__PYX_ERR(2, 125, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 122, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 122, __pyx_L5_except_error) + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -4375,21 +4348,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":123 + /* "reppy/robots.pyx":126 * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: */ - __Pyx_TraceLine(123,0,__PYX_ERR(2, 123, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __Pyx_TraceLine(126,0,__PYX_ERR(2, 126, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_2) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4398,27 +4371,27 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 123, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":124 + /* "reppy/robots.pyx":127 * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) # <<<<<<<<<<<<<< * except TooManyRedirects as exc: * wrap_exception(exceptions.ExcessiveRedirects, exc) */ - __Pyx_TraceLine(124,0,__PYX_ERR(2, 124, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 124, __pyx_L5_except_error) + __Pyx_TraceLine(127,0,__PYX_ERR(2, 127, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 127, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 124, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 127, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 124, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 127, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4428,41 +4401,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":125 + /* "reppy/robots.pyx":128 * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.ExcessiveRedirects, exc) * except ReadTimeout as exc: */ - __Pyx_TraceLine(125,0,__PYX_ERR(2, 125, __pyx_L5_except_error)) - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 125, __pyx_L5_except_error) + __Pyx_TraceLine(128,0,__PYX_ERR(2, 128, __pyx_L5_except_error)) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 128, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(2, 125, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(2, 128, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_7); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":126 + /* "reppy/robots.pyx":129 * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: * wrap_exception(exceptions.ExcessiveRedirects, exc) # <<<<<<<<<<<<<< * except ReadTimeout as exc: * wrap_exception(exceptions.ReadTimeout, exc) */ - __Pyx_TraceLine(126,0,__PYX_ERR(2, 126, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __Pyx_TraceLine(129,0,__PYX_ERR(2, 129, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 129, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 129, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 129, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4472,41 +4445,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":127 + /* "reppy/robots.pyx":130 * except TooManyRedirects as exc: * wrap_exception(exceptions.ExcessiveRedirects, exc) * except ReadTimeout as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.ReadTimeout, exc) * */ - __Pyx_TraceLine(127,0,__PYX_ERR(2, 127, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 127, __pyx_L5_except_error) + __Pyx_TraceLine(130,0,__PYX_ERR(2, 130, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 130, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 127, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 130, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":128 + /* "reppy/robots.pyx":131 * wrap_exception(exceptions.ExcessiveRedirects, exc) * except ReadTimeout as exc: * wrap_exception(exceptions.ReadTimeout, exc) # <<<<<<<<<<<<<< * * def RobotsUrlMethod(cls, url): */ - __Pyx_TraceLine(128,0,__PYX_ERR(2, 128, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __Pyx_TraceLine(131,0,__PYX_ERR(2, 131, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 131, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 131, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 131, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4518,7 +4491,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "reppy/robots.pyx":91 + /* "reppy/robots.pyx":94 * after_response_hook(wrapped) * raise wrapped * try: # <<<<<<<<<<<<<< @@ -4544,7 +4517,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L8_try_end:; } - /* "reppy/robots.pyx":81 + /* "reppy/robots.pyx":84 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -4579,7 +4552,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":130 +/* "reppy/robots.pyx":133 * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -4620,11 +4593,11 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(2, 130, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(2, 133, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(2, 130, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(2, 133, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4637,7 +4610,7 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 130, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 133, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4660,37 +4633,37 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject PyObject *__pyx_t_4 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__14) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); - __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[2], 130, 0, __PYX_ERR(2, 130, __pyx_L1_error)); + __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[2], 133, 0, __PYX_ERR(2, 133, __pyx_L1_error)); - /* "reppy/robots.pyx":132 + /* "reppy/robots.pyx":135 * def RobotsUrlMethod(cls, url): * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) # <<<<<<<<<<<<<< * * cdef class Robots: */ - __Pyx_TraceLine(132,0,__PYX_ERR(2, 132, __pyx_L1_error)) + __Pyx_TraceLine(135,0,__PYX_ERR(2, 135, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 132, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 135, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 132, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 135, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(2, 132, __pyx_L1_error) + __PYX_ERR(2, 135, __pyx_L1_error) } - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 132, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 135, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 132, __pyx_L1_error) + __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 135, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":130 + /* "reppy/robots.pyx":133 * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -4711,7 +4684,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":150 +/* "reppy/robots.pyx":153 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -4754,7 +4727,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(2, 150, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(2, 153, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: @@ -4764,7 +4737,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 150, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 153, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4777,12 +4750,12 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } __pyx_v_url = values[0]; - __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 150, __pyx_L3_error) + __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 153, __pyx_L3_error) __pyx_v_expires = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 150, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 153, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4803,43 +4776,43 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob std::string __pyx_t_2; Rep::Robots *__pyx_t_3; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[2], 150, 0, __PYX_ERR(2, 150, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 153, 0, __PYX_ERR(2, 153, __pyx_L1_error)); - /* "reppy/robots.pyx":151 + /* "reppy/robots.pyx":154 * * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) # <<<<<<<<<<<<<< * self.expires = expires * */ - __Pyx_TraceLine(151,0,__PYX_ERR(2, 151, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 151, __pyx_L1_error) + __Pyx_TraceLine(154,0,__PYX_ERR(2, 154, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 154, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 151, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 154, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = new Rep::Robots(__pyx_v_content, __pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(2, 151, __pyx_L1_error) + __PYX_ERR(2, 154, __pyx_L1_error) } __pyx_v_self->robots = __pyx_t_3; - /* "reppy/robots.pyx":152 + /* "reppy/robots.pyx":155 * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) * self.expires = expires # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(152,0,__PYX_ERR(2, 152, __pyx_L1_error)) + __Pyx_TraceLine(155,0,__PYX_ERR(2, 155, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); __Pyx_GOTREF(__pyx_v_self->expires); __Pyx_DECREF(__pyx_v_self->expires); __pyx_v_self->expires = __pyx_v_expires; - /* "reppy/robots.pyx":150 + /* "reppy/robots.pyx":153 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -4860,12 +4833,12 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob return __pyx_r; } -/* "reppy/robots.pyx":154 +/* "reppy/robots.pyx":157 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< - * return self.robots.str().decode('utf8') - * + * # Note: this could raise a UnicodeDecodeError in Python 3 if the + * # robots.txt had invalid UTF-8 */ /* Python wrapper */ @@ -4886,35 +4859,40 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[2], 154, 0, __PYX_ERR(2, 154, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[2], 157, 0, __PYX_ERR(2, 157, __pyx_L1_error)); - /* "reppy/robots.pyx":155 - * - * def __str__(self): - * return self.robots.str().decode('utf8') # <<<<<<<<<<<<<< + /* "reppy/robots.pyx":160 + * # Note: this could raise a UnicodeDecodeError in Python 3 if the + * # robots.txt had invalid UTF-8 + * return as_string(self.robots.str()) # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __Pyx_TraceLine(155,0,__PYX_ERR(2, 155, __pyx_L1_error)) + __Pyx_TraceLine(160,0,__PYX_ERR(2, 160, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_decode_cpp_string(__pyx_v_self->robots->str(), 0, PY_SSIZE_T_MAX, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 155, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 160, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; + __pyx_t_2 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 160, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":154 + /* "reppy/robots.pyx":157 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< - * return self.robots.str().decode('utf8') - * + * # Note: this could raise a UnicodeDecodeError in Python 3 if the + * # robots.txt had invalid UTF-8 */ /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); __Pyx_AddTraceback("reppy.robots.Robots.__str__", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; @@ -4924,8 +4902,8 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":157 - * return self.robots.str().decode('utf8') +/* "reppy/robots.pyx":162 + * return as_string(self.robots.str()) * * def __dealloc__(self): # <<<<<<<<<<<<<< * del self.robots @@ -4947,20 +4925,20 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - __Pyx_TraceCall("__dealloc__", __pyx_f[2], 157, 0, __PYX_ERR(2, 157, __pyx_L1_error)); + __Pyx_TraceCall("__dealloc__", __pyx_f[2], 162, 0, __PYX_ERR(2, 162, __pyx_L1_error)); - /* "reppy/robots.pyx":158 + /* "reppy/robots.pyx":163 * * def __dealloc__(self): * del self.robots # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(158,0,__PYX_ERR(2, 158, __pyx_L1_error)) + __Pyx_TraceLine(163,0,__PYX_ERR(2, 163, __pyx_L1_error)) delete __pyx_v_self->robots; - /* "reppy/robots.pyx":157 - * return self.robots.str().decode('utf8') + /* "reppy/robots.pyx":162 + * return as_string(self.robots.str()) * * def __dealloc__(self): # <<<<<<<<<<<<<< * del self.robots @@ -4976,7 +4954,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_RefNannyFinishContext(); } -/* "reppy/robots.pyx":161 +/* "reppy/robots.pyx":166 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -5005,22 +4983,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 161, 0, __PYX_ERR(2, 161, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 166, 0, __PYX_ERR(2, 166, __pyx_L1_error)); - /* "reppy/robots.pyx":163 + /* "reppy/robots.pyx":168 * def sitemaps(self): * '''Get all the sitemaps in this robots.txt.''' * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< * * def allowed(self, path, name): */ - __Pyx_TraceLine(163,0,__PYX_ERR(2, 163, __pyx_L1_error)) + __Pyx_TraceLine(168,0,__PYX_ERR(2, 168, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 163, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 168, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 163, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 168, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 163, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 168, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -5028,14 +5006,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 163, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 168, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":161 + /* "reppy/robots.pyx":166 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -5057,7 +5035,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ return __pyx_r; } -/* "reppy/robots.pyx":165 +/* "reppy/robots.pyx":170 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -5097,11 +5075,11 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(2, 165, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(2, 170, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(2, 165, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(2, 170, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -5114,7 +5092,7 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 165, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 170, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5135,32 +5113,32 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp std::string __pyx_t_2; std::string __pyx_t_3; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[2], 165, 0, __PYX_ERR(2, 165, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[2], 170, 0, __PYX_ERR(2, 170, __pyx_L1_error)); - /* "reppy/robots.pyx":167 + /* "reppy/robots.pyx":172 * def allowed(self, path, name): * '''Is the provided path allowed for the provided agent?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) # <<<<<<<<<<<<<< * * def agent(self, name): */ - __Pyx_TraceLine(167,0,__PYX_ERR(2, 167, __pyx_L1_error)) + __Pyx_TraceLine(172,0,__PYX_ERR(2, 172, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 167, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 172, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 167, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 172, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 167, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 172, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 167, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 172, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 167, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 172, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":165 + /* "reppy/robots.pyx":170 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -5180,7 +5158,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":169 +/* "reppy/robots.pyx":174 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -5213,20 +5191,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ int __pyx_t_5; PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("agent", 0); - __Pyx_TraceCall("agent", __pyx_f[2], 169, 0, __PYX_ERR(2, 169, __pyx_L1_error)); + __Pyx_TraceCall("agent", __pyx_f[2], 174, 0, __PYX_ERR(2, 174, __pyx_L1_error)); - /* "reppy/robots.pyx":176 + /* "reppy/robots.pyx":181 * Agent object. * ''' * return Agent.from_robots(self, as_bytes(name)) # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(176,0,__PYX_ERR(2, 176, __pyx_L1_error)) + __Pyx_TraceLine(181,0,__PYX_ERR(2, 181, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 176, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 176, __pyx_L1_error) + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; __pyx_t_5 = 0; @@ -5243,7 +5221,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 176, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -5252,14 +5230,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 176, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 176, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -5270,7 +5248,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 176, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -5279,7 +5257,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":169 + /* "reppy/robots.pyx":174 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -5303,7 +5281,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":179 +/* "reppy/robots.pyx":184 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -5332,20 +5310,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 179, 0, __PYX_ERR(2, 179, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 184, 0, __PYX_ERR(2, 184, __pyx_L1_error)); - /* "reppy/robots.pyx":181 + /* "reppy/robots.pyx":186 * def expired(self): * '''True if the current time is past its expiration.''' * return time.time() > self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(181,0,__PYX_ERR(2, 181, __pyx_L1_error)) + __Pyx_TraceLine(186,0,__PYX_ERR(2, 186, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 186, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 186, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -5359,20 +5337,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } } if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 186, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 186, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 186, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":179 + /* "reppy/robots.pyx":184 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -5394,7 +5372,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":184 +/* "reppy/robots.pyx":189 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -5420,22 +5398,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 184, 0, __PYX_ERR(2, 184, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 189, 0, __PYX_ERR(2, 189, __pyx_L1_error)); - /* "reppy/robots.pyx":186 + /* "reppy/robots.pyx":191 * def expires(self): * '''The expiration of this robots.txt.''' * return self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(186,0,__PYX_ERR(2, 186, __pyx_L1_error)) + __Pyx_TraceLine(191,0,__PYX_ERR(2, 191, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_self->expires); __pyx_r = __pyx_v_self->expires; goto __pyx_L0; - /* "reppy/robots.pyx":184 + /* "reppy/robots.pyx":189 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -5454,7 +5432,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":189 +/* "reppy/robots.pyx":194 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -5486,21 +5464,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 189, 0, __PYX_ERR(2, 189, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 194, 0, __PYX_ERR(2, 194, __pyx_L1_error)); - /* "reppy/robots.pyx":191 + /* "reppy/robots.pyx":196 * def ttl(self): * '''Remaining time for this response to be considered valid.''' * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(191,0,__PYX_ERR(2, 191, __pyx_L1_error)) + __Pyx_TraceLine(196,0,__PYX_ERR(2, 196, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 191, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 196, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 191, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 196, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -5514,24 +5492,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 191, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 196, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 191, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 196, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 191, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 196, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 191, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 196, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 191, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 196, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(2, 191, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(2, 196, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 191, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 196, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -5545,7 +5523,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":189 + /* "reppy/robots.pyx":194 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -5683,7 +5661,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_12__setstate_cython__(CYTHON_UN return __pyx_r; } -/* "reppy/robots.pyx":197 +/* "reppy/robots.pyx":202 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5727,7 +5705,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 197, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 202, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -5743,7 +5721,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 197, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 202, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5766,17 +5744,17 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[2], 197, 0, __PYX_ERR(2, 197, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 202, 0, __PYX_ERR(2, 202, __pyx_L1_error)); - /* "reppy/robots.pyx":198 + /* "reppy/robots.pyx":203 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(198,0,__PYX_ERR(2, 198, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 198, __pyx_L1_error) + __Pyx_TraceLine(203,0,__PYX_ERR(2, 203, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 203, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -5793,7 +5771,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 198, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 203, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -5801,13 +5779,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 198, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 203, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 198, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 203, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -5824,14 +5802,14 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 198, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 203, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":197 + /* "reppy/robots.pyx":202 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5970,7 +5948,7 @@ static PyObject *__pyx_pf_5reppy_6robots_9AllowNone_4__setstate_cython__(CYTHON_ return __pyx_r; } -/* "reppy/robots.pyx":204 +/* "reppy/robots.pyx":209 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -6013,7 +5991,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 204, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 209, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -6029,7 +6007,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 204, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 209, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -6052,15 +6030,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[2], 204, 0, __PYX_ERR(2, 204, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 209, 0, __PYX_ERR(2, 209, __pyx_L1_error)); - /* "reppy/robots.pyx":205 + /* "reppy/robots.pyx":210 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(205,0,__PYX_ERR(2, 205, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 205, __pyx_L1_error) + __Pyx_TraceLine(210,0,__PYX_ERR(2, 210, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 210, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -6077,7 +6055,7 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 205, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 210, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -6085,13 +6063,13 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 205, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 210, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 205, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 210, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -6108,14 +6086,14 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 205, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 210, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":204 + /* "reppy/robots.pyx":209 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -7501,7 +7479,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) __pyx_builtin_TypeError = __Pyx_GetBuiltinName(__pyx_n_s_TypeError); if (!__pyx_builtin_TypeError) __PYX_ERR(1, 2, __pyx_L1_error) - __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(2, 163, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(2, 168, __pyx_L1_error) __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(1, 61, __pyx_L1_error) return 0; __pyx_L1_error:; @@ -7517,20 +7495,20 @@ static int __Pyx_InitCachedConstants(void) { * return value * return value.encode('utf-8') # <<<<<<<<<<<<<< * - * cdef as_string(value): + * # For contexts which require a 'str' type, convert bytes to unicode if needed */ __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple_)) __PYX_ERR(2, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple_); __Pyx_GIVEREF(__pyx_tuple_); - /* "reppy/robots.pyx":30 + /* "reppy/robots.pyx":33 * if six.PY3: * if isinstance(value, bytes): * return value.decode('utf-8') # <<<<<<<<<<<<<< * return value * */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(2, 30, __pyx_L1_error) + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(2, 33, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__2); __Pyx_GIVEREF(__pyx_tuple__2); @@ -7553,51 +7531,51 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__5); __Pyx_GIVEREF(__pyx_tuple__5); - /* "reppy/robots.pyx":83 + /* "reppy/robots.pyx":86 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): */ - __pyx_tuple__8 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(2, 83, __pyx_L1_error) + __pyx_tuple__8 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(2, 86, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__8); __Pyx_GIVEREF(__pyx_tuple__8); - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":87 * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< * def wrap_exception(etype, cause): * wrapped = etype(cause) */ - __pyx_tuple__9 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(2, 84, __pyx_L1_error) + __pyx_tuple__9 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(2, 87, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__9); __Pyx_GIVEREF(__pyx_tuple__9); - /* "reppy/robots.pyx":85 + /* "reppy/robots.pyx":88 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< * wrapped = etype(cause) * wrapped.url = url */ - __pyx_tuple__10 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 85, __pyx_L1_error) + __pyx_tuple__10 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 88, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__10); __Pyx_GIVEREF(__pyx_tuple__10); - __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_wrap_exception, 85, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 85, __pyx_L1_error) + __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_wrap_exception, 88, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 88, __pyx_L1_error) - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":97 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __pyx_tuple__12 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 94, __pyx_L1_error) + __pyx_tuple__12 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 97, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__12); __Pyx_GIVEREF(__pyx_tuple__12); - __pyx_tuple__13 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(2, 94, __pyx_L1_error) + __pyx_tuple__13 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(2, 97, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__13); __Pyx_GIVEREF(__pyx_tuple__13); @@ -7670,53 +7648,53 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(__pyx_tuple__22); __pyx_codeobj__23 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__22, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__23)) __PYX_ERR(1, 65, __pyx_L1_error) - /* "reppy/robots.pyx":34 + /* "reppy/robots.pyx":37 * * * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< * '''Construct an Agent from a CppAgent.''' * agent = Agent() */ - __pyx_tuple__24 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(2, 34, __pyx_L1_error) + __pyx_tuple__24 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(2, 37, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__24); __Pyx_GIVEREF(__pyx_tuple__24); - __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__24, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 34, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(2, 34, __pyx_L1_error) + __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__24, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 37, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(2, 37, __pyx_L1_error) - /* "reppy/robots.pyx":77 + /* "reppy/robots.pyx":80 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __pyx_tuple__25 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(2, 77, __pyx_L1_error) + __pyx_tuple__25 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(2, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__25); __Pyx_GIVEREF(__pyx_tuple__25); - __pyx_codeobj__6 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_ParseMethod, 77, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__6)) __PYX_ERR(2, 77, __pyx_L1_error) + __pyx_codeobj__6 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_ParseMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__6)) __PYX_ERR(2, 80, __pyx_L1_error) - /* "reppy/robots.pyx":81 + /* "reppy/robots.pyx":84 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __pyx_tuple__26 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(2, 81, __pyx_L1_error) + __pyx_tuple__26 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(2, 84, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__26); __Pyx_GIVEREF(__pyx_tuple__26); - __pyx_codeobj__7 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__26, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FetchMethod, 81, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__7)) __PYX_ERR(2, 81, __pyx_L1_error) + __pyx_codeobj__7 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__26, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FetchMethod, 84, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__7)) __PYX_ERR(2, 84, __pyx_L1_error) - /* "reppy/robots.pyx":130 + /* "reppy/robots.pyx":133 * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __pyx_tuple__27 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(2, 130, __pyx_L1_error) + __pyx_tuple__27 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(2, 133, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__27); __Pyx_GIVEREF(__pyx_tuple__27); - __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__27, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 130, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) __PYX_ERR(2, 130, __pyx_L1_error) + __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__27, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 133, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) __PYX_ERR(2, 133, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -7880,29 +7858,29 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) /*--- Variable export code ---*/ /*--- Function export code ---*/ /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 44, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) __pyx_type_5reppy_6robots_Agent.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 44, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 44, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; - if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 134, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 137, __pyx_L1_error) __pyx_type_5reppy_6robots_Robots.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 134, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 134, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 137, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 137, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 194, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 199, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 194, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 194, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 199, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 199, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 201, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 206, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 201, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 201, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 206, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 206, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; - if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(2, 81, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(2, 84, __pyx_L1_error) __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_print = 0; __pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod = &__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod; if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(1, 64, __pyx_L1_error) @@ -8116,145 +8094,145 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":34 + /* "reppy/robots.pyx":37 * * * def FromRobotsMethod(cls, Robots robots, const string& name): # <<<<<<<<<<<<<< * '''Construct an Agent from a CppAgent.''' * agent = Agent() */ - __Pyx_TraceLine(34,0,__PYX_ERR(2, 34, __pyx_L1_error)) - __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_1FromRobotsMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 34, __pyx_L1_error) + __Pyx_TraceLine(37,0,__PYX_ERR(2, 37, __pyx_L1_error)) + __pyx_t_1 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_1FromRobotsMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 37, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FromRobotsMethod, __pyx_t_1) < 0) __PYX_ERR(2, 34, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FromRobotsMethod, __pyx_t_1) < 0) __PYX_ERR(2, 37, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":49 + /* "reppy/robots.pyx":52 * cdef CppAgent agent * * from_robots = classmethod(FromRobotsMethod) # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(49,0,__PYX_ERR(2, 49, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 49, __pyx_L1_error) + __Pyx_TraceLine(52,0,__PYX_ERR(2, 52, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 49, __pyx_L1_error) + __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Agent->tp_dict, __pyx_n_s_from_robots, __pyx_t_2) < 0) __PYX_ERR(2, 49, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Agent->tp_dict, __pyx_n_s_from_robots, __pyx_t_2) < 0) __PYX_ERR(2, 52, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Agent); - /* "reppy/robots.pyx":77 + /* "reppy/robots.pyx":80 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __Pyx_TraceLine(77,0,__PYX_ERR(2, 77, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 77, __pyx_L1_error) + __Pyx_TraceLine(80,0,__PYX_ERR(2, 80, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(2, 77, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(2, 80, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":81 + /* "reppy/robots.pyx":84 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __Pyx_TraceLine(81,0,__PYX_ERR(2, 81, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 81, __pyx_L1_error) + __Pyx_TraceLine(84,0,__PYX_ERR(2, 84, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 84, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(2, 81, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(2, 84, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":130 + /* "reppy/robots.pyx":133 * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __Pyx_TraceLine(130,0,__PYX_ERR(2, 130, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 130, __pyx_L1_error) + __Pyx_TraceLine(133,0,__PYX_ERR(2, 133, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 133, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(2, 130, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(2, 133, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":139 + /* "reppy/robots.pyx":142 * # The default TTL policy is to cache for 3600 seconds or what's provided in the * # headers, and a minimum of 600 seconds * DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) # <<<<<<<<<<<<<< * * # Class methods */ - __Pyx_TraceLine(139,0,__PYX_ERR(2, 139, __pyx_L1_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 139, __pyx_L1_error) + __Pyx_TraceLine(142,0,__PYX_ERR(2, 142, __pyx_L1_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 142, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 139, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 142, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(2, 139, __pyx_L1_error) - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(2, 139, __pyx_L1_error) - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 139, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(2, 142, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(2, 142, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 142, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(2, 139, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(2, 142, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":142 + /* "reppy/robots.pyx":145 * * # Class methods * parse = classmethod(ParseMethod) # <<<<<<<<<<<<<< * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) */ - __Pyx_TraceLine(142,0,__PYX_ERR(2, 142, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 142, __pyx_L1_error) + __Pyx_TraceLine(145,0,__PYX_ERR(2, 145, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 142, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(2, 142, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(2, 145, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":143 + /* "reppy/robots.pyx":146 * # Class methods * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) # <<<<<<<<<<<<<< * robots_url = classmethod(RobotsUrlMethod) * */ - __Pyx_TraceLine(143,0,__PYX_ERR(2, 143, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 143, __pyx_L1_error) + __Pyx_TraceLine(146,0,__PYX_ERR(2, 146, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 143, __pyx_L1_error) + __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 146, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(2, 143, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(2, 146, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":144 + /* "reppy/robots.pyx":147 * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) # <<<<<<<<<<<<<< * * # Data members */ - __Pyx_TraceLine(144,0,__PYX_ERR(2, 144, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 144, __pyx_L1_error) + __Pyx_TraceLine(147,0,__PYX_ERR(2, 147, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 147, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 144, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 147, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(2, 144, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(2, 147, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); @@ -8630,33 +8608,6 @@ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { return 0; } -/* decode_c_bytes */ - static CYTHON_INLINE PyObject* __Pyx_decode_c_bytes( - const char* cstring, Py_ssize_t length, Py_ssize_t start, Py_ssize_t stop, - const char* encoding, const char* errors, - PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { - if (unlikely((start < 0) | (stop < 0))) { - if (start < 0) { - start += length; - if (start < 0) - start = 0; - } - if (stop < 0) - stop += length; - } - if (stop > length) - stop = length; - length = stop - start; - if (unlikely(length <= 0)) - return PyUnicode_FromUnicode(NULL, 0); - cstring += start; - if (decode_func) { - return decode_func(cstring, length, errors); - } else { - return PyUnicode_Decode(cstring, length, encoding, errors); - } -} - /* PyErrFetchRestore */ #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { From a7763ec367786b4358557d3217ec1de2e81586b5 Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Tue, 9 Oct 2018 14:24:21 -0700 Subject: [PATCH 098/113] Add a test! --- tests/test_cache/test_cache.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_cache/test_cache.py b/tests/test_cache/test_cache.py index 476ca09..6ef5601 100644 --- a/tests/test_cache/test_cache.py +++ b/tests/test_cache/test_cache.py @@ -3,10 +3,13 @@ '''Tests about our caching utilities.''' import unittest - import mock +import sys + from reppy import cache +from reppy import logger +import reppy.exceptions from ..util import requests_fixtures @@ -119,6 +122,19 @@ def test_uses_default_expiration_on_failure(self): self.assertEqual( self.cache.cache['http://does-not-resolve/robots.txt'].expires, 17) + def test_logs_on_failure(self): + '''On fetch failure, it logs the exception.''' + logs = [] + def mock_logger(msg): + exc_type = sys.exc_info()[0] + logs.append( (exc_type, msg) ) + with mock.patch.object(logger, 'exception', mock_logger): + self.cache.get('http://does-not-resolve/') + + expected_err = reppy.exceptions.ConnectionException + expected_msg = 'Reppy error on http://does-not-resolve/robots.txt' + self.assertIn((expected_err, expected_msg), logs) + def test_agent_allowed(self): '''Can check for allowed.''' with requests_fixtures('test_agent_allowed'): From e651e310a69d1aa7e483c4903e8db098028cbc95 Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Tue, 9 Oct 2018 14:33:14 -0700 Subject: [PATCH 099/113] tweak logging message --- reppy/cache/__init__.py | 2 +- tests/test_cache/test_cache.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/reppy/cache/__init__.py b/reppy/cache/__init__.py index f15f0ec..e5cdaf1 100644 --- a/reppy/cache/__init__.py +++ b/reppy/cache/__init__.py @@ -62,7 +62,7 @@ def factory(self, url): try: return self.fetch(url) except BaseException as exc: - logger.exception('Reppy error on %s' % url) + logger.exception('Reppy cache fetch error on %s' % url) return self.cache_policy.exception(url, exc) def fetch(self, url): diff --git a/tests/test_cache/test_cache.py b/tests/test_cache/test_cache.py index 6ef5601..e2afa93 100644 --- a/tests/test_cache/test_cache.py +++ b/tests/test_cache/test_cache.py @@ -132,7 +132,7 @@ def mock_logger(msg): self.cache.get('http://does-not-resolve/') expected_err = reppy.exceptions.ConnectionException - expected_msg = 'Reppy error on http://does-not-resolve/robots.txt' + expected_msg = 'Reppy cache fetch error on http://does-not-resolve/robots.txt' self.assertIn((expected_err, expected_msg), logs) def test_agent_allowed(self): From 5b75e42711b56b389866d695ea27113ffb87bf31 Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Wed, 14 Nov 2018 14:05:13 -0800 Subject: [PATCH 100/113] Rewrite functionality removed from cachetools 3 For some reason, cachetools version 3 (released on 2018-11-04) removed the ability to pass in a `missing` function; apparently this made the implementation cleaner (see https://github.com/tkem/cachetools/issues/58). Unfortunately we use this functionality here. Rather than require version 2 of cachetools, I thought it would be better to allow reppy to be used with the new version. Unfortunately, that means a reimplementation of pretty much exactly what was removed from cachetools. I found this because we have a project which uses reppy and its tests suddenly broke, because pip automatically installed the newer version of cachetools. --- reppy/cache/__init__.py | 15 +++++++++++++- tests/test_cache/test_cache.py | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/reppy/cache/__init__.py b/reppy/cache/__init__.py index e5cdaf1..1617526 100644 --- a/reppy/cache/__init__.py +++ b/reppy/cache/__init__.py @@ -37,6 +37,19 @@ def get(self): return self.obj +class LRUCacheWithMissingHandler(LRUCache): + '''Same as LRUCache but we can pass in a "missing" handler which will + be called if the key is missing. The result is then stored in the + cache. cachetools's LRUCache used to support this functionality, but it + was inexplicably removed in cachetools 3.0.0.''' + def __init__(self, maxsize, missing): + LRUCache.__init__(self, maxsize=maxsize) + self._missing_handler = missing + + def __missing__(self, key): + self[key] = self._missing_handler(key) + return self[key] + class BaseCache(object): '''A base cache class.''' @@ -46,7 +59,7 @@ class BaseCache(object): def __init__(self, capacity, cache_policy=None, ttl_policy=None, *args, **kwargs): self.cache_policy = cache_policy or self.DEFAULT_CACHE_POLICY self.ttl_policy = ttl_policy or self.DEFAULT_TTL_POLICY - self.cache = LRUCache(maxsize=capacity, missing=self.missing) + self.cache = LRUCacheWithMissingHandler(maxsize=capacity, missing=self.missing) self.args = args self.kwargs = kwargs diff --git a/tests/test_cache/test_cache.py b/tests/test_cache/test_cache.py index e2afa93..366bf62 100644 --- a/tests/test_cache/test_cache.py +++ b/tests/test_cache/test_cache.py @@ -50,6 +50,42 @@ def test_does_not_implement_missing(self): with self.assertRaises(NotImplementedError): cache.BaseCache(10).fetch('http://example.com/robots.txt') +class TestBaseCacheLRUCacheWithMissingHandler(unittest.TestCase): + '''Tests LRUCacheWithMissingHandler''' + + def calls_the_missing_function(self): + # missing ... + def missing(item): + return item + item + cache = cache.LRUCacheWithMissingHandler(maxsize=123, missing=missing) + self.assertEqual(cache[5], 10) + self.assertEqual(cache[6], 12) + self.assertEqual(5 in cache, True) + self.assertEqual(6 in cache, True) + + def caches_the_result_of_the_missing_function(self): + calls = [] + def missing(item): + calls.append(item) + return item * item + cache = cache.LRUCacheWithMissingHandler(maxsize=123, missing=missing) + self.assertEqual(cache[5], 10) + self.assertEqual(cache[6], 12) + self.assertEqual(cache[6], 12) + self.assertEqual(cache[5], 10) + self.assertEqual(calls, [5, 6]) + + def fills_up_capactity(self): + def missing(item): + return item + item + cache = cache.LRUCacheWithMissingHandler(maxsize=2, missing=missing) + self.assertEqual(cache[5], 10) + self.assertEqual(cache[6], 12) + self.assertEqual(cache[7], 14) + self.assertEqual(5 in cache, False) + self.assertEqual(6 in cache, True) + self.assertEqual(7 in cache, True) + class TestRobotsCache(unittest.TestCase): '''Tests about RobotsCache.''' From 62543491dde8ae5b11a67923b884cab8f6ab52a6 Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Wed, 14 Nov 2018 14:11:00 -0800 Subject: [PATCH 101/113] bump cachetools in requirements.txt so we test reppy working with the new version of cachetools -- cachetools 3 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5798789..17b07a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -cachetools==2.0.0 +cachetools==3.0.0 requests==2.10.0 six==1.10.0 python-dateutil==2.5.3 From a90c338fc245c8fd379066f02f475c6bc739410e Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Fri, 16 Nov 2018 14:02:25 -0800 Subject: [PATCH 102/113] removed rogue comment --- tests/test_cache/test_cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cache/test_cache.py b/tests/test_cache/test_cache.py index 366bf62..eeae442 100644 --- a/tests/test_cache/test_cache.py +++ b/tests/test_cache/test_cache.py @@ -54,7 +54,6 @@ class TestBaseCacheLRUCacheWithMissingHandler(unittest.TestCase): '''Tests LRUCacheWithMissingHandler''' def calls_the_missing_function(self): - # missing ... def missing(item): return item + item cache = cache.LRUCacheWithMissingHandler(maxsize=123, missing=missing) From d884572cc009c8e2a160de278feaec95d1c365f9 Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Fri, 16 Nov 2018 15:01:35 -0800 Subject: [PATCH 103/113] Much simpler way of doing this. --- reppy/cache/__init__.py | 24 +++++------------------ tests/test_cache/test_cache.py | 35 ---------------------------------- 2 files changed, 5 insertions(+), 54 deletions(-) diff --git a/reppy/cache/__init__.py b/reppy/cache/__init__.py index 1617526..0dfd8bc 100644 --- a/reppy/cache/__init__.py +++ b/reppy/cache/__init__.py @@ -37,19 +37,6 @@ def get(self): return self.obj -class LRUCacheWithMissingHandler(LRUCache): - '''Same as LRUCache but we can pass in a "missing" handler which will - be called if the key is missing. The result is then stored in the - cache. cachetools's LRUCache used to support this functionality, but it - was inexplicably removed in cachetools 3.0.0.''' - def __init__(self, maxsize, missing): - LRUCache.__init__(self, maxsize=maxsize) - self._missing_handler = missing - - def __missing__(self, key): - self[key] = self._missing_handler(key) - return self[key] - class BaseCache(object): '''A base cache class.''' @@ -59,13 +46,16 @@ class BaseCache(object): def __init__(self, capacity, cache_policy=None, ttl_policy=None, *args, **kwargs): self.cache_policy = cache_policy or self.DEFAULT_CACHE_POLICY self.ttl_policy = ttl_policy or self.DEFAULT_TTL_POLICY - self.cache = LRUCacheWithMissingHandler(maxsize=capacity, missing=self.missing) + self.cache = LRUCache(maxsize=capacity) self.args = args self.kwargs = kwargs def get(self, url): '''Get the entity that corresponds to URL.''' - return self.cache[Robots.robots_url(url)].get() + robots_url = Robots.robots_url(url) + if robots_url not in self.cache: + self.cache[robots_url] = ExpiringObject(partial(self.factory, robots_url)) + return self.cache[robots_url].get() def factory(self, url): ''' @@ -82,10 +72,6 @@ def fetch(self, url): '''Return (expiration, obj) corresponding to provided url.''' raise NotImplementedError('BaseCache does not implement fetch.') - def missing(self, url): - '''Invoked on cache misses.''' - return ExpiringObject(partial(self.factory, url)) - class RobotsCache(BaseCache): '''A cache of Robots objects.''' diff --git a/tests/test_cache/test_cache.py b/tests/test_cache/test_cache.py index eeae442..e2afa93 100644 --- a/tests/test_cache/test_cache.py +++ b/tests/test_cache/test_cache.py @@ -50,41 +50,6 @@ def test_does_not_implement_missing(self): with self.assertRaises(NotImplementedError): cache.BaseCache(10).fetch('http://example.com/robots.txt') -class TestBaseCacheLRUCacheWithMissingHandler(unittest.TestCase): - '''Tests LRUCacheWithMissingHandler''' - - def calls_the_missing_function(self): - def missing(item): - return item + item - cache = cache.LRUCacheWithMissingHandler(maxsize=123, missing=missing) - self.assertEqual(cache[5], 10) - self.assertEqual(cache[6], 12) - self.assertEqual(5 in cache, True) - self.assertEqual(6 in cache, True) - - def caches_the_result_of_the_missing_function(self): - calls = [] - def missing(item): - calls.append(item) - return item * item - cache = cache.LRUCacheWithMissingHandler(maxsize=123, missing=missing) - self.assertEqual(cache[5], 10) - self.assertEqual(cache[6], 12) - self.assertEqual(cache[6], 12) - self.assertEqual(cache[5], 10) - self.assertEqual(calls, [5, 6]) - - def fills_up_capactity(self): - def missing(item): - return item + item - cache = cache.LRUCacheWithMissingHandler(maxsize=2, missing=missing) - self.assertEqual(cache[5], 10) - self.assertEqual(cache[6], 12) - self.assertEqual(cache[7], 14) - self.assertEqual(5 in cache, False) - self.assertEqual(6 in cache, True) - self.assertEqual(7 in cache, True) - class TestRobotsCache(unittest.TestCase): '''Tests about RobotsCache.''' From 1be251214892b2f15b2bc2e8c8c27491600e9d14 Mon Sep 17 00:00:00 2001 From: Evan Battaglia Date: Thu, 29 Nov 2018 12:34:20 -0800 Subject: [PATCH 104/113] release 0.4.12 I consider cachetools breaking current/old reppy to be enough cause to bump the version of reppy. For background, that was fixed in reppy here: https://github.com/seomoz/reppy/pull/103 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 911bcdf..1c5cc3c 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.11', + version='0.4.12', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From 1ffab1212aafd3bc729789531fa1bd285c380b17 Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Tue, 12 Mar 2019 15:13:54 -0600 Subject: [PATCH 105/113] Support len(agent) --- reppy/robots.cpp | 1137 +++++++++++++++++++++++-------------------- reppy/robots.pyx | 3 + tests/test_agent.py | 5 + 3 files changed, 615 insertions(+), 530 deletions(-) diff --git a/reppy/robots.cpp b/reppy/robots.cpp index 8f29741..6b69045 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -773,7 +773,7 @@ struct __pyx_obj_5reppy_6robots_Agent { }; -/* "reppy/robots.pyx":137 +/* "reppy/robots.pyx":140 * return as_string(CppRobots.robotsUrl(as_bytes(url))) * * cdef class Robots: # <<<<<<<<<<<<<< @@ -787,7 +787,7 @@ struct __pyx_obj_5reppy_6robots_Robots { }; -/* "reppy/robots.pyx":199 +/* "reppy/robots.pyx":202 * * * cdef class AllowNone(Robots): # <<<<<<<<<<<<<< @@ -799,7 +799,7 @@ struct __pyx_obj_5reppy_6robots_AllowNone { }; -/* "reppy/robots.pyx":206 +/* "reppy/robots.pyx":209 * * * cdef class AllowAll(Robots): # <<<<<<<<<<<<<< @@ -811,7 +811,7 @@ struct __pyx_obj_5reppy_6robots_AllowAll { }; -/* "reppy/robots.pyx":84 +/* "reppy/robots.pyx":87 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -1697,12 +1697,13 @@ static PyObject *__pyx_n_s_wrap_exception; static PyObject *__pyx_n_s_wrapped; static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject *__pyx_self, CYTHON_UNUSED PyObject *__pyx_v_cls, struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_robots, std::string __pyx_v_name); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ +static Py_ssize_t __pyx_pf_5reppy_6robots_5Agent_2__len__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_5Agent_8__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ -static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_4allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_6disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_8allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self); /* proto */ +static PyObject *__pyx_pf_5reppy_6robots_5Agent_12__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_content, PyObject *__pyx_v_expires); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject *__pyx_self, PyObject *__pyx_v_etype, PyObject *__pyx_v_cause); /* proto */ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_cls, PyObject *__pyx_v_url, PyObject *__pyx_v_ttl_policy, PyObject *__pyx_v_max_size, PyObject *__pyx_v_args, PyObject *__pyx_v_kwargs); /* proto */ @@ -2166,7 +2167,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ * def __str__(self): * return as_string(self.agent.str()) # <<<<<<<<<<<<<< * - * @property + * def __len__(self): */ __Pyx_TraceLine(55,0,__PYX_ERR(2, 55, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); @@ -2200,7 +2201,64 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent___str__(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":58 +/* "reppy/robots.pyx":57 + * return as_string(self.agent.str()) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self.agent.directives().size() + * + */ + +/* Python wrapper */ +static Py_ssize_t __pyx_pw_5reppy_6robots_5Agent_3__len__(PyObject *__pyx_v_self); /*proto*/ +static Py_ssize_t __pyx_pw_5reppy_6robots_5Agent_3__len__(PyObject *__pyx_v_self) { + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__ (wrapper)", 0); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_2__len__(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self)); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static Py_ssize_t __pyx_pf_5reppy_6robots_5Agent_2__len__(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self) { + Py_ssize_t __pyx_r; + __Pyx_TraceDeclarations + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__len__", 0); + __Pyx_TraceCall("__len__", __pyx_f[2], 57, 0, __PYX_ERR(2, 57, __pyx_L1_error)); + + /* "reppy/robots.pyx":58 + * + * def __len__(self): + * return self.agent.directives().size() # <<<<<<<<<<<<<< + * + * @property + */ + __Pyx_TraceLine(58,0,__PYX_ERR(2, 58, __pyx_L1_error)) + __pyx_r = __pyx_v_self->agent.directives().size(); + goto __pyx_L0; + + /* "reppy/robots.pyx":57 + * return as_string(self.agent.str()) + * + * def __len__(self): # <<<<<<<<<<<<<< + * return self.agent.directives().size() + * + */ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_AddTraceback("reppy.robots.Agent.__len__", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_TraceReturn(Py_None, 0); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "reppy/robots.pyx":61 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -2229,45 +2287,45 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ int __pyx_t_1; PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 58, 0, __PYX_ERR(2, 58, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 61, 0, __PYX_ERR(2, 61, __pyx_L1_error)); - /* "reppy/robots.pyx":60 + /* "reppy/robots.pyx":63 * def delay(self): * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() # <<<<<<<<<<<<<< * if value > 0: * return value */ - __Pyx_TraceLine(60,0,__PYX_ERR(2, 60, __pyx_L1_error)) + __Pyx_TraceLine(63,0,__PYX_ERR(2, 63, __pyx_L1_error)) __pyx_v_value = __pyx_v_self->agent.delay(); - /* "reppy/robots.pyx":61 + /* "reppy/robots.pyx":64 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< * return value * return None */ - __Pyx_TraceLine(61,0,__PYX_ERR(2, 61, __pyx_L1_error)) + __Pyx_TraceLine(64,0,__PYX_ERR(2, 64, __pyx_L1_error)) __pyx_t_1 = ((__pyx_v_value > 0.0) != 0); if (__pyx_t_1) { - /* "reppy/robots.pyx":62 + /* "reppy/robots.pyx":65 * cdef float value = self.agent.delay() * if value > 0: * return value # <<<<<<<<<<<<<< * return None * */ - __Pyx_TraceLine(62,0,__PYX_ERR(2, 62, __pyx_L1_error)) + __Pyx_TraceLine(65,0,__PYX_ERR(2, 65, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 62, __pyx_L1_error) + __pyx_t_2 = PyFloat_FromDouble(__pyx_v_value); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 65, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":61 + /* "reppy/robots.pyx":64 * '''The delay associated with this agent.''' * cdef float value = self.agent.delay() * if value > 0: # <<<<<<<<<<<<<< @@ -2276,20 +2334,20 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ */ } - /* "reppy/robots.pyx":63 + /* "reppy/robots.pyx":66 * if value > 0: * return value * return None # <<<<<<<<<<<<<< * * def allow(self, path): */ - __Pyx_TraceLine(63,0,__PYX_ERR(2, 63, __pyx_L1_error)) + __Pyx_TraceLine(66,0,__PYX_ERR(2, 66, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(Py_None); __pyx_r = Py_None; goto __pyx_L0; - /* "reppy/robots.pyx":58 + /* "reppy/robots.pyx":61 * * @property * def delay(self): # <<<<<<<<<<<<<< @@ -2309,7 +2367,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ return __pyx_r; } -/* "reppy/robots.pyx":65 +/* "reppy/robots.pyx":68 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -2318,56 +2376,56 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_3allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ -static char __pyx_doc_5reppy_6robots_5Agent_2allow[] = "Allow the provided path."; -static PyObject *__pyx_pw_5reppy_6robots_5Agent_3allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pw_5reppy_6robots_5Agent_5allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ +static char __pyx_doc_5reppy_6robots_5Agent_4allow[] = "Allow the provided path."; +static PyObject *__pyx_pw_5reppy_6robots_5Agent_5allow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("allow (wrapper)", 0); - __pyx_r = __pyx_pf_5reppy_6robots_5Agent_2allow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_4allow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pf_5reppy_6robots_5Agent_4allow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allow", 0); - __Pyx_TraceCall("allow", __pyx_f[2], 65, 0, __PYX_ERR(2, 65, __pyx_L1_error)); + __Pyx_TraceCall("allow", __pyx_f[2], 68, 0, __PYX_ERR(2, 68, __pyx_L1_error)); - /* "reppy/robots.pyx":67 + /* "reppy/robots.pyx":70 * def allow(self, path): * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(67,0,__PYX_ERR(2, 67, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 67, __pyx_L1_error) + __Pyx_TraceLine(70,0,__PYX_ERR(2, 70, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 70, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 67, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 70, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.allow(__pyx_t_2); - /* "reppy/robots.pyx":68 + /* "reppy/robots.pyx":71 * '''Allow the provided path.''' * self.agent.allow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def disallow(self, path): */ - __Pyx_TraceLine(68,0,__PYX_ERR(2, 68, __pyx_L1_error)) + __Pyx_TraceLine(71,0,__PYX_ERR(2, 71, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":65 + /* "reppy/robots.pyx":68 * return None * * def allow(self, path): # <<<<<<<<<<<<<< @@ -2387,7 +2445,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 return __pyx_r; } -/* "reppy/robots.pyx":70 +/* "reppy/robots.pyx":73 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2396,56 +2454,56 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_2allow(struct __pyx_obj_5reppy_6 */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_5disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ -static char __pyx_doc_5reppy_6robots_5Agent_4disallow[] = "Disallow the provided path."; -static PyObject *__pyx_pw_5reppy_6robots_5Agent_5disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pw_5reppy_6robots_5Agent_7disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ +static char __pyx_doc_5reppy_6robots_5Agent_6disallow[] = "Disallow the provided path."; +static PyObject *__pyx_pw_5reppy_6robots_5Agent_7disallow(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("disallow (wrapper)", 0); - __pyx_r = __pyx_pf_5reppy_6robots_5Agent_4disallow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_6disallow(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pf_5reppy_6robots_5Agent_6disallow(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("disallow", 0); - __Pyx_TraceCall("disallow", __pyx_f[2], 70, 0, __PYX_ERR(2, 70, __pyx_L1_error)); + __Pyx_TraceCall("disallow", __pyx_f[2], 73, 0, __PYX_ERR(2, 73, __pyx_L1_error)); - /* "reppy/robots.pyx":72 + /* "reppy/robots.pyx":75 * def disallow(self, path): * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) # <<<<<<<<<<<<<< * return self * */ - __Pyx_TraceLine(72,0,__PYX_ERR(2, 72, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 72, __pyx_L1_error) + __Pyx_TraceLine(75,0,__PYX_ERR(2, 75, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 75, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 72, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 75, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_v_self->agent.disallow(__pyx_t_2); - /* "reppy/robots.pyx":73 + /* "reppy/robots.pyx":76 * '''Disallow the provided path.''' * self.agent.disallow(as_bytes(path)) * return self # <<<<<<<<<<<<<< * * def allowed(self, path): */ - __Pyx_TraceLine(73,0,__PYX_ERR(2, 73, __pyx_L1_error)) + __Pyx_TraceLine(76,0,__PYX_ERR(2, 76, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(((PyObject *)__pyx_v_self)); __pyx_r = ((PyObject *)__pyx_v_self); goto __pyx_L0; - /* "reppy/robots.pyx":70 + /* "reppy/robots.pyx":73 * return self * * def disallow(self, path): # <<<<<<<<<<<<<< @@ -2465,7 +2523,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":75 +/* "reppy/robots.pyx":78 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2474,48 +2532,48 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4disallow(struct __pyx_obj_5repp */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_7allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ -static char __pyx_doc_5reppy_6robots_5Agent_6allowed[] = "Is the provided URL allowed?"; -static PyObject *__pyx_pw_5reppy_6robots_5Agent_7allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pw_5reppy_6robots_5Agent_9allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path); /*proto*/ +static char __pyx_doc_5reppy_6robots_5Agent_8allowed[] = "Is the provided URL allowed?"; +static PyObject *__pyx_pw_5reppy_6robots_5Agent_9allowed(PyObject *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("allowed (wrapper)", 0); - __pyx_r = __pyx_pf_5reppy_6robots_5Agent_6allowed(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_8allowed(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v_path)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { +static PyObject *__pyx_pf_5reppy_6robots_5Agent_8allowed(struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, PyObject *__pyx_v_path) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; std::string __pyx_t_2; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[2], 75, 0, __PYX_ERR(2, 75, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[2], 78, 0, __PYX_ERR(2, 78, __pyx_L1_error)); - /* "reppy/robots.pyx":77 + /* "reppy/robots.pyx":80 * def allowed(self, path): * '''Is the provided URL allowed?''' * return self.agent.allowed(as_bytes(path)) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(77,0,__PYX_ERR(2, 77, __pyx_L1_error)) + __Pyx_TraceLine(80,0,__PYX_ERR(2, 80, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 77, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 77, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 80, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 77, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->agent.allowed(__pyx_t_2)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 80, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":75 + /* "reppy/robots.pyx":78 * return self * * def allowed(self, path): # <<<<<<<<<<<<<< @@ -2542,19 +2600,19 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6allowed(struct __pyx_obj_5reppy */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_9__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_9__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { +static PyObject *__pyx_pw_5reppy_6robots_5Agent_11__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_11__reduce_cython__(PyObject *__pyx_v_self, CYTHON_UNUSED PyObject *unused) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__reduce_cython__ (wrapper)", 0); - __pyx_r = __pyx_pf_5reppy_6robots_5Agent_8__reduce_cython__(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self)); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_10__reduce_cython__(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_5Agent_8__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self) { +static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__reduce_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations @@ -2600,19 +2658,19 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_8__reduce_cython__(CYTHON_UNUSED */ /* Python wrapper */ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_11__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ -static PyObject *__pyx_pw_5reppy_6robots_5Agent_11__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { +static PyObject *__pyx_pw_5reppy_6robots_5Agent_13__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state); /*proto*/ +static PyObject *__pyx_pw_5reppy_6robots_5Agent_13__setstate_cython__(PyObject *__pyx_v_self, PyObject *__pyx_v___pyx_state) { PyObject *__pyx_r = 0; __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__setstate_cython__ (wrapper)", 0); - __pyx_r = __pyx_pf_5reppy_6robots_5Agent_10__setstate_cython__(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); + __pyx_r = __pyx_pf_5reppy_6robots_5Agent_12__setstate_cython__(((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_v_self), ((PyObject *)__pyx_v___pyx_state)); /* function exit code */ __Pyx_RefNannyFinishContext(); return __pyx_r; } -static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { +static PyObject *__pyx_pf_5reppy_6robots_5Agent_12__setstate_cython__(CYTHON_UNUSED struct __pyx_obj_5reppy_6robots_Agent *__pyx_v_self, CYTHON_UNUSED PyObject *__pyx_v___pyx_state) { PyObject *__pyx_r = NULL; __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations @@ -2650,7 +2708,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__setstate_cython__(CYTHON_UNU return __pyx_r; } -/* "reppy/robots.pyx":80 +/* "reppy/robots.pyx":83 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2698,13 +2756,13 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(2, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(2, 83, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(2, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(2, 83, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 3: @@ -2714,7 +2772,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(2, 80, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "ParseMethod") < 0)) __PYX_ERR(2, 83, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2734,7 +2792,7 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 80, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 83, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.ParseMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -2759,18 +2817,18 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_6 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__6) __Pyx_RefNannySetupContext("ParseMethod", 0); - __Pyx_TraceCall("ParseMethod", __pyx_f[2], 80, 0, __PYX_ERR(2, 80, __pyx_L1_error)); + __Pyx_TraceCall("ParseMethod", __pyx_f[2], 83, 0, __PYX_ERR(2, 83, __pyx_L1_error)); - /* "reppy/robots.pyx":82 + /* "reppy/robots.pyx":85 * def ParseMethod(cls, url, content, expires=None): * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) # <<<<<<<<<<<<<< * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): */ - __Pyx_TraceLine(82,0,__PYX_ERR(2, 82, __pyx_L1_error)) + __Pyx_TraceLine(85,0,__PYX_ERR(2, 85, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 82, __pyx_L1_error) + __pyx_t_2 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_content); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_v_cls); __pyx_t_3 = __pyx_v_cls; __pyx_t_4 = NULL; @@ -2788,7 +2846,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 82, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -2797,14 +2855,14 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_v_url, __pyx_t_2, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 82, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_5, 3+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 82, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(3+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -2818,7 +2876,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_5, __pyx_v_expires); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 82, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 85, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -2827,7 +2885,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":80 + /* "reppy/robots.pyx":83 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< @@ -2851,7 +2909,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":84 +/* "reppy/robots.pyx":87 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -2915,7 +2973,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(2, 84, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(2, 87, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: @@ -2932,7 +2990,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } if (unlikely(kw_args > 0)) { const Py_ssize_t used_pos_args = (pos_args < 4) ? pos_args : 4; - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(2, 84, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, __pyx_v_kwargs, values, used_pos_args, "FetchMethod") < 0)) __PYX_ERR(2, 87, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -2956,7 +3014,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 84, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 87, __pyx_L3_error) __pyx_L3_error:; __Pyx_DECREF(__pyx_v_args); __pyx_v_args = 0; __Pyx_DECREF(__pyx_v_kwargs); __pyx_v_kwargs = 0; @@ -2973,7 +3031,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb return __pyx_r; } -/* "reppy/robots.pyx":88 +/* "reppy/robots.pyx":91 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< @@ -3013,11 +3071,11 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cause)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(2, 88, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(2, 91, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "wrap_exception") < 0)) __PYX_ERR(2, 88, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "wrap_exception") < 0)) __PYX_ERR(2, 91, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -3030,7 +3088,7 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 88, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 91, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.FetchMethod.wrap_exception", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -3059,16 +3117,16 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_RefNannySetupContext("wrap_exception", 0); __pyx_outer_scope = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *) __Pyx_CyFunction_GetClosure(__pyx_self); __pyx_cur_scope = __pyx_outer_scope; - __Pyx_TraceCall("wrap_exception", __pyx_f[2], 88, 0, __PYX_ERR(2, 88, __pyx_L1_error)); + __Pyx_TraceCall("wrap_exception", __pyx_f[2], 91, 0, __PYX_ERR(2, 91, __pyx_L1_error)); - /* "reppy/robots.pyx":89 + /* "reppy/robots.pyx":92 * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): * wrapped = etype(cause) # <<<<<<<<<<<<<< * wrapped.url = url * if after_response_hook is not None: */ - __Pyx_TraceLine(89,0,__PYX_ERR(2, 89, __pyx_L1_error)) + __Pyx_TraceLine(92,0,__PYX_ERR(2, 92, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_etype); __pyx_t_2 = __pyx_v_etype; __pyx_t_3 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { @@ -3081,13 +3139,13 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * } } if (!__pyx_t_3) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3095,19 +3153,19 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __pyx_t_3 = NULL; __Pyx_INCREF(__pyx_v_cause); __Pyx_GIVEREF(__pyx_v_cause); PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_cause); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; } @@ -3116,39 +3174,39 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __pyx_v_wrapped = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":90 + /* "reppy/robots.pyx":93 * def wrap_exception(etype, cause): * wrapped = etype(cause) * wrapped.url = url # <<<<<<<<<<<<<< * if after_response_hook is not None: * after_response_hook(wrapped) */ - __Pyx_TraceLine(90,0,__PYX_ERR(2, 90, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_url)) { __Pyx_RaiseClosureNameError("url"); __PYX_ERR(2, 90, __pyx_L1_error) } - if (__Pyx_PyObject_SetAttrStr(__pyx_v_wrapped, __pyx_n_s_url, __pyx_cur_scope->__pyx_v_url) < 0) __PYX_ERR(2, 90, __pyx_L1_error) + __Pyx_TraceLine(93,0,__PYX_ERR(2, 93, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_url)) { __Pyx_RaiseClosureNameError("url"); __PYX_ERR(2, 93, __pyx_L1_error) } + if (__Pyx_PyObject_SetAttrStr(__pyx_v_wrapped, __pyx_n_s_url, __pyx_cur_scope->__pyx_v_url) < 0) __PYX_ERR(2, 93, __pyx_L1_error) - /* "reppy/robots.pyx":91 + /* "reppy/robots.pyx":94 * wrapped = etype(cause) * wrapped.url = url * if after_response_hook is not None: # <<<<<<<<<<<<<< * after_response_hook(wrapped) * raise wrapped */ - __Pyx_TraceLine(91,0,__PYX_ERR(2, 91, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 91, __pyx_L1_error) } + __Pyx_TraceLine(94,0,__PYX_ERR(2, 94, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 94, __pyx_L1_error) } __pyx_t_5 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); __pyx_t_6 = (__pyx_t_5 != 0); if (__pyx_t_6) { - /* "reppy/robots.pyx":92 + /* "reppy/robots.pyx":95 * wrapped.url = url * if after_response_hook is not None: * after_response_hook(wrapped) # <<<<<<<<<<<<<< * raise wrapped * try: */ - __Pyx_TraceLine(92,0,__PYX_ERR(2, 92, __pyx_L1_error)) - if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 92, __pyx_L1_error) } + __Pyx_TraceLine(95,0,__PYX_ERR(2, 95, __pyx_L1_error)) + if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 95, __pyx_L1_error) } __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); __pyx_t_2 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_4 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { @@ -3161,13 +3219,13 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * } } if (!__pyx_t_4) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3175,19 +3233,19 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 92, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); __pyx_t_4 = NULL; __Pyx_INCREF(__pyx_v_wrapped); __Pyx_GIVEREF(__pyx_v_wrapped); PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_v_wrapped); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } @@ -3195,7 +3253,7 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":91 + /* "reppy/robots.pyx":94 * wrapped = etype(cause) * wrapped.url = url * if after_response_hook is not None: # <<<<<<<<<<<<<< @@ -3204,18 +3262,18 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * */ } - /* "reppy/robots.pyx":93 + /* "reppy/robots.pyx":96 * if after_response_hook is not None: * after_response_hook(wrapped) * raise wrapped # <<<<<<<<<<<<<< * try: * # Limit the size of the request */ - __Pyx_TraceLine(93,0,__PYX_ERR(2, 93, __pyx_L1_error)) + __Pyx_TraceLine(96,0,__PYX_ERR(2, 96, __pyx_L1_error)) __Pyx_Raise(__pyx_v_wrapped, 0, 0, 0); - __PYX_ERR(2, 93, __pyx_L1_error) + __PYX_ERR(2, 96, __pyx_L1_error) - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":91 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< @@ -3238,7 +3296,7 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * return __pyx_r; } -/* "reppy/robots.pyx":84 +/* "reppy/robots.pyx":87 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -3281,69 +3339,69 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (unlikely(!__pyx_cur_scope)) { __pyx_cur_scope = ((struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)Py_None); __Pyx_INCREF(Py_None); - __PYX_ERR(2, 84, __pyx_L1_error) + __PYX_ERR(2, 87, __pyx_L1_error) } else { __Pyx_GOTREF(__pyx_cur_scope); } - __Pyx_TraceCall("FetchMethod", __pyx_f[2], 84, 0, __PYX_ERR(2, 84, __pyx_L1_error)); + __Pyx_TraceCall("FetchMethod", __pyx_f[2], 87, 0, __PYX_ERR(2, 87, __pyx_L1_error)); __pyx_cur_scope->__pyx_v_url = __pyx_v_url; __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); - /* "reppy/robots.pyx":86 + /* "reppy/robots.pyx":89 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): */ - __Pyx_TraceLine(86,0,__PYX_ERR(2, 86, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 86, __pyx_L1_error) + __Pyx_TraceLine(89,0,__PYX_ERR(2, 89, __pyx_L1_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 86, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GIVEREF(__pyx_t_2); __pyx_cur_scope->__pyx_v_after_response_hook = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":87 + /* "reppy/robots.pyx":90 * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< * def wrap_exception(etype, cause): * wrapped = etype(cause) */ - __Pyx_TraceLine(87,0,__PYX_ERR(2, 87, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 87, __pyx_L1_error) + __Pyx_TraceLine(90,0,__PYX_ERR(2, 90, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 90, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 87, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 90, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_after_parse_hook = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":91 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< * wrapped = etype(cause) * wrapped.url = url */ - __Pyx_TraceLine(88,0,__PYX_ERR(2, 88, __pyx_L1_error)) - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 88, __pyx_L1_error) + __Pyx_TraceLine(91,0,__PYX_ERR(2, 91, __pyx_L1_error)) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_wrap_exception = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":97 * after_response_hook(wrapped) * raise wrapped * try: # <<<<<<<<<<<<<< * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_TraceLine(94,0,__PYX_ERR(2, 94, __pyx_L3_error)) + __Pyx_TraceLine(97,0,__PYX_ERR(2, 97, __pyx_L3_error)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign @@ -3353,41 +3411,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGOTREF(__pyx_t_5); /*try:*/ { - /* "reppy/robots.pyx":96 + /* "reppy/robots.pyx":99 * try: * # Limit the size of the request * kwargs['stream'] = True # <<<<<<<<<<<<<< * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) */ - __Pyx_TraceLine(96,0,__PYX_ERR(2, 96, __pyx_L3_error)) - if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(2, 96, __pyx_L3_error) + __Pyx_TraceLine(99,0,__PYX_ERR(2, 99, __pyx_L3_error)) + if (unlikely(PyDict_SetItem(__pyx_v_kwargs, __pyx_n_s_stream, Py_True) < 0)) __PYX_ERR(2, 99, __pyx_L3_error) - /* "reppy/robots.pyx":97 + /* "reppy/robots.pyx":100 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __Pyx_TraceLine(97,0,__PYX_ERR(2, 97, __pyx_L3_error)) + __Pyx_TraceLine(100,0,__PYX_ERR(2, 100, __pyx_L3_error)) /*with:*/ { - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_cur_scope->__pyx_v_url); - __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_8 = PyNumber_Add(__pyx_t_6, __pyx_v_args); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_6 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, __pyx_v_kwargs); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -3402,14 +3460,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; @@ -3418,28 +3476,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { - __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); __pyx_t_6 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(2, 97, __pyx_L3_error) + __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L9_error) + __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L9_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { @@ -3452,10 +3510,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (__pyx_t_6) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L9_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L9_error) __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 97, __pyx_L9_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L9_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -3474,78 +3532,78 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_res = __pyx_t_7; __pyx_t_7 = 0; - /* "reppy/robots.pyx":98 + /* "reppy/robots.pyx":101 * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: * content = res.raw.read(amt=max_size, decode_content=True) # <<<<<<<<<<<<<< * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): */ - __Pyx_TraceLine(98,0,__PYX_ERR(2, 98, __pyx_L13_error)) - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 98, __pyx_L13_error) + __Pyx_TraceLine(101,0,__PYX_ERR(2, 101, __pyx_L13_error)) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 98, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 98, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(2, 98, __pyx_L13_error) - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 98, __pyx_L13_error) - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 98, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(2, 101, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_v_content = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":100 + /* "reppy/robots.pyx":103 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) */ - __Pyx_TraceLine(100,0,__PYX_ERR(2, 100, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L13_error) + __Pyx_TraceLine(103,0,__PYX_ERR(2, 103, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 103, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 103, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 103, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(2, 100, __pyx_L13_error) - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 100, __pyx_L13_error) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(2, 103, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 103, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 103, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 100, __pyx_L13_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 103, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (__pyx_t_13) { - /* "reppy/robots.pyx":101 + /* "reppy/robots.pyx":104 * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( # <<<<<<<<<<<<<< * 'Content larger than %s bytes' % max_size) * */ - __Pyx_TraceLine(101,0,__PYX_ERR(2, 101, __pyx_L13_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 101, __pyx_L13_error) + __Pyx_TraceLine(104,0,__PYX_ERR(2, 104, __pyx_L13_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":102 + /* "reppy/robots.pyx":105 * if res.raw.read(amt=1, decode_content=True): * raise exceptions.ContentTooLong( * 'Content larger than %s bytes' % max_size) # <<<<<<<<<<<<<< * * if after_response_hook is not None: */ - __Pyx_TraceLine(102,0,__PYX_ERR(2, 102, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 102, __pyx_L13_error) + __Pyx_TraceLine(105,0,__PYX_ERR(2, 105, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 105, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_6 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -3558,14 +3616,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_6) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; @@ -3574,20 +3632,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_2); __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3595,9 +3653,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __PYX_ERR(2, 101, __pyx_L13_error) + __PYX_ERR(2, 104, __pyx_L13_error) - /* "reppy/robots.pyx":100 + /* "reppy/robots.pyx":103 * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big * if res.raw.read(amt=1, decode_content=True): # <<<<<<<<<<<<<< @@ -3606,26 +3664,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":104 + /* "reppy/robots.pyx":107 * 'Content larger than %s bytes' % max_size) * * if after_response_hook is not None: # <<<<<<<<<<<<<< * after_response_hook(res) * */ - __Pyx_TraceLine(104,0,__PYX_ERR(2, 104, __pyx_L13_error)) + __Pyx_TraceLine(107,0,__PYX_ERR(2, 107, __pyx_L13_error)) __pyx_t_13 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { - /* "reppy/robots.pyx":105 + /* "reppy/robots.pyx":108 * * if after_response_hook is not None: * after_response_hook(res) # <<<<<<<<<<<<<< * * # Get the TTL policy's ruling on the ttl */ - __Pyx_TraceLine(105,0,__PYX_ERR(2, 105, __pyx_L13_error)) + __Pyx_TraceLine(108,0,__PYX_ERR(2, 108, __pyx_L13_error)) __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); __pyx_t_7 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_8 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { @@ -3638,13 +3696,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3652,19 +3710,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_8); __pyx_t_8 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_2, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 105, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } @@ -3672,7 +3730,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":104 + /* "reppy/robots.pyx":107 * 'Content larger than %s bytes' % max_size) * * if after_response_hook is not None: # <<<<<<<<<<<<<< @@ -3681,28 +3739,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":108 + /* "reppy/robots.pyx":111 * * # Get the TTL policy's ruling on the ttl * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) # <<<<<<<<<<<<<< * * if res.status_code == 200: */ - __Pyx_TraceLine(108,0,__PYX_ERR(2, 108, __pyx_L13_error)) - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 108, __pyx_L13_error) + __Pyx_TraceLine(111,0,__PYX_ERR(2, 111, __pyx_L13_error)) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_v_ttl_policy); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 111, __pyx_L13_error) if (!__pyx_t_14) { } else { __Pyx_INCREF(__pyx_v_ttl_policy); __pyx_t_7 = __pyx_v_ttl_policy; goto __pyx_L21_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_2); __pyx_t_7 = __pyx_t_2; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_L21_bool_binop_done:; - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __pyx_t_7 = NULL; @@ -3716,13 +3774,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -3730,19 +3788,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_res); __Pyx_GIVEREF(__pyx_v_res); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3751,32 +3809,32 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_expires = __pyx_t_1; __pyx_t_1 = 0; - /* "reppy/robots.pyx":110 + /* "reppy/robots.pyx":113 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: */ - __Pyx_TraceLine(110,0,__PYX_ERR(2, 110, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 110, __pyx_L13_error) + __Pyx_TraceLine(113,0,__PYX_ERR(2, 113, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 110, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 110, __pyx_L13_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_14) { - /* "reppy/robots.pyx":111 + /* "reppy/robots.pyx":114 * * if res.status_code == 200: * robots = cls.parse(url, content, expires) # <<<<<<<<<<<<<< * if after_parse_hook is not None: * after_parse_hook(robots) */ - __Pyx_TraceLine(111,0,__PYX_ERR(2, 111, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) + __Pyx_TraceLine(114,0,__PYX_ERR(2, 114, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_8 = NULL; __pyx_t_15 = 0; @@ -3793,7 +3851,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else @@ -3801,13 +3859,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 111, __pyx_L13_error) + __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); if (__pyx_t_8) { __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; @@ -3821,7 +3879,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_15, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } @@ -3829,26 +3887,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_v_robots = __pyx_t_2; __pyx_t_2 = 0; - /* "reppy/robots.pyx":112 + /* "reppy/robots.pyx":115 * if res.status_code == 200: * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: # <<<<<<<<<<<<<< * after_parse_hook(robots) * return robots */ - __Pyx_TraceLine(112,0,__PYX_ERR(2, 112, __pyx_L13_error)) + __Pyx_TraceLine(115,0,__PYX_ERR(2, 115, __pyx_L13_error)) __pyx_t_14 = (__pyx_v_after_parse_hook != Py_None); __pyx_t_13 = (__pyx_t_14 != 0); if (__pyx_t_13) { - /* "reppy/robots.pyx":113 + /* "reppy/robots.pyx":116 * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: * after_parse_hook(robots) # <<<<<<<<<<<<<< * return robots * elif res.status_code in (401, 403): */ - __Pyx_TraceLine(113,0,__PYX_ERR(2, 113, __pyx_L13_error)) + __Pyx_TraceLine(116,0,__PYX_ERR(2, 116, __pyx_L13_error)) __Pyx_INCREF(__pyx_v_after_parse_hook); __pyx_t_1 = __pyx_v_after_parse_hook; __pyx_t_7 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_1))) { @@ -3861,13 +3919,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } } if (!__pyx_t_7) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); } else { #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else @@ -3875,19 +3933,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_GOTREF(__pyx_t_2); } else #endif { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; __Pyx_INCREF(__pyx_v_robots); __Pyx_GIVEREF(__pyx_v_robots); PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_robots); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } @@ -3895,7 +3953,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":112 + /* "reppy/robots.pyx":115 * if res.status_code == 200: * robots = cls.parse(url, content, expires) * if after_parse_hook is not None: # <<<<<<<<<<<<<< @@ -3904,20 +3962,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":114 + /* "reppy/robots.pyx":117 * if after_parse_hook is not None: * after_parse_hook(robots) * return robots # <<<<<<<<<<<<<< * elif res.status_code in (401, 403): * return AllowNone(url, expires) */ - __Pyx_TraceLine(114,0,__PYX_ERR(2, 114, __pyx_L13_error)) + __Pyx_TraceLine(117,0,__PYX_ERR(2, 117, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_robots); __pyx_r = __pyx_v_robots; goto __pyx_L17_try_return; - /* "reppy/robots.pyx":110 + /* "reppy/robots.pyx":113 * expires = (ttl_policy or cls.DEFAULT_TTL_POLICY).expires(res) * * if res.status_code == 200: # <<<<<<<<<<<<<< @@ -3926,28 +3984,28 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":115 + /* "reppy/robots.pyx":118 * after_parse_hook(robots) * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: */ - __Pyx_TraceLine(115,0,__PYX_ERR(2, 115, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 115, __pyx_L13_error) + __Pyx_TraceLine(118,0,__PYX_ERR(2, 118, __pyx_L13_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 115, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 115, __pyx_L13_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (!__pyx_t_14) { } else { __pyx_t_13 = __pyx_t_14; goto __pyx_L25_bool_binop_done; } - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 115, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 115, __pyx_L13_error) + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_13 = __pyx_t_14; __pyx_L25_bool_binop_done:; @@ -3955,16 +4013,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { - /* "reppy/robots.pyx":116 + /* "reppy/robots.pyx":119 * return robots * elif res.status_code in (401, 403): * return AllowNone(url, expires) # <<<<<<<<<<<<<< * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) */ - __Pyx_TraceLine(116,0,__PYX_ERR(2, 116, __pyx_L13_error)) + __Pyx_TraceLine(119,0,__PYX_ERR(2, 119, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) + __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 119, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); @@ -3972,14 +4030,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 116, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 119, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L17_try_return; - /* "reppy/robots.pyx":115 + /* "reppy/robots.pyx":118 * after_parse_hook(robots) * return robots * elif res.status_code in (401, 403): # <<<<<<<<<<<<<< @@ -3988,45 +4046,45 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":117 + /* "reppy/robots.pyx":120 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< * return AllowAll(url, expires) * else: */ - __Pyx_TraceLine(117,0,__PYX_ERR(2, 117, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) + __Pyx_TraceLine(120,0,__PYX_ERR(2, 120, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_13) { } else { __pyx_t_14 = __pyx_t_13; goto __pyx_L27_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 117, __pyx_L13_error) + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_14 = __pyx_t_13; __pyx_L27_bool_binop_done:; if (__pyx_t_14) { - /* "reppy/robots.pyx":118 + /* "reppy/robots.pyx":121 * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: * return AllowAll(url, expires) # <<<<<<<<<<<<<< * else: * raise exceptions.BadStatusCode( */ - __Pyx_TraceLine(118,0,__PYX_ERR(2, 118, __pyx_L13_error)) + __Pyx_TraceLine(121,0,__PYX_ERR(2, 121, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) + __pyx_t_1 = PyTuple_New(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 121, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); @@ -4034,14 +4092,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 118, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 121, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L17_try_return; - /* "reppy/robots.pyx":117 + /* "reppy/robots.pyx":120 * elif res.status_code in (401, 403): * return AllowNone(url, expires) * elif res.status_code >= 400 and res.status_code < 500: # <<<<<<<<<<<<<< @@ -4050,32 +4108,32 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } - /* "reppy/robots.pyx":120 + /* "reppy/robots.pyx":123 * return AllowAll(url, expires) * else: * raise exceptions.BadStatusCode( # <<<<<<<<<<<<<< * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: */ - __Pyx_TraceLine(120,0,__PYX_ERR(2, 120, __pyx_L13_error)) + __Pyx_TraceLine(123,0,__PYX_ERR(2, 123, __pyx_L13_error)) /*else*/ { - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 120, __pyx_L13_error) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 123, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 120, __pyx_L13_error) + __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 123, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":121 + /* "reppy/robots.pyx":124 * else: * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) # <<<<<<<<<<<<<< * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) */ - __Pyx_TraceLine(121,0,__PYX_ERR(2, 121, __pyx_L13_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 121, __pyx_L13_error) + __Pyx_TraceLine(124,0,__PYX_ERR(2, 124, __pyx_L13_error)) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 124, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 121, __pyx_L13_error) + __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 124, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); @@ -4083,10 +4141,10 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_cur_scope->__pyx_v_url); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 121, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 124, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 121, __pyx_L13_error) + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 124, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_6 = NULL; __pyx_t_15 = 0; @@ -4103,7 +4161,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 123, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -4113,7 +4171,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 123, __pyx_L13_error) __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -4121,7 +4179,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } else #endif { - __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 120, __pyx_L13_error) + __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 123, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_16); if (__pyx_t_6) { __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; @@ -4132,17 +4190,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_15, __pyx_t_7); __pyx_t_1 = 0; __pyx_t_7 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 123, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_Raise(__pyx_t_2, 0, 0, 0); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __PYX_ERR(2, 120, __pyx_L13_error) + __PYX_ERR(2, 123, __pyx_L13_error) } - /* "reppy/robots.pyx":97 + /* "reppy/robots.pyx":100 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< @@ -4159,20 +4217,20 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; /*except:*/ { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 97, __pyx_L15_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 100, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); - __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 97, __pyx_L15_except_error) + __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_7, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - if (unlikely(!__pyx_t_17)) __PYX_ERR(2, 97, __pyx_L15_except_error) + if (unlikely(!__pyx_t_17)) __PYX_ERR(2, 100, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_17); __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_17); __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; - if (__pyx_t_14 < 0) __PYX_ERR(2, 97, __pyx_L15_except_error) + if (__pyx_t_14 < 0) __PYX_ERR(2, 100, __pyx_L15_except_error) __pyx_t_13 = ((!(__pyx_t_14 != 0)) != 0); if (__pyx_t_13) { __Pyx_GIVEREF(__pyx_t_2); @@ -4180,7 +4238,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGIVEREF(__pyx_t_16); __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_8, __pyx_t_16); __pyx_t_2 = 0; __pyx_t_8 = 0; __pyx_t_16 = 0; - __PYX_ERR(2, 97, __pyx_L15_except_error) + __PYX_ERR(2, 100, __pyx_L15_except_error) } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; @@ -4211,7 +4269,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__12, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_12)) __PYX_ERR(2, 97, __pyx_L3_error) + if (unlikely(!__pyx_t_12)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_12); __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; } @@ -4223,7 +4281,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_9) { __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__13, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - if (unlikely(!__pyx_t_11)) __PYX_ERR(2, 97, __pyx_L3_error) + if (unlikely(!__pyx_t_11)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_11); __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } @@ -4240,7 +4298,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L32:; } - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":97 * after_response_hook(wrapped) * raise wrapped * try: # <<<<<<<<<<<<<< @@ -4260,41 +4318,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; - /* "reppy/robots.pyx":122 + /* "reppy/robots.pyx":125 * raise exceptions.BadStatusCode( * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: */ - __Pyx_TraceLine(122,0,__PYX_ERR(2, 122, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 122, __pyx_L5_except_error) + __Pyx_TraceLine(125,0,__PYX_ERR(2, 125, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(2, 122, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(2, 125, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":123 + /* "reppy/robots.pyx":126 * 'Got %i for %s' % (res.status_code, url), res.status_code) * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) # <<<<<<<<<<<<<< * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) */ - __Pyx_TraceLine(123,0,__PYX_ERR(2, 123, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __Pyx_TraceLine(126,0,__PYX_ERR(2, 126, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 123, __pyx_L5_except_error) + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -4304,41 +4362,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":124 + /* "reppy/robots.pyx":127 * except SSLError as exc: * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: */ - __Pyx_TraceLine(124,0,__PYX_ERR(2, 124, __pyx_L5_except_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 124, __pyx_L5_except_error) + __Pyx_TraceLine(127,0,__PYX_ERR(2, 127, __pyx_L5_except_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 127, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_2); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 124, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 127, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; - /* "reppy/robots.pyx":125 + /* "reppy/robots.pyx":128 * wrap_exception(exceptions.SSLException, exc) * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) # <<<<<<<<<<<<<< * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) */ - __Pyx_TraceLine(125,0,__PYX_ERR(2, 125, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 125, __pyx_L5_except_error) + __Pyx_TraceLine(128,0,__PYX_ERR(2, 128, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 128, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 125, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 128, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 125, __pyx_L5_except_error) + __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 128, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; @@ -4348,21 +4406,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":126 + /* "reppy/robots.pyx":129 * except ConnectionError as exc: * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: */ - __Pyx_TraceLine(126,0,__PYX_ERR(2, 126, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __Pyx_TraceLine(129,0,__PYX_ERR(2, 129, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 129, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 129, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 129, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 129, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_2) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4371,27 +4429,27 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 126, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 129, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":127 + /* "reppy/robots.pyx":130 * wrap_exception(exceptions.ConnectionException, exc) * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) # <<<<<<<<<<<<<< * except TooManyRedirects as exc: * wrap_exception(exceptions.ExcessiveRedirects, exc) */ - __Pyx_TraceLine(127,0,__PYX_ERR(2, 127, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 127, __pyx_L5_except_error) + __Pyx_TraceLine(130,0,__PYX_ERR(2, 130, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 130, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 127, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 130, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 127, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 130, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4401,41 +4459,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":128 + /* "reppy/robots.pyx":131 * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.ExcessiveRedirects, exc) * except ReadTimeout as exc: */ - __Pyx_TraceLine(128,0,__PYX_ERR(2, 128, __pyx_L5_except_error)) - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __Pyx_TraceLine(131,0,__PYX_ERR(2, 131, __pyx_L5_except_error)) + __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 131, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(2, 128, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(2, 131, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_7); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":129 + /* "reppy/robots.pyx":132 * wrap_exception(exceptions.MalformedUrl, exc) * except TooManyRedirects as exc: * wrap_exception(exceptions.ExcessiveRedirects, exc) # <<<<<<<<<<<<<< * except ReadTimeout as exc: * wrap_exception(exceptions.ReadTimeout, exc) */ - __Pyx_TraceLine(129,0,__PYX_ERR(2, 129, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 129, __pyx_L5_except_error) + __Pyx_TraceLine(132,0,__PYX_ERR(2, 132, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 132, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 129, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 132, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 129, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 132, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4445,41 +4503,41 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L4_exception_handled; } - /* "reppy/robots.pyx":130 + /* "reppy/robots.pyx":133 * except TooManyRedirects as exc: * wrap_exception(exceptions.ExcessiveRedirects, exc) * except ReadTimeout as exc: # <<<<<<<<<<<<<< * wrap_exception(exceptions.ReadTimeout, exc) * */ - __Pyx_TraceLine(130,0,__PYX_ERR(2, 130, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 130, __pyx_L5_except_error) + __Pyx_TraceLine(133,0,__PYX_ERR(2, 133, __pyx_L5_except_error)) + __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 133, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_7); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 130, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 133, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); __Pyx_GOTREF(__pyx_t_2); __Pyx_GOTREF(__pyx_t_8); __Pyx_INCREF(__pyx_t_2); __pyx_v_exc = __pyx_t_2; - /* "reppy/robots.pyx":131 + /* "reppy/robots.pyx":134 * wrap_exception(exceptions.ExcessiveRedirects, exc) * except ReadTimeout as exc: * wrap_exception(exceptions.ReadTimeout, exc) # <<<<<<<<<<<<<< * * def RobotsUrlMethod(cls, url): */ - __Pyx_TraceLine(131,0,__PYX_ERR(2, 131, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 131, __pyx_L5_except_error) + __Pyx_TraceLine(134,0,__PYX_ERR(2, 134, __pyx_L5_except_error)) + __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 134, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 131, __pyx_L5_except_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 134, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 131, __pyx_L5_except_error) + __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 134, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_16); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; @@ -4491,7 +4549,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L5_except_error; __pyx_L5_except_error:; - /* "reppy/robots.pyx":94 + /* "reppy/robots.pyx":97 * after_response_hook(wrapped) * raise wrapped * try: # <<<<<<<<<<<<<< @@ -4517,7 +4575,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_L8_try_end:; } - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":87 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< @@ -4552,7 +4610,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ return __pyx_r; } -/* "reppy/robots.pyx":133 +/* "reppy/robots.pyx":136 * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -4593,11 +4651,11 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(2, 133, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(2, 136, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(2, 133, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "RobotsUrlMethod") < 0)) __PYX_ERR(2, 136, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -4610,7 +4668,7 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 133, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 136, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.RobotsUrlMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4633,37 +4691,37 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject PyObject *__pyx_t_4 = NULL; __Pyx_TraceFrameInit(__pyx_codeobj__14) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); - __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[2], 133, 0, __PYX_ERR(2, 133, __pyx_L1_error)); + __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[2], 136, 0, __PYX_ERR(2, 136, __pyx_L1_error)); - /* "reppy/robots.pyx":135 + /* "reppy/robots.pyx":138 * def RobotsUrlMethod(cls, url): * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) # <<<<<<<<<<<<<< * * cdef class Robots: */ - __Pyx_TraceLine(135,0,__PYX_ERR(2, 135, __pyx_L1_error)) + __Pyx_TraceLine(138,0,__PYX_ERR(2, 138, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 135, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 135, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 138, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(2, 135, __pyx_L1_error) + __PYX_ERR(2, 138, __pyx_L1_error) } - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 135, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 135, __pyx_L1_error) + __pyx_t_4 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 138, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_4; __pyx_t_4 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":133 + /* "reppy/robots.pyx":136 * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< @@ -4684,7 +4742,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject return __pyx_r; } -/* "reppy/robots.pyx":153 +/* "reppy/robots.pyx":156 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -4727,7 +4785,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(2, 153, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(2, 156, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: @@ -4737,7 +4795,7 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 153, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 156, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -4750,12 +4808,12 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO } } __pyx_v_url = values[0]; - __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 153, __pyx_L3_error) + __pyx_v_content = __pyx_convert_string_from_py_std__in_string(values[1]); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 156, __pyx_L3_error) __pyx_v_expires = values[2]; } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 153, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 156, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -4776,43 +4834,43 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob std::string __pyx_t_2; Rep::Robots *__pyx_t_3; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[2], 153, 0, __PYX_ERR(2, 153, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 156, 0, __PYX_ERR(2, 156, __pyx_L1_error)); - /* "reppy/robots.pyx":154 + /* "reppy/robots.pyx":157 * * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) # <<<<<<<<<<<<<< * self.expires = expires * */ - __Pyx_TraceLine(154,0,__PYX_ERR(2, 154, __pyx_L1_error)) - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 154, __pyx_L1_error) + __Pyx_TraceLine(157,0,__PYX_ERR(2, 157, __pyx_L1_error)) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_url); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 157, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 154, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 157, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; try { __pyx_t_3 = new Rep::Robots(__pyx_v_content, __pyx_t_2); } catch(...) { try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } - __PYX_ERR(2, 154, __pyx_L1_error) + __PYX_ERR(2, 157, __pyx_L1_error) } __pyx_v_self->robots = __pyx_t_3; - /* "reppy/robots.pyx":155 + /* "reppy/robots.pyx":158 * def __init__(self, url, const string& content, expires=None): * self.robots = new CppRobots(content, as_bytes(url)) * self.expires = expires # <<<<<<<<<<<<<< * * def __str__(self): */ - __Pyx_TraceLine(155,0,__PYX_ERR(2, 155, __pyx_L1_error)) + __Pyx_TraceLine(158,0,__PYX_ERR(2, 158, __pyx_L1_error)) __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); __Pyx_GOTREF(__pyx_v_self->expires); __Pyx_DECREF(__pyx_v_self->expires); __pyx_v_self->expires = __pyx_v_expires; - /* "reppy/robots.pyx":153 + /* "reppy/robots.pyx":156 * cdef object expires * * def __init__(self, url, const string& content, expires=None): # <<<<<<<<<<<<<< @@ -4833,7 +4891,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob return __pyx_r; } -/* "reppy/robots.pyx":157 +/* "reppy/robots.pyx":160 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4861,27 +4919,27 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; __Pyx_RefNannySetupContext("__str__", 0); - __Pyx_TraceCall("__str__", __pyx_f[2], 157, 0, __PYX_ERR(2, 157, __pyx_L1_error)); + __Pyx_TraceCall("__str__", __pyx_f[2], 160, 0, __PYX_ERR(2, 160, __pyx_L1_error)); - /* "reppy/robots.pyx":160 + /* "reppy/robots.pyx":163 * # Note: this could raise a UnicodeDecodeError in Python 3 if the * # robots.txt had invalid UTF-8 * return as_string(self.robots.str()) # <<<<<<<<<<<<<< * * def __dealloc__(self): */ - __Pyx_TraceLine(160,0,__PYX_ERR(2, 160, __pyx_L1_error)) + __Pyx_TraceLine(163,0,__PYX_ERR(2, 163, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 160, __pyx_L1_error) + __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_v_self->robots->str()); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 163, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 160, __pyx_L1_error) + __pyx_t_2 = __pyx_f_5reppy_6robots_as_string(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 163, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":157 + /* "reppy/robots.pyx":160 * self.expires = expires * * def __str__(self): # <<<<<<<<<<<<<< @@ -4902,7 +4960,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_2__str__(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":162 +/* "reppy/robots.pyx":165 * return as_string(self.robots.str()) * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4925,19 +4983,19 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__dealloc__", 0); - __Pyx_TraceCall("__dealloc__", __pyx_f[2], 162, 0, __PYX_ERR(2, 162, __pyx_L1_error)); + __Pyx_TraceCall("__dealloc__", __pyx_f[2], 165, 0, __PYX_ERR(2, 165, __pyx_L1_error)); - /* "reppy/robots.pyx":163 + /* "reppy/robots.pyx":166 * * def __dealloc__(self): * del self.robots # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(163,0,__PYX_ERR(2, 163, __pyx_L1_error)) + __Pyx_TraceLine(166,0,__PYX_ERR(2, 166, __pyx_L1_error)) delete __pyx_v_self->robots; - /* "reppy/robots.pyx":162 + /* "reppy/robots.pyx":165 * return as_string(self.robots.str()) * * def __dealloc__(self): # <<<<<<<<<<<<<< @@ -4954,7 +5012,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy __Pyx_RefNannyFinishContext(); } -/* "reppy/robots.pyx":166 +/* "reppy/robots.pyx":169 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -4983,22 +5041,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 166, 0, __PYX_ERR(2, 166, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 169, 0, __PYX_ERR(2, 169, __pyx_L1_error)); - /* "reppy/robots.pyx":168 + /* "reppy/robots.pyx":171 * def sitemaps(self): * '''Get all the sitemaps in this robots.txt.''' * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< * * def allowed(self, path, name): */ - __Pyx_TraceLine(168,0,__PYX_ERR(2, 168, __pyx_L1_error)) + __Pyx_TraceLine(171,0,__PYX_ERR(2, 171, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 168, __pyx_L1_error) + __pyx_t_1 = __Pyx_CFunc_object____object___to_py(__pyx_f_5reppy_6robots_as_string); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 171, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 168, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_vector_to_py_std_3a__3a_string(__pyx_v_self->robots->sitemaps()); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 171, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 168, __pyx_L1_error) + __pyx_t_3 = PyTuple_New(2); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 171, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); @@ -5006,14 +5064,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ PyTuple_SET_ITEM(__pyx_t_3, 1, __pyx_t_2); __pyx_t_1 = 0; __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 168, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 171, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_r = __pyx_t_2; __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":166 + /* "reppy/robots.pyx":169 * * @property * def sitemaps(self): # <<<<<<<<<<<<<< @@ -5035,7 +5093,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ return __pyx_r; } -/* "reppy/robots.pyx":170 +/* "reppy/robots.pyx":173 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -5075,11 +5133,11 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self case 1: if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(2, 170, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(2, 173, __pyx_L3_error) } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(2, 170, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "allowed") < 0)) __PYX_ERR(2, 173, __pyx_L3_error) } } else if (PyTuple_GET_SIZE(__pyx_args) != 2) { goto __pyx_L5_argtuple_error; @@ -5092,7 +5150,7 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 170, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 173, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.Robots.allowed", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5113,32 +5171,32 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp std::string __pyx_t_2; std::string __pyx_t_3; __Pyx_RefNannySetupContext("allowed", 0); - __Pyx_TraceCall("allowed", __pyx_f[2], 170, 0, __PYX_ERR(2, 170, __pyx_L1_error)); + __Pyx_TraceCall("allowed", __pyx_f[2], 173, 0, __PYX_ERR(2, 173, __pyx_L1_error)); - /* "reppy/robots.pyx":172 + /* "reppy/robots.pyx":175 * def allowed(self, path, name): * '''Is the provided path allowed for the provided agent?''' * return self.robots.allowed(as_bytes(path), as_bytes(name)) # <<<<<<<<<<<<<< * * def agent(self, name): */ - __Pyx_TraceLine(172,0,__PYX_ERR(2, 172, __pyx_L1_error)) + __Pyx_TraceLine(175,0,__PYX_ERR(2, 175, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 172, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_path); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 175, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 172, __pyx_L1_error) + __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 175, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 172, __pyx_L1_error) + __pyx_t_1 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 175, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 172, __pyx_L1_error) + __pyx_t_3 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 175, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 172, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyBool_FromLong(__pyx_v_self->robots->allowed(__pyx_t_2, __pyx_t_3)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 175, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":170 + /* "reppy/robots.pyx":173 * return map(as_string, self.robots.sitemaps()) * * def allowed(self, path, name): # <<<<<<<<<<<<<< @@ -5158,7 +5216,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp return __pyx_r; } -/* "reppy/robots.pyx":174 +/* "reppy/robots.pyx":177 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -5191,20 +5249,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ int __pyx_t_5; PyObject *__pyx_t_6 = NULL; __Pyx_RefNannySetupContext("agent", 0); - __Pyx_TraceCall("agent", __pyx_f[2], 174, 0, __PYX_ERR(2, 174, __pyx_L1_error)); + __Pyx_TraceCall("agent", __pyx_f[2], 177, 0, __PYX_ERR(2, 177, __pyx_L1_error)); - /* "reppy/robots.pyx":181 + /* "reppy/robots.pyx":184 * Agent object. * ''' * return Agent.from_robots(self, as_bytes(name)) # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(181,0,__PYX_ERR(2, 181, __pyx_L1_error)) + __Pyx_TraceLine(184,0,__PYX_ERR(2, 184, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_n_s_from_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_3 = __pyx_f_5reppy_6robots_as_bytes(__pyx_v_name); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; __pyx_t_5 = 0; @@ -5221,7 +5279,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 184, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; @@ -5230,14 +5288,14 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[3] = {__pyx_t_4, ((PyObject *)__pyx_v_self), __pyx_t_3}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_5, 2+__pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 184, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else #endif { - __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_6 = PyTuple_New(2+__pyx_t_5); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_6); if (__pyx_t_4) { __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; @@ -5248,7 +5306,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_5, __pyx_t_3); __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 181, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 184, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } @@ -5257,7 +5315,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ __pyx_t_1 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":174 + /* "reppy/robots.pyx":177 * return self.robots.allowed(as_bytes(path), as_bytes(name)) * * def agent(self, name): # <<<<<<<<<<<<<< @@ -5281,7 +5339,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8agent(struct __pyx_obj_5reppy_ return __pyx_r; } -/* "reppy/robots.pyx":184 +/* "reppy/robots.pyx":187 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -5310,20 +5368,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 184, 0, __PYX_ERR(2, 184, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 187, 0, __PYX_ERR(2, 187, __pyx_L1_error)); - /* "reppy/robots.pyx":186 + /* "reppy/robots.pyx":189 * def expired(self): * '''True if the current time is past its expiration.''' * return time.time() > self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(186,0,__PYX_ERR(2, 186, __pyx_L1_error)) + __Pyx_TraceLine(189,0,__PYX_ERR(2, 189, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 186, __pyx_L1_error) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 186, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; @@ -5337,20 +5395,20 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o } } if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 186, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 189, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 186, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 189, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 186, __pyx_L1_error) + __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 189, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_r = __pyx_t_3; __pyx_t_3 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":184 + /* "reppy/robots.pyx":187 * * @property * def expired(self): # <<<<<<<<<<<<<< @@ -5372,7 +5430,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":189 +/* "reppy/robots.pyx":192 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -5398,22 +5456,22 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 189, 0, __PYX_ERR(2, 189, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 192, 0, __PYX_ERR(2, 192, __pyx_L1_error)); - /* "reppy/robots.pyx":191 + /* "reppy/robots.pyx":194 * def expires(self): * '''The expiration of this robots.txt.''' * return self.expires # <<<<<<<<<<<<<< * * @property */ - __Pyx_TraceLine(191,0,__PYX_ERR(2, 191, __pyx_L1_error)) + __Pyx_TraceLine(194,0,__PYX_ERR(2, 194, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __Pyx_INCREF(__pyx_v_self->expires); __pyx_r = __pyx_v_self->expires; goto __pyx_L0; - /* "reppy/robots.pyx":189 + /* "reppy/robots.pyx":192 * * @property * def expires(self): # <<<<<<<<<<<<<< @@ -5432,7 +5490,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expires___get__(struct __pyx_o return __pyx_r; } -/* "reppy/robots.pyx":194 +/* "reppy/robots.pyx":197 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -5464,21 +5522,21 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 PyObject *__pyx_t_5 = NULL; int __pyx_t_6; __Pyx_RefNannySetupContext("__get__", 0); - __Pyx_TraceCall("__get__", __pyx_f[2], 194, 0, __PYX_ERR(2, 194, __pyx_L1_error)); + __Pyx_TraceCall("__get__", __pyx_f[2], 197, 0, __PYX_ERR(2, 197, __pyx_L1_error)); - /* "reppy/robots.pyx":196 + /* "reppy/robots.pyx":199 * def ttl(self): * '''Remaining time for this response to be considered valid.''' * return max(self.expires - time.time(), 0) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(196,0,__PYX_ERR(2, 196, __pyx_L1_error)) + __Pyx_TraceLine(199,0,__PYX_ERR(2, 199, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 196, __pyx_L1_error) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 196, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = NULL; @@ -5492,24 +5550,24 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 } } if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 196, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 196, __pyx_L1_error) + __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 199, __pyx_L1_error) } __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 196, __pyx_L1_error) + __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 196, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 196, __pyx_L1_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_3, __pyx_t_4, Py_GT); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(2, 196, __pyx_L1_error) + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_6 < 0)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_6) { - __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 196, __pyx_L1_error) + __pyx_t_5 = __Pyx_PyInt_From_long(__pyx_t_1); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); __pyx_t_2 = __pyx_t_5; __pyx_t_5 = 0; @@ -5523,7 +5581,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; goto __pyx_L0; - /* "reppy/robots.pyx":194 + /* "reppy/robots.pyx":197 * * @property * def ttl(self): # <<<<<<<<<<<<<< @@ -5661,7 +5719,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_12__setstate_cython__(CYTHON_UN return __pyx_r; } -/* "reppy/robots.pyx":202 +/* "reppy/robots.pyx":205 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5705,7 +5763,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 202, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 205, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -5721,7 +5779,7 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 202, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 205, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowNone.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -5744,17 +5802,17 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[2], 202, 0, __PYX_ERR(2, 202, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 205, 0, __PYX_ERR(2, 205, __pyx_L1_error)); - /* "reppy/robots.pyx":203 + /* "reppy/robots.pyx":206 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'User-agent: *\nDisallow: /', expires) # <<<<<<<<<<<<<< * * */ - __Pyx_TraceLine(203,0,__PYX_ERR(2, 203, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 203, __pyx_L1_error) + __Pyx_TraceLine(206,0,__PYX_ERR(2, 206, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 206, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -5771,7 +5829,7 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 203, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 206, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -5779,13 +5837,13 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b_User_agent_Disallow, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 203, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 206, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 203, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 206, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -5802,14 +5860,14 @@ static int __pyx_pf_5reppy_6robots_9AllowNone___init__(struct __pyx_obj_5reppy_6 __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 203, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 206, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":202 + /* "reppy/robots.pyx":205 * '''No requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5948,7 +6006,7 @@ static PyObject *__pyx_pf_5reppy_6robots_9AllowNone_4__setstate_cython__(CYTHON_ return __pyx_r; } -/* "reppy/robots.pyx":209 +/* "reppy/robots.pyx":212 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -5991,7 +6049,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } } if (unlikely(kw_args > 0)) { - if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 209, __pyx_L3_error) + if (unlikely(__Pyx_ParseOptionalKeywords(__pyx_kwds, __pyx_pyargnames, 0, values, pos_args, "__init__") < 0)) __PYX_ERR(2, 212, __pyx_L3_error) } } else { switch (PyTuple_GET_SIZE(__pyx_args)) { @@ -6007,7 +6065,7 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P } goto __pyx_L4_argument_unpacking_done; __pyx_L5_argtuple_error:; - __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 209, __pyx_L3_error) + __Pyx_RaiseArgtupleInvalid("__init__", 0, 1, 2, PyTuple_GET_SIZE(__pyx_args)); __PYX_ERR(2, 212, __pyx_L3_error) __pyx_L3_error:; __Pyx_AddTraceback("reppy.robots.AllowAll.__init__", __pyx_clineno, __pyx_lineno, __pyx_filename); __Pyx_RefNannyFinishContext(); @@ -6030,15 +6088,15 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r int __pyx_t_4; PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__init__", 0); - __Pyx_TraceCall("__init__", __pyx_f[2], 209, 0, __PYX_ERR(2, 209, __pyx_L1_error)); + __Pyx_TraceCall("__init__", __pyx_f[2], 212, 0, __PYX_ERR(2, 212, __pyx_L1_error)); - /* "reppy/robots.pyx":210 + /* "reppy/robots.pyx":213 * * def __init__(self, url, expires=None): * Robots.__init__(self, url, b'', expires) # <<<<<<<<<<<<<< */ - __Pyx_TraceLine(210,0,__PYX_ERR(2, 210, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 210, __pyx_L1_error) + __Pyx_TraceLine(213,0,__PYX_ERR(2, 213, __pyx_L1_error)) + __pyx_t_2 = __Pyx_PyObject_GetAttrStr(((PyObject *)__pyx_ptype_5reppy_6robots_Robots), __pyx_n_s_init); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 213, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; __pyx_t_4 = 0; @@ -6055,7 +6113,7 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 210, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 213, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else @@ -6063,13 +6121,13 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 210, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 213, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); } else #endif { - __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 210, __pyx_L1_error) + __pyx_t_5 = PyTuple_New(4+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 213, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_5); if (__pyx_t_3) { __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; @@ -6086,14 +6144,14 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 210, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 213, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "reppy/robots.pyx":209 + /* "reppy/robots.pyx":212 * '''All requests are allowed.''' * * def __init__(self, url, expires=None): # <<<<<<<<<<<<<< @@ -6782,11 +6840,11 @@ static PyObject *__pyx_getprop_5reppy_6robots_5Agent_delay(PyObject *o, CYTHON_U } static PyMethodDef __pyx_methods_5reppy_6robots_Agent[] = { - {"allow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_3allow, METH_O, __pyx_doc_5reppy_6robots_5Agent_2allow}, - {"disallow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_5disallow, METH_O, __pyx_doc_5reppy_6robots_5Agent_4disallow}, - {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_7allowed, METH_O, __pyx_doc_5reppy_6robots_5Agent_6allowed}, - {"__reduce_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_9__reduce_cython__, METH_NOARGS, 0}, - {"__setstate_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_11__setstate_cython__, METH_O, 0}, + {"allow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_5allow, METH_O, __pyx_doc_5reppy_6robots_5Agent_4allow}, + {"disallow", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_7disallow, METH_O, __pyx_doc_5reppy_6robots_5Agent_6disallow}, + {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_9allowed, METH_O, __pyx_doc_5reppy_6robots_5Agent_8allowed}, + {"__reduce_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_11__reduce_cython__, METH_NOARGS, 0}, + {"__setstate_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_5Agent_13__setstate_cython__, METH_O, 0}, {0, 0, 0, 0} }; @@ -6795,6 +6853,25 @@ static struct PyGetSetDef __pyx_getsets_5reppy_6robots_Agent[] = { {0, 0, 0, 0, 0} }; +static PySequenceMethods __pyx_tp_as_sequence_Agent = { + __pyx_pw_5reppy_6robots_5Agent_3__len__, /*sq_length*/ + 0, /*sq_concat*/ + 0, /*sq_repeat*/ + 0, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + 0, /*sq_contains*/ + 0, /*sq_inplace_concat*/ + 0, /*sq_inplace_repeat*/ +}; + +static PyMappingMethods __pyx_tp_as_mapping_Agent = { + __pyx_pw_5reppy_6robots_5Agent_3__len__, /*mp_length*/ + 0, /*mp_subscript*/ + 0, /*mp_ass_subscript*/ +}; + static PyTypeObject __pyx_type_5reppy_6robots_Agent = { PyVarObject_HEAD_INIT(0, 0) "reppy.robots.Agent", /*tp_name*/ @@ -6812,8 +6889,8 @@ static PyTypeObject __pyx_type_5reppy_6robots_Agent = { #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ + &__pyx_tp_as_sequence_Agent, /*tp_as_sequence*/ + &__pyx_tp_as_mapping_Agent, /*tp_as_mapping*/ 0, /*tp_hash*/ 0, /*tp_call*/ __pyx_pw_5reppy_6robots_5Agent_1__str__, /*tp_str*/ @@ -7479,7 +7556,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { static int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) __pyx_builtin_TypeError = __Pyx_GetBuiltinName(__pyx_n_s_TypeError); if (!__pyx_builtin_TypeError) __PYX_ERR(1, 2, __pyx_L1_error) - __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(2, 168, __pyx_L1_error) + __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(2, 171, __pyx_L1_error) __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_n_s_range); if (!__pyx_builtin_range) __PYX_ERR(1, 61, __pyx_L1_error) return 0; __pyx_L1_error:; @@ -7531,51 +7608,51 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GOTREF(__pyx_tuple__5); __Pyx_GIVEREF(__pyx_tuple__5); - /* "reppy/robots.pyx":86 + /* "reppy/robots.pyx":89 * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): */ - __pyx_tuple__8 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(2, 86, __pyx_L1_error) + __pyx_tuple__8 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__8); __Pyx_GIVEREF(__pyx_tuple__8); - /* "reppy/robots.pyx":87 + /* "reppy/robots.pyx":90 * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< * def wrap_exception(etype, cause): * wrapped = etype(cause) */ - __pyx_tuple__9 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(2, 87, __pyx_L1_error) + __pyx_tuple__9 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(2, 90, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__9); __Pyx_GIVEREF(__pyx_tuple__9); - /* "reppy/robots.pyx":88 + /* "reppy/robots.pyx":91 * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< * wrapped = etype(cause) * wrapped.url = url */ - __pyx_tuple__10 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 88, __pyx_L1_error) + __pyx_tuple__10 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__10); __Pyx_GIVEREF(__pyx_tuple__10); - __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_wrap_exception, 88, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 88, __pyx_L1_error) + __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_wrap_exception, 91, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 91, __pyx_L1_error) - /* "reppy/robots.pyx":97 + /* "reppy/robots.pyx":100 * # Limit the size of the request * kwargs['stream'] = True * with closing(requests.get(url, *args, **kwargs)) as res: # <<<<<<<<<<<<<< * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __pyx_tuple__12 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 97, __pyx_L1_error) + __pyx_tuple__12 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__12); __Pyx_GIVEREF(__pyx_tuple__12); - __pyx_tuple__13 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(2, 97, __pyx_L1_error) + __pyx_tuple__13 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(2, 100, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__13); __Pyx_GIVEREF(__pyx_tuple__13); @@ -7660,41 +7737,41 @@ static int __Pyx_InitCachedConstants(void) { __Pyx_GIVEREF(__pyx_tuple__24); __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__24, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 37, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(2, 37, __pyx_L1_error) - /* "reppy/robots.pyx":80 + /* "reppy/robots.pyx":83 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __pyx_tuple__25 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(2, 80, __pyx_L1_error) + __pyx_tuple__25 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(2, 83, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__25); __Pyx_GIVEREF(__pyx_tuple__25); - __pyx_codeobj__6 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_ParseMethod, 80, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__6)) __PYX_ERR(2, 80, __pyx_L1_error) + __pyx_codeobj__6 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_ParseMethod, 83, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__6)) __PYX_ERR(2, 83, __pyx_L1_error) - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":87 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __pyx_tuple__26 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(2, 84, __pyx_L1_error) + __pyx_tuple__26 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(2, 87, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__26); __Pyx_GIVEREF(__pyx_tuple__26); - __pyx_codeobj__7 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__26, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FetchMethod, 84, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__7)) __PYX_ERR(2, 84, __pyx_L1_error) + __pyx_codeobj__7 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__26, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FetchMethod, 87, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__7)) __PYX_ERR(2, 87, __pyx_L1_error) - /* "reppy/robots.pyx":133 + /* "reppy/robots.pyx":136 * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __pyx_tuple__27 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(2, 133, __pyx_L1_error) + __pyx_tuple__27 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(2, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_tuple__27); __Pyx_GIVEREF(__pyx_tuple__27); - __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__27, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 133, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) __PYX_ERR(2, 133, __pyx_L1_error) + __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__27, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 136, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) __PYX_ERR(2, 136, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -7863,24 +7940,24 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; - if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 137, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 140, __pyx_L1_error) __pyx_type_5reppy_6robots_Robots.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 137, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 137, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 140, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 140, __pyx_L1_error) __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 199, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 202, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 199, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 199, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 202, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 202, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 206, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 209, __pyx_L1_error) __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 206, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 206, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 209, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 209, __pyx_L1_error) __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; - if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(2, 84, __pyx_L1_error) + if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(2, 87, __pyx_L1_error) __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_print = 0; __pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod = &__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod; if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(1, 64, __pyx_L1_error) @@ -8124,115 +8201,115 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Agent); - /* "reppy/robots.pyx":80 + /* "reppy/robots.pyx":83 * * * def ParseMethod(cls, url, content, expires=None): # <<<<<<<<<<<<<< * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __Pyx_TraceLine(80,0,__PYX_ERR(2, 80, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 80, __pyx_L1_error) + __Pyx_TraceLine(83,0,__PYX_ERR(2, 83, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_3ParseMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 83, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(2, 80, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_ParseMethod, __pyx_t_2) < 0) __PYX_ERR(2, 83, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":84 + /* "reppy/robots.pyx":87 * return cls(url, as_bytes(content), expires) * * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): # <<<<<<<<<<<<<< * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __Pyx_TraceLine(84,0,__PYX_ERR(2, 84, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 84, __pyx_L1_error) + __Pyx_TraceLine(87,0,__PYX_ERR(2, 87, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_5FetchMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 87, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(2, 84, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_FetchMethod, __pyx_t_2) < 0) __PYX_ERR(2, 87, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":133 + /* "reppy/robots.pyx":136 * wrap_exception(exceptions.ReadTimeout, exc) * * def RobotsUrlMethod(cls, url): # <<<<<<<<<<<<<< * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __Pyx_TraceLine(133,0,__PYX_ERR(2, 133, __pyx_L1_error)) - __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 133, __pyx_L1_error) + __Pyx_TraceLine(136,0,__PYX_ERR(2, 136, __pyx_L1_error)) + __pyx_t_2 = PyCFunction_NewEx(&__pyx_mdef_5reppy_6robots_7RobotsUrlMethod, NULL, __pyx_n_s_reppy_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 136, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(2, 133, __pyx_L1_error) + if (PyDict_SetItem(__pyx_d, __pyx_n_s_RobotsUrlMethod, __pyx_t_2) < 0) __PYX_ERR(2, 136, __pyx_L1_error) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - /* "reppy/robots.pyx":142 + /* "reppy/robots.pyx":145 * # The default TTL policy is to cache for 3600 seconds or what's provided in the * # headers, and a minimum of 600 seconds * DEFAULT_TTL_POLICY = HeaderWithDefaultPolicy(default=3600, minimum=600) # <<<<<<<<<<<<<< * * # Class methods */ - __Pyx_TraceLine(142,0,__PYX_ERR(2, 142, __pyx_L1_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 142, __pyx_L1_error) + __Pyx_TraceLine(145,0,__PYX_ERR(2, 145, __pyx_L1_error)) + __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 142, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(2, 142, __pyx_L1_error) - if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(2, 142, __pyx_L1_error) - __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 142, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_default, __pyx_int_3600) < 0) __PYX_ERR(2, 145, __pyx_L1_error) + if (PyDict_SetItem(__pyx_t_1, __pyx_n_s_minimum, __pyx_int_600) < 0) __PYX_ERR(2, 145, __pyx_L1_error) + __pyx_t_3 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_empty_tuple, __pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(2, 142, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_DEFAULT_TTL_POLICY, __pyx_t_3) < 0) __PYX_ERR(2, 145, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":145 + /* "reppy/robots.pyx":148 * * # Class methods * parse = classmethod(ParseMethod) # <<<<<<<<<<<<<< * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) */ - __Pyx_TraceLine(145,0,__PYX_ERR(2, 145, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 145, __pyx_L1_error) + __Pyx_TraceLine(148,0,__PYX_ERR(2, 148, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 145, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(2, 145, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_parse, __pyx_t_1) < 0) __PYX_ERR(2, 148, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":146 + /* "reppy/robots.pyx":149 * # Class methods * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) # <<<<<<<<<<<<<< * robots_url = classmethod(RobotsUrlMethod) * */ - __Pyx_TraceLine(146,0,__PYX_ERR(2, 146, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 146, __pyx_L1_error) + __Pyx_TraceLine(149,0,__PYX_ERR(2, 149, __pyx_L1_error)) + __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 146, __pyx_L1_error) + __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(2, 146, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_fetch, __pyx_t_3) < 0) __PYX_ERR(2, 149, __pyx_L1_error) __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); - /* "reppy/robots.pyx":147 + /* "reppy/robots.pyx":150 * parse = classmethod(ParseMethod) * fetch = classmethod(FetchMethod) * robots_url = classmethod(RobotsUrlMethod) # <<<<<<<<<<<<<< * * # Data members */ - __Pyx_TraceLine(147,0,__PYX_ERR(2, 147, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 147, __pyx_L1_error) + __Pyx_TraceLine(150,0,__PYX_ERR(2, 150, __pyx_L1_error)) + __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); - __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 147, __pyx_L1_error) + __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(2, 147, __pyx_L1_error) + if (PyDict_SetItem((PyObject *)__pyx_ptype_5reppy_6robots_Robots->tp_dict, __pyx_n_s_robots_url, __pyx_t_1) < 0) __PYX_ERR(2, 150, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; PyType_Modified(__pyx_ptype_5reppy_6robots_Robots); diff --git a/reppy/robots.pyx b/reppy/robots.pyx index 65e4c02..87983de 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -54,6 +54,9 @@ cdef class Agent: def __str__(self): return as_string(self.agent.str()) + def __len__(self): + return self.agent.directives().size() + @property def delay(self): '''The delay associated with this agent.''' diff --git a/tests/test_agent.py b/tests/test_agent.py index a125c81..22fe792 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -10,6 +10,11 @@ def parse(self, content, name): '''Parse the robots.txt in content and return the agent of the provided name.''' return Robots.parse('http://example.com', content).agent(name) + def test_length(self): + '''An agent knows how many directives it has.''' + agent = Agent().disallow('/path').allow('/path/') + self.assertEqual(len(agent), 2) + def test_make_allowed(self): '''Make an agent that allows a path.''' agent = Agent().disallow('/path').allow('/path/') From bac6b055687f255ba115b67c1c2e23083008033c Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Tue, 12 Mar 2019 15:14:46 -0600 Subject: [PATCH 106/113] Release version 0.4.13 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1c5cc3c..e19cfb2 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.12', + version='0.4.13', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From 2554d8d5d76c3c79a63934a2550d8c04c8ffe136 Mon Sep 17 00:00:00 2001 From: Tammy Bailey Date: Tue, 28 May 2019 11:34:08 -0700 Subject: [PATCH 107/113] Updating rep-cpp submodule --- reppy/rep-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index a2f304e..f406b76 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit a2f304e3731287be1e6b45c53815460db175c448 +Subproject commit f406b7658bf9929450e53508a3dd9d2e3102a845 From 722b5dc703e06f9c0221fe8df550991dd8c391fe Mon Sep 17 00:00:00 2001 From: Greg Lindahl Date: Sun, 8 Sep 2019 22:31:00 -0700 Subject: [PATCH 108/113] fix API to match py2 and py3 --- reppy/robots.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reppy/robots.pyx b/reppy/robots.pyx index 87983de..cbb634d 100644 --- a/reppy/robots.pyx +++ b/reppy/robots.pyx @@ -168,7 +168,7 @@ cdef class Robots: @property def sitemaps(self): '''Get all the sitemaps in this robots.txt.''' - return map(as_string, self.robots.sitemaps()) + return list(map(as_string, self.robots.sitemaps())) def allowed(self, path, name): '''Is the provided path allowed for the provided agent?''' From 5a7a3a7f7cdd71c0e994dfe547a905f093293bd4 Mon Sep 17 00:00:00 2001 From: Greg Lindahl Date: Sun, 8 Sep 2019 22:34:40 -0700 Subject: [PATCH 109/113] test to catch previous api difference --- tests/test_robots.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_robots.py b/tests/test_robots.py index ae81649..9448672 100644 --- a/tests/test_robots.py +++ b/tests/test_robots.py @@ -132,7 +132,7 @@ def test_exposes_sitemaps(self): Sitemap: http://a.com/sitemap.xml Sitemap: http://b.com/sitemap.xml ''') - self.assertEqual(list(robot.sitemaps), [ + self.assertEqual(robot.sitemaps, [ 'http://a.com/sitemap.xml', 'http://b.com/sitemap.xml' ]) @@ -148,7 +148,7 @@ def test_case_insensitivity(self): def test_empty(self): '''Makes sure we can parse an empty robots.txt''' robot = robots.Robots.parse('http://example.com/robots.txt', '') - self.assertEqual(list(robot.sitemaps), []) + self.assertEqual(robot.sitemaps, []) self.assertTrue(robot.allowed('/', 'agent')) def test_comments(self): From fd6098015c6ff0ed7ba055645876987faaf3e1cd Mon Sep 17 00:00:00 2001 From: Greg Lindahl Date: Sun, 8 Sep 2019 22:49:32 -0700 Subject: [PATCH 110/113] apparently I have to check this in when I change robots.pyx --- reppy/rep-cpp | 2 +- reppy/robots.cpp | 3733 ++++++++++++++++++++++++++-------------------- 2 files changed, 2117 insertions(+), 1618 deletions(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index f406b76..3da0bbb 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit f406b7658bf9929450e53508a3dd9d2e3102a845 +Subproject commit 3da0bbbb6fb3621ae446359a54dfdd237b22c76c diff --git a/reppy/robots.cpp b/reppy/robots.cpp index 6b69045..3fa7461 100644 --- a/reppy/robots.cpp +++ b/reppy/robots.cpp @@ -1,4 +1,4 @@ -/* Generated by Cython 0.27.3 */ +/* Generated by Cython 0.29.7 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -7,7 +7,8 @@ #elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000) #error Cython requires Python 2.6+ or Python 3.3+. #else -#define CYTHON_ABI "0_27_3" +#define CYTHON_ABI "0_29_7" +#define CYTHON_HEX_VERSION 0x001D07F0 #define CYTHON_FUTURE_DIVISION 0 #include #ifndef offsetof @@ -78,6 +79,10 @@ #define CYTHON_PEP489_MULTI_PHASE_INIT 0 #undef CYTHON_USE_TP_FINALIZE #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 #elif defined(PYSTON_VERSION) #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 1 @@ -115,6 +120,10 @@ #define CYTHON_PEP489_MULTI_PHASE_INIT 0 #undef CYTHON_USE_TP_FINALIZE #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 #else #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 0 @@ -168,11 +177,17 @@ #define CYTHON_FAST_PYCALL 1 #endif #ifndef CYTHON_PEP489_MULTI_PHASE_INIT - #define CYTHON_PEP489_MULTI_PHASE_INIT (0 && PY_VERSION_HEX >= 0x03050000) + #define CYTHON_PEP489_MULTI_PHASE_INIT (PY_VERSION_HEX >= 0x03050000) #endif #ifndef CYTHON_USE_TP_FINALIZE #define CYTHON_USE_TP_FINALIZE (PY_VERSION_HEX >= 0x030400a1) #endif + #ifndef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX >= 0x030600B1) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK (PY_VERSION_HEX >= 0x030700A3) + #endif #endif #if !defined(CYTHON_FAST_PYCCALL) #define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) @@ -182,7 +197,121 @@ #undef SHIFT #undef BASE #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_MAYBE_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_MAYBE_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int32 uint32_t; + #endif + #endif +#else + #include +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) && __cplusplus >= 201103L + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #elif __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__ ) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif + +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif #endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(U other) { return *ptr == other; } + template bool operator !=(U other) { return *ptr != other; } + private: + T *ptr; +}; + #if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) #define Py_OptimizeFlag 0 #endif @@ -211,12 +340,15 @@ #ifndef Py_TPFLAGS_HAVE_FINALIZE #define Py_TPFLAGS_HAVE_FINALIZE 0 #endif -#if PY_VERSION_HEX < 0x030700A0 || !defined(METH_FASTCALL) +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#if PY_VERSION_HEX <= 0x030700A3 || !defined(METH_FASTCALL) #ifndef METH_FASTCALL #define METH_FASTCALL 0x80 #endif - typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, Py_ssize_t nargs); - typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject **args, + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames); #else #define __Pyx_PyCFunctionFast _PyCFunctionFast @@ -224,10 +356,27 @@ #endif #if CYTHON_FAST_PYCCALL #define __Pyx_PyFastCFunction_Check(func)\ - ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS))))) + ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))))) #else #define __Pyx_PyFastCFunction_Check(func) 0 #endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030400A1 + #define PyMem_RawMalloc(n) PyMem_Malloc(n) + #define PyMem_RawRealloc(p, n) PyMem_Realloc(p, n) + #define PyMem_RawFree(p) PyMem_Free(p) +#endif +#if CYTHON_COMPILING_IN_PYSTON + #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif #if !CYTHON_FAST_THREAD_STATE || PY_VERSION_HEX < 0x02070000 #define __Pyx_PyThreadState_Current PyThreadState_GET() #elif PY_VERSION_HEX >= 0x03060000 @@ -237,6 +386,36 @@ #else #define __Pyx_PyThreadState_Current _PyThreadState_Current #endif +#if PY_VERSION_HEX < 0x030700A2 && !defined(PyThread_tss_create) && !defined(Py_tss_NEEDS_INIT) +#include "pythread.h" +#define Py_tss_NEEDS_INIT 0 +typedef int Py_tss_t; +static CYTHON_INLINE int PyThread_tss_create(Py_tss_t *key) { + *key = PyThread_create_key(); + return 0; +} +static CYTHON_INLINE Py_tss_t * PyThread_tss_alloc(void) { + Py_tss_t *key = (Py_tss_t *)PyObject_Malloc(sizeof(Py_tss_t)); + *key = Py_tss_NEEDS_INIT; + return key; +} +static CYTHON_INLINE void PyThread_tss_free(Py_tss_t *key) { + PyObject_Free(key); +} +static CYTHON_INLINE int PyThread_tss_is_created(Py_tss_t *key) { + return *key != Py_tss_NEEDS_INIT; +} +static CYTHON_INLINE void PyThread_tss_delete(Py_tss_t *key) { + PyThread_delete_key(*key); + *key = Py_tss_NEEDS_INIT; +} +static CYTHON_INLINE int PyThread_tss_set(Py_tss_t *key, void *value) { + return PyThread_set_key_value(*key, value); +} +static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { + return PyThread_get_key_value(*key); +} +#endif #if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) #define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) #else @@ -249,6 +428,11 @@ #define __Pyx_PyNumber_Divide(x,y) PyNumber_Divide(x,y) #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) #endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStr(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +#else +#define __Pyx_PyDict_GetItemStr(dict, name) PyDict_GetItem(dict, name) +#endif #if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ @@ -293,20 +477,8 @@ #if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) #endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) - #define PyObject_Malloc(s) PyMem_Malloc(s) - #define PyObject_Free(p) PyMem_Free(p) - #define PyObject_Realloc(p) PyMem_Realloc(p) -#endif -#if CYTHON_COMPILING_IN_PYSTON - #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) -#else - #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) -#endif -#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) -#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyString_Check(b) && !PyString_CheckExact(b)))) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyString_Format(a, b) PyUnicode_Format(a, b) #else @@ -321,6 +493,7 @@ #define PyString_Type PyUnicode_Type #define PyString_Check PyUnicode_Check #define PyString_CheckExact PyUnicode_CheckExact + #define PyObject_Unicode PyObject_Str #endif #if PY_MAJOR_VERSION >= 3 #define __Pyx_PyBaseString_Check(obj) PyUnicode_Check(obj) @@ -332,7 +505,11 @@ #ifndef PySet_CheckExact #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) #endif -#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) +#else + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) +#endif #if PY_MAJOR_VERSION >= 3 #define PyIntObject PyLongObject #define PyInt_Type PyLong_Type @@ -367,16 +544,10 @@ #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t #endif #if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : PyInstanceMethod_New(func)) + #define __Pyx_PyMethod_New(func, self, klass) ((self) ? PyMethod_New(func, self) : (Py_INCREF(func), func)) #else #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) #endif -#ifndef __has_attribute - #define __has_attribute(x) 0 -#endif -#ifndef __has_cpp_attribute - #define __has_cpp_attribute(x) 0 -#endif #if CYTHON_USE_ASYNC_SLOTS #if PY_VERSION_HEX >= 0x030500B1 #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods @@ -394,128 +565,24 @@ unaryfunc am_anext; } __Pyx_PyAsyncMethodsStruct; #endif -#ifndef CYTHON_RESTRICT - #if defined(__GNUC__) - #define CYTHON_RESTRICT __restrict__ - #elif defined(_MSC_VER) && _MSC_VER >= 1400 - #define CYTHON_RESTRICT __restrict - #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L - #define CYTHON_RESTRICT restrict - #else - #define CYTHON_RESTRICT - #endif -#endif -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -#ifndef CYTHON_MAYBE_UNUSED_VAR -# if defined(__cplusplus) - template void CYTHON_MAYBE_UNUSED_VAR( const T& ) { } -# else -# define CYTHON_MAYBE_UNUSED_VAR(x) (void)(x) -# endif -#endif -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif + +#if defined(WIN32) || defined(MS_WINDOWS) + #define _USE_MATH_DEFINES #endif -#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) -#ifdef _MSC_VER - #ifndef _MSC_STDINT_H_ - #if _MSC_VER < 1300 - typedef unsigned char uint8_t; - typedef unsigned int uint32_t; - #else - typedef unsigned __int8 uint8_t; - typedef unsigned __int32 uint32_t; - #endif - #endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) #else - #include +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} #endif -#ifndef CYTHON_FALLTHROUGH - #if defined(__cplusplus) && __cplusplus >= 201103L - #if __has_cpp_attribute(fallthrough) - #define CYTHON_FALLTHROUGH [[fallthrough]] - #elif __has_cpp_attribute(clang::fallthrough) - #define CYTHON_FALLTHROUGH [[clang::fallthrough]] - #elif __has_cpp_attribute(gnu::fallthrough) - #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] - #endif - #endif - #ifndef CYTHON_FALLTHROUGH - #if __has_attribute(fallthrough) - #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) - #else - #define CYTHON_FALLTHROUGH - #endif - #endif - #if defined(__clang__ ) && defined(__apple_build_version__) - #if __apple_build_version__ < 7000000 - #undef CYTHON_FALLTHROUGH - #define CYTHON_FALLTHROUGH - #endif - #endif -#endif - -#ifndef __cplusplus - #error "Cython files generated with the C++ option must be compiled with a C++ compiler." -#endif -#ifndef CYTHON_INLINE - #if defined(__clang__) - #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) - #else - #define CYTHON_INLINE inline - #endif -#endif -template -void __Pyx_call_destructor(T& x) { - x.~T(); -} -template -class __Pyx_FakeReference { - public: - __Pyx_FakeReference() : ptr(NULL) { } - __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } - T *operator->() { return ptr; } - T *operator&() { return ptr; } - operator T&() { return *ptr; } - template bool operator ==(U other) { return *ptr == other; } - template bool operator !=(U other) { return *ptr != other; } - private: - T *ptr; -}; - -#if defined(WIN32) || defined(MS_WINDOWS) - #define _USE_MATH_DEFINES -#endif -#include -#ifdef NAN -#define __PYX_NAN() ((float) NAN) -#else -static CYTHON_INLINE float __PYX_NAN() { - float value; - memset(&value, 0xFF, sizeof(value)); - return value; -} -#endif -#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) -#define __Pyx_truncl trunc -#else -#define __Pyx_truncl truncl +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl #endif @@ -534,12 +601,13 @@ static CYTHON_INLINE float __PYX_NAN() { #define __PYX_HAVE__reppy__robots #define __PYX_HAVE_API__reppy__robots +/* Early includes */ #include -#include #include "ios" #include "new" #include "stdexcept" #include "typeinfo" +#include #include #include "rep-cpp/include/directive.h" #include "rep-cpp/include/agent.h" @@ -556,7 +624,8 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; #define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 -#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT (PY_MAJOR_VERSION >= 3 && __PYX_DEFAULT_STRING_ENCODING_IS_UTF8) #define __PYX_DEFAULT_STRING_ENCODING "" #define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString #define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize @@ -572,6 +641,9 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc (sizeof(type) == sizeof(Py_ssize_t) &&\ (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} #if defined (__cplusplus) && __cplusplus >= 201103L #include #define __Pyx_sst_abs(value) std::abs(value) @@ -628,8 +700,9 @@ static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { #define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode #define __Pyx_NewRef(obj) (Py_INCREF(obj), obj) #define __Pyx_Owned_Py_None(b) __Pyx_NewRef(Py_None) -#define __Pyx_PyBool_FromLong(b) ((b) ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False)) +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); #define __Pyx_PySequence_Tuple(obj)\ (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) @@ -710,7 +783,7 @@ static int __Pyx_init_sys_getdefaultencoding_params(void) { if (!default_encoding) goto bad; default_encoding_c = PyBytes_AsString(default_encoding); if (!default_encoding_c) goto bad; - __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c)); + __PYX_DEFAULT_STRING_ENCODING = (char*) malloc(strlen(default_encoding_c) + 1); if (!__PYX_DEFAULT_STRING_ENCODING) goto bad; strcpy(__PYX_DEFAULT_STRING_ENCODING, default_encoding_c); Py_DECREF(default_encoding); @@ -736,7 +809,7 @@ static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } static PyObject *__pyx_m = NULL; static PyObject *__pyx_d; static PyObject *__pyx_b; -static PyObject *__pyx_cython_runtime; +static PyObject *__pyx_cython_runtime = NULL; static PyObject *__pyx_empty_tuple; static PyObject *__pyx_empty_bytes; static PyObject *__pyx_empty_unicode; @@ -904,16 +977,7 @@ struct __pyx_obj___pyx_scope_struct____Pyx_CFunc_object____object___to_py { /* PyObjectGetAttrStr.proto */ #if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { - PyTypeObject* tp = Py_TYPE(obj); - if (likely(tp->tp_getattro)) - return tp->tp_getattro(obj, attr_name); -#if PY_MAJOR_VERSION < 3 - if (likely(tp->tp_getattr)) - return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); -#endif - return PyObject_GetAttr(obj, attr_name); -} +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); #else #define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) #endif @@ -921,6 +985,42 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject /* GetBuiltinName.proto */ static PyObject *__Pyx_GetBuiltinName(PyObject *name); +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#define __Pyx_PyErr_Occurred() __pyx_tstate->curexc_type +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + /* Profile.proto */ #ifndef CYTHON_PROFILE #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON @@ -1016,7 +1116,7 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); } static void __Pyx_call_return_trace_func(PyThreadState *tstate, PyFrameObject *frame, PyObject *result) { PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); + __Pyx_ErrFetchInState(tstate, &type, &value, &traceback); tstate->tracing++; tstate->use_tracing = 0; if (CYTHON_TRACE && tstate->c_tracefunc) @@ -1026,7 +1126,7 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); CYTHON_FRAME_DEL(frame); tstate->use_tracing = 1; tstate->tracing--; - PyErr_Restore(type, value, traceback); + __Pyx_ErrRestoreInState(tstate, type, value, traceback); } #ifdef WITH_THREAD #define __Pyx_TraceReturn(result, nogil)\ @@ -1070,7 +1170,7 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); static int __Pyx_call_line_trace_func(PyThreadState *tstate, PyFrameObject *frame, int lineno) { int ret; PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); + __Pyx_ErrFetchInState(tstate, &type, &value, &traceback); __Pyx_PyFrame_SetLineNumber(frame, lineno); tstate->tracing++; tstate->use_tracing = 0; @@ -1078,7 +1178,7 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); tstate->use_tracing = 1; tstate->tracing--; if (likely(!ret)) { - PyErr_Restore(type, value, traceback); + __Pyx_ErrRestoreInState(tstate, type, value, traceback); } else { Py_XDECREF(type); Py_XDECREF(value); @@ -1123,6 +1223,36 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name); #define __Pyx_TraceLine(lineno, nogil, goto_error) if ((1)); else goto_error; #endif +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif + /* PyObjectCall.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); @@ -1130,8 +1260,63 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg #define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) #endif +/* PyObjectCall2Args.proto */ +static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* PyDictVersioning.proto */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __pyx_dict_cached_value;\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + /* GetModuleGlobalName.proto */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} +#define __Pyx_GetModuleGlobalNameUncached(var, name) {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif /* RaiseArgTupleInvalid.proto */ static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, @@ -1151,98 +1336,61 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ __Pyx__ArgTypeTest(obj, type, name, exact)) static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); -/* PyThreadStateGet.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; -#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; -#define __Pyx_PyErr_Occurred() __pyx_tstate->curexc_type -#else -#define __Pyx_PyThreadState_declare -#define __Pyx_PyThreadState_assign -#define __Pyx_PyErr_Occurred() PyErr_Occurred() -#endif - -/* PyErrFetchRestore.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) -#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +/* PyObjectCallNoArg.proto */ #if CYTHON_COMPILING_IN_CPYTHON -#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) -#else -#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) -#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); #else -#define __Pyx_PyErr_Clear() PyErr_Clear() -#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) -#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) -#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) -#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) #endif /* RaiseException.proto */ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); -/* PyFunctionFastCall.proto */ -#if CYTHON_FAST_PYCALL -#define __Pyx_PyFunction_FastCall(func, args, nargs)\ - __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) -#endif -#endif - -/* PyCFunctionFastCall.proto */ -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/* PyObjectCallMethO.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); -#endif - -/* PyObjectCallOneArg.proto */ -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); - /* None.proto */ static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname); /* PyObjectSetAttrStr.proto */ #if CYTHON_USE_TYPE_SLOTS -#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_SetAttrStr(o,n,NULL) -static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value) { - PyTypeObject* tp = Py_TYPE(obj); - if (likely(tp->tp_setattro)) - return tp->tp_setattro(obj, attr_name, value); -#if PY_MAJOR_VERSION < 3 - if (likely(tp->tp_setattr)) - return tp->tp_setattr(obj, PyString_AS_STRING(attr_name), value); -#endif - return PyObject_SetAttr(obj, attr_name, value); -} +#define __Pyx_PyObject_DelAttrStr(o,n) __Pyx_PyObject_SetAttrStr(o, n, NULL) +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value); #else #define __Pyx_PyObject_DelAttrStr(o,n) PyObject_DelAttr(o,n) #define __Pyx_PyObject_SetAttrStr(o,n,v) PyObject_SetAttr(o,n,v) #endif +/* py_dict_pop.proto */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_Pop(PyObject *d, PyObject *key, PyObject *default_value); + +/* UnpackUnboundCMethod.proto */ +typedef struct { + PyObject *type; + PyObject **method_name; + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; + +/* CallUnboundCMethod2.proto */ +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030600B1 +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* CallUnboundCMethod1.proto */ +static PyObject* __Pyx__CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg); +#else +#define __Pyx_CallUnboundCMethod1(cfunc, self, arg) __Pyx__CallUnboundCMethod1(cfunc, self, arg) +#endif + /* FetchCommonType.proto */ static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); /* CythonFunction.proto */ #define __Pyx_CyFunction_USED 1 -#include #define __Pyx_CYFUNCTION_STATICMETHOD 0x01 #define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 #define __Pyx_CYFUNCTION_CCLASS 0x04 @@ -1276,6 +1424,7 @@ typedef struct { PyObject *func_annotations; } __pyx_CyFunctionObject; static PyTypeObject *__pyx_CyFunctionType = 0; +#define __Pyx_CyFunction_Check(obj) (__Pyx_TypeCheck(obj, __pyx_CyFunctionType)) #define __Pyx_CyFunction_NewEx(ml, flags, qualname, self, module, globals, code)\ __Pyx_CyFunction_New(__pyx_CyFunctionType, ml, flags, qualname, self, module, globals, code) static PyObject *__Pyx_CyFunction_New(PyTypeObject *, PyMethodDef *ml, @@ -1320,20 +1469,13 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObj #define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) #endif -/* PyObjectCallNoArg.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); -#else -#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) -#endif +/* PyIntCompare.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, long inplace); -/* PyIntBinop.proto */ -#if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); -#else -#define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ - PyObject_RichCompare(op1, op2, Py_EQ) - #endif +/* GetTopmostException.proto */ +#if CYTHON_USE_EXC_INFO_STACK +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif /* SaveResetException.proto */ #if CYTHON_FAST_THREAD_STATE @@ -1354,13 +1496,18 @@ static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); #endif -/* PyErrExceptionMatches.proto */ -#if CYTHON_FAST_THREAD_STATE -#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) -static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); #else -#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +#define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) #endif +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) /* WriteUnraisableException.proto */ static void __Pyx_WriteUnraisable(const char *name, int clineno, @@ -1387,6 +1534,20 @@ static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { /* IncludeStringH.proto */ #include +/* PyObject_GenericGetAttrNoDict.proto */ +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GenericGetAttrNoDict PyObject_GenericGetAttr +#endif + +/* PyObject_GenericGetAttr.proto */ +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static PyObject* __Pyx_PyObject_GenericGetAttr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GenericGetAttr PyObject_GenericGetAttr +#endif + /* SetupReduce.proto */ static int __Pyx_setup_reduce(PyObject* type_obj); @@ -1398,7 +1559,7 @@ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); /* ClassMethod.proto */ #include "descrobject.h" -static PyObject* __Pyx_Method_ClassMethod(PyObject *method); +static CYTHON_UNUSED PyObject* __Pyx_Method_ClassMethod(PyObject *method); /* CLineInTraceback.proto */ #ifdef CYTHON_CLINE_IN_TRACEBACK @@ -1441,18 +1602,6 @@ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); /* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); -/* FastTypeChecks.proto */ -#if CYTHON_COMPILING_IN_CPYTHON -#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) -static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); -static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); -static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); -#else -#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) -#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) -#define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) -#endif - /* CheckBinaryVersion.proto */ static int __Pyx_check_binary_version(void); @@ -1495,7 +1644,7 @@ static PyObject *__pyx_builtin_TypeError; static PyObject *__pyx_builtin_map; static PyObject *__pyx_builtin_range; static const char __pyx_k_PY3[] = "PY3"; -static const char __pyx_k__19[] = ""; +static const char __pyx_k__14[] = ""; static const char __pyx_k_amt[] = "amt"; static const char __pyx_k_cls[] = "cls"; static const char __pyx_k_exc[] = "exc"; @@ -1518,6 +1667,7 @@ static const char __pyx_k_test[] = "__test__"; static const char __pyx_k_time[] = "time"; static const char __pyx_k_util[] = "util"; static const char __pyx_k_wrap[] = "wrap"; +static const char __pyx_k_Agent[] = "Agent"; static const char __pyx_k_agent[] = "agent"; static const char __pyx_k_cause[] = "cause"; static const char __pyx_k_enter[] = "__enter__"; @@ -1527,6 +1677,7 @@ static const char __pyx_k_parse[] = "parse"; static const char __pyx_k_range[] = "range"; static const char __pyx_k_utf_8[] = "utf-8"; static const char __pyx_k_value[] = "value"; +static const char __pyx_k_Robots[] = "Robots"; static const char __pyx_k_decode[] = "decode"; static const char __pyx_k_encode[] = "encode"; static const char __pyx_k_import[] = "__import__"; @@ -1542,11 +1693,13 @@ static const char __pyx_k_default[] = "default"; static const char __pyx_k_expires[] = "expires"; static const char __pyx_k_minimum[] = "minimum"; static const char __pyx_k_wrapped[] = "wrapped"; +static const char __pyx_k_AllowAll[] = "AllowAll"; static const char __pyx_k_SSLError[] = "SSLError"; static const char __pyx_k_getstate[] = "__getstate__"; static const char __pyx_k_max_size[] = "max_size"; static const char __pyx_k_requests[] = "requests"; static const char __pyx_k_setstate[] = "__setstate__"; +static const char __pyx_k_AllowNone[] = "AllowNone"; static const char __pyx_k_TypeError[] = "TypeError"; static const char __pyx_k_reduce_ex[] = "__reduce_ex__"; static const char __pyx_k_InvalidURL[] = "InvalidURL"; @@ -1594,6 +1747,9 @@ static const char __pyx_k_Pyx_CFunc_object____object___t[] = "__Pyx_CFunc_object static const char __pyx_k_self_robots_cannot_be_converted[] = "self.robots cannot be converted to a Python object for pickling"; static const char __pyx_k_FetchMethod_locals_wrap_exceptio[] = "FetchMethod..wrap_exception"; static const char __pyx_k_self_agent_cannot_be_converted_t[] = "self.agent cannot be converted to a Python object for pickling"; +static PyObject *__pyx_n_s_Agent; +static PyObject *__pyx_n_s_AllowAll; +static PyObject *__pyx_n_s_AllowNone; static PyObject *__pyx_n_s_BadStatusCode; static PyObject *__pyx_n_s_ConnectionError; static PyObject *__pyx_n_s_ConnectionException; @@ -1614,6 +1770,7 @@ static PyObject *__pyx_n_s_PY3; static PyObject *__pyx_n_s_ParseMethod; static PyObject *__pyx_n_s_Pyx_CFunc_object____object___t; static PyObject *__pyx_n_s_ReadTimeout; +static PyObject *__pyx_n_s_Robots; static PyObject *__pyx_n_s_RobotsUrlMethod; static PyObject *__pyx_n_s_SSLError; static PyObject *__pyx_n_s_SSLException; @@ -1622,8 +1779,8 @@ static PyObject *__pyx_n_s_TypeError; static PyObject *__pyx_n_s_URLRequired; static PyObject *__pyx_kp_b_User_agent_Disallow; static PyObject *__pyx_n_s_ValueError; -static PyObject *__pyx_n_s__19; -static PyObject *__pyx_kp_b__19; +static PyObject *__pyx_n_s__14; +static PyObject *__pyx_kp_b__14; static PyObject *__pyx_n_s_after_parse_hook; static PyObject *__pyx_n_s_after_response_hook; static PyObject *__pyx_n_s_agent; @@ -1732,6 +1889,7 @@ static PyObject *__pyx_tp_new_5reppy_6robots_AllowNone(PyTypeObject *t, PyObject static PyObject *__pyx_tp_new_5reppy_6robots_AllowAll(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new_5reppy_6robots___pyx_scope_struct__FetchMethod(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ static PyObject *__pyx_tp_new___pyx_scope_struct____Pyx_CFunc_object____object___to_py(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/ +static __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop = {0, &__pyx_n_s_pop, 0, 0, 0}; static PyObject *__pyx_int_1; static PyObject *__pyx_int_200; static PyObject *__pyx_int_400; @@ -1741,32 +1899,28 @@ static PyObject *__pyx_int_500; static PyObject *__pyx_int_600; static PyObject *__pyx_int_3600; static PyObject *__pyx_int_1048576; -static PyObject *__pyx_tuple_; +static PyObject *__pyx_codeobj_; static PyObject *__pyx_tuple__2; -static PyObject *__pyx_tuple__4; -static PyObject *__pyx_tuple__5; +static PyObject *__pyx_tuple__3; +static PyObject *__pyx_tuple__6; static PyObject *__pyx_tuple__8; -static PyObject *__pyx_tuple__9; static PyObject *__pyx_tuple__10; +static PyObject *__pyx_tuple__11; static PyObject *__pyx_tuple__12; static PyObject *__pyx_tuple__13; static PyObject *__pyx_tuple__15; static PyObject *__pyx_tuple__16; static PyObject *__pyx_tuple__17; -static PyObject *__pyx_tuple__18; +static PyObject *__pyx_tuple__19; static PyObject *__pyx_tuple__20; static PyObject *__pyx_tuple__21; static PyObject *__pyx_tuple__22; -static PyObject *__pyx_tuple__24; -static PyObject *__pyx_tuple__25; -static PyObject *__pyx_tuple__26; -static PyObject *__pyx_tuple__27; -static PyObject *__pyx_codeobj__3; -static PyObject *__pyx_codeobj__6; +static PyObject *__pyx_codeobj__4; +static PyObject *__pyx_codeobj__5; static PyObject *__pyx_codeobj__7; -static PyObject *__pyx_codeobj__11; -static PyObject *__pyx_codeobj__14; -static PyObject *__pyx_codeobj__23; +static PyObject *__pyx_codeobj__9; +static PyObject *__pyx_codeobj__18; +/* Late includes */ /* "reppy/robots.pyx":22 * from . import util, logger, exceptions @@ -1784,6 +1938,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { int __pyx_t_2; PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("as_bytes", 0); __Pyx_TraceCall("as_bytes", __pyx_f[2], 22, 0, __PYX_ERR(2, 22, __pyx_L1_error)); @@ -1830,13 +1985,25 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { */ __Pyx_TraceLine(25,0,__PYX_ERR(2, 25, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_encode); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 25, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_tuple_, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 25, __pyx_L1_error) + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_encode); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 25, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_r = __pyx_t_4; - __pyx_t_4 = 0; + __pyx_t_5 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); + if (likely(__pyx_t_5)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_4, function); + } + } + __pyx_t_3 = (__pyx_t_5) ? __Pyx_PyObject_Call2Args(__pyx_t_4, __pyx_t_5, __pyx_kp_s_utf_8) : __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_kp_s_utf_8); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 25, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; goto __pyx_L0; /* "reppy/robots.pyx":22 @@ -1851,6 +2018,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_bytes(PyObject *__pyx_v_value) { __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_3); __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); __Pyx_AddTraceback("reppy.robots.as_bytes", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; @@ -1876,6 +2044,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { PyObject *__pyx_t_2 = NULL; int __pyx_t_3; int __pyx_t_4; + PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("as_string", 0); __Pyx_TraceCall("as_string", __pyx_f[2], 30, 0, __PYX_ERR(2, 30, __pyx_L1_error)); @@ -1887,7 +2056,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { * return value.decode('utf-8') */ __Pyx_TraceLine(31,0,__PYX_ERR(2, 31, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_six); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 31, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_six); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 31, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_PY3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 31, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); @@ -1917,13 +2086,25 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { */ __Pyx_TraceLine(33,0,__PYX_ERR(2, 33, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_decode); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 33, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 33, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_value, __pyx_n_s_decode); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 33, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_r = __pyx_t_1; - __pyx_t_1 = 0; + __pyx_t_5 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_1))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_1); + if (likely(__pyx_t_5)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_1); + __Pyx_INCREF(__pyx_t_5); + __Pyx_INCREF(function); + __Pyx_DECREF_SET(__pyx_t_1, function); + } + } + __pyx_t_2 = (__pyx_t_5) ? __Pyx_PyObject_Call2Args(__pyx_t_1, __pyx_t_5, __pyx_kp_s_utf_8) : __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_kp_s_utf_8); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 33, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; goto __pyx_L0; /* "reppy/robots.pyx":32 @@ -1969,6 +2150,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); __Pyx_AddTraceback("reppy.robots.as_string", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; @@ -1989,7 +2171,7 @@ static PyObject *__pyx_f_5reppy_6robots_as_string(PyObject *__pyx_v_value) { /* Python wrapper */ static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static char __pyx_doc_5reppy_6robots_FromRobotsMethod[] = "Construct an Agent from a CppAgent."; -static PyMethodDef __pyx_mdef_5reppy_6robots_1FromRobotsMethod = {"FromRobotsMethod", (PyCFunction)__pyx_pw_5reppy_6robots_1FromRobotsMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_FromRobotsMethod}; +static PyMethodDef __pyx_mdef_5reppy_6robots_1FromRobotsMethod = {"FromRobotsMethod", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_5reppy_6robots_1FromRobotsMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_FromRobotsMethod}; static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { CYTHON_UNUSED PyObject *__pyx_v_cls = 0; struct __pyx_obj_5reppy_6robots_Robots *__pyx_v_robots = 0; @@ -2016,17 +2198,17 @@ static PyObject *__pyx_pw_5reppy_6robots_1FromRobotsMethod(PyObject *__pyx_self, kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_robots)) != 0)) kw_args--; + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_robots)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 1); __PYX_ERR(2, 37, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("FromRobotsMethod", 1, 3, 3, 2); __PYX_ERR(2, 37, __pyx_L3_error) } @@ -2071,7 +2253,7 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject __Pyx_TraceDeclarations __Pyx_RefNannyDeclarations PyObject *__pyx_t_1 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__3) + __Pyx_TraceFrameInit(__pyx_codeobj_) __Pyx_RefNannySetupContext("FromRobotsMethod", 0); __Pyx_TraceCall("FromRobotsMethod", __pyx_f[2], 37, 0, __PYX_ERR(2, 37, __pyx_L1_error)); @@ -2083,7 +2265,7 @@ static PyObject *__pyx_pf_5reppy_6robots_FromRobotsMethod(CYTHON_UNUSED PyObject * # required to be copied because we often toss the containing */ __Pyx_TraceLine(39,0,__PYX_ERR(2, 39, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_Agent), __pyx_empty_tuple, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 39, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_CallNoArg(((PyObject *)__pyx_ptype_5reppy_6robots_Agent)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 39, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_agent = ((struct __pyx_obj_5reppy_6robots_Agent *)__pyx_t_1); __pyx_t_1 = 0; @@ -2343,8 +2525,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_5delay___get__(struct __pyx_obj_ */ __Pyx_TraceLine(66,0,__PYX_ERR(2, 66, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __Pyx_INCREF(Py_None); - __pyx_r = Py_None; + __pyx_r = Py_None; __Pyx_INCREF(Py_None); goto __pyx_L0; /* "reppy/robots.pyx":61 @@ -2410,7 +2591,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_4allow(struct __pyx_obj_5reppy_6 __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 70, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_self->agent.allow(__pyx_t_2); + (void)(__pyx_v_self->agent.allow(__pyx_t_2)); /* "reppy/robots.pyx":71 * '''Allow the provided path.''' @@ -2488,7 +2669,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_6disallow(struct __pyx_obj_5repp __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_convert_string_from_py_std__in_string(__pyx_t_1); if (unlikely(PyErr_Occurred())) __PYX_ERR(2, 75, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_self->agent.disallow(__pyx_t_2); + (void)(__pyx_v_self->agent.disallow(__pyx_t_2)); /* "reppy/robots.pyx":76 * '''Disallow the provided path.''' @@ -2627,7 +2808,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_10__reduce_cython__(CYTHON_UNUSE * raise TypeError("self.agent cannot be converted to a Python object for pickling") */ __Pyx_TraceLine(2,0,__PYX_ERR(1, 2, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -2684,7 +2865,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_12__setstate_cython__(CYTHON_UNU * raise TypeError("self.agent cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< */ __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -2719,7 +2900,7 @@ static PyObject *__pyx_pf_5reppy_6robots_5Agent_12__setstate_cython__(CYTHON_UNU /* Python wrapper */ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static char __pyx_doc_5reppy_6robots_2ParseMethod[] = "Parse a robots.txt file."; -static PyMethodDef __pyx_mdef_5reppy_6robots_3ParseMethod = {"ParseMethod", (PyCFunction)__pyx_pw_5reppy_6robots_3ParseMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_2ParseMethod}; +static PyMethodDef __pyx_mdef_5reppy_6robots_3ParseMethod = {"ParseMethod", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_5reppy_6robots_3ParseMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_2ParseMethod}; static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_cls = 0; PyObject *__pyx_v_url = 0; @@ -2750,24 +2931,24 @@ static PyObject *__pyx_pw_5reppy_6robots_3ParseMethod(PyObject *__pyx_self, PyOb kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 1); __PYX_ERR(2, 83, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: - if (likely((values[2] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; + if (likely((values[2] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("ParseMethod", 0, 3, 4, 2); __PYX_ERR(2, 83, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 3: if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_expires); if (value) { values[3] = value; kw_args--; } } } @@ -2815,7 +2996,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ PyObject *__pyx_t_4 = NULL; int __pyx_t_5; PyObject *__pyx_t_6 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__6) + __Pyx_TraceFrameInit(__pyx_codeobj__4) __Pyx_RefNannySetupContext("ParseMethod", 0); __Pyx_TraceCall("ParseMethod", __pyx_f[2], 83, 0, __PYX_ERR(2, 83, __pyx_L1_error)); @@ -2920,7 +3101,7 @@ static PyObject *__pyx_pf_5reppy_6robots_2ParseMethod(CYTHON_UNUSED PyObject *__ /* Python wrapper */ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static char __pyx_doc_5reppy_6robots_4FetchMethod[] = "Get the robots.txt at the provided URL."; -static PyMethodDef __pyx_mdef_5reppy_6robots_5FetchMethod = {"FetchMethod", (PyCFunction)__pyx_pw_5reppy_6robots_5FetchMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_4FetchMethod}; +static PyMethodDef __pyx_mdef_5reppy_6robots_5FetchMethod = {"FetchMethod", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_5reppy_6robots_5FetchMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_4FetchMethod}; static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_cls = 0; PyObject *__pyx_v_url = 0; @@ -2967,24 +3148,24 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("FetchMethod", 0, 2, 4, 1); __PYX_ERR(2, 87, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_ttl_policy); + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_ttl_policy); if (value) { values[2] = value; kw_args--; } } CYTHON_FALLTHROUGH; case 3: if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_max_size); + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_max_size); if (value) { values[3] = value; kw_args--; } } } @@ -3041,7 +3222,7 @@ static PyObject *__pyx_pw_5reppy_6robots_5FetchMethod(PyObject *__pyx_self, PyOb /* Python wrapper */ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ -static PyMethodDef __pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception = {"wrap_exception", (PyCFunction)__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception, METH_VARARGS|METH_KEYWORDS, 0}; +static PyMethodDef __pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception = {"wrap_exception", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception, METH_VARARGS|METH_KEYWORDS, 0}; static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { PyObject *__pyx_v_etype = 0; PyObject *__pyx_v_cause = 0; @@ -3065,11 +3246,11 @@ static PyObject *__pyx_pw_5reppy_6robots_11FetchMethod_1wrap_exception(PyObject kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_etype)) != 0)) kw_args--; + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_etype)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cause)) != 0)) kw_args--; + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_cause)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("wrap_exception", 1, 2, 2, 1); __PYX_ERR(2, 91, __pyx_L3_error) } @@ -3111,9 +3292,8 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; - PyObject *__pyx_t_4 = NULL; + int __pyx_t_4; int __pyx_t_5; - int __pyx_t_6; __Pyx_RefNannySetupContext("wrap_exception", 0); __pyx_outer_scope = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *) __Pyx_CyFunction_GetClosure(__pyx_self); __pyx_cur_scope = __pyx_outer_scope; @@ -3138,38 +3318,10 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_DECREF_SET(__pyx_t_2, function); } } - if (!__pyx_t_3) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_v_cause}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 92, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_3); __pyx_t_3 = NULL; - __Pyx_INCREF(__pyx_v_cause); - __Pyx_GIVEREF(__pyx_v_cause); - PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_v_cause); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - } - } + __pyx_t_1 = (__pyx_t_3) ? __Pyx_PyObject_Call2Args(__pyx_t_2, __pyx_t_3, __pyx_v_cause) : __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_cause); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 92, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_wrapped = __pyx_t_1; __pyx_t_1 = 0; @@ -3194,9 +3346,9 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * */ __Pyx_TraceLine(94,0,__PYX_ERR(2, 94, __pyx_L1_error)) if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 94, __pyx_L1_error) } - __pyx_t_5 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); - __pyx_t_6 = (__pyx_t_5 != 0); - if (__pyx_t_6) { + __pyx_t_4 = (__pyx_cur_scope->__pyx_v_after_response_hook != Py_None); + __pyx_t_5 = (__pyx_t_4 != 0); + if (__pyx_t_5) { /* "reppy/robots.pyx":95 * wrapped.url = url @@ -3208,48 +3360,20 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_TraceLine(95,0,__PYX_ERR(2, 95, __pyx_L1_error)) if (unlikely(!__pyx_cur_scope->__pyx_v_after_response_hook)) { __Pyx_RaiseClosureNameError("after_response_hook"); __PYX_ERR(2, 95, __pyx_L1_error) } __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); - __pyx_t_2 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_4 = NULL; + __pyx_t_2 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_3 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_2); - if (likely(__pyx_t_4)) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); + if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); - __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_2, function); } } - if (!__pyx_t_4) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_v_wrapped}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L1_error) - __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - { - __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 95, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_4); __pyx_t_4 = NULL; - __Pyx_INCREF(__pyx_v_wrapped); - __Pyx_GIVEREF(__pyx_v_wrapped); - PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_v_wrapped); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - } - } + __pyx_t_1 = (__pyx_t_3) ? __Pyx_PyObject_Call2Args(__pyx_t_2, __pyx_t_3, __pyx_v_wrapped) : __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_wrapped); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 95, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -3286,7 +3410,6 @@ static PyObject *__pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(PyObject * __Pyx_XDECREF(__pyx_t_1); __Pyx_XDECREF(__pyx_t_2); __Pyx_XDECREF(__pyx_t_3); - __Pyx_XDECREF(__pyx_t_4); __Pyx_AddTraceback("reppy.robots.FetchMethod.wrap_exception", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __Pyx_XDECREF(__pyx_v_wrapped); @@ -3333,7 +3456,8 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ int __pyx_t_15; PyObject *__pyx_t_16 = NULL; PyObject *__pyx_t_17 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__7) + PyObject *__pyx_t_18 = NULL; + __Pyx_TraceFrameInit(__pyx_codeobj__5) __Pyx_RefNannySetupContext("FetchMethod", 0); __pyx_cur_scope = (struct __pyx_obj_5reppy_6robots___pyx_scope_struct__FetchMethod *)__pyx_tp_new_5reppy_6robots___pyx_scope_struct__FetchMethod(__pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { @@ -3356,14 +3480,11 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * def wrap_exception(etype, cause): */ __Pyx_TraceLine(89,0,__PYX_ERR(2, 89, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyDict_Pop(__pyx_v_kwargs, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 89, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_tuple__8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 89, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_GIVEREF(__pyx_t_2); - __pyx_cur_scope->__pyx_v_after_response_hook = __pyx_t_2; - __pyx_t_2 = 0; + __Pyx_GIVEREF(__pyx_t_1); + __pyx_cur_scope->__pyx_v_after_response_hook = __pyx_t_1; + __pyx_t_1 = 0; /* "reppy/robots.pyx":90 * '''Get the robots.txt at the provided URL.''' @@ -3373,11 +3494,8 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrapped = etype(cause) */ __Pyx_TraceLine(90,0,__PYX_ERR(2, 90, __pyx_L1_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_kwargs, __pyx_n_s_pop); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 90, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_tuple__9, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 90, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyDict_Pop(__pyx_v_kwargs, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 90, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_v_after_parse_hook = __pyx_t_1; __pyx_t_1 = 0; @@ -3389,7 +3507,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrapped.url = url */ __Pyx_TraceLine(91,0,__PYX_ERR(2, 91, __pyx_L1_error)) - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__11)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 91, __pyx_L1_error) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_5reppy_6robots_11FetchMethod_1wrap_exception, 0, __pyx_n_s_FetchMethod_locals_wrap_exceptio, ((PyObject*)__pyx_cur_scope), __pyx_n_s_reppy_robots, __pyx_d, ((PyObject *)__pyx_codeobj__7)); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 91, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_wrap_exception = __pyx_t_1; __pyx_t_1 = 0; @@ -3401,14 +3519,14 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * # Limit the size of the request * kwargs['stream'] = True */ - __Pyx_TraceLine(97,0,__PYX_ERR(2, 97, __pyx_L3_error)) + __Pyx_TraceLine(97,0,__PYX_ERR(2, 97, __pyx_L1_error)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign - __Pyx_ExceptionSave(&__pyx_t_3, &__pyx_t_4, &__pyx_t_5); + __Pyx_ExceptionSave(&__pyx_t_2, &__pyx_t_3, &__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_2); __Pyx_XGOTREF(__pyx_t_3); __Pyx_XGOTREF(__pyx_t_4); - __Pyx_XGOTREF(__pyx_t_5); /*try:*/ { /* "reppy/robots.pyx":99 @@ -3430,9 +3548,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ __Pyx_TraceLine(100,0,__PYX_ERR(2, 100, __pyx_L3_error)) /*with:*/ { - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_closing); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L3_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = __Pyx_GetModuleGlobalName(__pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 100, __pyx_L3_error) + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_n_s_closing); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 100, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_requests); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_get); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_7); @@ -3450,75 +3568,42 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; __pyx_t_8 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_2); + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_5); if (likely(__pyx_t_8)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); __Pyx_INCREF(__pyx_t_8); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_2, function); - } - } - if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L3_error) - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L3_error) - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_t_6}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L3_error) - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } else - #endif - { - __pyx_t_7 = PyTuple_New(1+1); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L3_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_7, 0+1, __pyx_t_6); - __pyx_t_6 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L3_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF_SET(__pyx_t_5, function); } } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_1 = (__pyx_t_8) ? __Pyx_PyObject_Call2Args(__pyx_t_5, __pyx_t_8, __pyx_t_6) : __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 100, __pyx_L3_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_t_9 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_exit); if (unlikely(!__pyx_t_9)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_9); - __pyx_t_7 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L9_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_7))) { - __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_7); - if (likely(__pyx_t_6)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); - __Pyx_INCREF(__pyx_t_6); + __pyx_t_6 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 100, __pyx_L9_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_8 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_8)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_8); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_7, function); + __Pyx_DECREF_SET(__pyx_t_6, function); } } - if (__pyx_t_6) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_6); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L9_error) - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; - } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 100, __pyx_L9_error) - } - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_t_2; - __pyx_t_2 = 0; + __pyx_t_5 = (__pyx_t_8) ? __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_t_8) : __Pyx_PyObject_CallNoArg(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 100, __pyx_L9_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __pyx_t_5; + __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /*try:*/ { { @@ -3529,8 +3614,8 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_XGOTREF(__pyx_t_11); __Pyx_XGOTREF(__pyx_t_12); /*try:*/ { - __pyx_v_res = __pyx_t_7; - __pyx_t_7 = 0; + __pyx_v_res = __pyx_t_6; + __pyx_t_6 = 0; /* "reppy/robots.pyx":101 * kwargs['stream'] = True @@ -3540,21 +3625,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * if res.raw.read(amt=1, decode_content=True): */ __Pyx_TraceLine(101,0,__PYX_ERR(2, 101, __pyx_L13_error)) - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 101, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 101, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_read); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 101, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 101, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_7); - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(2, 101, __pyx_L13_error) - if (PyDict_SetItem(__pyx_t_7, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 101, __pyx_L13_error) - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_7); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 101, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 101, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_6); + if (PyDict_SetItem(__pyx_t_6, __pyx_n_s_amt, __pyx_v_max_size) < 0) __PYX_ERR(2, 101, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_6, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 101, __pyx_L13_error) + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_empty_tuple, __pyx_t_6); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 101, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_v_content = __pyx_t_2; - __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_v_content = __pyx_t_5; + __pyx_t_5 = 0; /* "reppy/robots.pyx":103 * content = res.raw.read(amt=max_size, decode_content=True) @@ -3564,22 +3649,22 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * 'Content larger than %s bytes' % max_size) */ __Pyx_TraceLine(103,0,__PYX_ERR(2, 103, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 103, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_read); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 103, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __pyx_t_2 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 103, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(2, 103, __pyx_L13_error) - if (PyDict_SetItem(__pyx_t_2, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 103, __pyx_L13_error) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_empty_tuple, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 103, __pyx_L13_error) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_raw); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 103, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_read); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 103, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __pyx_t_5 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 103, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_amt, __pyx_int_1) < 0) __PYX_ERR(2, 103, __pyx_L13_error) + if (PyDict_SetItem(__pyx_t_5, __pyx_n_s_decode_content, Py_True) < 0) __PYX_ERR(2, 103, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_empty_tuple, __pyx_t_5); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 103, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 103, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - if (__pyx_t_13) { + if (unlikely(__pyx_t_13)) { /* "reppy/robots.pyx":104 * # Try to read an additional byte, to see if the response is too big @@ -3589,11 +3674,11 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * */ __Pyx_TraceLine(104,0,__PYX_ERR(2, 104, __pyx_L13_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 104, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 104, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 104, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_t_5, __pyx_n_s_ContentTooLong); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 104, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; /* "reppy/robots.pyx":105 * if res.raw.read(amt=1, decode_content=True): @@ -3603,54 +3688,24 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * if after_response_hook is not None: */ __Pyx_TraceLine(105,0,__PYX_ERR(2, 105, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyString_Format(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 105, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_6 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { - __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_7); - if (likely(__pyx_t_6)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); - __Pyx_INCREF(__pyx_t_6); + __pyx_t_5 = __Pyx_PyString_FormatSafe(__pyx_kp_s_Content_larger_than_s_bytes, __pyx_v_max_size); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 105, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_8 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_8)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_8); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_7, function); - } - } - if (!__pyx_t_6) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_7)) { - PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { - PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_2}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else - #endif - { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 104, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; - __Pyx_GIVEREF(__pyx_t_2); - PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_t_2); - __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF_SET(__pyx_t_6, function); } } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_1 = (__pyx_t_8) ? __Pyx_PyObject_Call2Args(__pyx_t_6, __pyx_t_8, __pyx_t_5) : __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_t_5); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 104, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __PYX_ERR(2, 104, __pyx_L13_error) @@ -3685,49 +3740,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ __Pyx_TraceLine(108,0,__PYX_ERR(2, 108, __pyx_L13_error)) __Pyx_INCREF(__pyx_cur_scope->__pyx_v_after_response_hook); - __pyx_t_7 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_8 = NULL; - if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_7))) { - __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_7); - if (likely(__pyx_t_8)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_7); - __Pyx_INCREF(__pyx_t_8); + __pyx_t_6 = __pyx_cur_scope->__pyx_v_after_response_hook; __pyx_t_5 = NULL; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { + __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_6); + if (likely(__pyx_t_5)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); + __Pyx_INCREF(__pyx_t_5); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_7, function); + __Pyx_DECREF_SET(__pyx_t_6, function); } } - if (!__pyx_t_8) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_7, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_1); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_7)) { - PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_7)) { - PyObject *__pyx_temp[2] = {__pyx_t_8, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_7, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - { - __pyx_t_2 = PyTuple_New(1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 108, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_t_8); __pyx_t_8 = NULL; - __Pyx_INCREF(__pyx_v_res); - __Pyx_GIVEREF(__pyx_v_res); - PyTuple_SET_ITEM(__pyx_t_2, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_7, __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } - } - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __pyx_t_1 = (__pyx_t_5) ? __Pyx_PyObject_Call2Args(__pyx_t_6, __pyx_t_5, __pyx_v_res) : __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_v_res); + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 108, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; /* "reppy/robots.pyx":107 @@ -3751,61 +3778,33 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (!__pyx_t_14) { } else { __Pyx_INCREF(__pyx_v_ttl_policy); - __pyx_t_7 = __pyx_v_ttl_policy; + __pyx_t_6 = __pyx_v_ttl_policy; goto __pyx_L21_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_INCREF(__pyx_t_2); - __pyx_t_7 = __pyx_t_2; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_DEFAULT_TTL_POLICY); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 111, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_INCREF(__pyx_t_5); + __pyx_t_6 = __pyx_t_5; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_L21_bool_binop_done:; - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_expires); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 111, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = NULL; - if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { - __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_2); - if (likely(__pyx_t_7)) { - PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); - __Pyx_INCREF(__pyx_t_7); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_expires); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 111, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = NULL; + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_5); + if (likely(__pyx_t_6)) { + PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_6); __Pyx_INCREF(function); - __Pyx_DECREF_SET(__pyx_t_2, function); + __Pyx_DECREF_SET(__pyx_t_5, function); } } - if (!__pyx_t_7) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_2, __pyx_v_res); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_1); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_res}; - __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_GOTREF(__pyx_t_1); - } else - #endif - { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 111, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; - __Pyx_INCREF(__pyx_v_res); - __Pyx_GIVEREF(__pyx_v_res); - PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_res); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_8, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - } - } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_1 = (__pyx_t_6) ? __Pyx_PyObject_Call2Args(__pyx_t_5, __pyx_t_6, __pyx_v_res) : __Pyx_PyObject_CallOneArg(__pyx_t_5, __pyx_v_res); + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 111, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_v_expires = __pyx_t_1; __pyx_t_1 = 0; @@ -3819,11 +3818,11 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_TraceLine(113,0,__PYX_ERR(2, 113, __pyx_L13_error)) __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 113, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 113, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); + __pyx_t_5 = __Pyx_PyInt_EqObjC(__pyx_t_1, __pyx_int_200, 0xC8, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 113, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 113, __pyx_L13_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 113, __pyx_L13_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_14) { /* "reppy/robots.pyx":114 @@ -3836,13 +3835,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_TraceLine(114,0,__PYX_ERR(2, 114, __pyx_L13_error)) __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_cls, __pyx_n_s_parse); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 114, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_8 = NULL; + __pyx_t_6 = NULL; __pyx_t_15 = 0; if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_1))) { - __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_1); - if (likely(__pyx_t_8)) { + __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_1); + if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_1); - __Pyx_INCREF(__pyx_t_8); + __Pyx_INCREF(__pyx_t_6); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_1, function); __pyx_t_15 = 1; @@ -3850,42 +3849,42 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_1)) { - PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_GOTREF(__pyx_t_2); + PyObject *__pyx_temp[4] = {__pyx_t_6, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; + __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 114, __pyx_L13_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_5); } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { - PyObject *__pyx_temp[4] = {__pyx_t_8, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_GOTREF(__pyx_t_2); + PyObject *__pyx_temp[4] = {__pyx_t_6, __pyx_cur_scope->__pyx_v_url, __pyx_v_content, __pyx_v_expires}; + __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-__pyx_t_15, 3+__pyx_t_15); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 114, __pyx_L13_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_5); } else #endif { - __pyx_t_7 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 114, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_7); - if (__pyx_t_8) { - __Pyx_GIVEREF(__pyx_t_8); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_8); __pyx_t_8 = NULL; + __pyx_t_8 = PyTuple_New(3+__pyx_t_15); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 114, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_8); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_6); __pyx_t_6 = NULL; } __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_7, 0+__pyx_t_15, __pyx_cur_scope->__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_8, 0+__pyx_t_15, __pyx_cur_scope->__pyx_v_url); __Pyx_INCREF(__pyx_v_content); __Pyx_GIVEREF(__pyx_v_content); - PyTuple_SET_ITEM(__pyx_t_7, 1+__pyx_t_15, __pyx_v_content); + PyTuple_SET_ITEM(__pyx_t_8, 1+__pyx_t_15, __pyx_v_content); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); - PyTuple_SET_ITEM(__pyx_t_7, 2+__pyx_t_15, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_7, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 114, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + PyTuple_SET_ITEM(__pyx_t_8, 2+__pyx_t_15, __pyx_v_expires); + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 114, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; } __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_v_robots = __pyx_t_2; - __pyx_t_2 = 0; + __pyx_v_robots = __pyx_t_5; + __pyx_t_5 = 0; /* "reppy/robots.pyx":115 * if res.status_code == 200: @@ -3908,50 +3907,22 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ __Pyx_TraceLine(116,0,__PYX_ERR(2, 116, __pyx_L13_error)) __Pyx_INCREF(__pyx_v_after_parse_hook); - __pyx_t_1 = __pyx_v_after_parse_hook; __pyx_t_7 = NULL; + __pyx_t_1 = __pyx_v_after_parse_hook; __pyx_t_8 = NULL; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_1))) { - __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_1); - if (likely(__pyx_t_7)) { + __pyx_t_8 = PyMethod_GET_SELF(__pyx_t_1); + if (likely(__pyx_t_8)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_1); - __Pyx_INCREF(__pyx_t_7); + __Pyx_INCREF(__pyx_t_8); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_1, function); } } - if (!__pyx_t_7) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - } else { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(__pyx_t_1)) { - PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_GOTREF(__pyx_t_2); - } else - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(__pyx_t_1)) { - PyObject *__pyx_temp[2] = {__pyx_t_7, __pyx_v_robots}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_1, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_GOTREF(__pyx_t_2); - } else - #endif - { - __pyx_t_8 = PyTuple_New(1+1); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 116, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_8); - __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_8, 0, __pyx_t_7); __pyx_t_7 = NULL; - __Pyx_INCREF(__pyx_v_robots); - __Pyx_GIVEREF(__pyx_v_robots); - PyTuple_SET_ITEM(__pyx_t_8, 0+1, __pyx_v_robots); - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_8, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 116, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - } - } + __pyx_t_5 = (__pyx_t_8) ? __Pyx_PyObject_Call2Args(__pyx_t_1, __pyx_t_8, __pyx_v_robots) : __Pyx_PyObject_CallOneArg(__pyx_t_1, __pyx_v_robots); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 116, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; /* "reppy/robots.pyx":115 * if res.status_code == 200: @@ -3992,9 +3963,9 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * elif res.status_code >= 400 and res.status_code < 500: */ __Pyx_TraceLine(118,0,__PYX_ERR(2, 118, __pyx_L13_error)) - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 118, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 118, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_5, __pyx_int_401, 0x191, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -4003,13 +3974,13 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_13 = __pyx_t_14; goto __pyx_L25_bool_binop_done; } - __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_2, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyInt_EqObjC(__pyx_t_5, __pyx_int_403, 0x193, 0); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_14 < 0)) __PYX_ERR(2, 118, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_13 = __pyx_t_14; __pyx_L25_bool_binop_done:; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_t_14 = (__pyx_t_13 != 0); if (__pyx_t_14) { @@ -4022,17 +3993,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ __Pyx_TraceLine(119,0,__PYX_ERR(2, 119, __pyx_L13_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 119, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); + __pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 119, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_cur_scope->__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_cur_scope->__pyx_v_url); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); - PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_expires); - __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_2, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 119, __pyx_L13_error) + PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_expires); + __pyx_t_1 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowNone), __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 119, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_r = __pyx_t_1; __pyx_t_1 = 0; goto __pyx_L17_try_return; @@ -4056,24 +4027,24 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_TraceLine(120,0,__PYX_ERR(2, 120, __pyx_L13_error)) __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_2 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) + __pyx_t_5 = PyObject_RichCompare(__pyx_t_1, __pyx_int_400, Py_GE); __Pyx_XGOTREF(__pyx_t_5); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 120, __pyx_L13_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_5); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 120, __pyx_L13_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; if (__pyx_t_13) { } else { __pyx_t_14 = __pyx_t_13; goto __pyx_L27_bool_binop_done; } - __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 120, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_1 = PyObject_RichCompare(__pyx_t_2, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 120, __pyx_L13_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 120, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_1 = PyObject_RichCompare(__pyx_t_5, __pyx_int_500, Py_LT); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 120, __pyx_L13_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __pyx_t_13 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_13 < 0)) __PYX_ERR(2, 120, __pyx_L13_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_14 = __pyx_t_13; __pyx_L27_bool_binop_done:; - if (__pyx_t_14) { + if (likely(__pyx_t_14)) { /* "reppy/robots.pyx":121 * return AllowNone(url, expires) @@ -4092,11 +4063,11 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_v_expires); - __pyx_t_2 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 121, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); + __pyx_t_5 = __Pyx_PyObject_Call(((PyObject *)__pyx_ptype_5reppy_6robots_AllowAll), __pyx_t_1, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 121, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; + __pyx_r = __pyx_t_5; + __pyx_t_5 = 0; goto __pyx_L17_try_return; /* "reppy/robots.pyx":120 @@ -4117,7 +4088,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ __Pyx_TraceLine(123,0,__PYX_ERR(2, 123, __pyx_L13_error)) /*else*/ { - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 123, __pyx_L13_error) + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 123, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_8 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_n_s_BadStatusCode); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 123, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_8); @@ -4133,26 +4104,26 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __Pyx_TraceLine(124,0,__PYX_ERR(2, 124, __pyx_L13_error)) __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 124, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = PyTuple_New(2); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 124, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_7); + __pyx_t_6 = PyTuple_New(2); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 124, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_6); __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_1); __Pyx_INCREF(__pyx_cur_scope->__pyx_v_url); __Pyx_GIVEREF(__pyx_cur_scope->__pyx_v_url); - PyTuple_SET_ITEM(__pyx_t_7, 1, __pyx_cur_scope->__pyx_v_url); + PyTuple_SET_ITEM(__pyx_t_6, 1, __pyx_cur_scope->__pyx_v_url); __pyx_t_1 = 0; - __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_7); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 124, __pyx_L13_error) + __pyx_t_1 = __Pyx_PyString_Format(__pyx_kp_s_Got_i_for_s, __pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 124, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 124, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_6 = NULL; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __Pyx_PyObject_GetAttrStr(__pyx_v_res, __pyx_n_s_status_code); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 124, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = NULL; __pyx_t_15 = 0; if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_8))) { - __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_8); - if (likely(__pyx_t_6)) { + __pyx_t_7 = PyMethod_GET_SELF(__pyx_t_8); + if (likely(__pyx_t_7)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_8); - __Pyx_INCREF(__pyx_t_6); + __Pyx_INCREF(__pyx_t_7); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_8, function); __pyx_t_15 = 1; @@ -4160,43 +4131,43 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_8)) { - PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 123, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_2); + PyObject *__pyx_temp[3] = {__pyx_t_7, __pyx_t_1, __pyx_t_6}; + __pyx_t_5 = __Pyx_PyFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 123, __pyx_L13_error) + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_8)) { - PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_1, __pyx_t_7}; - __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 123, __pyx_L13_error) - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; - __Pyx_GOTREF(__pyx_t_2); + PyObject *__pyx_temp[3] = {__pyx_t_7, __pyx_t_1, __pyx_t_6}; + __pyx_t_5 = __Pyx_PyCFunction_FastCall(__pyx_t_8, __pyx_temp+1-__pyx_t_15, 2+__pyx_t_15); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 123, __pyx_L13_error) + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } else #endif { __pyx_t_16 = PyTuple_New(2+__pyx_t_15); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 123, __pyx_L13_error) __Pyx_GOTREF(__pyx_t_16); - if (__pyx_t_6) { - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_6); __pyx_t_6 = NULL; + if (__pyx_t_7) { + __Pyx_GIVEREF(__pyx_t_7); PyTuple_SET_ITEM(__pyx_t_16, 0, __pyx_t_7); __pyx_t_7 = NULL; } __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_16, 0+__pyx_t_15, __pyx_t_1); - __Pyx_GIVEREF(__pyx_t_7); - PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_15, __pyx_t_7); + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_16, 1+__pyx_t_15, __pyx_t_6); __pyx_t_1 = 0; - __pyx_t_7 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 123, __pyx_L13_error) - __Pyx_GOTREF(__pyx_t_2); + __pyx_t_6 = 0; + __pyx_t_5 = __Pyx_PyObject_Call(__pyx_t_8, __pyx_t_16, NULL); if (unlikely(!__pyx_t_5)) __PYX_ERR(2, 123, __pyx_L13_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; } __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_Raise(__pyx_t_2, 0, 0, 0); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_Raise(__pyx_t_5, 0, 0, 0); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __PYX_ERR(2, 123, __pyx_L13_error) } @@ -4209,23 +4180,23 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ */ } __pyx_L13_error:; - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; /*except:*/ { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 100, __pyx_L15_except_error) - __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 100, __pyx_L15_except_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); - __pyx_t_7 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 100, __pyx_L15_except_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_7, NULL); + __pyx_t_6 = PyTuple_Pack(3, __pyx_t_5, __pyx_t_8, __pyx_t_16); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 100, __pyx_L15_except_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_17 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_t_6, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; if (unlikely(!__pyx_t_17)) __PYX_ERR(2, 100, __pyx_L15_except_error) __Pyx_GOTREF(__pyx_t_17); __pyx_t_14 = __Pyx_PyObject_IsTrue(__pyx_t_17); @@ -4233,16 +4204,16 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ if (__pyx_t_14 < 0) __PYX_ERR(2, 100, __pyx_L15_except_error) __pyx_t_13 = ((!(__pyx_t_14 != 0)) != 0); if (__pyx_t_13) { - __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_5); __Pyx_GIVEREF(__pyx_t_8); __Pyx_XGIVEREF(__pyx_t_16); - __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_8, __pyx_t_16); - __pyx_t_2 = 0; __pyx_t_8 = 0; __pyx_t_16 = 0; + __Pyx_ErrRestoreWithState(__pyx_t_5, __pyx_t_8, __pyx_t_16); + __pyx_t_5 = 0; __pyx_t_8 = 0; __pyx_t_16 = 0; __PYX_ERR(2, 100, __pyx_L15_except_error) } - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; goto __pyx_L14_exception_handled; } __pyx_L15_except_error:; @@ -4267,7 +4238,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ /*finally:*/ { /*normal exit:*/{ if (__pyx_t_9) { - __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__12, NULL); + __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__8, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; if (unlikely(!__pyx_t_12)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_12); @@ -4279,7 +4250,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ __pyx_t_12 = __pyx_r; __pyx_r = 0; if (__pyx_t_9) { - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__13, NULL); + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_9, __pyx_tuple__8, NULL); __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; if (unlikely(!__pyx_t_11)) __PYX_ERR(2, 100, __pyx_L3_error) __Pyx_GOTREF(__pyx_t_11); @@ -4306,17 +4277,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * kwargs['stream'] = True */ } + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; - __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; goto __pyx_L8_try_end; __pyx_L3_error:; - __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_XDECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; /* "reppy/robots.pyx":125 * raise exceptions.BadStatusCode( @@ -4326,16 +4297,19 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * except ConnectionError as exc: */ __Pyx_TraceLine(125,0,__PYX_ERR(2, 125, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_SSLError); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 125, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16); - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_ErrFetch(&__pyx_t_16, &__pyx_t_8, &__pyx_t_5); + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_SSLError); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 125, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_15 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_16, __pyx_t_6); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_ErrRestore(__pyx_t_16, __pyx_t_8, __pyx_t_5); + __pyx_t_16 = 0; __pyx_t_8 = 0; __pyx_t_5 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_16, &__pyx_t_8, &__pyx_t_2) < 0) __PYX_ERR(2, 125, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 125, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_GOTREF(__pyx_t_8); - __Pyx_GOTREF(__pyx_t_2); + __Pyx_GOTREF(__pyx_t_16); __Pyx_INCREF(__pyx_t_8); __pyx_v_exc = __pyx_t_8; @@ -4347,18 +4321,18 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrap_exception(exceptions.ConnectionException, exc) */ __Pyx_TraceLine(126,0,__PYX_ERR(2, 126, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 126, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_SSLException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 126, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 126, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 126, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; goto __pyx_L4_exception_handled; } @@ -4370,14 +4344,17 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * except (URLRequired, MissingSchema, InvalidSchema, InvalidURL) as exc: */ __Pyx_TraceLine(127,0,__PYX_ERR(2, 127, __pyx_L5_except_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 127, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_2); - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_ErrFetch(&__pyx_t_16, &__pyx_t_8, &__pyx_t_5); + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_ConnectionError); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 127, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_15 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_16, __pyx_t_6); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_ErrRestore(__pyx_t_16, __pyx_t_8, __pyx_t_5); + __pyx_t_16 = 0; __pyx_t_8 = 0; __pyx_t_5 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 127, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 127, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_GOTREF(__pyx_t_8); __Pyx_GOTREF(__pyx_t_16); __Pyx_INCREF(__pyx_t_8); @@ -4391,18 +4368,18 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrap_exception(exceptions.MalformedUrl, exc) */ __Pyx_TraceLine(128,0,__PYX_ERR(2, 128, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 128, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_7, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_6, __pyx_n_s_ConnectionException); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 128, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __pyx_t_7 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 128, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_6 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 128, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; goto __pyx_L4_exception_handled; } @@ -4414,27 +4391,30 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * except TooManyRedirects as exc: */ __Pyx_TraceLine(129,0,__PYX_ERR(2, 129, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_URLRequired); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 129, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 129, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 129, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_2); - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 129, __pyx_L5_except_error) + __Pyx_ErrFetch(&__pyx_t_16, &__pyx_t_8, &__pyx_t_5); + __Pyx_GetModuleGlobalName(__pyx_t_6, __pyx_n_s_URLRequired); if (unlikely(!__pyx_t_6)) __PYX_ERR(2, 129, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_MissingSchema); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 129, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GetModuleGlobalName(__pyx_t_7, __pyx_n_s_InvalidSchema); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 129, __pyx_L5_except_error) __Pyx_GOTREF(__pyx_t_7); - __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_16) || __Pyx_PyErr_ExceptionMatches(__pyx_t_8) || __Pyx_PyErr_ExceptionMatches(__pyx_t_2) || __Pyx_PyErr_ExceptionMatches(__pyx_t_7); - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_18, __pyx_n_s_InvalidURL); if (unlikely(!__pyx_t_18)) __PYX_ERR(2, 129, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __pyx_t_15 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_16, __pyx_t_6) || __Pyx_PyErr_GivenExceptionMatches(__pyx_t_16, __pyx_t_1) || __Pyx_PyErr_GivenExceptionMatches(__pyx_t_16, __pyx_t_7) || __Pyx_PyErr_GivenExceptionMatches(__pyx_t_16, __pyx_t_18); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_ErrRestore(__pyx_t_16, __pyx_t_8, __pyx_t_5); + __pyx_t_16 = 0; __pyx_t_8 = 0; __pyx_t_5 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 129, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 129, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_GOTREF(__pyx_t_8); - __Pyx_INCREF(__pyx_t_2); - __pyx_v_exc = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_16); + __Pyx_INCREF(__pyx_t_8); + __pyx_v_exc = __pyx_t_8; /* "reppy/robots.pyx":130 * wrap_exception(exceptions.ConnectionException, exc) @@ -4444,18 +4424,18 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrap_exception(exceptions.ExcessiveRedirects, exc) */ __Pyx_TraceLine(130,0,__PYX_ERR(2, 130, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 130, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 130, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 130, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_18, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(2, 130, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_MalformedUrl); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 130, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __pyx_t_18 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_7, __pyx_v_exc); if (unlikely(!__pyx_t_18)) __PYX_ERR(2, 130, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; goto __pyx_L4_exception_handled; } @@ -4467,18 +4447,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * except ReadTimeout as exc: */ __Pyx_TraceLine(131,0,__PYX_ERR(2, 131, __pyx_L5_except_error)) - __pyx_t_8 = __Pyx_GetModuleGlobalName(__pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_8)) __PYX_ERR(2, 131, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_8); - __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_8); - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_ErrFetch(&__pyx_t_16, &__pyx_t_8, &__pyx_t_5); + __Pyx_GetModuleGlobalName(__pyx_t_18, __pyx_n_s_TooManyRedirects); if (unlikely(!__pyx_t_18)) __PYX_ERR(2, 131, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __pyx_t_15 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_16, __pyx_t_18); + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_ErrRestore(__pyx_t_16, __pyx_t_8, __pyx_t_5); + __pyx_t_16 = 0; __pyx_t_8 = 0; __pyx_t_5 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_8, &__pyx_t_2, &__pyx_t_7) < 0) __PYX_ERR(2, 131, __pyx_L5_except_error) + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 131, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_GOTREF(__pyx_t_8); - __Pyx_GOTREF(__pyx_t_2); - __Pyx_GOTREF(__pyx_t_7); - __Pyx_INCREF(__pyx_t_2); - __pyx_v_exc = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_16); + __Pyx_INCREF(__pyx_t_8); + __pyx_v_exc = __pyx_t_8; /* "reppy/robots.pyx":132 * wrap_exception(exceptions.MalformedUrl, exc) @@ -4488,18 +4471,18 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * wrap_exception(exceptions.ReadTimeout, exc) */ __Pyx_TraceLine(132,0,__PYX_ERR(2, 132, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 132, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 132, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 132, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_18, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(2, 132, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_ExcessiveRedirects); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 132, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __pyx_t_18 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_7, __pyx_v_exc); if (unlikely(!__pyx_t_18)) __PYX_ERR(2, 132, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; goto __pyx_L4_exception_handled; } @@ -4511,18 +4494,21 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * */ __Pyx_TraceLine(133,0,__PYX_ERR(2, 133, __pyx_L5_except_error)) - __pyx_t_7 = __Pyx_GetModuleGlobalName(__pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 133, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - __pyx_t_15 = __Pyx_PyErr_ExceptionMatches(__pyx_t_7); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; + __Pyx_ErrFetch(&__pyx_t_16, &__pyx_t_8, &__pyx_t_5); + __Pyx_GetModuleGlobalName(__pyx_t_18, __pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_18)) __PYX_ERR(2, 133, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __pyx_t_15 = __Pyx_PyErr_GivenExceptionMatches(__pyx_t_16, __pyx_t_18); + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_ErrRestore(__pyx_t_16, __pyx_t_8, __pyx_t_5); + __pyx_t_16 = 0; __pyx_t_8 = 0; __pyx_t_5 = 0; if (__pyx_t_15) { __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); - if (__Pyx_GetException(&__pyx_t_7, &__pyx_t_2, &__pyx_t_8) < 0) __PYX_ERR(2, 133, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_7); - __Pyx_GOTREF(__pyx_t_2); + if (__Pyx_GetException(&__pyx_t_5, &__pyx_t_8, &__pyx_t_16) < 0) __PYX_ERR(2, 133, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_5); __Pyx_GOTREF(__pyx_t_8); - __Pyx_INCREF(__pyx_t_2); - __pyx_v_exc = __pyx_t_2; + __Pyx_GOTREF(__pyx_t_16); + __Pyx_INCREF(__pyx_t_8); + __pyx_v_exc = __pyx_t_8; /* "reppy/robots.pyx":134 * wrap_exception(exceptions.ExcessiveRedirects, exc) @@ -4532,18 +4518,18 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * def RobotsUrlMethod(cls, url): */ __Pyx_TraceLine(134,0,__PYX_ERR(2, 134, __pyx_L5_except_error)) - __pyx_t_16 = __Pyx_GetModuleGlobalName(__pyx_n_s_exceptions); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 134, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __pyx_t_1 = __Pyx_PyObject_GetAttrStr(__pyx_t_16, __pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 134, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; - __pyx_t_16 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_1, __pyx_v_exc); if (unlikely(!__pyx_t_16)) __PYX_ERR(2, 134, __pyx_L5_except_error) - __Pyx_GOTREF(__pyx_t_16); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - __Pyx_DECREF(__pyx_t_16); __pyx_t_16 = 0; + __Pyx_GetModuleGlobalName(__pyx_t_18, __pyx_n_s_exceptions); if (unlikely(!__pyx_t_18)) __PYX_ERR(2, 134, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); + __pyx_t_7 = __Pyx_PyObject_GetAttrStr(__pyx_t_18, __pyx_n_s_ReadTimeout); if (unlikely(!__pyx_t_7)) __PYX_ERR(2, 134, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_7); + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __pyx_t_18 = __pyx_pf_5reppy_6robots_11FetchMethod_wrap_exception(__pyx_v_wrap_exception, __pyx_t_7, __pyx_v_exc); if (unlikely(!__pyx_t_18)) __PYX_ERR(2, 134, __pyx_L5_except_error) + __Pyx_GOTREF(__pyx_t_18); __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_18); __pyx_t_18 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_XDECREF(__pyx_t_16); __pyx_t_16 = 0; goto __pyx_L4_exception_handled; } goto __pyx_L5_except_error; @@ -4556,22 +4542,22 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ * # Limit the size of the request * kwargs['stream'] = True */ + __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_XGIVEREF(__pyx_t_4); - __Pyx_XGIVEREF(__pyx_t_5); - __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); goto __pyx_L1_error; __pyx_L7_try_return:; + __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_XGIVEREF(__pyx_t_4); - __Pyx_XGIVEREF(__pyx_t_5); - __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); goto __pyx_L0; __pyx_L4_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_2); __Pyx_XGIVEREF(__pyx_t_3); __Pyx_XGIVEREF(__pyx_t_4); - __Pyx_XGIVEREF(__pyx_t_5); - __Pyx_ExceptionReset(__pyx_t_3, __pyx_t_4, __pyx_t_5); + __Pyx_ExceptionReset(__pyx_t_2, __pyx_t_3, __pyx_t_4); __pyx_L8_try_end:; } @@ -4588,11 +4574,12 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ goto __pyx_L0; __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); __Pyx_XDECREF(__pyx_t_6); __Pyx_XDECREF(__pyx_t_7); __Pyx_XDECREF(__pyx_t_8); __Pyx_XDECREF(__pyx_t_16); + __Pyx_XDECREF(__pyx_t_18); __Pyx_AddTraceback("reppy.robots.FetchMethod", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = NULL; __pyx_L0:; @@ -4621,7 +4608,7 @@ static PyObject *__pyx_pf_5reppy_6robots_4FetchMethod(CYTHON_UNUSED PyObject *__ /* Python wrapper */ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/ static char __pyx_doc_5reppy_6robots_6RobotsUrlMethod[] = "Get the robots.txt URL that corresponds to the provided one."; -static PyMethodDef __pyx_mdef_5reppy_6robots_7RobotsUrlMethod = {"RobotsUrlMethod", (PyCFunction)__pyx_pw_5reppy_6robots_7RobotsUrlMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_6RobotsUrlMethod}; +static PyMethodDef __pyx_mdef_5reppy_6robots_7RobotsUrlMethod = {"RobotsUrlMethod", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_5reppy_6robots_7RobotsUrlMethod, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_6RobotsUrlMethod}; static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, PyObject *__pyx_args, PyObject *__pyx_kwds) { CYTHON_UNUSED PyObject *__pyx_v_cls = 0; PyObject *__pyx_v_url = 0; @@ -4645,11 +4632,11 @@ static PyObject *__pyx_pw_5reppy_6robots_7RobotsUrlMethod(PyObject *__pyx_self, kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_cls)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("RobotsUrlMethod", 1, 2, 2, 1); __PYX_ERR(2, 136, __pyx_L3_error) } @@ -4689,7 +4676,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject std::string __pyx_t_2; std::string __pyx_t_3; PyObject *__pyx_t_4 = NULL; - __Pyx_TraceFrameInit(__pyx_codeobj__14) + __Pyx_TraceFrameInit(__pyx_codeobj__9) __Pyx_RefNannySetupContext("RobotsUrlMethod", 0); __Pyx_TraceCall("RobotsUrlMethod", __pyx_f[2], 136, 0, __PYX_ERR(2, 136, __pyx_L1_error)); @@ -4709,7 +4696,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6RobotsUrlMethod(CYTHON_UNUSED PyObject try { __pyx_t_3 = Rep::Robots::robotsUrl(__pyx_t_2); } catch(...) { - try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } + try { throw; } catch(const std::exception& exn) {PyErr_SetString(__pyx_builtin_ValueError, exn.what());} catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } __PYX_ERR(2, 138, __pyx_L1_error) } __pyx_t_1 = __pyx_convert_PyBytes_string_to_py_std__in_string(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 138, __pyx_L1_error) @@ -4779,18 +4766,18 @@ static int __pyx_pw_5reppy_6robots_6Robots_1__init__(PyObject *__pyx_v_self, PyO kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_content)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("__init__", 0, 2, 3, 1); __PYX_ERR(2, 156, __pyx_L3_error) } CYTHON_FALLTHROUGH; case 2: if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_expires); if (value) { values[2] = value; kw_args--; } } } @@ -4851,7 +4838,7 @@ static int __pyx_pf_5reppy_6robots_6Robots___init__(struct __pyx_obj_5reppy_6rob try { __pyx_t_3 = new Rep::Robots(__pyx_v_content, __pyx_t_2); } catch(...) { - try { throw; } catch(const std::exception& exn) { PyErr_SetString(__pyx_builtin_ValueError, exn.what()); } catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } + try { throw; } catch(const std::exception& exn) {PyErr_SetString(__pyx_builtin_ValueError, exn.what());} catch(...) { PyErr_SetNone(__pyx_builtin_ValueError); } __PYX_ERR(2, 157, __pyx_L1_error) } __pyx_v_self->robots = __pyx_t_3; @@ -5017,7 +5004,7 @@ static void __pyx_pf_5reppy_6robots_6Robots_4__dealloc__(struct __pyx_obj_5reppy * @property * def sitemaps(self): # <<<<<<<<<<<<<< * '''Get all the sitemaps in this robots.txt.''' - * return map(as_string, self.robots.sitemaps()) + * return list(map(as_string, self.robots.sitemaps())) */ /* Python wrapper */ @@ -5046,7 +5033,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ /* "reppy/robots.pyx":171 * def sitemaps(self): * '''Get all the sitemaps in this robots.txt.''' - * return map(as_string, self.robots.sitemaps()) # <<<<<<<<<<<<<< + * return list(map(as_string, self.robots.sitemaps())) # <<<<<<<<<<<<<< * * def allowed(self, path, name): */ @@ -5067,8 +5054,11 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ __pyx_t_2 = __Pyx_PyObject_Call(__pyx_builtin_map, __pyx_t_3, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 171, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - __pyx_r = __pyx_t_2; - __pyx_t_2 = 0; + __pyx_t_3 = PySequence_List(__pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_r = __pyx_t_3; + __pyx_t_3 = 0; goto __pyx_L0; /* "reppy/robots.pyx":169 @@ -5076,7 +5066,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ * @property * def sitemaps(self): # <<<<<<<<<<<<<< * '''Get all the sitemaps in this robots.txt.''' - * return map(as_string, self.robots.sitemaps()) + * return list(map(as_string, self.robots.sitemaps())) */ /* function exit code */ @@ -5094,7 +5084,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_8sitemaps___get__(struct __pyx_ } /* "reppy/robots.pyx":173 - * return map(as_string, self.robots.sitemaps()) + * return list(map(as_string, self.robots.sitemaps())) * * def allowed(self, path, name): # <<<<<<<<<<<<<< * '''Is the provided path allowed for the provided agent?''' @@ -5127,11 +5117,11 @@ static PyObject *__pyx_pw_5reppy_6robots_6Robots_7allowed(PyObject *__pyx_v_self kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_path)) != 0)) kw_args--; + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_path)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: - if (likely((values[1] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; + if (likely((values[1] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_name)) != 0)) kw_args--; else { __Pyx_RaiseArgtupleInvalid("allowed", 1, 2, 2, 1); __PYX_ERR(2, 173, __pyx_L3_error) } @@ -5197,7 +5187,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_6allowed(struct __pyx_obj_5repp goto __pyx_L0; /* "reppy/robots.pyx":173 - * return map(as_string, self.robots.sitemaps()) + * return list(map(as_string, self.robots.sitemaps())) * * def allowed(self, path, name): # <<<<<<<<<<<<<< * '''Is the provided path allowed for the provided agent?''' @@ -5379,7 +5369,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o */ __Pyx_TraceLine(189,0,__PYX_ERR(2, 189, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 189, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); @@ -5394,12 +5384,9 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_7expired___get__(struct __pyx_o __Pyx_DECREF_SET(__pyx_t_3, function); } } - if (__pyx_t_2) { - __pyx_t_1 = __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 189, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; - } else { - __pyx_t_1 = __Pyx_PyObject_CallNoArg(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 189, __pyx_L1_error) - } + __pyx_t_1 = (__pyx_t_2) ? __Pyx_PyObject_CallOneArg(__pyx_t_3, __pyx_t_2) : __Pyx_PyObject_CallNoArg(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 189, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __pyx_t_3 = PyObject_RichCompare(__pyx_t_1, __pyx_v_self->expires, Py_GT); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 189, __pyx_L1_error) @@ -5534,7 +5521,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_TraceLine(199,0,__PYX_ERR(2, 199, __pyx_L1_error)) __Pyx_XDECREF(__pyx_r); __pyx_t_1 = 0; - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 199, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_n_s_time); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_4); @@ -5549,12 +5536,9 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_3ttl___get__(struct __pyx_obj_5 __Pyx_DECREF_SET(__pyx_t_4, function); } } - if (__pyx_t_3) { - __pyx_t_2 = __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 199, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; - } else { - __pyx_t_2 = __Pyx_PyObject_CallNoArg(__pyx_t_4); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 199, __pyx_L1_error) - } + __pyx_t_2 = (__pyx_t_3) ? __Pyx_PyObject_CallOneArg(__pyx_t_4, __pyx_t_3) : __Pyx_PyObject_CallNoArg(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 199, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_t_4 = PyNumber_Subtract(__pyx_v_self->expires, __pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(2, 199, __pyx_L1_error) @@ -5638,7 +5622,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_10__reduce_cython__(CYTHON_UNUS * raise TypeError("self.robots cannot be converted to a Python object for pickling") */ __Pyx_TraceLine(2,0,__PYX_ERR(1, 2, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__15, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__10, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -5695,7 +5679,7 @@ static PyObject *__pyx_pf_5reppy_6robots_6Robots_12__setstate_cython__(CYTHON_UN * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< */ __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__16, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__11, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -5753,12 +5737,12 @@ static int __pyx_pw_5reppy_6robots_9AllowNone_1__init__(PyObject *__pyx_v_self, kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_expires); if (value) { values[1] = value; kw_args--; } } } @@ -5925,7 +5909,7 @@ static PyObject *__pyx_pf_5reppy_6robots_9AllowNone_2__reduce_cython__(CYTHON_UN * raise TypeError("self.robots cannot be converted to a Python object for pickling") */ __Pyx_TraceLine(2,0,__PYX_ERR(1, 2, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__17, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__12, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -5982,7 +5966,7 @@ static PyObject *__pyx_pf_5reppy_6robots_9AllowNone_4__setstate_cython__(CYTHON_ * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< */ __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__18, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__13, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -6039,12 +6023,12 @@ static int __pyx_pw_5reppy_6robots_8AllowAll_1__init__(PyObject *__pyx_v_self, P kw_args = PyDict_Size(__pyx_kwds); switch (pos_args) { case 0: - if (likely((values[0] = PyDict_GetItem(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; + if (likely((values[0] = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_url)) != 0)) kw_args--; else goto __pyx_L5_argtuple_error; CYTHON_FALLTHROUGH; case 1: if (kw_args > 0) { - PyObject* value = PyDict_GetItem(__pyx_kwds, __pyx_n_s_expires); + PyObject* value = __Pyx_PyDict_GetItemStr(__pyx_kwds, __pyx_n_s_expires); if (value) { values[1] = value; kw_args--; } } } @@ -6112,7 +6096,7 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r } #if CYTHON_FAST_PYCALL if (PyFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__14, __pyx_v_expires}; __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 213, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); @@ -6120,7 +6104,7 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r #endif #if CYTHON_FAST_PYCCALL if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { - PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__19, __pyx_v_expires}; + PyObject *__pyx_temp[5] = {__pyx_t_3, ((PyObject *)__pyx_v_self), __pyx_v_url, __pyx_kp_b__14, __pyx_v_expires}; __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 4+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 213, __pyx_L1_error) __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_1); @@ -6138,9 +6122,9 @@ static int __pyx_pf_5reppy_6robots_8AllowAll___init__(struct __pyx_obj_5reppy_6r __Pyx_INCREF(__pyx_v_url); __Pyx_GIVEREF(__pyx_v_url); PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_4, __pyx_v_url); - __Pyx_INCREF(__pyx_kp_b__19); - __Pyx_GIVEREF(__pyx_kp_b__19); - PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b__19); + __Pyx_INCREF(__pyx_kp_b__14); + __Pyx_GIVEREF(__pyx_kp_b__14); + PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_kp_b__14); __Pyx_INCREF(__pyx_v_expires); __Pyx_GIVEREF(__pyx_v_expires); PyTuple_SET_ITEM(__pyx_t_5, 3+__pyx_t_4, __pyx_v_expires); @@ -6208,7 +6192,7 @@ static PyObject *__pyx_pf_5reppy_6robots_8AllowAll_2__reduce_cython__(CYTHON_UNU * raise TypeError("self.robots cannot be converted to a Python object for pickling") */ __Pyx_TraceLine(2,0,__PYX_ERR(1, 2, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__20, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__15, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 2, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -6265,7 +6249,7 @@ static PyObject *__pyx_pf_5reppy_6robots_8AllowAll_4__setstate_cython__(CYTHON_U * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< */ __Pyx_TraceLine(4,0,__PYX_ERR(1, 4, __pyx_L1_error)) - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__21, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_builtin_TypeError, __pyx_tuple__16, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 4, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -6706,7 +6690,7 @@ static PyObject *__Pyx_CFunc_object____object___to_py(PyObject *(*__pyx_v_f)(PyO * return f(value) */ __Pyx_TraceLine(65,0,__PYX_ERR(1, 65, __pyx_L1_error)) - __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__23)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 65, __pyx_L1_error) + __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_11cfunc_dot_to_py_36__Pyx_CFunc_object____object___to_py_1wrap, 0, __pyx_n_s_Pyx_CFunc_object____object___t, ((PyObject*)__pyx_cur_scope), __pyx_n_s_cfunc_to_py, __pyx_d, ((PyObject *)__pyx_codeobj__18)); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 65, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_v_wrap = __pyx_t_1; __pyx_t_1 = 0; @@ -6762,7 +6746,8 @@ static PyObject *__pyx_convert_vector_to_py_std_3a__3a_string(const std::vector< PyObject *__pyx_t_1 = NULL; size_t __pyx_t_2; size_t __pyx_t_3; - PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_4; + PyObject *__pyx_t_5 = NULL; __Pyx_RefNannySetupContext("__pyx_convert_vector_to_py_std_3a__3a_string", 0); __Pyx_TraceCall("__pyx_convert_vector_to_py_std_3a__3a_string", __pyx_f[1], 60, 0, __PYX_ERR(1, 60, __pyx_L1_error)); @@ -6778,12 +6763,13 @@ static PyObject *__pyx_convert_vector_to_py_std_3a__3a_string(const std::vector< __pyx_t_1 = PyList_New(0); if (unlikely(!__pyx_t_1)) __PYX_ERR(1, 61, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __pyx_v_v.size(); - for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { - __pyx_v_i = __pyx_t_3; - __pyx_t_4 = __pyx_convert_PyBytes_string_to_py_std__in_string((__pyx_v_v[__pyx_v_i])); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 61, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_4); - if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_4))) __PYX_ERR(1, 61, __pyx_L1_error) - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_3 = __pyx_t_2; + for (__pyx_t_4 = 0; __pyx_t_4 < __pyx_t_3; __pyx_t_4+=1) { + __pyx_v_i = __pyx_t_4; + __pyx_t_5 = __pyx_convert_PyBytes_string_to_py_std__in_string((__pyx_v_v[__pyx_v_i])); if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (unlikely(__Pyx_ListComp_Append(__pyx_t_1, (PyObject*)__pyx_t_5))) __PYX_ERR(1, 61, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } __pyx_r = __pyx_t_1; __pyx_t_1 = 0; @@ -6800,7 +6786,7 @@ static PyObject *__pyx_convert_vector_to_py_std_3a__3a_string(const std::vector< /* function exit code */ __pyx_L1_error:; __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); __Pyx_AddTraceback("vector.to_py.__pyx_convert_vector_to_py_std_3a__3a_string", __pyx_clineno, __pyx_lineno, __pyx_filename); __pyx_r = 0; __pyx_L0:; @@ -6999,7 +6985,7 @@ static PyObject *__pyx_getprop_5reppy_6robots_6Robots_ttl(PyObject *o, CYTHON_UN } static PyMethodDef __pyx_methods_5reppy_6robots_Robots[] = { - {"allowed", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_7allowed, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_6Robots_6allowed}, + {"allowed", (PyCFunction)(void*)(PyCFunctionWithKeywords)__pyx_pw_5reppy_6robots_6Robots_7allowed, METH_VARARGS|METH_KEYWORDS, __pyx_doc_5reppy_6robots_6Robots_6allowed}, {"agent", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_9agent, METH_O, __pyx_doc_5reppy_6robots_6Robots_8agent}, {"__reduce_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_11__reduce_cython__, METH_NOARGS, 0}, {"__setstate_cython__", (PyCFunction)__pyx_pw_5reppy_6robots_6Robots_13__setstate_cython__, METH_O, 0}, @@ -7448,8 +7434,20 @@ static struct PyModuleDef __pyx_moduledef = { NULL /* m_free */ }; #endif +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif static __Pyx_StringTabEntry __pyx_string_tab[] = { + {&__pyx_n_s_Agent, __pyx_k_Agent, sizeof(__pyx_k_Agent), 0, 0, 1, 1}, + {&__pyx_n_s_AllowAll, __pyx_k_AllowAll, sizeof(__pyx_k_AllowAll), 0, 0, 1, 1}, + {&__pyx_n_s_AllowNone, __pyx_k_AllowNone, sizeof(__pyx_k_AllowNone), 0, 0, 1, 1}, {&__pyx_n_s_BadStatusCode, __pyx_k_BadStatusCode, sizeof(__pyx_k_BadStatusCode), 0, 0, 1, 1}, {&__pyx_n_s_ConnectionError, __pyx_k_ConnectionError, sizeof(__pyx_k_ConnectionError), 0, 0, 1, 1}, {&__pyx_n_s_ConnectionException, __pyx_k_ConnectionException, sizeof(__pyx_k_ConnectionException), 0, 0, 1, 1}, @@ -7470,6 +7468,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_ParseMethod, __pyx_k_ParseMethod, sizeof(__pyx_k_ParseMethod), 0, 0, 1, 1}, {&__pyx_n_s_Pyx_CFunc_object____object___t, __pyx_k_Pyx_CFunc_object____object___t, sizeof(__pyx_k_Pyx_CFunc_object____object___t), 0, 0, 1, 1}, {&__pyx_n_s_ReadTimeout, __pyx_k_ReadTimeout, sizeof(__pyx_k_ReadTimeout), 0, 0, 1, 1}, + {&__pyx_n_s_Robots, __pyx_k_Robots, sizeof(__pyx_k_Robots), 0, 0, 1, 1}, {&__pyx_n_s_RobotsUrlMethod, __pyx_k_RobotsUrlMethod, sizeof(__pyx_k_RobotsUrlMethod), 0, 0, 1, 1}, {&__pyx_n_s_SSLError, __pyx_k_SSLError, sizeof(__pyx_k_SSLError), 0, 0, 1, 1}, {&__pyx_n_s_SSLException, __pyx_k_SSLException, sizeof(__pyx_k_SSLException), 0, 0, 1, 1}, @@ -7478,8 +7477,8 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_URLRequired, __pyx_k_URLRequired, sizeof(__pyx_k_URLRequired), 0, 0, 1, 1}, {&__pyx_kp_b_User_agent_Disallow, __pyx_k_User_agent_Disallow, sizeof(__pyx_k_User_agent_Disallow), 0, 0, 0, 0}, {&__pyx_n_s_ValueError, __pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 0, 1, 1}, - {&__pyx_n_s__19, __pyx_k__19, sizeof(__pyx_k__19), 0, 0, 1, 1}, - {&__pyx_kp_b__19, __pyx_k__19, sizeof(__pyx_k__19), 0, 0, 0, 0}, + {&__pyx_n_s__14, __pyx_k__14, sizeof(__pyx_k__14), 0, 0, 1, 1}, + {&__pyx_kp_b__14, __pyx_k__14, sizeof(__pyx_k__14), 0, 0, 0, 0}, {&__pyx_n_s_after_parse_hook, __pyx_k_after_parse_hook, sizeof(__pyx_k_after_parse_hook), 0, 0, 1, 1}, {&__pyx_n_s_after_response_hook, __pyx_k_after_response_hook, sizeof(__pyx_k_after_response_hook), 0, 0, 1, 1}, {&__pyx_n_s_agent, __pyx_k_agent, sizeof(__pyx_k_agent), 0, 0, 1, 1}, @@ -7553,7 +7552,7 @@ static __Pyx_StringTabEntry __pyx_string_tab[] = { {&__pyx_n_s_wrapped, __pyx_k_wrapped, sizeof(__pyx_k_wrapped), 0, 0, 1, 1}, {0, 0, 0, 0, 0, 0, 0} }; -static int __Pyx_InitCachedBuiltins(void) { +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) { __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_n_s_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 34, __pyx_L1_error) __pyx_builtin_TypeError = __Pyx_GetBuiltinName(__pyx_n_s_TypeError); if (!__pyx_builtin_TypeError) __PYX_ERR(1, 2, __pyx_L1_error) __pyx_builtin_map = __Pyx_GetBuiltinName(__pyx_n_s_map); if (!__pyx_builtin_map) __PYX_ERR(2, 171, __pyx_L1_error) @@ -7563,84 +7562,40 @@ static int __Pyx_InitCachedBuiltins(void) { return -1; } -static int __Pyx_InitCachedConstants(void) { +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) { __Pyx_RefNannyDeclarations __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); - /* "reppy/robots.pyx":25 - * if isinstance(value, bytes): - * return value - * return value.encode('utf-8') # <<<<<<<<<<<<<< - * - * # For contexts which require a 'str' type, convert bytes to unicode if needed - */ - __pyx_tuple_ = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple_)) __PYX_ERR(2, 25, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple_); - __Pyx_GIVEREF(__pyx_tuple_); - - /* "reppy/robots.pyx":33 - * if six.PY3: - * if isinstance(value, bytes): - * return value.decode('utf-8') # <<<<<<<<<<<<<< - * return value - * - */ - __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_utf_8); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(2, 33, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__2); - __Pyx_GIVEREF(__pyx_tuple__2); - /* "(tree fragment)":2 * def __reduce_cython__(self): * raise TypeError("self.agent cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< * def __setstate_cython__(self, __pyx_state): * raise TypeError("self.agent cannot be converted to a Python object for pickling") */ - __pyx_tuple__4 = PyTuple_Pack(1, __pyx_kp_s_self_agent_cannot_be_converted_t); if (unlikely(!__pyx_tuple__4)) __PYX_ERR(1, 2, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__4); - __Pyx_GIVEREF(__pyx_tuple__4); + __pyx_tuple__2 = PyTuple_Pack(1, __pyx_kp_s_self_agent_cannot_be_converted_t); if (unlikely(!__pyx_tuple__2)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__2); + __Pyx_GIVEREF(__pyx_tuple__2); /* "(tree fragment)":4 * raise TypeError("self.agent cannot be converted to a Python object for pickling") * def __setstate_cython__(self, __pyx_state): * raise TypeError("self.agent cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< */ - __pyx_tuple__5 = PyTuple_Pack(1, __pyx_kp_s_self_agent_cannot_be_converted_t); if (unlikely(!__pyx_tuple__5)) __PYX_ERR(1, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__5); - __Pyx_GIVEREF(__pyx_tuple__5); + __pyx_tuple__3 = PyTuple_Pack(1, __pyx_kp_s_self_agent_cannot_be_converted_t); if (unlikely(!__pyx_tuple__3)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__3); + __Pyx_GIVEREF(__pyx_tuple__3); - /* "reppy/robots.pyx":89 - * def FetchMethod(cls, url, ttl_policy=None, max_size=1048576, *args, **kwargs): - * '''Get the robots.txt at the provided URL.''' - * after_response_hook = kwargs.pop('after_response_hook', None) # <<<<<<<<<<<<<< + /* "reppy/robots.pyx":91 + * after_response_hook = kwargs.pop('after_response_hook', None) * after_parse_hook = kwargs.pop('after_parse_hook', None) - * def wrap_exception(etype, cause): + * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< + * wrapped = etype(cause) + * wrapped.url = url */ - __pyx_tuple__8 = PyTuple_Pack(2, __pyx_n_s_after_response_hook, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(2, 89, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__8); - __Pyx_GIVEREF(__pyx_tuple__8); - - /* "reppy/robots.pyx":90 - * '''Get the robots.txt at the provided URL.''' - * after_response_hook = kwargs.pop('after_response_hook', None) - * after_parse_hook = kwargs.pop('after_parse_hook', None) # <<<<<<<<<<<<<< - * def wrap_exception(etype, cause): - * wrapped = etype(cause) - */ - __pyx_tuple__9 = PyTuple_Pack(2, __pyx_n_s_after_parse_hook, Py_None); if (unlikely(!__pyx_tuple__9)) __PYX_ERR(2, 90, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__9); - __Pyx_GIVEREF(__pyx_tuple__9); - - /* "reppy/robots.pyx":91 - * after_response_hook = kwargs.pop('after_response_hook', None) - * after_parse_hook = kwargs.pop('after_parse_hook', None) - * def wrap_exception(etype, cause): # <<<<<<<<<<<<<< - * wrapped = etype(cause) - * wrapped.url = url - */ - __pyx_tuple__10 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(2, 91, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__10); - __Pyx_GIVEREF(__pyx_tuple__10); - __pyx_codeobj__11 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__10, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_wrap_exception, 91, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__11)) __PYX_ERR(2, 91, __pyx_L1_error) + __pyx_tuple__6 = PyTuple_Pack(3, __pyx_n_s_etype, __pyx_n_s_cause, __pyx_n_s_wrapped); if (unlikely(!__pyx_tuple__6)) __PYX_ERR(2, 91, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__6); + __Pyx_GIVEREF(__pyx_tuple__6); + __pyx_codeobj__7 = (PyObject*)__Pyx_PyCode_New(2, 0, 3, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__6, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_wrap_exception, 91, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__7)) __PYX_ERR(2, 91, __pyx_L1_error) /* "reppy/robots.pyx":100 * # Limit the size of the request @@ -7649,12 +7604,9 @@ static int __Pyx_InitCachedConstants(void) { * content = res.raw.read(amt=max_size, decode_content=True) * # Try to read an additional byte, to see if the response is too big */ - __pyx_tuple__12 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(2, 100, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__12); - __Pyx_GIVEREF(__pyx_tuple__12); - __pyx_tuple__13 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(2, 100, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__13); - __Pyx_GIVEREF(__pyx_tuple__13); + __pyx_tuple__8 = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_tuple__8)) __PYX_ERR(2, 100, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__8); + __Pyx_GIVEREF(__pyx_tuple__8); /* "(tree fragment)":2 * def __reduce_cython__(self): @@ -7662,18 +7614,18 @@ static int __Pyx_InitCachedConstants(void) { * def __setstate_cython__(self, __pyx_state): * raise TypeError("self.robots cannot be converted to a Python object for pickling") */ - __pyx_tuple__15 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 2, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__15); - __Pyx_GIVEREF(__pyx_tuple__15); + __pyx_tuple__10 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__10)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__10); + __Pyx_GIVEREF(__pyx_tuple__10); /* "(tree fragment)":4 * raise TypeError("self.robots cannot be converted to a Python object for pickling") * def __setstate_cython__(self, __pyx_state): * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< */ - __pyx_tuple__16 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__16); - __Pyx_GIVEREF(__pyx_tuple__16); + __pyx_tuple__11 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__11)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__11); + __Pyx_GIVEREF(__pyx_tuple__11); /* "(tree fragment)":2 * def __reduce_cython__(self): @@ -7681,18 +7633,18 @@ static int __Pyx_InitCachedConstants(void) { * def __setstate_cython__(self, __pyx_state): * raise TypeError("self.robots cannot be converted to a Python object for pickling") */ - __pyx_tuple__17 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 2, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__17); - __Pyx_GIVEREF(__pyx_tuple__17); + __pyx_tuple__12 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__12)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__12); + __Pyx_GIVEREF(__pyx_tuple__12); /* "(tree fragment)":4 * raise TypeError("self.robots cannot be converted to a Python object for pickling") * def __setstate_cython__(self, __pyx_state): * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< */ - __pyx_tuple__18 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__18)) __PYX_ERR(1, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__18); - __Pyx_GIVEREF(__pyx_tuple__18); + __pyx_tuple__13 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__13)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__13); + __Pyx_GIVEREF(__pyx_tuple__13); /* "(tree fragment)":2 * def __reduce_cython__(self): @@ -7700,18 +7652,18 @@ static int __Pyx_InitCachedConstants(void) { * def __setstate_cython__(self, __pyx_state): * raise TypeError("self.robots cannot be converted to a Python object for pickling") */ - __pyx_tuple__20 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__20)) __PYX_ERR(1, 2, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__20); - __Pyx_GIVEREF(__pyx_tuple__20); + __pyx_tuple__15 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__15)) __PYX_ERR(1, 2, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__15); + __Pyx_GIVEREF(__pyx_tuple__15); /* "(tree fragment)":4 * raise TypeError("self.robots cannot be converted to a Python object for pickling") * def __setstate_cython__(self, __pyx_state): * raise TypeError("self.robots cannot be converted to a Python object for pickling") # <<<<<<<<<<<<<< */ - __pyx_tuple__21 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__21)) __PYX_ERR(1, 4, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__21); - __Pyx_GIVEREF(__pyx_tuple__21); + __pyx_tuple__16 = PyTuple_Pack(1, __pyx_kp_s_self_robots_cannot_be_converted); if (unlikely(!__pyx_tuple__16)) __PYX_ERR(1, 4, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__16); + __Pyx_GIVEREF(__pyx_tuple__16); /* "cfunc.to_py":65 * @cname("__Pyx_CFunc_object____object___to_py") @@ -7720,10 +7672,10 @@ static int __Pyx_InitCachedConstants(void) { * """wrap(value)""" * return f(value) */ - __pyx_tuple__22 = PyTuple_Pack(1, __pyx_n_s_value); if (unlikely(!__pyx_tuple__22)) __PYX_ERR(1, 65, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__22); - __Pyx_GIVEREF(__pyx_tuple__22); - __pyx_codeobj__23 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__22, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__23)) __PYX_ERR(1, 65, __pyx_L1_error) + __pyx_tuple__17 = PyTuple_Pack(1, __pyx_n_s_value); if (unlikely(!__pyx_tuple__17)) __PYX_ERR(1, 65, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__17); + __Pyx_GIVEREF(__pyx_tuple__17); + __pyx_codeobj__18 = (PyObject*)__Pyx_PyCode_New(1, 0, 1, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__17, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_stringsource, __pyx_n_s_wrap, 65, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__18)) __PYX_ERR(1, 65, __pyx_L1_error) /* "reppy/robots.pyx":37 * @@ -7732,10 +7684,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Construct an Agent from a CppAgent.''' * agent = Agent() */ - __pyx_tuple__24 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__24)) __PYX_ERR(2, 37, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__24); - __Pyx_GIVEREF(__pyx_tuple__24); - __pyx_codeobj__3 = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__24, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 37, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__3)) __PYX_ERR(2, 37, __pyx_L1_error) + __pyx_tuple__19 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_robots, __pyx_n_s_name, __pyx_n_s_agent); if (unlikely(!__pyx_tuple__19)) __PYX_ERR(2, 37, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__19); + __Pyx_GIVEREF(__pyx_tuple__19); + __pyx_codeobj_ = (PyObject*)__Pyx_PyCode_New(3, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__19, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FromRobotsMethod, 37, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj_)) __PYX_ERR(2, 37, __pyx_L1_error) /* "reppy/robots.pyx":83 * @@ -7744,10 +7696,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Parse a robots.txt file.''' * return cls(url, as_bytes(content), expires) */ - __pyx_tuple__25 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__25)) __PYX_ERR(2, 83, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__25); - __Pyx_GIVEREF(__pyx_tuple__25); - __pyx_codeobj__6 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__25, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_ParseMethod, 83, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__6)) __PYX_ERR(2, 83, __pyx_L1_error) + __pyx_tuple__20 = PyTuple_Pack(4, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_content, __pyx_n_s_expires); if (unlikely(!__pyx_tuple__20)) __PYX_ERR(2, 83, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__20); + __Pyx_GIVEREF(__pyx_tuple__20); + __pyx_codeobj__4 = (PyObject*)__Pyx_PyCode_New(4, 0, 4, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__20, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_ParseMethod, 83, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__4)) __PYX_ERR(2, 83, __pyx_L1_error) /* "reppy/robots.pyx":87 * return cls(url, as_bytes(content), expires) @@ -7756,10 +7708,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Get the robots.txt at the provided URL.''' * after_response_hook = kwargs.pop('after_response_hook', None) */ - __pyx_tuple__26 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__26)) __PYX_ERR(2, 87, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__26); - __Pyx_GIVEREF(__pyx_tuple__26); - __pyx_codeobj__7 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__26, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FetchMethod, 87, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__7)) __PYX_ERR(2, 87, __pyx_L1_error) + __pyx_tuple__21 = PyTuple_Pack(15, __pyx_n_s_cls, __pyx_n_s_url, __pyx_n_s_ttl_policy, __pyx_n_s_max_size, __pyx_n_s_args, __pyx_n_s_kwargs, __pyx_n_s_after_response_hook, __pyx_n_s_after_parse_hook, __pyx_n_s_wrap_exception, __pyx_n_s_wrap_exception, __pyx_n_s_res, __pyx_n_s_content, __pyx_n_s_expires, __pyx_n_s_robots, __pyx_n_s_exc); if (unlikely(!__pyx_tuple__21)) __PYX_ERR(2, 87, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__21); + __Pyx_GIVEREF(__pyx_tuple__21); + __pyx_codeobj__5 = (PyObject*)__Pyx_PyCode_New(4, 0, 15, 0, CO_OPTIMIZED|CO_NEWLOCALS|CO_VARARGS|CO_VARKEYWORDS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__21, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_FetchMethod, 87, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__5)) __PYX_ERR(2, 87, __pyx_L1_error) /* "reppy/robots.pyx":136 * wrap_exception(exceptions.ReadTimeout, exc) @@ -7768,10 +7720,10 @@ static int __Pyx_InitCachedConstants(void) { * '''Get the robots.txt URL that corresponds to the provided one.''' * return as_string(CppRobots.robotsUrl(as_bytes(url))) */ - __pyx_tuple__27 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__27)) __PYX_ERR(2, 136, __pyx_L1_error) - __Pyx_GOTREF(__pyx_tuple__27); - __Pyx_GIVEREF(__pyx_tuple__27); - __pyx_codeobj__14 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__27, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 136, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__14)) __PYX_ERR(2, 136, __pyx_L1_error) + __pyx_tuple__22 = PyTuple_Pack(2, __pyx_n_s_cls, __pyx_n_s_url); if (unlikely(!__pyx_tuple__22)) __PYX_ERR(2, 136, __pyx_L1_error) + __Pyx_GOTREF(__pyx_tuple__22); + __Pyx_GIVEREF(__pyx_tuple__22); + __pyx_codeobj__9 = (PyObject*)__Pyx_PyCode_New(2, 0, 2, 0, CO_OPTIMIZED|CO_NEWLOCALS, __pyx_empty_bytes, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_tuple__22, __pyx_empty_tuple, __pyx_empty_tuple, __pyx_kp_s_reppy_robots_pyx, __pyx_n_s_RobotsUrlMethod, 136, __pyx_empty_bytes); if (unlikely(!__pyx_codeobj__9)) __PYX_ERR(2, 136, __pyx_L1_error) __Pyx_RefNannyFinishContext(); return 0; __pyx_L1_error:; @@ -7779,7 +7731,8 @@ static int __Pyx_InitCachedConstants(void) { return -1; } -static int __Pyx_InitGlobals(void) { +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) { + __pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; if (__Pyx_InitStrings(__pyx_string_tab) < 0) __PYX_ERR(2, 1, __pyx_L1_error); __pyx_int_1 = PyInt_FromLong(1); if (unlikely(!__pyx_int_1)) __PYX_ERR(2, 1, __pyx_L1_error) __pyx_int_200 = PyInt_FromLong(200); if (unlikely(!__pyx_int_200)) __PYX_ERR(2, 1, __pyx_L1_error) @@ -7795,21 +7748,175 @@ static int __Pyx_InitGlobals(void) { return -1; } +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(void); /*proto*/ + +static int __Pyx_modinit_global_init_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_init_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) + __pyx_type_5reppy_6robots_Agent.tp_print = 0; + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type_5reppy_6robots_Agent.tp_dictoffset && __pyx_type_5reppy_6robots_Agent.tp_getattro == PyObject_GenericGetAttr)) { + __pyx_type_5reppy_6robots_Agent.tp_getattro = __Pyx_PyObject_GenericGetAttr; + } + if (PyObject_SetAttr(__pyx_m, __pyx_n_s_Agent, (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) + __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; + if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 140, __pyx_L1_error) + __pyx_type_5reppy_6robots_Robots.tp_print = 0; + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type_5reppy_6robots_Robots.tp_dictoffset && __pyx_type_5reppy_6robots_Robots.tp_getattro == PyObject_GenericGetAttr)) { + __pyx_type_5reppy_6robots_Robots.tp_getattro = __Pyx_PyObject_GenericGetAttr; + } + if (PyObject_SetAttr(__pyx_m, __pyx_n_s_Robots, (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 140, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 140, __pyx_L1_error) + __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; + __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 202, __pyx_L1_error) + __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type_5reppy_6robots_AllowNone.tp_dictoffset && __pyx_type_5reppy_6robots_AllowNone.tp_getattro == PyObject_GenericGetAttr)) { + __pyx_type_5reppy_6robots_AllowNone.tp_getattro = __Pyx_PyObject_GenericGetAttr; + } + if (PyObject_SetAttr(__pyx_m, __pyx_n_s_AllowNone, (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 202, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 202, __pyx_L1_error) + __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; + __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; + if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 209, __pyx_L1_error) + __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type_5reppy_6robots_AllowAll.tp_dictoffset && __pyx_type_5reppy_6robots_AllowAll.tp_getattro == PyObject_GenericGetAttr)) { + __pyx_type_5reppy_6robots_AllowAll.tp_getattro = __Pyx_PyObject_GenericGetAttr; + } + if (PyObject_SetAttr(__pyx_m, __pyx_n_s_AllowAll, (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 209, __pyx_L1_error) + if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 209, __pyx_L1_error) + __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; + if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(2, 87, __pyx_L1_error) + __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_print = 0; + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_dictoffset && __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_getattro == PyObject_GenericGetAttr)) { + __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; + } + __pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod = &__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod; + if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(1, 64, __pyx_L1_error) + __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; + if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) && likely(!__pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_dictoffset && __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro == PyObject_GenericGetAttr)) { + __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; + } + __pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = &__pyx_scope_struct____Pyx_CFunc_object____object___to_py; + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_import_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(void) { + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + + +#if PY_MAJOR_VERSION < 3 +#ifdef CYTHON_NO_PYINIT_EXPORT +#define __Pyx_PyMODINIT_FUNC void +#else +#define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#endif +#else +#ifdef CYTHON_NO_PYINIT_EXPORT +#define __Pyx_PyMODINIT_FUNC PyObject * +#else +#define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#endif +#endif + + #if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC initrobots(void); /*proto*/ -PyMODINIT_FUNC initrobots(void) +__Pyx_PyMODINIT_FUNC initrobots(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC initrobots(void) #else -PyMODINIT_FUNC PyInit_robots(void); /*proto*/ -PyMODINIT_FUNC PyInit_robots(void) +__Pyx_PyMODINIT_FUNC PyInit_robots(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_robots(void) #if CYTHON_PEP489_MULTI_PHASE_INIT { return PyModuleDef_Init(&__pyx_moduledef); } -static int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name) { +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + #if PY_VERSION_HEX >= 0x030700A1 + static PY_INT64_T main_interpreter_id = -1; + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return (unlikely(current_id == -1)) ? -1 : 0; + } else if (unlikely(main_interpreter_id != current_id)) + #else + static PyInterpreterState *main_interpreter = NULL; + PyInterpreterState *current_interpreter = PyThreadState_Get()->interp; + if (!main_interpreter) { + main_interpreter = current_interpreter; + } else if (unlikely(main_interpreter != current_interpreter)) + #endif + { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) { PyObject *value = PyObject_GetAttrString(spec, from_name); int result = 0; if (likely(value)) { - result = PyDict_SetItemString(moddict, to_name, value); + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } Py_DECREF(value); } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Clear(); @@ -7818,8 +7925,10 @@ static int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const ch } return result; } -static PyObject* __pyx_pymod_create(PyObject *spec, CYTHON_UNUSED PyModuleDef *def) { +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, CYTHON_UNUSED PyModuleDef *def) { PyObject *module = NULL, *moddict, *modname; + if (__Pyx_check_single_interpreter()) + return NULL; if (__pyx_m) return __Pyx_NewRef(__pyx_m); modname = PyObject_GetAttrString(spec, "name"); @@ -7829,10 +7938,10 @@ static PyObject* __pyx_pymod_create(PyObject *spec, CYTHON_UNUSED PyModuleDef *d if (unlikely(!module)) goto bad; moddict = PyModule_GetDict(module); if (unlikely(!moddict)) goto bad; - if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__") < 0)) goto bad; - if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__") < 0)) goto bad; - if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__") < 0)) goto bad; - if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__") < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; return module; bad: Py_XDECREF(module); @@ -7840,7 +7949,7 @@ static PyObject* __pyx_pymod_create(PyObject *spec, CYTHON_UNUSED PyModuleDef *d } -static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) +static CYTHON_SMALL_CODE int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) #endif #endif { @@ -7850,19 +7959,28 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) PyObject *__pyx_t_3 = NULL; __Pyx_RefNannyDeclarations #if CYTHON_PEP489_MULTI_PHASE_INIT - if (__pyx_m && __pyx_m == __pyx_pyinit_module) return 0; - #endif - #if CYTHON_REFNANNY - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); - if (!__Pyx_RefNanny) { - PyErr_Clear(); - __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); - if (!__Pyx_RefNanny) - Py_FatalError("failed to import 'refnanny' module"); + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'robots' has already been imported. Re-initialisation is not supported."); + return -1; } + #elif PY_MAJOR_VERSION >= 3 + if (__pyx_m) return __Pyx_NewRef(__pyx_m); #endif - __Pyx_RefNannySetupContext("PyMODINIT_FUNC PyInit_robots(void)", 0); + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + __Pyx_RefNannySetupContext("__Pyx_PyMODINIT_FUNC PyInit_robots(void)", 0); if (__Pyx_check_binary_version() < 0) __PYX_ERR(2, 1, __pyx_L1_error) + #ifdef __Pxy_PyFrame_Initialize_Offsets + __Pxy_PyFrame_Initialize_Offsets(); + #endif __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(2, 1, __pyx_L1_error) __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(2, 1, __pyx_L1_error) __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(2, 1, __pyx_L1_error) @@ -7906,10 +8024,9 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) __pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_d)) __PYX_ERR(2, 1, __pyx_L1_error) Py_INCREF(__pyx_d); __pyx_b = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_b)) __PYX_ERR(2, 1, __pyx_L1_error) - __pyx_cython_runtime = PyImport_AddModule((char *) "cython_runtime"); if (unlikely(!__pyx_cython_runtime)) __PYX_ERR(2, 1, __pyx_L1_error) - #if CYTHON_COMPILING_IN_PYPY Py_INCREF(__pyx_b); - #endif + __pyx_cython_runtime = PyImport_AddModule((char *) "cython_runtime"); if (unlikely(!__pyx_cython_runtime)) __PYX_ERR(2, 1, __pyx_L1_error) + Py_INCREF(__pyx_cython_runtime); if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_b) < 0) __PYX_ERR(2, 1, __pyx_L1_error); /*--- Initialize various global constants etc. ---*/ if (__Pyx_InitGlobals() < 0) __PYX_ERR(2, 1, __pyx_L1_error) @@ -7917,7 +8034,7 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) if (__Pyx_init_sys_getdefaultencoding_params() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #endif if (__pyx_module_is_main_reppy__robots) { - if (PyObject_SetAttrString(__pyx_m, "__name__", __pyx_n_s_main) < 0) __PYX_ERR(2, 1, __pyx_L1_error) + if (PyObject_SetAttr(__pyx_m, __pyx_n_s_name_2, __pyx_n_s_main) < 0) __PYX_ERR(2, 1, __pyx_L1_error) } #if PY_MAJOR_VERSION >= 3 { @@ -7931,46 +8048,19 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) if (__Pyx_InitCachedBuiltins() < 0) __PYX_ERR(2, 1, __pyx_L1_error) /*--- Constants init code ---*/ if (__Pyx_InitCachedConstants() < 0) __PYX_ERR(2, 1, __pyx_L1_error) - /*--- Global init code ---*/ - /*--- Variable export code ---*/ - /*--- Function export code ---*/ - /*--- Type init code ---*/ - if (PyType_Ready(&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) - __pyx_type_5reppy_6robots_Agent.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Agent", (PyObject *)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Agent) < 0) __PYX_ERR(2, 47, __pyx_L1_error) - __pyx_ptype_5reppy_6robots_Agent = &__pyx_type_5reppy_6robots_Agent; - if (PyType_Ready(&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 140, __pyx_L1_error) - __pyx_type_5reppy_6robots_Robots.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "Robots", (PyObject *)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 140, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_Robots) < 0) __PYX_ERR(2, 140, __pyx_L1_error) - __pyx_ptype_5reppy_6robots_Robots = &__pyx_type_5reppy_6robots_Robots; - __pyx_type_5reppy_6robots_AllowNone.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 202, __pyx_L1_error) - __pyx_type_5reppy_6robots_AllowNone.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowNone", (PyObject *)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 202, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowNone) < 0) __PYX_ERR(2, 202, __pyx_L1_error) - __pyx_ptype_5reppy_6robots_AllowNone = &__pyx_type_5reppy_6robots_AllowNone; - __pyx_type_5reppy_6robots_AllowAll.tp_base = __pyx_ptype_5reppy_6robots_Robots; - if (PyType_Ready(&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 209, __pyx_L1_error) - __pyx_type_5reppy_6robots_AllowAll.tp_print = 0; - if (PyObject_SetAttrString(__pyx_m, "AllowAll", (PyObject *)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 209, __pyx_L1_error) - if (__Pyx_setup_reduce((PyObject*)&__pyx_type_5reppy_6robots_AllowAll) < 0) __PYX_ERR(2, 209, __pyx_L1_error) - __pyx_ptype_5reppy_6robots_AllowAll = &__pyx_type_5reppy_6robots_AllowAll; - if (PyType_Ready(&__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod) < 0) __PYX_ERR(2, 87, __pyx_L1_error) - __pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod.tp_print = 0; - __pyx_ptype_5reppy_6robots___pyx_scope_struct__FetchMethod = &__pyx_type_5reppy_6robots___pyx_scope_struct__FetchMethod; - if (PyType_Ready(&__pyx_scope_struct____Pyx_CFunc_object____object___to_py) < 0) __PYX_ERR(1, 64, __pyx_L1_error) - __pyx_scope_struct____Pyx_CFunc_object____object___to_py.tp_print = 0; - __pyx_ptype___pyx_scope_struct____Pyx_CFunc_object____object___to_py = &__pyx_scope_struct____Pyx_CFunc_object____object___to_py; - /*--- Type import code ---*/ - /*--- Variable import code ---*/ - /*--- Function import code ---*/ + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(); + (void)__Pyx_modinit_variable_export_code(); + (void)__Pyx_modinit_function_export_code(); + if (unlikely(__Pyx_modinit_type_init_code() != 0)) goto __pyx_L1_error; + (void)__Pyx_modinit_type_import_code(); + (void)__Pyx_modinit_variable_import_code(); + (void)__Pyx_modinit_function_import_code(); /*--- Execution code ---*/ #if defined(__Pyx_Generator_USED) || defined(__Pyx_Coroutine_USED) if (__Pyx_patch_abc() < 0) __PYX_ERR(2, 1, __pyx_L1_error) #endif - __Pyx_TraceCall("PyMODINIT_FUNC PyInit_robots(void)", __pyx_f[2], 1, 0, __PYX_ERR(2, 1, __pyx_L1_error)); + __Pyx_TraceCall("__Pyx_PyMODINIT_FUNC PyInit_robots(void)", __pyx_f[2], 1, 0, __PYX_ERR(2, 1, __pyx_L1_error)); /* "reppy/robots.pyx":4 * # distutils: define_macros=CYTHON_TRACE=1 @@ -8154,7 +8244,7 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) __Pyx_INCREF(__pyx_n_s_exceptions); __Pyx_GIVEREF(__pyx_n_s_exceptions); PyList_SET_ITEM(__pyx_t_2, 2, __pyx_n_s_exceptions); - __pyx_t_1 = __Pyx_Import(__pyx_n_s__19, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 20, __pyx_L1_error) + __pyx_t_1 = __Pyx_Import(__pyx_n_s__14, __pyx_t_2, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 20, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_1, __pyx_n_s_util); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 20, __pyx_L1_error) @@ -8171,6 +8261,26 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + /* "reppy/robots.pyx":22 + * from . import util, logger, exceptions + * + * cdef as_bytes(value): # <<<<<<<<<<<<<< + * if isinstance(value, bytes): + * return value + */ + __Pyx_TraceLine(22,0,__PYX_ERR(2, 22, __pyx_L1_error)) + + + /* "reppy/robots.pyx":30 + * # (i.e., Python 3). Note: could raise UnicodeDecodeError in Python 3 if input + * # is invalid UTF-8 + * cdef as_string(value): # <<<<<<<<<<<<<< + * if six.PY3: + * if isinstance(value, bytes): + */ + __Pyx_TraceLine(30,0,__PYX_ERR(2, 30, __pyx_L1_error)) + + /* "reppy/robots.pyx":37 * * @@ -8192,7 +8302,7 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) * def __str__(self): */ __Pyx_TraceLine(52,0,__PYX_ERR(2, 52, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 52, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_FromRobotsMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_2 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 52, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); @@ -8248,7 +8358,7 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) * # Class methods */ __Pyx_TraceLine(145,0,__PYX_ERR(2, 145, __pyx_L1_error)) - __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 145, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_n_s_HeaderWithDefaultPolicy); if (unlikely(!__pyx_t_2)) __PYX_ERR(2, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_1 = __Pyx_PyDict_NewPresized(2); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 145, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); @@ -8270,7 +8380,7 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) * robots_url = classmethod(RobotsUrlMethod) */ __Pyx_TraceLine(148,0,__PYX_ERR(2, 148, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 148, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_ParseMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 148, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); @@ -8287,7 +8397,7 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) * */ __Pyx_TraceLine(149,0,__PYX_ERR(2, 149, __pyx_L1_error)) - __pyx_t_1 = __Pyx_GetModuleGlobalName(__pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 149, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_1, __pyx_n_s_FetchMethod); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = __Pyx_Method_ClassMethod(__pyx_t_1); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 149, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); @@ -8304,7 +8414,7 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) * # Data members */ __Pyx_TraceLine(150,0,__PYX_ERR(2, 150, __pyx_L1_error)) - __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 150, __pyx_L1_error) + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_n_s_RobotsUrlMethod); if (unlikely(!__pyx_t_3)) __PYX_ERR(2, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_1 = __Pyx_Method_ClassMethod(__pyx_t_3); if (unlikely(!__pyx_t_1)) __PYX_ERR(2, 150, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); @@ -8324,40 +8434,112 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) if (PyDict_SetItem(__pyx_d, __pyx_n_s_test, __pyx_t_1) < 0) __PYX_ERR(2, 1, __pyx_L1_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; - /* "vector.to_py":60 - * - * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") - * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): # <<<<<<<<<<<<<< - * return [v[i] for i in range(v.size())] + /* "string.from_py":13 * + * @cname("__pyx_convert_string_from_py_std__in_string") + * cdef string __pyx_convert_string_from_py_std__in_string(object o) except *: # <<<<<<<<<<<<<< + * cdef Py_ssize_t length + * cdef const char* data = __Pyx_PyObject_AsStringAndSize(o, &length) */ - __Pyx_TraceReturn(Py_None, 0); + __Pyx_TraceLine(13,0,__PYX_ERR(1, 13, __pyx_L1_error)) - /*--- Wrapped vars code ---*/ - goto __pyx_L0; - __pyx_L1_error:; - __Pyx_XDECREF(__pyx_t_1); - __Pyx_XDECREF(__pyx_t_2); - __Pyx_XDECREF(__pyx_t_3); - if (__pyx_m) { - if (__pyx_d) { - __Pyx_AddTraceback("init reppy.robots", 0, __pyx_lineno, __pyx_filename); - } - Py_DECREF(__pyx_m); __pyx_m = 0; - } else if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ImportError, "init reppy.robots"); - } - __pyx_L0:; - __Pyx_RefNannyFinishContext(); - #if CYTHON_PEP489_MULTI_PHASE_INIT - return (__pyx_m != NULL) ? 0 : -1; - #elif PY_MAJOR_VERSION >= 3 - return __pyx_m; - #else - return; - #endif -} + /* "string.to_py":31 + * + * @cname("__pyx_convert_PyObject_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyObject_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyObject_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + __Pyx_TraceLine(31,0,__PYX_ERR(1, 31, __pyx_L1_error)) + + + /* "string.to_py":37 + * + * @cname("__pyx_convert_PyUnicode_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyUnicode_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyUnicode_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + __Pyx_TraceLine(37,0,__PYX_ERR(1, 37, __pyx_L1_error)) + + + /* "string.to_py":43 + * + * @cname("__pyx_convert_PyStr_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyStr_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyStr_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + __Pyx_TraceLine(43,0,__PYX_ERR(1, 43, __pyx_L1_error)) + + + /* "string.to_py":49 + * + * @cname("__pyx_convert_PyBytes_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyBytes_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyBytes_FromStringAndSize(s.data(), s.size()) + * cdef extern from *: + */ + __Pyx_TraceLine(49,0,__PYX_ERR(1, 49, __pyx_L1_error)) + + + /* "string.to_py":55 + * + * @cname("__pyx_convert_PyByteArray_string_to_py_std__in_string") + * cdef inline object __pyx_convert_PyByteArray_string_to_py_std__in_string(const string& s): # <<<<<<<<<<<<<< + * return __Pyx_PyByteArray_FromStringAndSize(s.data(), s.size()) + * + */ + __Pyx_TraceLine(55,0,__PYX_ERR(1, 55, __pyx_L1_error)) + + + /* "cfunc.to_py":64 + * + * @cname("__Pyx_CFunc_object____object___to_py") + * cdef object __Pyx_CFunc_object____object___to_py(object (*f)(object) ): # <<<<<<<<<<<<<< + * def wrap(object value): + * """wrap(value)""" + */ + __Pyx_TraceLine(64,0,__PYX_ERR(1, 64, __pyx_L1_error)) + + + /* "vector.to_py":60 + * + * @cname("__pyx_convert_vector_to_py_std_3a__3a_string") + * cdef object __pyx_convert_vector_to_py_std_3a__3a_string(vector[X]& v): # <<<<<<<<<<<<<< + * return [v[i] for i in range(v.size())] + * + */ + __Pyx_TraceLine(60,0,__PYX_ERR(1, 60, __pyx_L1_error)) + + __Pyx_TraceReturn(Py_None, 0); + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + if (__pyx_m) { + if (__pyx_d) { + __Pyx_AddTraceback("init reppy.robots", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + Py_CLEAR(__pyx_m); + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init reppy.robots"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #elif PY_MAJOR_VERSION >= 3 + return __pyx_m; + #else + return; + #endif +} /* --- Runtime support code --- */ /* Refnanny */ @@ -8365,9 +8547,9 @@ static int __pyx_pymod_exec_robots(PyObject *__pyx_pyinit_module) static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { PyObject *m = NULL, *p = NULL; void *r = NULL; - m = PyImport_ImportModule((char *)modname); + m = PyImport_ImportModule(modname); if (!m) goto end; - p = PyObject_GetAttrString(m, (char *)"RefNannyAPI"); + p = PyObject_GetAttrString(m, "RefNannyAPI"); if (!p) goto end; r = PyLong_AsVoidPtr(p); end: @@ -8377,6 +8559,20 @@ static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { } #endif +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_getattr)) + return tp->tp_getattr(obj, PyString_AS_STRING(attr_name)); +#endif + return PyObject_GetAttr(obj, attr_name); +} +#endif + /* GetBuiltinName */ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); @@ -8391,6 +8587,30 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { return result; } +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +} +#endif + /* Profile */ #if CYTHON_PROFILE static int __Pyx_TraceSetupAndCall(PyCodeObject** code, @@ -8426,7 +8646,7 @@ static int __Pyx_TraceSetupAndCall(PyCodeObject** code, retval = 1; tstate->tracing++; tstate->use_tracing = 0; - PyErr_Fetch(&type, &value, &traceback); + __Pyx_ErrFetchInState(tstate, &type, &value, &traceback); #if CYTHON_TRACE if (tstate->c_tracefunc) retval = tstate->c_tracefunc(tstate->c_traceobj, *frame, PyTrace_CALL, NULL) == 0; @@ -8437,7 +8657,7 @@ static int __Pyx_TraceSetupAndCall(PyCodeObject** code, (CYTHON_TRACE && tstate->c_tracefunc)); tstate->tracing--; if (retval) { - PyErr_Restore(type, value, traceback); + __Pyx_ErrRestoreInState(tstate, type, value, traceback); return tstate->use_tracing && retval; } else { Py_XDECREF(type); @@ -8484,6 +8704,148 @@ static PyCodeObject *__Pyx_createFrameCodeObject(const char *funcname, const cha } #endif +/* PyCFunctionFastCall */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + int flags = PyCFunction_GET_FLAGS(func); + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { + return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); + } else { + return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); + } +} +#endif + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif +#endif + /* PyObjectCall */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { @@ -8504,26 +8866,158 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg } #endif +/* PyObjectCall2Args */ +static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args, *result = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(function)) { + PyObject *args[2] = {arg1, arg2}; + return __Pyx_PyFunction_FastCall(function, args, 2); + } + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(function)) { + PyObject *args[2] = {arg1, arg2}; + return __Pyx_PyCFunction_FastCall(function, args, 2); + } + #endif + args = PyTuple_New(2); + if (unlikely(!args)) goto done; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + Py_INCREF(function); + result = __Pyx_PyObject_Call(function, args, NULL); + Py_DECREF(args); + Py_DECREF(function); +done: + return result; +} + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallOneArg */ +#if CYTHON_COMPILING_IN_CPYTHON +static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_New(1); + if (unlikely(!args)) return NULL; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif + if (likely(PyCFunction_Check(func))) { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif + } + } + return __Pyx__PyObject_CallOneArg(func, arg); +} +#else +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *result; + PyObject *args = PyTuple_Pack(1, arg); + if (unlikely(!args)) return NULL; + result = __Pyx_PyObject_Call(func, args, NULL); + Py_DECREF(args); + return result; +} +#endif + +/* PyDictVersioning */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + /* GetModuleGlobalName */ -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ PyObject *result; #if !CYTHON_AVOID_BORROWED_REFS +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 + result = _PyDict_GetItem_KnownHash(__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } else if (unlikely(PyErr_Occurred())) { + return NULL; + } +#else result = PyDict_GetItem(__pyx_d, name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) if (likely(result)) { - Py_INCREF(result); - } else { + return __Pyx_NewRef(result); + } +#endif #else result = PyObject_GetItem(__pyx_d, name); - if (!result) { - PyErr_Clear(); -#endif - result = __Pyx_GetBuiltinName(name); + __PYX_UPDATE_DICT_CACHE(__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); } - return result; + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); } /* RaiseArgTupleInvalid */ - static void __Pyx_RaiseArgtupleInvalid( +static void __Pyx_RaiseArgtupleInvalid( const char* func_name, int exact, Py_ssize_t num_min, @@ -8549,7 +9043,7 @@ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { } /* RaiseDoubleKeywords */ - static void __Pyx_RaiseDoubleKeywordsError( +static void __Pyx_RaiseDoubleKeywordsError( const char* func_name, PyObject* kw_name) { @@ -8563,7 +9057,7 @@ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { } /* ParseKeywords */ - static int __Pyx_ParseOptionalKeywords( +static int __Pyx_ParseOptionalKeywords( PyObject *kwds, PyObject **argnames[], PyObject *kwds2, @@ -8665,7 +9159,7 @@ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { } /* ArgTypeTest */ - static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) { if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); @@ -8685,32 +9179,30 @@ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { return 0; } -/* PyErrFetchRestore */ - #if CYTHON_FAST_THREAD_STATE -static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { - PyObject *tmp_type, *tmp_value, *tmp_tb; - tmp_type = tstate->curexc_type; - tmp_value = tstate->curexc_value; - tmp_tb = tstate->curexc_traceback; - tstate->curexc_type = type; - tstate->curexc_value = value; - tstate->curexc_traceback = tb; - Py_XDECREF(tmp_type); - Py_XDECREF(tmp_value); - Py_XDECREF(tmp_tb); -} -static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - *type = tstate->curexc_type; - *value = tstate->curexc_value; - *tb = tstate->curexc_traceback; - tstate->curexc_type = 0; - tstate->curexc_value = 0; - tstate->curexc_traceback = 0; +/* PyObjectCallNoArg */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, NULL, 0); + } +#endif +#ifdef __Pyx_CyFunction_USED + if (likely(PyCFunction_Check(func) || __Pyx_CyFunction_Check(func))) +#else + if (likely(PyCFunction_Check(func))) +#endif + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); + } + } + return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); } #endif /* RaiseException */ - #if PY_MAJOR_VERSION < 3 +#if PY_MAJOR_VERSION < 3 static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, CYTHON_UNUSED PyObject *cause) { __Pyx_PyThreadState_declare @@ -8868,216 +9360,169 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject } #endif -/* PyFunctionFastCall */ - #if CYTHON_FAST_PYCALL -#include "frameobject.h" -static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, - PyObject *globals) { - PyFrameObject *f; - PyThreadState *tstate = __Pyx_PyThreadState_Current; - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) { - return NULL; - } - fastlocals = f->f_localsplus; - for (i = 0; i < na; i++) { - Py_INCREF(*args); - fastlocals[i] = *args++; - } - result = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return result; +/* None */ +static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); } -#if 1 || PY_VERSION_HEX < 0x030600B1 -static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { - PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); - PyObject *globals = PyFunction_GET_GLOBALS(func); - PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *closure; -#if PY_MAJOR_VERSION >= 3 - PyObject *kwdefs; -#endif - PyObject *kwtuple, **k; - PyObject **d; - Py_ssize_t nd; - Py_ssize_t nk; - PyObject *result; - assert(kwargs == NULL || PyDict_Check(kwargs)); - nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { - return NULL; - } - if ( -#if PY_MAJOR_VERSION >= 3 - co->co_kwonlyargcount == 0 && + +/* PyObjectSetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE int __Pyx_PyObject_SetAttrStr(PyObject* obj, PyObject* attr_name, PyObject* value) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_setattro)) + return tp->tp_setattro(obj, attr_name, value); +#if PY_MAJOR_VERSION < 3 + if (likely(tp->tp_setattr)) + return tp->tp_setattr(obj, PyString_AS_STRING(attr_name), value); #endif - likely(kwargs == NULL || nk == 0) && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - if (argdefs == NULL && co->co_argcount == nargs) { - result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); - goto done; - } - else if (nargs == 0 && argdefs != NULL - && co->co_argcount == Py_SIZE(argdefs)) { - /* function called with no arguments, but all parameters have - a default value: use default values as arguments .*/ - args = &PyTuple_GET_ITEM(argdefs, 0); - result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); - goto done; - } - } - if (kwargs != NULL) { - Py_ssize_t pos, i; - kwtuple = PyTuple_New(2 * nk); - if (kwtuple == NULL) { - result = NULL; - goto done; - } - k = &PyTuple_GET_ITEM(kwtuple, 0); - pos = i = 0; - while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { - Py_INCREF(k[i]); - Py_INCREF(k[i+1]); - i += 2; - } - nk = i / 2; - } - else { - kwtuple = NULL; - k = NULL; - } - closure = PyFunction_GET_CLOSURE(func); -#if PY_MAJOR_VERSION >= 3 - kwdefs = PyFunction_GET_KW_DEFAULTS(func); + return PyObject_SetAttr(obj, attr_name, value); +} #endif - if (argdefs != NULL) { - d = &PyTuple_GET_ITEM(argdefs, 0); - nd = Py_SIZE(argdefs); - } - else { - d = NULL; - nd = 0; + +/* UnpackUnboundCMethod */ +static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { + PyObject *method; + method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name); + if (unlikely(!method)) + return -1; + target->method = method; +#if CYTHON_COMPILING_IN_CPYTHON + #if PY_MAJOR_VERSION >= 3 + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + #endif + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); } -#if PY_MAJOR_VERSION >= 3 - result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, kwdefs, closure); -#else - result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, - args, nargs, - k, (int)nk, - d, (int)nd, closure); #endif - Py_XDECREF(kwtuple); -done: - Py_LeaveRecursiveCall(); - return result; + return 0; } -#endif -#endif -/* PyCFunctionFastCall */ - #if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - int flags = PyCFunction_GET_FLAGS(func); - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { - return (*((__Pyx_PyCFunctionFastWithKeywords)meth)) (self, args, nargs, NULL); - } else { - return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs); +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030600B1 +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + if (likely(cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + #if PY_VERSION_HEX >= 0x030700A0 + return (*(__Pyx_PyCFunctionFast)(void*)(PyCFunction)cfunc->func)(self, args, 2); + #else + return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, args, 2, NULL); + #endif + } + #if PY_VERSION_HEX >= 0x030700A0 + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, args, 2, NULL); + #endif } + return __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); } #endif - -/* PyObjectCallMethO */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { - PyObject *self, *result; - PyCFunction cfunc; - cfunc = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) - return NULL; - result = cfunc(self, arg); - Py_LeaveRecursiveCall(); - if (unlikely(!result) && unlikely(!PyErr_Occurred())) { - PyErr_SetString( - PyExc_SystemError, - "NULL result without error in PyObject_Call"); +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + PyObject *args, *result = NULL; + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + args = PyTuple_New(2); + if (unlikely(!args)) goto bad; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = (*(PyCFunctionWithKeywords)(void*)(PyCFunction)cfunc->func)(self, args, NULL); + else + result = (*cfunc->func)(self, args); + } else { + args = PyTuple_New(3); + if (unlikely(!args)) goto bad; + Py_INCREF(self); + PyTuple_SET_ITEM(args, 0, self); + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 1, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 2, arg2); + result = __Pyx_PyObject_Call(cfunc->method, args, NULL); } - return result; -} +#else + args = PyTuple_Pack(3, self, arg1, arg2); + if (unlikely(!args)) goto bad; + result = __Pyx_PyObject_Call(cfunc->method, args, NULL); #endif - -/* PyObjectCallOneArg */ - #if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); +bad: + Py_XDECREF(args); return result; } -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif - if (likely(PyCFunction_Check(func))) { - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif + +/* CallUnboundCMethod1 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg) { + if (likely(cfunc->func)) { + int flag = cfunc->flag; + if (flag == METH_O) { + return (*(cfunc->func))(self, arg); + } else if (PY_VERSION_HEX >= 0x030600B1 && flag == METH_FASTCALL) { + if (PY_VERSION_HEX >= 0x030700A0) { + return (*(__Pyx_PyCFunctionFast)(void*)(PyCFunction)cfunc->func)(self, &arg, 1); + } else { + return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); + } + } else if (PY_VERSION_HEX >= 0x030700A0 && flag == (METH_FASTCALL | METH_KEYWORDS)) { + return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); } } - return __Pyx__PyObject_CallOneArg(func, arg); + return __Pyx__CallUnboundCMethod1(cfunc, self, arg); } +#endif +static PyObject* __Pyx__CallUnboundCMethod1(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg){ + PyObject *args, *result = NULL; + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + args = PyTuple_New(1); + if (unlikely(!args)) goto bad; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 0, arg); + if (cfunc->flag & METH_KEYWORDS) + result = (*(PyCFunctionWithKeywords)(void*)(PyCFunction)cfunc->func)(self, args, NULL); + else + result = (*cfunc->func)(self, args); + } else { + args = PyTuple_New(2); + if (unlikely(!args)) goto bad; + Py_INCREF(self); + PyTuple_SET_ITEM(args, 0, self); + Py_INCREF(arg); + PyTuple_SET_ITEM(args, 1, arg); + result = __Pyx_PyObject_Call(cfunc->method, args, NULL); + } #else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); + args = PyTuple_Pack(2, self, arg); + if (unlikely(!args)) goto bad; + result = __Pyx_PyObject_Call(cfunc->method, args, NULL); +#endif +bad: + Py_XDECREF(args); return result; } -#endif -/* None */ - static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { - PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); +/* py_dict_pop */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_Pop(PyObject *d, PyObject *key, PyObject *default_value) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX > 0x030600B3 + if ((1)) { + return _PyDict_Pop(d, key, default_value); + } else +#endif + if (default_value) { + return __Pyx_CallUnboundCMethod2(&__pyx_umethod_PyDict_Type_pop, d, key, default_value); + } else { + return __Pyx_CallUnboundCMethod1(&__pyx_umethod_PyDict_Type_pop, d, key); + } } /* FetchCommonType */ - static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { PyObject* fake_module; PyTypeObject* cached_type = NULL; fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); @@ -9116,7 +9561,8 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObjec } /* CythonFunction */ - static PyObject * +#include +static PyObject * __Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) { if (unlikely(op->func_doc == NULL)) { @@ -9137,7 +9583,7 @@ __Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure return op->func_doc; } static int -__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value) +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) { PyObject *tmp = op->func_doc; if (value == NULL) { @@ -9149,7 +9595,7 @@ __Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value) return 0; } static PyObject * -__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { if (unlikely(op->func_name == NULL)) { #if PY_MAJOR_VERSION >= 3 @@ -9164,14 +9610,15 @@ __Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op) return op->func_name; } static int -__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value) +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) { PyObject *tmp; #if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { + if (unlikely(value == NULL || !PyUnicode_Check(value))) #else - if (unlikely(value == NULL || !PyString_Check(value))) { + if (unlikely(value == NULL || !PyString_Check(value))) #endif + { PyErr_SetString(PyExc_TypeError, "__name__ must be set to a string object"); return -1; @@ -9183,20 +9630,21 @@ __Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value) return 0; } static PyObject * -__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { Py_INCREF(op->func_qualname); return op->func_qualname; } static int -__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value) +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) { PyObject *tmp; #if PY_MAJOR_VERSION >= 3 - if (unlikely(value == NULL || !PyUnicode_Check(value))) { + if (unlikely(value == NULL || !PyUnicode_Check(value))) #else - if (unlikely(value == NULL || !PyString_Check(value))) { + if (unlikely(value == NULL || !PyString_Check(value))) #endif + { PyErr_SetString(PyExc_TypeError, "__qualname__ must be set to a string object"); return -1; @@ -9218,7 +9666,7 @@ __Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure return self; } static PyObject * -__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { if (unlikely(op->func_dict == NULL)) { op->func_dict = PyDict_New(); @@ -9229,7 +9677,7 @@ __Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op) return op->func_dict; } static int -__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value) +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) { PyObject *tmp; if (unlikely(value == NULL)) { @@ -9249,19 +9697,19 @@ __Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value) return 0; } static PyObject * -__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { Py_INCREF(op->func_globals); return op->func_globals; } static PyObject * -__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { Py_INCREF(Py_None); return Py_None; } static PyObject * -__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op) +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { PyObject* result = (op->func_code) ? op->func_code : Py_None; Py_INCREF(result); @@ -9290,7 +9738,7 @@ __Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { return result; } static int -__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value) { +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { PyObject* tmp; if (!value) { value = Py_None; @@ -9306,7 +9754,7 @@ __Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value) { return 0; } static PyObject * -__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op) { +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { PyObject* result = op->defaults_tuple; if (unlikely(!result)) { if (op->defaults_getter) { @@ -9320,7 +9768,7 @@ __Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op) { return result; } static int -__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value) { +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { PyObject* tmp; if (!value) { value = Py_None; @@ -9336,7 +9784,7 @@ __Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value) { return 0; } static PyObject * -__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op) { +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { PyObject* result = op->defaults_kwdict; if (unlikely(!result)) { if (op->defaults_getter) { @@ -9350,7 +9798,7 @@ __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op) { return result; } static int -__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value) { +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { PyObject* tmp; if (!value || value == Py_None) { value = NULL; @@ -9366,7 +9814,7 @@ __Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value) { return 0; } static PyObject * -__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op) { +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { PyObject* result = op->func_annotations; if (unlikely(!result)) { result = PyDict_New(); @@ -9546,7 +9994,7 @@ static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, Py return (*meth)(self, arg); break; case METH_VARARGS | METH_KEYWORDS: - return (*(PyCFunctionWithKeywords)meth)(self, arg, kw); + return (*(PyCFunctionWithKeywords)(void*)meth)(self, arg, kw); case METH_NOARGS: if (likely(kw == NULL || PyDict_Size(kw) == 0)) { size = PyTuple_GET_SIZE(arg); @@ -9708,30 +10156,8 @@ static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, Py Py_INCREF(dict); } -/* PyObjectCallNoArg */ - #if CYTHON_COMPILING_IN_CPYTHON -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, NULL, 0); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || __Pyx_TypeCheck(func, __pyx_CyFunctionType))) { -#else - if (likely(PyCFunction_Check(func))) { -#endif - if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { - return __Pyx_PyObject_CallMethO(func, NULL); - } - } - return __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL); -} -#endif - -/* PyIntBinop */ - #if !CYTHON_COMPILING_IN_PYPY -static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { +/* PyIntCompare */ +static CYTHON_INLINE PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED long inplace) { if (op1 == op2) { Py_RETURN_TRUE; } @@ -9739,88 +10165,87 @@ static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED if (likely(PyInt_CheckExact(op1))) { const long b = intval; long a = PyInt_AS_LONG(op1); - if (a == b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } + if (a == b) Py_RETURN_TRUE; else Py_RETURN_FALSE; } #endif #if CYTHON_USE_PYLONG_INTERNALS if (likely(PyLong_CheckExact(op1))) { - const long b = intval; - long a; + int unequal; + unsigned long uintval; + Py_ssize_t size = Py_SIZE(op1); const digit* digits = ((PyLongObject*)op1)->ob_digit; - const Py_ssize_t size = Py_SIZE(op1); - if (likely(__Pyx_sst_abs(size) <= 1)) { - a = likely(size) ? digits[0] : 0; - if (size == -1) a = -a; + if (intval == 0) { + if (size == 0) Py_RETURN_TRUE; else Py_RETURN_FALSE; + } else if (intval < 0) { + if (size >= 0) + Py_RETURN_FALSE; + intval = -intval; + size = -size; } else { - switch (size) { - case -2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 2: - if (8 * sizeof(long) - 1 > 2 * PyLong_SHIFT) { - a = (long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case -3: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - a = -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 3: - if (8 * sizeof(long) - 1 > 3 * PyLong_SHIFT) { - a = (long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case -4: - if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - a = -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - case 4: - if (8 * sizeof(long) - 1 > 4 * PyLong_SHIFT) { - a = (long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0])); - break; - } - #if PyLong_SHIFT < 30 && PyLong_SHIFT != 15 - default: return PyLong_Type.tp_richcompare(op1, op2, Py_EQ); - #else - default: Py_RETURN_FALSE; - #endif - } - } - if (a == b) { - Py_RETURN_TRUE; - } else { + if (size <= 0) Py_RETURN_FALSE; - } + } + uintval = (unsigned long) intval; +#if PyLong_SHIFT * 4 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 4)) { + unequal = (size != 5) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[4] != ((uintval >> (4 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 3 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 3)) { + unequal = (size != 4) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[3] != ((uintval >> (3 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 2 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 2)) { + unequal = (size != 3) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)) | (digits[2] != ((uintval >> (2 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif +#if PyLong_SHIFT * 1 < SIZEOF_LONG*8 + if (uintval >> (PyLong_SHIFT * 1)) { + unequal = (size != 2) || (digits[0] != (uintval & (unsigned long) PyLong_MASK)) + | (digits[1] != ((uintval >> (1 * PyLong_SHIFT)) & (unsigned long) PyLong_MASK)); + } else +#endif + unequal = (size != 1) || (((unsigned long) digits[0]) != (uintval & (unsigned long) PyLong_MASK)); + if (unequal == 0) Py_RETURN_TRUE; else Py_RETURN_FALSE; } #endif if (PyFloat_CheckExact(op1)) { const long b = intval; double a = PyFloat_AS_DOUBLE(op1); - if ((double)a == (double)b) { - Py_RETURN_TRUE; - } else { - Py_RETURN_FALSE; - } + if ((double)a == (double)b) Py_RETURN_TRUE; else Py_RETURN_FALSE; + } + return ( + PyObject_RichCompare(op1, op2, Py_EQ)); +} + +/* GetTopmostException */ +#if CYTHON_USE_EXC_INFO_STACK +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_type == NULL || exc_info->exc_type == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; } - return PyObject_RichCompare(op1, op2, Py_EQ); + return exc_info; } #endif /* SaveResetException */ - #if CYTHON_FAST_THREAD_STATE +#if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - #if PY_VERSION_HEX >= 0x030700A2 - *type = tstate->exc_state.exc_type; - *value = tstate->exc_state.exc_value; - *tb = tstate->exc_state.exc_traceback; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; #else *type = tstate->exc_type; *value = tstate->exc_value; @@ -9832,13 +10257,14 @@ static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject * } static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; - #if PY_VERSION_HEX >= 0x030700A2 - tmp_type = tstate->exc_state.exc_type; - tmp_value = tstate->exc_state.exc_value; - tmp_tb = tstate->exc_state.exc_traceback; - tstate->exc_state.exc_type = type; - tstate->exc_state.exc_value = value; - tstate->exc_state.exc_traceback = tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; #else tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; @@ -9854,11 +10280,12 @@ static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject #endif /* GetException */ - #if CYTHON_FAST_THREAD_STATE -static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) #else -static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) #endif +{ PyObject *local_type, *local_value, *local_tb; #if CYTHON_FAST_THREAD_STATE PyObject *tmp_type, *tmp_value, *tmp_tb; @@ -9891,13 +10318,16 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) *value = local_value; *tb = local_tb; #if CYTHON_FAST_THREAD_STATE - #if PY_VERSION_HEX >= 0x030700A2 - tmp_type = tstate->exc_state.exc_type; - tmp_value = tstate->exc_state.exc_value; - tmp_tb = tstate->exc_state.exc_traceback; - tstate->exc_state.exc_type = local_type; - tstate->exc_state.exc_value = local_value; - tstate->exc_state.exc_traceback = local_tb; + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + } #else tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; @@ -9923,10 +10353,65 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) return -1; } -/* PyErrExceptionMatches */ - #if CYTHON_FAST_THREAD_STATE -static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = a->tp_base; + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +#if PY_MAJOR_VERSION == 2 +static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { + PyObject *exception, *value, *tb; + int res; + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ErrFetch(&exception, &value, &tb); + res = exc_type1 ? PyObject_IsSubclass(err, exc_type1) : 0; + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + if (!res) { + res = PyObject_IsSubclass(err, exc_type2); + if (unlikely(res == -1)) { + PyErr_WriteUnraisable(err); + res = 0; + } + } + __Pyx_ErrRestore(exception, value, tb); + return res; +} +#else +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + int res = exc_type1 ? __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type1) : 0; + if (!res) { + res = __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } + return res; +} +#endif +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); n = PyTuple_GET_SIZE(tuple); #if PY_MAJOR_VERSION >= 3 for (i=0; icurexc_type; - if (exc_type == err) return 1; - if (unlikely(!exc_type)) return 0; - if (unlikely(PyTuple_Check(err))) - return __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); - return __Pyx_PyErr_GivenExceptionMatches(exc_type, err); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject* exc_type) { + if (likely(err == exc_type)) return 1; + if (likely(PyExceptionClass_Check(err))) { + if (likely(PyExceptionClass_Check(exc_type))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, NULL, exc_type); + } else if (likely(PyTuple_Check(exc_type))) { + return __Pyx_PyErr_GivenExceptionMatchesTuple(err, exc_type); + } else { + } + } + return PyErr_GivenExceptionMatches(err, exc_type); +} +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *exc_type1, PyObject *exc_type2) { + assert(PyExceptionClass_Check(exc_type1)); + assert(PyExceptionClass_Check(exc_type2)); + if (likely(err == exc_type1 || err == exc_type2)) return 1; + if (likely(PyExceptionClass_Check(err))) { + return __Pyx_inner_PyErr_GivenExceptionMatches2(err, exc_type1, exc_type2); + } + return (PyErr_GivenExceptionMatches(err, exc_type1) || PyErr_GivenExceptionMatches(err, exc_type2)); } #endif /* WriteUnraisableException */ - static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, +static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, int full_traceback, CYTHON_UNUSED int nogil) { PyObject *old_exc, *old_val, *old_tb; @@ -9990,8 +10495,58 @@ static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tsta #endif } +/* PyObject_GenericGetAttrNoDict */ +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { + PyErr_Format(PyExc_AttributeError, +#if PY_MAJOR_VERSION >= 3 + "'%.50s' object has no attribute '%U'", + tp->tp_name, attr_name); +#else + "'%.50s' object has no attribute '%.400s'", + tp->tp_name, PyString_AS_STRING(attr_name)); +#endif + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { + PyObject *descr; + PyTypeObject *tp = Py_TYPE(obj); + if (unlikely(!PyString_Check(attr_name))) { + return PyObject_GenericGetAttr(obj, attr_name); + } + assert(!tp->tp_dictoffset); + descr = _PyType_Lookup(tp, attr_name); + if (unlikely(!descr)) { + return __Pyx_RaiseGenericGetAttributeError(tp, attr_name); + } + Py_INCREF(descr); + #if PY_MAJOR_VERSION < 3 + if (likely(PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_HAVE_CLASS))) + #endif + { + descrgetfunc f = Py_TYPE(descr)->tp_descr_get; + if (unlikely(f)) { + PyObject *res = f(descr, obj, (PyObject *)tp); + Py_DECREF(descr); + return res; + } + } + return descr; +} +#endif + +/* PyObject_GenericGetAttr */ +#if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 +static PyObject* __Pyx_PyObject_GenericGetAttr(PyObject* obj, PyObject* attr_name) { + if (unlikely(Py_TYPE(obj)->tp_dictoffset)) { + return PyObject_GenericGetAttr(obj, attr_name); + } + return __Pyx_PyObject_GenericGetAttrNoDict(obj, attr_name); +} +#endif + /* SetupReduce */ - static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { +static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { int ret; PyObject *name_attr; name_attr = __Pyx_PyObject_GetAttrStr(meth, __pyx_n_s_name_2); @@ -10067,7 +10622,7 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { } /* Import */ - static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { PyObject *empty_list = 0; PyObject *module = 0; PyObject *global_dict = 0; @@ -10114,7 +10669,7 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { if (!py_level) goto bad; module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, NULL); + name, global_dict, empty_dict, list, py_level, (PyObject *)NULL); Py_DECREF(py_level); #else module = PyImport_ImportModuleLevelObject( @@ -10132,7 +10687,7 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { } /* ImportFrom */ - static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Format(PyExc_ImportError, @@ -10146,14 +10701,14 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { } /* ClassMethod */ - static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { +static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { #if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM <= 0x05080000 if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) { return PyClassMethod_New(method); } #else #if CYTHON_COMPILING_IN_PYSTON || CYTHON_COMPILING_IN_PYPY - if (PyMethodDescr_Check(method)) { + if (PyMethodDescr_Check(method)) #else static PyTypeObject *methoddescr_type = NULL; if (methoddescr_type == NULL) { @@ -10162,8 +10717,9 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { methoddescr_type = Py_TYPE(meth); Py_DECREF(meth); } - if (__Pyx_TypeCheck(method, methoddescr_type)) { + if (__Pyx_TypeCheck(method, methoddescr_type)) #endif + { PyMethodDescrObject *descr = (PyMethodDescrObject *)method; #if PY_VERSION_HEX < 0x03020000 PyTypeObject *d_type = descr->d_type; @@ -10180,7 +10736,7 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { return PyClassMethod_New(method); } #ifdef __Pyx_CyFunction_USED - else if (__Pyx_TypeCheck(method, __pyx_CyFunctionType)) { + else if (__Pyx_CyFunction_Check(method)) { return PyClassMethod_New(method); } #endif @@ -10191,18 +10747,23 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { } /* CLineInTraceback */ - #ifndef CYTHON_CLINE_IN_TRACEBACK -static int __Pyx_CLineForTraceback(CYTHON_UNUSED PyThreadState *tstate, int c_line) { +#ifndef CYTHON_CLINE_IN_TRACEBACK +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { PyObject *use_cline; PyObject *ptype, *pvalue, *ptraceback; #if CYTHON_COMPILING_IN_CPYTHON PyObject **cython_runtime_dict; #endif + if (unlikely(!__pyx_cython_runtime)) { + return c_line; + } __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); #if CYTHON_COMPILING_IN_CPYTHON cython_runtime_dict = _PyObject_GetDictPtr(__pyx_cython_runtime); if (likely(cython_runtime_dict)) { - use_cline = PyDict_GetItem(*cython_runtime_dict, __pyx_n_s_cline_in_traceback); + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_n_s_cline_in_traceback)) } else #endif { @@ -10219,7 +10780,7 @@ static int __Pyx_CLineForTraceback(CYTHON_UNUSED PyThreadState *tstate, int c_li c_line = 0; PyObject_SetAttr(__pyx_cython_runtime, __pyx_n_s_cline_in_traceback, Py_False); } - else if (PyObject_Not(use_cline) != 0) { + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { c_line = 0; } __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); @@ -10228,7 +10789,7 @@ static int __Pyx_CLineForTraceback(CYTHON_UNUSED PyThreadState *tstate, int c_li #endif /* CodeObjectCache */ - static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { return count; @@ -10308,7 +10869,7 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { } /* AddTraceback */ - #include "compile.h" +#include "compile.h" #include "frameobject.h" #include "traceback.h" static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( @@ -10393,8 +10954,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntToPy */ - static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { - const long neg_one = (long) -1, const_zero = (long) 0; +static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { + const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { if (sizeof(long) < sizeof(long)) { @@ -10424,7 +10985,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntFromPyVerify */ - #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) #define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) @@ -10446,8 +11007,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntFromPy */ - static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { - const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; +static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { + const size_t neg_one = (size_t) ((size_t) 0 - (size_t) 1), const_zero = (size_t) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { @@ -10635,8 +11196,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntFromPy */ - static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { - const long neg_one = (long) -1, const_zero = (long) 0; +static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { + const long neg_one = (long) ((long) 0 - (long) 1), const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { @@ -10824,8 +11385,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, } /* CIntFromPy */ - static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { - const int neg_one = (int) -1, const_zero = (int) 0; +static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { + const int neg_one = (int) ((int) 0 - (int) 1), const_zero = (int) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { @@ -11012,80 +11573,8 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, return (int) -1; } -/* FastTypeChecks */ - #if CYTHON_COMPILING_IN_CPYTHON -static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { - while (a) { - a = a->tp_base; - if (a == b) - return 1; - } - return b == &PyBaseObject_Type; -} -static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { - PyObject *mro; - if (a == b) return 1; - mro = a->tp_mro; - if (likely(mro)) { - Py_ssize_t i, n; - n = PyTuple_GET_SIZE(mro); - for (i = 0; i < n; i++) { - if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) - return 1; - } - return 0; - } - return __Pyx_InBases(a, b); -} -#if PY_MAJOR_VERSION == 2 -static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { - PyObject *exception, *value, *tb; - int res; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - __Pyx_ErrFetch(&exception, &value, &tb); - res = exc_type1 ? PyObject_IsSubclass(err, exc_type1) : 0; - if (unlikely(res == -1)) { - PyErr_WriteUnraisable(err); - res = 0; - } - if (!res) { - res = PyObject_IsSubclass(err, exc_type2); - if (unlikely(res == -1)) { - PyErr_WriteUnraisable(err); - res = 0; - } - } - __Pyx_ErrRestore(exception, value, tb); - return res; -} -#else -static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { - int res = exc_type1 ? __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type1) : 0; - if (!res) { - res = __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); - } - return res; -} -#endif -static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject* exc_type) { - if (likely(err == exc_type)) return 1; - if (likely(PyExceptionClass_Check(err))) { - return __Pyx_inner_PyErr_GivenExceptionMatches2(err, NULL, exc_type); - } - return PyErr_GivenExceptionMatches(err, exc_type); -} -static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *exc_type1, PyObject *exc_type2) { - if (likely(err == exc_type1 || err == exc_type2)) return 1; - if (likely(PyExceptionClass_Check(err))) { - return __Pyx_inner_PyErr_GivenExceptionMatches2(err, exc_type1, exc_type2); - } - return (PyErr_GivenExceptionMatches(err, exc_type1) || PyErr_GivenExceptionMatches(err, exc_type2)); -} -#endif - /* CheckBinaryVersion */ - static int __Pyx_check_binary_version(void) { +static int __Pyx_check_binary_version(void) { char ctversion[4], rtversion[4]; PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); @@ -11101,7 +11590,7 @@ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObj } /* InitStrings */ - static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { +static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { #if PY_MAJOR_VERSION < 3 if (t->is_unicode) { @@ -11127,7 +11616,7 @@ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObj if (!*t->p) return -1; if (PyObject_Hash(*t->p) == -1) - PyErr_Clear(); + return -1; ++t; } return 0; @@ -11210,6 +11699,13 @@ static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { if (is_true | (x == Py_False) | (x == Py_None)) return is_true; else return PyObject_IsTrue(x); } +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const char* type_name) { #if PY_MAJOR_VERSION >= 3 if (PyLong_Check(result)) { @@ -11287,7 +11783,7 @@ static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { if (sizeof(Py_ssize_t) >= sizeof(long)) return PyInt_AS_LONG(b); else - return PyInt_AsSsize_t(x); + return PyInt_AsSsize_t(b); } #endif if (likely(PyLong_CheckExact(b))) { @@ -11341,6 +11837,9 @@ static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { Py_DECREF(x); return ival; } +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { return PyInt_FromSize_t(ival); } From e76d41e78ea523d5287e3f68ce5cb5034139a6aa Mon Sep 17 00:00:00 2001 From: Dan Lecocq Date: Wed, 11 Sep 2019 10:18:32 -0600 Subject: [PATCH 111/113] Version bump to 0.4.14 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e19cfb2..f78ab4e 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.13', + version='0.4.14', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like From cb57131799ff2ba7e263a12c82861c8169dc8c70 Mon Sep 17 00:00:00 2001 From: J W <3443995+evil-d20@users.noreply.github.com> Date: Thu, 12 Mar 2020 15:31:33 -0700 Subject: [PATCH 112/113] Version & Contact Info Update. (#123) * Updates to requirements.txt to include all necessary parts & pin versions. * Updated version number for fresh target, & made contact info consistent. --- requirements.txt | 7 +++++++ setup.py | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 17b07a3..df609be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,10 @@ cachetools==3.0.0 requests==2.10.0 six==1.10.0 python-dateutil==2.5.3 +Cython==0.29.14 +mock==4.0.1 +requests_mock==1.7.0 +nose==1.3.7 +colorama==0.4.3 +python-termstyle==0.1.10 +rednose==1.2.1 diff --git a/setup.py b/setup.py index f78ab4e..60eb3a9 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ setup( name='reppy', - version='0.4.14', + version='0.4.16', description='Replacement robots.txt Parser', long_description='''Replaces the built-in robotsparser with a RFC-conformant implementation that supports modern robots.txt constructs like @@ -69,8 +69,8 @@ - Configurable user agent for fetching robots.txt - Automatic refetching based on expiration ''', - maintainer='Lindsey Reno', - maintainer_email='lindsey@moz.com', + maintainer='Moz, Inc.', + maintainer_email='turbo@moz.com', url='http://github.com/seomoz/reppy', license='MIT', platforms='Posix; MacOS X', From 1de42671db9e8aeae90a4cd6634cd929a12cdb21 Mon Sep 17 00:00:00 2001 From: Tim Dufala <5989116+tdufala@users.noreply.github.com> Date: Thu, 11 Jan 2024 20:56:23 -0800 Subject: [PATCH 113/113] Point to new rep-cpp, which points to new url-cpp, fixing A case of not including what we use - relying on other headers to include what intend to include, when what they include should be considered opaque, to us. --- reppy/rep-cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reppy/rep-cpp b/reppy/rep-cpp index 3da0bbb..9c1278b 160000 --- a/reppy/rep-cpp +++ b/reppy/rep-cpp @@ -1 +1 @@ -Subproject commit 3da0bbbb6fb3621ae446359a54dfdd237b22c76c +Subproject commit 9c1278b754a7284304ee263f86adadeb0297b02f