Fixed sitemap.xml parsing (Fixes #107)

photon.py

@@ -341,7 +341,7 @@ def xmlParser(response):
     return findall(r'<loc>(.*?)</loc>', response)
 
 
-def zap(url):
+def zap(inputUrl):
     """Extract links from robots.txt and sitemap.xml."""
     if args.archive:
         from plugins.wayback import time_machine
@@ -356,7 +356,7 @@ def zap(url):
             verb('Internal page', url)
             internal.add(url)
     # Makes request to robots.txt
-    response = requests.get(url + '/robots.txt', verify=False).text
+    response = requests.get(inputUrl + '/robots.txt', verify=False).text
     # Making sure robots.txt isn't some fancy 404 page
     if '<body' not in response:
         # If you know it, you know it
@@ -376,15 +376,15 @@ def zap(url):
                     robots.add(url)
             print('%s URLs retrieved from robots.txt: %s' % (good, len(robots)))
     # Makes request to sitemap.xml
-    response = requests.get(url + '/sitemap.xml', verify=False).text
+    response = requests.get(inputUrl + '/sitemap.xml', verify=False).text
     # Making sure robots.txt isn't some fancy 404 page
     if '<body' not in response:
         matches = xmlParser(response)
         if matches: # if there are any matches
             print('%s URLs retrieved from sitemap.xml: %s' % (
                 good, len(matches)))
             for match in matches:
-                verb('Internal page', url)
+                verb('Internal page', match)
                 # Cleaning up the URL and adding it to the internal list for
                 # crawling
                 internal.add(match)