Merge pull request #161 from xmendez/use_raw_data

Use raw data

.travis.yml

@@ -7,6 +7,7 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
+  - "3.7"
 before_install:
   - docker-compose -f tests/server_dir/docker-compose.yml up -d
 install:
@@ -32,3 +33,7 @@ deploy:
       - /^v.*$/
     tags: true
     python: 3.6
+addons:
+  apt:
+    packages:
+    - libcurl4-openssl-dev

docs/user/advanced.rst

@@ -508,15 +508,16 @@ headers.request.<<name>>     Specified HTTP request given header
 headers.response.<<name>>    Specified HTTP response given header
 params.all                   All HTTP request GET and POST parameters
 params.get                   All HTTP request GET parameters
-params.post                  All HTTP request POST parameters
+params.post                  HTTP request POST parameters in returned as a dictionary
+params.raw_post              HTTP request POST parameters payload
 params.get.<<name>>          Spcified HTTP request GET parameter
 params.post.<<name>>         Spcified HTTP request POST parameter
 pstrip                       Returns a signature of the HTTP request using the parameter's names without values (useful for unique operations)
 is_path                      Returns true when the HTTP request path refers to a directory.
 reqtime                      Returns the total time that HTTP request took to be retrieved
 ============================ =============================================
 
-It is worth noting that Wfuzz will try to parse the POST parameters according to the specified content type header. Currently, application/x-www-form-urlencoded, multipart/form-dat and application/json are supported.
+It is worth noting that Wfuzz will try to parse the POST parameters according to the specified content type header. Currently, application/x-www-form-urlencoded, multipart/form-dat and application/json are supported. This is prone to error depending on the data format, raw_post will not try to do any processing.
 
 FuzzRequest URL field is broken in smaller (read only) parts using the urlparse Python's module in the urlp attribute.
 

src/wfuzz/__init__.py

@@ -1,5 +1,5 @@
 __title__ = 'wfuzz'
-__version__ = "2.4"
+__version__ = "2.4.1"
 __build__ = 0x023000
 __author__ = 'Xavier Mendez'
 __license__ = 'GPL 2.0'

src/wfuzz/externals/reqresp/Request.py

@@ -50,7 +50,7 @@ def __init__(self):
         self.multiPOSThead = {}
 
         self.__variablesGET = VariablesSet()
-        self.__variablesPOST = VariablesSet()
+        self._variablesPOST = VariablesSet()
         self._non_parsed_post = None
 
         # diccionario, por ejemplo headers["Cookie"]
@@ -89,7 +89,7 @@ def __init__(self):
     @property
     def method(self):
         if self._method is None:
-            return "POST" if (self.getPOSTVars() or self._non_parsed_post is not None) else "GET"
+            return "POST" if self._non_parsed_post is not None else "GET"
 
         return self._method
 
@@ -148,17 +148,14 @@ def __getattr__(self, name):
         elif name == "path":
             return self.__path
         elif name == "postdata":
-            if self._non_parsed_post is not None:
-                return self._non_parsed_post
-
             if self.ContentType == "application/x-www-form-urlencoded":
-                return self.__variablesPOST.urlEncoded()
+                return self._variablesPOST.urlEncoded()
             elif self.ContentType == "multipart/form-data":
-                return self.__variablesPOST.multipartEncoded()
+                return self._variablesPOST.multipartEncoded()
             elif self.ContentType == 'application/json':
-                return self.__variablesPOST.json_encoded()
+                return self._variablesPOST.json_encoded()
             else:
-                return self.__variablesPOST.urlEncoded()
+                return self._variablesPOST.urlEncoded()
         else:
             raise AttributeError
 
@@ -210,10 +207,10 @@ def existsGETVar(self, key):
         return self.__variablesGET.existsVar(key)
 
     def existPOSTVar(self, key):
-        return self.__variablesPOST.existsVar(key)
+        return self._variablesPOST.existsVar(key)
 
     def setVariablePOST(self, key, value):
-        v = self.__variablesPOST.getVariable(key)
+        v = self._variablesPOST.getVariable(key)
         v.update(value)
 #       self._headers["Content-Length"] = str(len(self.postdata))
 
@@ -225,21 +222,25 @@ def getGETVars(self):
         return self.__variablesGET.variables
 
     def getPOSTVars(self):
-        return self.__variablesPOST.variables
+        return self._variablesPOST.variables
 
     def setPostData(self, pd, boundary=None):
+        self._non_parsed_post = pd
+        self._variablesPOST = VariablesSet()
+
         try:
-            self.__variablesPOST = VariablesSet()
-            if self.ContentType == "application/x-www-form-urlencoded":
-                self.__variablesPOST.parseUrlEncoded(pd)
-            elif self.ContentType == "multipart/form-data":
-                self.__variablesPOST.parseMultipart(pd, boundary)
+            if self.ContentType == "multipart/form-data":
+                self._variablesPOST.parseMultipart(pd, boundary)
             elif self.ContentType == 'application/json':
-                self.__variablesPOST.parse_json_encoded(pd)
+                self._variablesPOST.parse_json_encoded(pd)
             else:
-                self.__variablesPOST.parseUrlEncoded(pd)
+                self._variablesPOST.parseUrlEncoded(pd)
         except Exception:
-            self._non_parsed_post = pd
+            try:
+                self._variablesPOST.parseUrlEncoded(pd)
+            except Exception:
+                print("Warning: POST parameters not parsed")
+                pass
 
 ############################################################################
 
@@ -345,8 +346,8 @@ def to_pycurl_object(c, req):
         else:
             c.setopt(pycurl.CUSTOMREQUEST, req.method)
 
-        if req.getPOSTVars() or req._non_parsed_post is not None:
-            c.setopt(pycurl.POSTFIELDS, python2_3_convert_to_unicode(req.postdata))
+        if req._non_parsed_post is not None:
+            c.setopt(pycurl.POSTFIELDS, python2_3_convert_to_unicode(req._non_parsed_post))
 
         c.setopt(pycurl.FOLLOWLOCATION, 1 if req.followLocation else 0)
 
@@ -393,7 +394,7 @@ def perform(self):
     # ######## ESTE conjunto de funciones no es necesario para el uso habitual de la clase
 
     def getAll(self):
-        pd = self.postdata
+        pd = self._non_parsed_post if self._non_parsed_post else ''
         string = str(self.method) + " " + str(self.pathWithVariables) + " " + str(self.protocol) + "\n"
         for i, j in self._headers.items():
             string += i + ": " + j + "\n"
@@ -421,7 +422,7 @@ def parseRequest(self, rawRequest, prot="http"):
         tp = TextParser()
         tp.setSource("string", rawRequest)
 
-        self.__variablesPOST = VariablesSet()
+        self._variablesPOST = VariablesSet()
         self._headers = {}                # diccionario, por ejemplo headers["Cookie"]
 
         tp.readLine()

src/wfuzz/externals/reqresp/Variables.py

@@ -83,11 +83,11 @@ def parseUrlEncoded(self, cad):
 
         for i in cad.split("&"):
             if i:
-                list = i.split("=", 1)
-                if len(list) == 1:
-                    dicc.append(Variable(list[0], None))
-                elif len(list) == 2:
-                    dicc.append(Variable(list[0], list[1]))
+                var_list = i.split("=", 1)
+                if len(var_list) == 1:
+                    dicc.append(Variable(var_list[0], None))
+                elif len(var_list) == 2:
+                    dicc.append(Variable(var_list[0], var_list[1]))
 
         self.variables = dicc
 

src/wfuzz/fuzzobjects.py

@@ -18,7 +18,7 @@
 
 from .filter import FuzzResFilter
 from .externals.reqresp import Request, Response
-from .exception import FuzzExceptBadAPI, FuzzExceptBadOptions, FuzzExceptInternalError
+from .exception import FuzzExceptBadAPI, FuzzExceptBadOptions, FuzzExceptInternalError, FuzzException
 from .facade import Facade, ERROR_CODE
 from .mixins import FuzzRequestUrlMixing, FuzzRequestSoupMixing
 
@@ -114,19 +114,23 @@ def get(self, values):
 
     @property
     def post(self):
-        if self._req._non_parsed_post is None:
-            return params.param([(x.name, x.value) for x in self._req.getPOSTVars()])
-        else:
-            return self._req.postdata
+        return params.param([(x.name, x.value) for x in self._req.getPOSTVars()])
 
     @post.setter
     def post(self, pp):
         if isinstance(pp, dict):
             for key, value in pp.items():
                 self._req.setVariablePOST(key, str(value) if value is not None else value)
+
+            self._req._non_parsed_post = self._req._variablesPOST.urlEncoded()
+
         elif isinstance(pp, str):
             self._req.setPostData(pp)
 
+    @property
+    def raw_post(self):
+        return self._req._non_parsed_post
+
     @property
     def all(self):
         return params.param(self.get + self.post)
@@ -330,8 +334,8 @@ def update_from_raw_http(self, raw, scheme, raw_response=None, raw_content=None)
         self._request.parseRequest(raw, scheme)
 
         # Parse request sets postdata = '' when there's POST request without data
-        if self.method == "POST" and not self.params.post:
-            self.params.post = {'': None}
+        if self.method == "POST" and self.params.raw_post is None:
+            self.params.post = ''
 
         if raw_response:
             rp = Response()
@@ -394,7 +398,7 @@ def from_copy(self):
         newreq.wf_ip = self.wf_ip
 
         newreq.headers.request = self.headers.request
-        newreq.params.post = self.params.post
+        newreq.params.post = self.params.raw_post
 
         newreq.follow = self.follow
         newreq.auth = self.auth

src/wfuzz/plugins/payloads/wfuzzp.py

@@ -60,4 +60,4 @@ def _gen_wfuzz(self, output_fn):
         except IOError as e:
             raise FuzzExceptBadFile("Error opening wfuzz payload file. %s" % str(e))
         except EOFError:
-            raise StopIteration
+            return

tests/test_acceptance.py

@@ -49,6 +49,8 @@
     ("test_novalue_post_fuzz", "-z list --zD a -u {}/anything -d FUZZ".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --filter r.params.post.a:=1 --field r.params.post.a", ["1"], None),
     ("test_json_post_fuzz2", "-z list --zD anything -u {}/FUZZ -d {{\"a\":\"2\"}} -H Content-Type:application/json".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --field r.params.post.a", ["2"], None),
     ("test_json_post_fuzz3", "-z list --zD anything -u {}/FUZZ -d {{\"a\":\"2\"}} -H Content-Type:application/json".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --filter r.params.post.a:=1 --field r.params.post.a", ["1"], None),
+    ("test_json_nested", "-z list --zD anything -u {}/FUZZ -d {{\"test\":\"me\",\"another\":1,\"nested\":{{\"this\":2}}}} -H Content-Type:application/json".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --field r.params.post.nested.this", [2], None),
+    ("test_json_nested2", "-z list --zD anything -u {}/FUZZ -d {{\"test\":\"me\",\"another\":1,\"nested\":{{\"this\":2}}}} -H Content-Type:application/json".format(HTTPBIN_URL), "-z wfuzzp --zD $$PREVFILE$$ -u FUZZ --field r.params.post.another", [1], None),
 
     # field fuzz values
     ("test_desc_fuzz", "-z range,1-1 {}/FUZZ".format(HTTPBIN_URL), "-z wfuzzp,$$PREVFILE$$ FUZZ", ["http://localhost:9000/1"], None),
@@ -443,7 +445,7 @@ def duplicate_tests(test_list, group, test_gen_fun):
 
 def create_savedsession_tests(test_list, test_gen_fun):
     """
-    generates wfuzz tests that run 2 times with recipe input, expecting same results.
+    generates wfuzz tests that run 2 times with a saved session, expecting same results.
 
     """
     for test_name, prev_cli, next_cli, expected_res, exception_str in test_list: