diff --git a/README.md b/README.md index 35110c3..09b8e3d 100644 --- a/README.md +++ b/README.md @@ -9,22 +9,33 @@ o888o `8. o88' 888o .8' -* Crawl URL from GHDB. -* Let user defined and saved dorks as per severity/category so dorks can be quickly executed to gather intel -* Generate Report and JSON file of extracted urls -* Require python3 +* Build on python3 +* Fetch exposed URLs using google dorks. +* Support HTML Report to feed results to crawler. +* Support JSON & XML output of extracted urls to feed results to external apis. +* Support grep-able output generation. +* Suport proxies. +* Multithread & optimised for better performance. +* Easy dorks updation. +* Uses huge dork collection. +* Dorks db is updating frequently. +* Easily integrable with other tools. +* Can be easily optimized to generate low noise and fetch quality results. +* Can be used for checking leaked info for tagreted domain. +* Integrate in ecosystem to search for urls for targeted domain and check for any blacklisted/unwanted exposed info. +* Out of the box search queries can be easily extendable to check extra information. ## usage - f0x.py [-h] [-s SITE] [-q QUERY] [-i] [-A EX_QUERY] [-C CATEGORY] [-S {1,2,3,4,5,6,7,8,9,10}] - [--only] [--upper] [-a] [-Q] [-r PAGE_SIZE] [-t DORK_SIZE] [-T MAX_RESULTS] [-m MIN] [-M MAX] - [-d DELAY] [-p PARALLEL] [-U UA] [-o OUTPUT] [-j] [-R] [--update] [-L] [-v] + $f0x.py [-h] [-d DOMAIN] [-q QUERY] [-n] [-Q EX_QUERY] [-c CATEGORY] [-cA] [-S SEVERITY] [-SQ] [-SA] [-t THREADS] [-p PROXY] + [-pF PROXY_FILE] [-pO] [-pC PROXY_COUNT] [-C PROXY_CONN] [--no-ssl-check] [--timeout TIME_OUT] [-m DELAY_MIN] + [-M DELAY_MAX] [-w DELAY] [-U UA] [--update] [-v] [-V] [-r PAGE_SIZE] [-R NO_OF_PAGES] [-T MAX_RESULTS] [-l] [-L] + [-o OUT_DIR] [-oJ] [-oX] [-oR] [--silent] ## example $ python3 f0x.py --update - $ python3 f0x.py --L - $ python3 f0x.py -C 'Files_Containing_Juicy_Info' -o /ghdb/juicyfiles -T 60 -v - $ python3 f0x.py -S 9 -o /ghdb/juicyfiles -T 60 -v + $ python3 f0x.py -L + $ python3 f0x.py --any --quality -v -p "http://10.10.10.10:4444" --no-ssl-check -t 3 diff --git a/f0x.config b/f0x.config new file mode 100644 index 0000000..f1799e7 --- /dev/null +++ b/f0x.config @@ -0,0 +1,4 @@ +[defaults] + +repo_url=https://github.com/dineshsaini/dorks.git +dork_path=~/.f0x/dorks diff --git a/f0x.py b/f0x.py index 2faa190..ddbba99 100755 --- a/f0x.py +++ b/f0x.py @@ -1,753 +1,982 @@ -#!/usr/bin/env pyton3 +#!/usr/bin/env python3 # -*- coding: utf-8 -*- +__NAME__ = "f0x.py" +__VERSION__ = "2.0" + import argparse import re import os -import errno -import urllib.parse as urlparse -import random import time -import mechanize -import requests import json +import threading +import asyncio +import random +from proxybroker import Broker +from concurrent.futures import ThreadPoolExecutor from git import Repo -import shutil + +from lib.prettyPrint import Prettify as pp +from lib.config import ConfigManager as conf +from lib.utils import DirUtil as dutil +from lib.utils import FileUtil as futil +from lib.google import GoogleSearch as gsrch +from lib.utils import Random as rand +from lib.useragents import UA + def banner(): - print (''' - .o88o. .o o. - 888 `" .8' `8. - o888oo .8' oooo ooo `8. - 888 88 `88b..8P' 88 - 888 88 Y888' 88 - 888 `8. .o8"'88b .8' - o888o `8. o88' 888o .8' - - '''); + print(pp.green(' .o88o. .o o.')) + print(pp.green(' 888 `" .8\' `8.')) + print(pp.green(' o888oo .8\' ' + pp.yellow('oooo ooo') + ' `8.')) + print(pp.green(' 888 88 ' + pp.yellow('`88b..8P') + ' 88')) + print(pp.green(' 888 88 ' + pp.yellow('Y888') + ' 88')) + print(pp.green(' 888 `8. ' + pp.yellow('.o8"\'88b') + ' .8\'')) + print(pp.green(' o888o `8. ' + pp.yellow('o88\' 888o') + ' .8\'')) + print() + parser = argparse.ArgumentParser() -parser.add_argument('-s', '--site', help='Specify target site.', dest='site') +# input +parser.add_argument('-d', '--domain', help='Specify Target domain.', + dest='domain') + +parser.add_argument('-q', '--query', help='Specify query/dork manually, and' + + ' don\'t use more dorks from dorks-db.', dest='query') + +parser.add_argument('-n', '--no-stop', help='Works with `--query`, If ' + + 'specified dorks from dorks-db will also be used along ' + + 'with manually supplied dork.', dest='query_nostop', + action="store_true") + +parser.add_argument('-Q', '--query-args', help='Specify extra query to ' + + 'supply with each dorks.', dest='ex_query') + +# dork selection +parser.add_argument('-c', '--category', help='Comma (,) separated dorks ' + + 'categories to use.', dest='category') + +parser.add_argument('-cA', '--any', help='Use all available categories.', + dest='cat_any', action="store_true") + +parser.add_argument('-S', '--severity', help='Comma (,) separated severity ' + + 'range(from 1 to 10). eg. consider range expansion as : ' + + '"-4" will become "1,2,3,4", or ' + + '"2-5" will become "2,3,4,5", or ' + + '"7-" will become "7,8,9,10", or ' + + '"1-4,7" will become "1,2,3,4,7".', dest='severity') + +parser.add_argument('-SQ', '--quality', help='Only use quality dorks ' + + 'i.e. with `severity >= 7`.', dest='sev_quality', + action="store_true") + +parser.add_argument('-SA', '--all', help='Use all available severities.', + dest='sev_all', action="store_true") + +# optimize fox +parser.add_argument('-t', '--threads', help='Max parallel threads ' + + '(Default: 3).', type=int, dest='threads') + +parser.add_argument('-p', '--proxy', help="Specify proxy to use.", dest='proxy') + +parser.add_argument('-pF', '--proxy-file', help='Specify file to read proxies ' + +'list (one per line).', dest='proxy_file') + +parser.add_argument('-pO', '--open-proxies', help='Make use of open public ' + + 'proxies.', dest='proxy_open', action='store_true') + +parser.add_argument('-pC', '--open-proxies-count', help='Try collecting ' + + '`n` open proxies. (Default: 20)', dest='proxy_count', + type=int) + +parser.add_argument('-C', '--conn', help='Max connections per proxy ' + + '(Default: 2).', dest='proxy_conn', type=int) + +parser.add_argument('--no-ssl-check', help='Disable certificate check.', + dest='no_ssl_check', action='store_true') + +parser.add_argument('--timeout', help='Set request timeout.', dest='time_out', + type=int) + +parser.add_argument('-m', '--min-delay', help='Specify minimum delay(sec) ' + + 'between requests. (Default: `max-delay` - 3)s.', + dest='delay_min', type=int) + +parser.add_argument('-M', '--max-delay', help='Specify maximum delay(sec) ' + + 'between requests. (Default: `min-delay` + 3)s.', + dest='delay_max', type=int) + +parser.add_argument('-w', '--wait', help='Specify fix delay(sec) between ' + + 'requests (Default: 1s).', dest='delay', type=int) -parser.add_argument('-q', '--query', help='Dork to use. If specified, other files will not be read.', dest='query') -parser.add_argument('-i', '--inclusive', help='This works with `query` option only, if used, will also read dorks from file. ', dest='inc', action="store_true") -parser.add_argument('-A', '--args', help='Specify extra query to supply with each dorks.', dest='ex_query') +parser.add_argument('-U', '--user-agent', help='Specify User Agent.', dest='ua') -parser.add_argument('-C', '--category', help='Use dorks from this category only.', dest='category') -parser.add_argument('-S', '--severity', help='Specify minimum severity(inclusive) dork file to read, range is [0, 10], defalut: 5.', dest='severity', type=int, choices=range(1, 11)) -parser.add_argument( '--only', help='Use along with severity, to select only a particular value.', dest='s_only', action='store_true') -parser.add_argument( '--upper', help='Use along with severity, to mark provided value as upper limit (exclusive).', dest='s_upper', action='store_true') -parser.add_argument('-a', '--all', help='Use all the dork files to fetch result (overrides --only, --upper flags).', dest='s_all', action='store_true') -parser.add_argument('-Q', '--quality', help='Use only top severity(>=8) dork files (overrides --only, --upper flags). ', dest='s_qual', action='store_true') +parser.add_argument('--update', help='Update dorks repo and exit.', + dest='repo_update', action="store_true") -parser.add_argument('-r', '--results', help='Total results to fetch in one request, default is 30.', dest='page_size', type=int) -parser.add_argument('-t', '--total', help='Total results to fetch for each dork, default is 150.', dest='dork_size', type=int) -parser.add_argument('-T', '--max', help='Maximum results to fetch for all the dorks combined.', dest='max_results', type=int) +parser.add_argument('-v', '--verbose', help='Be verbose.', dest='verbose', + action="store_true") -#parser.add_argument('-P', '--proxy', help='proxy', dest='') -#parser.add_argument('-f', '--proxy-file', help='list of proxies', dest='') -#parser.add_argument('-c', '--conn', help='connections per proxy', dest='') +parser.add_argument('-V', '--version', help='Display version info and exit.', + dest='version', action="store_true") -parser.add_argument('-m', '--mintime', help='Specify minimum sec to wait between requests, If not specified, default 5 sec range is assumed', dest='min', type=int) -parser.add_argument('-M', '--maxtime', help='Specify maximum sec to wait between requests, if not specified, default 5 sec range is assumed.', dest='max', type=int) -parser.add_argument('-d', '--delay', help='Specify fix delay(in sec), if specified, took priority over variable delay.', dest='delay', type=int) -parser.add_argument('-p', '--parallel', help='Specify total no of parallel requests, default is 5.', dest='parallel', type=int) -parser.add_argument('-U', '--user-agent', help='Specify User Agent ', dest='UA') +parser.add_argument('-r', '--results', help='Dork results to fetch in one ' + + 'page request (Default: 30).', dest='page_size', type=int) -parser.add_argument('-o', '--output', help='Specify output directory', dest='output') -parser.add_argument('-j', '--json', help='Save output in JSON format only', dest='json', action="store_true") -parser.add_argument('-R', '--report', help='Create Report along with JSON format ouput, default', dest='report', action="store_true") +parser.add_argument('-R', '--requests', help='Pages to request for each ' + + 'dork (Default: 5).', dest='no_of_pages', type=int) -parser.add_argument('--update', help='Update Dorks Repo, and exit', dest='updaterepo', action="store_true") -parser.add_argument('-L', '--list', help='List Repo categories, total dorks and exit', dest='listrepo', action="store_true") -parser.add_argument('-v', '--verbose', help='Be verbose.', dest='verbose', action="store_true") +parser.add_argument('-T', '--max-results', help='Maximum results to fetch ' + + 'for all the dorks combined.', dest='max_results', type=int) + +# output filters +parser.add_argument('-l', '--list-dorks', help='List all dorks to be used ' + + 'and exit. Specify `category` or `severity` to narrow ' + + 'down the list. ', dest='list_dorks', action="store_true") + +parser.add_argument('-L', '--categories', help='List available categories ' + + 'and exit.', dest='list_cat', action="store_true") + +parser.add_argument('-o', '--outdir', help='Specify output directory.', + dest='out_dir') + +parser.add_argument('-oJ', '--out-json', help='Save output in JSON format.', + dest='out_fmt_json', action="store_true") + +parser.add_argument('-oX', '--out-xml', help='Save output in XML format.', + dest='out_fmt_xml', action="store_true") + +parser.add_argument('-oR', '--out-report', help='Create html report with ' + + 'JSON format results.', dest='out_report', + action="store_true") + +parser.add_argument('--silent', help='Do not print fetched links to stdout, ' + + 'just save them in file.', dest='out_silent', + action='store_true') args = parser.parse_args() +if args.version: + print("{} v{}".format(pp.as_bold(pp.red(__NAME__)), pp.blue(__VERSION__))) + quit() + banner() -if not (args.site or \ - args.query or \ - args.category or \ - args.severity or \ - args.s_all or \ - args.s_qual or \ - args.page_size or \ - args.dork_size or \ - args.max_results or \ - args.min or \ - args.max or \ - args.delay or \ - args.parallel or \ - args.UA or \ - args.output or \ - args.updaterepo or \ - args.listrepo): - print ("[ERROR]: no options are specified\n\n") - print (parser.format_help()) - quit() -#---------------------------------------------------------------------------------- +class F0x: + + def __init__(self, verbose=False): + self._conn_per_proxy_count = None + self._proxy_ptr = -1 + self._out_fmt_json = 1 << 0 + self._out_fmt_xml = 1 << 1 + self._out_report = 1 << 2 + self._outmode = 0 + self._proxy_lock = threading.Lock() + self._count_lock = threading.Lock() + + self.verbose = verbose + self.severities = None + self.categories = None + self.domain = None + self.query = None + self.ex_args = None + self.process_dorksdb_flag = True + self.useragent = None + self.threads = 3 + self.delay_min = 1 + self.delay_max = 1 + self.connection_per_proxy = 2 + self.proxies_list = None + self.request_timeout = None + self.ssl_check = True + self.open_proxy_count = 20 + self.page_size = gsrch.correct_page_size(30) + self.no_of_pages = 5 + self.max_results = None + self.outdir = None + self.out_silent = False + self.results_count = 0 + + self._load_conf() -#read config file -def get_value(key): - with open(getFileName(os.path.dirname(os.path.realpath(__file__)), 'f0x.config'), 'r') as config: - for line in config: - if line.startswith(key): - return line.split('=')[1].strip('\n') + def _load_conf(self): + conf.load('./f0x.config') + if self.is_verbose(): + pp.p_debug("Loaded Config file, keys: {}" + .format(self.get_conf().getKeys())) + + def get_conf(self): + return conf.getConfig() -def getNewDir(o, dn=''): - out_dir = o - if out_dir.endswith('/'): - out_dir += dn - else: - out_dir += '/' + dn + def is_verbose(self): + return self.verbose + + def set_categories(self, categories=None): + self.categories = categories + + def get_categories(self): + return self.categories + + def set_severities(self, severities=None): + self.severities = severities + + def get_severities(self): + return self.severities + + def set_domain(self, domain): + self.domain = domain + + def get_domain(self): + return self.domain + + def set_query(self, query): + self.query = query + + def get_query(self): + return self.query + + def set_ex_args(self, ex_args): + self.ex_args = ex_args + + def get_ex_args(self): + return self.ex_args + + def set_useragent(self, useragent): + self.useragent = useragent + + def get_useragent(self): + return self.useragent + + def set_process_dorksdb_flag(self, flag): + self.process_dorksdb_flag = flag + + def get_process_dorksdb_flag(self): + return self.process_dorksdb_flag + + def set_threads(self, threads): + self.threads = threads + + def get_threads(self): + return self.threads + + def set_delay_min(self, delay_min): + self.delay_min = delay_min + + def get_delay_min(self): + return self.delay_min + + def set_delay_max(self, delay_max): + self.delay_max = delay_max + + def get_delay_max(self): + return self.delay_max + + def set_connection_per_proxy(self, conn): + self.connection_per_proxy = conn + + def get_connection_per_proxy(self): + return self.connection_per_proxy + + def add_proxy(self, proxy): + if not self.proxies_list: + self.proxies_list = [] + self._conn_per_proxy_count = [] + + self.proxies_list += [proxy] + self._conn_per_proxy_count += [0] + + def get_proxy_list(self): + return self.proxies_list + + def set_page_size(self, page_size): + self.page_size = page_size + + def get_page_size(self): + return self.page_size + + def set_no_of_pages(self, no_of_pages): + self.no_of_pages = no_of_pages + + def get_no_of_pages(self): + return self.no_of_pages + + def set_max_results(self, max_results): + self.max_results = max_results + + def get_max_results(self): + return self.max_results + + def set_outdir(self, outdir): + self.outdir = outdir + + def get_outdir(self): + return self.outdir + + def set_outmode_json(self): + self._outmode |= self._out_fmt_json + + def set_outmode_xml(self): + self._outmode |= self._out_fmt_xml + + def set_outmode_report(self): + self._outmode |= self._out_report + + def get_outmode_json(self): + return (self._outmode & self._out_fmt_json) == self._out_fmt_json + + def get_outmode_xml(self): + return (self._outmode & self._out_fmt_xml) == self._out_fmt_xml + + def get_outmode_report(self): + return (self._outmode & self._out_report) == self._out_report + + def set_output_silent(self, flag): + self.out_silent = flag + + def is_output_silent(self): + return self.out_silent + + def set_results_count(self, count): + self.results_count = count + + def get_results_count(self): + return self.results_count + + def set_request_timeout(self, timeout): + self.request_timeout = timeout + + def get_request_timeout(self): + return self.request_timeout + + def set_ssl_check(self, ssl_check): + self.ssl_check = ssl_check + + def do_ssl_check(self): + return self.ssl_check + + def set_open_proxy_count(self, count): + self.open_proxy_count = count + + def get_open_proxy_count(self): + return self.open_proxy_count + + async def _record_proxy(self, proxies): + while True: + proxy = await proxies.get() + if proxy is None: + break + + proto = 'https' if 'HTTPS' in proxy.types else 'http' + proxy_url = '%s://%s:%d' % (proto, proxy.host, proxy.port) + self.add_proxy(proxy_url) + + if fox.is_verbose(): + pp.p_log("Found proxy: {}".format(pp.light_green(proxy_url))) + + def collect_open_proxies(self): + proxies = asyncio.Queue() + broker = Broker(proxies) + tasks = asyncio.gather(broker.find(types=['HTTP', 'HTTPS'], + limit=self.get_open_proxy_count()), + self._record_proxy(proxies)) + + loop = asyncio.get_event_loop() + loop.run_until_complete(tasks) + + def update_dorks_repo(self): + pp.p_log("Building Dork Repo.") + repo_url = self.get_conf().get('repo_url') + pp.p_log("Fetching from '{}'".format(repo_url)) + + tmpdir = dutil.create_temp_dir('f0x', 'repo_') + Repo.clone_from(repo_url, tmpdir) + pp.p_log("Done Fetching.") + + rmdirs = ['.git'] + rmfiles = ['README.md', 'LICENSE'] + + for i in rmdirs: + try: + g = dutil.get_dir(tmpdir, i) + except: + pass + else: + dutil.rmdir(g) + + for i in rmfiles: + try: + f = futil.get_file(tmpdir, i) + except: + pass + else: + os.remove(f) + try: + dutil.merge_dirs(tmpdir, self.get_dork_path()) + except Exception as e: + pp.p_error(e) + quit() + + pp.p_log("Dork Repo updated.") + + def get_dork_path(self): + dp = '' + flag = False - if not os.path.exists(out_dir): try: - os.makedirs(out_dir) - except OSError as e: - if e.errno != errno.EEXIST: - raise - return out_dir - -def getDir(o, dn = ''): - return getNewDir(o, dn) - -def query_encode(query): - return urlparse.quote_plus(query) - -# query, site, extra_query_string -def createURL(q, s, eqs): - u = 'https://www.google.com/search?gbv=1&q=' - if q == '': - print ("Query cannot be empty") - return - u += query_encode(q) - if eqs != '': - u += '+' + query_encode(eqs) - if s != '': - u += '+' + query_encode(s) - u += '&btnG=Google+Search' - return u - -def getSeverities(): - sev = [] - if severity_flag == 0: - sev = list (range (severity, 11)) - elif severity_flag == 1: - sev = [severity] - elif severity_flag == 2: - sev = list (range (1, severity)) #if severity = 1, return empty set - return sev - -def getFiles(f): - l = [] - for j in os.listdir(f): - t = f - if t.endswith('/'): - t += j + dp = self.get_conf().get('dork_path') + except: + pp.p_error("Dorks path not exists.") + flag = True else: - t += '/' + j + if dp == '': + pp.p_error("Dorks path not defined.") + flag = True + + if flag: + raise Exception("Error in Config file.") - if os.path.isfile(t): - l += [t] + if dp.startswith('~'): + dp = os.path.expanduser(dp) + elif dp.startswith('/'): + pass + elif dp.startswith('./'): + cwd = os.path.realpath('.') + dp = dutil.join_names(cwd, dp[2:]) else: - l += getFiles(t) - return l - -def getDirs(f): - l = [] - for j in os.listdir(f): - t = f - if t.endswith('/'): - t += j + cwd = os.path.realpath('.') + dp = dutil.join_names(cwd, dp) + + return dp + + def list_repo_categories(self): + dp = None + try: + dp = self.get_dork_path() + except Exception as e: + pp.p_error(e) + quit() + + dl = dutil.get_dir_list(dp, True) + if len(dl) == 0: + pp.p_log("No Dorks available, Update dork repo.") + return + + for i in dl: + dc = re.sub('^{}[/]?'.format(dp), '', i) + dc = re.sub('/', '.', dc) + td = len(dutil.get_files_list(i, True)) + pp.p_log("Category: {}".format(dc)) + pp.p_log("Total Dorks: {}\n".format(td), '**') + + def list_dorks(self): + cat = None + sev = None + + if self.get_categories(): + cat = self.get_categories() else: - t += '/' + j - - if os.path.isdir(t): - l += [t] - l += getDirs(t) - return l - -def getDorks(rq, inc, svr, cat): - dorks = [] - if rq: - if svr == 10: - dorks += [rq] - if not inc: + cat = [""] + + if self.get_severities(): + sev = self.get_severities() + else: + sev = range(1, 11) + + for c in cat: + for d in self.get_dorks(c, sev): + pp.p_log(d) + + def get_dorks(self, category, sev_list): + dorks = [] + dpath = None + chome = '' + + if not sev_list or len(sev_list) == 0: return dorks + + try: + dpath = self.get_dork_path() + except Exception as e: + pp.p_error(e) + return [] + + if category: + chome = re.sub('\.', '/', category) + + dpath = dutil.get_dir(dpath, chome) + + for i in dutil.get_files_list(dpath, True): + with open (i, 'r') as dfile: + d = None + j = None + for l in dfile: + if l.lstrip().lower().startswith('googledork:'): + d = re.sub('^googledork:', '', l.lstrip().lower()) + d = d.strip() + elif l.lstrip().lower().startswith('severity:'): + j = re.sub('^severity:', '', l.lstrip().lower()) + j = j.strip() + elif (not d) and l.lstrip().lower().startswith('dork:'): + d = re.sub('^dork:', '', l.lstrip().lower()) + d = d.strip() + + if d and j: + break + + if j and int(j) in sev_list and d: + dorks.append(d) + + return dorks - dpath = get_value('dork_path') - chome = '' - - if cat != '': - chome = re.sub('\.', '/', cat) - - dpath = getDir(dpath, chome) - - for i in getFiles(dpath): - with open (i, 'r') as dfile: - d = '' - j = '' - for l in dfile: - if l.lstrip().lower().startswith('dork:'): - d = re.sub('^[dD][oO][rR][kK]:', '', l.lstrip()) - d = d.strip() - elif l.lstrip().lower().startswith('severity:'): - j = re.sub('^severity:', '', l.lstrip().lower()) - j = j.strip() - - if int(j) == svr: - dorks.append(d) - - return dorks - -def getUserAgents(): - uaf = get_value('useragents') - if not uaf.startswith('/'): #relative path - uaf = getFileName(os.path.dirname(os.path.realpath(__file__)), uaf) - - useragents = [] - with open(uaf, 'r') as uas: - useragents = [ua.strip('\n') for ua in uas] - return useragents - - -def wget(u): - hdrs = { - 'Host': 'www.google.com', - 'User-Agent': random.choice(useragents), - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - 'Accept-Encoding': 'gzip, deflate', - 'Referer': 'https://www.google.com/', - 'DNT': '1', - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', - 'Cache-Control': 'max-age=0, no-cache', - 'Pragma': 'no-cache', - 'TE': 'Trailers' - } - req = requests.get(u, headers=hdrs) - return req.text - -def getNewRandomDir(o, dn=''): - return getNewDir(getNewDir(o, dn), str(time.time_ns())) - -def getFileName(o, n): - if o.endswith('/'): - return o + n - else: - return o + '/' + n - -def persist(r, d, s, o, fc): - if fc == 0: - fd = open(getFileName(o, 'dork.info'), 'w') - fd.write("dork: {}\n".format(d)) - fd.write("severity: {}\n".format(s)) - fd.close() - fd = open(getFileName(o, 'dork_' + str(fc + 1)), 'w') - fd.write(r) - fd.close() - -def getDelay(): - return delay + (random.random() * var_delay) - -def pageHasMoreResults(r): - o = re.search('aria-label="Next page"[^>]*>((Next >)|(]*>>))', r, re.I) - if o: - return True - else: - return False - -mr_achived = 0 -def canFetchMore(): - return (max_results - mr_achived) > 0 - -# TODO: make it synchronised later -def updateResultsCount(c): - global mr_achived - mr_achived += c - -# TODO: make it async later -def extractURLs(o, res): - fd = open(getFileName(o, 'urls.txt'), 'a') - - for pat in re.findall('\s+href="/url\?q=([^"&]*)[^"]*"[^>]*>', res, re.M|re.I): - if not re.search('^http(s)?://(www\.)?[^.]*\.google\.com', pat, re.I): - u = urlparse.unquote(pat) - fd.write("{}\n".format(u)) - - fd.close() - -def processDork(d, s, qe, ps, ds, o, sev): - if not canFetchMore(): - return - - u = createURL(d, s, qe) - i = -1 - rFlag = True - dd = getNewRandomDir(o, 'dorks') - r = 0 - print("[*] Processing dork: {}".format(d)) - while rFlag and canFetchMore() and ((ps * (i + 1)) <= ds): - url = '' - i += 1 - if i == 0: - url = "{}&start=&num={}".format(u, ps) + def fetch_page_response(self, dork, pagenum, proxy): + gurl = gsrch.prepare_URL(dork, self.get_domain(), self.get_query(), + pagenum, self.get_page_size()) + + ua = None + if self.get_useragent(): + ua = self.get_useragent() else: - url = "{}&start={}&num={}".format(u, ps * i, ps) - t = getDelay() - print("[*] Sleeping for {} sec.".format(t)) - time.sleep(t) - print("[*] Processing now.") - print("[*] Next Page Request: {}".format(r + 1)) - response = wget(url) - print("[*] Got Response.") - persist(response, d, sev, dd, r) - r += 1 - updateResultsCount(ps) - rFlag = pageHasMoreResults(response) - extractURLs(dd, response) - -# TODO: implement thread -def dbBuilder(): - for s in getSeverities(): - for i in getDorks(r_query, inclusive, s, category): - processDork(i, site, query_extra, page_size, dork_size, out_dir, s) - print("[*] Finished fetching results.") - -def jsonBuilder(o): - for f in os.listdir(o): - i = getFileName(o, f) - - if os.path.isdir(i): - if os.path.isfile(getFileName(i, 'urls.txt')): - l = [] - s = '' - d = '' - - with open(getFileName(i, 'urls.txt'), 'r') as urls: - for u in urls: - l += [u.strip('\n')] - - with open(getFileName(i, 'dork.info'), 'r') as infos: - for line in infos: - if line.startswith('dork: '): - d = re.sub('dork: ', '', line) - d = d.strip('\n') - elif line.startswith('severity: '): - s = re.sub('severity: ', '', line) - s = s.strip('\n') - - fd = open(getFileName(i, 'result.json'), 'w') - - data = { - 'severity' : s, - 'dork' : d, - 'urls' : l - } - - fd.write(json.dumps(data)) - fd.close() - -def buildReportObj(dd): - data = [] - - for i in range(1, 11): - data += [{ - 'severity': i, - 'lists': [] - }] - - for d in os.listdir(dd): - f = getFileName(dd, d) - - if os.path.isdir(f): - if os.path.isfile(getFileName(f, 'result.json')): - jd = {} - - with open(getFileName(f, 'result.json'), 'r') as jfile: - jd = json.load(jfile) + ua = UA.get_random_ua() + + return gsrch.fetch(gurl, ua, proxy, self.get_request_timeout(), + self.do_ssl_check()) + + def save_links(self, dork, dname, pnum, links_list): + if not self.get_outdir(): + for l in links_list: + pp.p_log(l) + return dname + + append = True + if pnum == 1: + append = False + dname = dutil.create_random_dir(self.get_outdir(), dname) + dname = re.sub('^{}[/]?'.format(self.get_outdir()), '', dname) + + futil.dump_list(futil. + join_names(dutil.get_dir(self.get_outdir(), + dname), + "{}.info".format(dname)), + ["dork: {}".format(dork)], append) + + futil.dump_list(futil. + join_names(dutil.get_dir(self.get_outdir(), dname), + "{}.txt".format(dname)), links_list, append) + + if not self.is_output_silent(): + for l in links_list: + pp.p_log(l) - data[int(jd['severity']) - 1]['lists'] += [{ - 'dork': jd['dork'], - 'path': './dorks/' + d + '/result.json', - 'count': len(jd['urls']) - }] - return data - -def reportBuilder(o, d): - of = getFileName(o, 'report.html') - fd = open(of, 'w') - banner = ''' - -''' - - do = buildReportObj(getDir(o, d)) - - css= ''' - .banner{ - font-weight: 600; - } - - .banner-footer { - font-style: italic; - } - - .severity { - font-size: 2.4em; - } - - .label { - font-size: 1.4em; - } - - .label-value { - font-style: italic; - font-weight: 600; - } - - ''' - - fd.write("OSINT Report - [GHDB]".format(css)) - fd.write(banner) - - for i in do: - fd.write('

Severity {}

'.format(i['severity'], i['severity'])) - for j in i['lists']: - fd.write('

Dork Used: {}

'.format(j['dork'])) - fd.write('

URLs Retrived: {}

'.format(j['count'])) - fd.write('

JSON File: {}

'.format(j['path'], j['path'])) - fd.write('
') - fd.write("
") - - fd.write("") - fd.close() - -def getDorkRepoUrl(): - return get_value('repo_url') - -def listStats(): - dp = '' - try: - dp = get_value('dork_path') - except: - pass - if dp == '': - print("[ERROR]: No Dorks Available. Check Config file.") - return - - for i in getDirs(dp): - dc = re.sub('^{}[/]?'.format(dp), '', i) - dc = re.sub('/', '.', dc) - td = len (getFiles(i)) - print("[*] Category: {}".format(dc)) - print("[**] Total Dork: {}\n".format(td)) - -def mergedir(s, d): - for i in os.listdir(s): - if os.path.isfile(getFileName(s, i)): - shutil.move(getFileName(s,i), getFileName(d, i)) - else: - mergedir(getDir(s, i), getDir(d, i)) - -def pullDorksRepo(): - print("[*] Building Dork Repo") - print("[*] Fetching from '{}'".format(getDorkRepoUrl())) - - tmpdir = '/tmp/f0x/' - if os.path.exists(tmpdir): - try: - shutil.rmtree(tmpdir) - except: - tmpdir = getNewRandomDir(tmpdir) + return dname - Repo.clone_from(getDorkRepoUrl(), tmpdir) + def update_results_stats(self, c): + with self._count_lock: + self.set_results_count(self.get_results_count() + c) - print("[*] Fetching done.".format(getDorkRepoUrl())) + def can_fetch_more(self): + if self.get_max_results(): + return self.get_results_count() < self.get_max_results() - g = getDir(tmpdir, '.git') - if os.path.exists(g): + return True + + def get_proxy_object(self): + proxy = {'proxy': None, 'loc': None} + p = None + l = None + + if self.get_proxy_list(): + pl = len(self.get_proxy_list()) + c = 0 + + with self._proxy_lock: + while True: + c += 1 + self._proxy_ptr += 1 + if self._proxy_ptr == pl: + self._proxy_ptr = 0 + + if self._conn_per_proxy_count[self._proxy_ptr] < \ + self.get_connection_per_proxy(): + p = self.get_proxy_list()[self._proxy_ptr] + self._conn_per_proxy_count[self._proxy_ptr] += 1 + l = self._proxy_ptr + break + + if c >= pl: + c = 0 + time.sleep((self.get_delay_min() * + self.get_no_of_pages()) / 4) + + if p: + proxy = {'proxy': {'http': p, 'https': p}, 'loc': l} + return proxy + + def release_proxy(self, proxyobj): + l = proxyobj['loc'] try: - shutil.rmtree(g) + if (l or int(l) == 0) and self._conn_per_proxy_count[l] != 0: + self._conn_per_proxy_count[l] -= 1 except: pass - - r = getFileName(tmpdir, 'README.md') - if os.path.isfile(r): - os.remove(r) - - l = getFileName(tmpdir, 'LICENSE') - if os.path.isfile(l): - os.remove(l) - mergedir(tmpdir, getDir(get_value('dork_path'))) + def process_dork(self, dork): + if self.is_verbose(): + pp.p_debug("Processing dork: {}".format(dork)) + + dname = "dork{}".format(int(random.random() * 1000)) + proxy = self.get_proxy_object() - print("[*] Dork Repo Updated, dork location: {}".format(get_value('dork_path'))) + for p in range(1, self.get_no_of_pages() + 1): + if not self.can_fetch_more(): + break + + time.sleep(rand.rand_between(self.get_delay_min(), + self.get_delay_max())) + response = None + try: + response = self.fetch_page_response(dork, p, proxy['proxy']) + except Exception as e: + self.release_proxy(proxy) + gsrch.session_cleanup() + pp.p_error(e) + return + + if self.is_verbose(): + pp.p_debug("Fetched page : {}".format(p)) + + links = gsrch.extract_urls(response) + if self.is_verbose(): + pp.p_debug("Found {} url(s).".format(len(links))) + + dname = self.save_links(dork, dname, p, links) + + self.update_results_stats(len(links)) + + if not gsrch.has_next(response): + break + + self.release_proxy(proxy) + gsrch.session_cleanup() + + def execute(self): + dorks = [] + if self.get_query(): + dorks += [self.get_query()] + + if self.get_process_dorksdb_flag(): + cat = [] + if self.get_categories(): + cat = self.get_categories() + + for c in cat: + dorks += self.get_dorks(c, self.get_severities()) + + if self.is_verbose(): + pp.p_debug("{} dorks to fetch.".format(len(dorks))) + + with ThreadPoolExecutor(max_workers=self.get_threads()) as exec: + exec.map(self.process_dork, dorks) + + self.make_report() + + def make_report(self): + if not self.get_outdir(): + return + + if not (self.get_outmode_json() or self.get_outmode_report() or + self.get_outmode_xml()): + return + + fdr = None + if self.get_outmode_report(): + fdr = open(futil.join_names(self.get_outdir(), 'index.html') , 'w') + fdr.write(' f0x Report: links' + + '') + + for ddir in dutil.get_dir_list(self.get_outdir()): + dname = re.sub('^{}[/]?'.format(self.get_outdir()), '', ddir) -def buildConfFile(cpath): - r = '' - d = '' - u = '' - try: - r = get_value('repo_url') - except: - pass + links = futil.get_file_aslist( + futil.get_file(ddir, "{}.txt".format(dname))) + + if self.get_outmode_json(): + with open(futil.join_names(ddir, '{}.json'.format(dname)), + 'w') as fd: + fd.write(json.dumps({'urls': links})) + + try: + fdh = None + fdx = None + + if self.get_outmode_report(): + fdh = open(futil.join_names(ddir, '{}.html'.format(dname)), + 'w') + + if self.get_outmode_xml(): + fdx = open(futil.join_names(ddir, '{}.xml'.format(dname)), + 'w') + + except Exception as e: + pp.p_error(e) + else: + if fdh: + fdh.write(' dork urls' + + '') + if fdx: + fdx.write('') + + for link in links: + if fdh: + fdh.write('{}
'.format(link, link)) + + if fdx: + fdx.write('{}'.format(link)) + + if fdx: + fdx.write('') + if fdh: + fdh.write('') + + if fdr: + fdr.write(('dork: {} urls ' + + 'fetched: {}
').format(dname, dname, dname, + len(links))) + + finally: + if fdh: + fdh.close() + if fdx: + fdx.close() + + if fdr: + fdr.write('') + fdr.close() - try: - d = get_value('dork_path') - except: - pass + +fox = F0x(verbose=args.verbose) - try: - u = get_value('useragents') - except: - pass +if args.repo_update: + fox.update_dorks_repo() + quit() - fd = open(cpath, 'w') - if r == '': - fd.write('repo_url={}\n'.format('https://github.com/em-corp/dorks.git')) - else: - fd.write('repo_url={}\n'.format(r)) - if d == '': - fd.write('dork_path={}\n'.format(getDir(os.path.expanduser('~'), '.f0x/dorks'))) - else: - fd.write('dork_path={}\n'.format(d)) - if u == '': - fd.write('useragents=./user-agents\n') - else: - fd.write('useragents={}\n'.format(u)) - fd.close() +if args.list_cat: + fox.list_repo_categories() + quit() -def configure(): - cpath = getFileName(os.path.dirname(os.path.realpath(__file__)), 'f0x.config') +flag_dork_selector = False + +if args.severity: + flag_dork_selector = True + s = [] + l = "1" + m = "10" + + if re.search("[^0-9,-]", args.severity): + pp.p_error("Severity value can only contains numbers or numeral " + + "range, separated by comma (,).") + quit() + + for i in args.severity.split(','): + j = i + if i.startswith('-'): + j = l + i + elif i.endswith('-'): + j = i + m + + k = j.split('-') + for x in range(int(k[0]), int(k[-1]) + 1): + s += [x] + + s = list(set(s)) + fox.set_severities(s) - if not os.path.isfile(cpath): - print ("[*] Creating Configuration file.") - buildConfFile(cpath) - print('[*] Done.') +if args.sev_all: + flag_dork_selector = True + + s = [] + for x in range(1, 11): + s += [x] + + fox.set_severities(s) - if get_value('repo_url') == '' or \ - get_value('dork_path') == '' or \ - get_value('useragents') == '': - print("[*] Error Reading Conf file.") - print("[*] Creating new Configuration file.") - buildConfFile(cpath) - print('[*] Done.') +if args.severity and args.sev_all and fox.is_verbose(): + pp.p_debug("Provided severity range is overridden by `all` switch.") -#---------------------------------------------------------------------------------- +if args.sev_quality: + flag_dork_selector = True + + s = [] + for x in range(7, 11): + s += [x] + fox.set_severities(s) + +if (args.severity or args.sev_all) and args.sev_quality and fox.is_verbose(): + pp.p_debug("Provided severity (range | `all` switch) is overridden by " + + "`quality` switch.") + +if args.category: + flag_dork_selector = True + fox.set_categories(args.category.split(',')) -verbose = args.verbose +if args.cat_any: + flag_dork_selector = True + fox.set_categories(["."]) -if args.listrepo: - listStats() - quit() +if args.category and args.cat_any and fox.is_verbose(): + pp.p_debug("Provided categories value is overridden by `any` switch.") -configure() +if args.list_dorks: + fox.list_dorks() + quit() + +if args.query: + flag_dork_selector = True + fox.set_query(args.query.strip()) + fox.set_process_dorksdb_flag(args.query_nostop) -if args.updaterepo: - pullDorksRepo() +if not flag_dork_selector: + pp.p_error('Please provide atleast one dork selector from ' + + '`category`, `severity` or `query`.') quit() -site='' -# if site provided -if args.site: - site = args.site.strip() - site = re.sub(r'^http(s)?://(www\.)?', '', site) - site = re.sub('/.*(/)?', '', site) +if args.domain: + domain = args.domain.strip() + domain = re.sub(r'^http(s)?://(www\.)?', '', domain) + domain = re.sub('/.*(/)?', '', domain) -if verbose and args.site: - print ("[DEBUG]: Target recieved ==> {}".format(site)) + fox.set_domain(domain) -query_extra = '' if args.ex_query: - query_extra = args.ex_query.strip() + fox.set_ex_args(args.ex_query.strip()) -if verbose and args.ex_query: - print ("[DEBUG]: Extra query parameters to use ==> {}".format(query_extra)) +if args.ua: + fox.set_useragent(args.ua.strip()) -r_query='' -# if raw query provided -if args.query: - r_query = args.query.strip() +if args.threads: + if args.threads > 0: + fox.set_threads(args.threads) + else: + pp.p_error("Please provide some +ve value for threads.") + quit() -if verbose and args.query: - print ("[DEBUG]: Query provided ==> {}".format(r_query)) +if args.delay: + if args.delay > 0: + fox.set_delay_min(args.delay) + fox.set_delay_max(args.delay) + else: + pp.p_error("Please provide some +ve value for delay.") + quit() -inclusive = args.inc +if args.delay_min: + if args.delay_min > 0: + fox.set_delay_min(args.delay_min) + fox.set_delay_max(args.delay_min + 3) + else: + pp.p_error("Please provide some +ve value for delay_min.") + quit() + +if args.delay_max: + if args.delay_max > 0: + fox.set_delay_max(args.delay_max) + if args.delay_max - 3 > 0: + fox.set_delay_min(args.delay_max - 3) + else: + fox.set_delay_min(0) + else: + pp.p_error("Please provide some +ve value for delay_max.") + quit() + +if args.delay_min and args.delay_max: + fox.set_delay_min(args.delay_min) + fox.set_delay_max(args.delay_max) -if inclusive and not args.query: - print ("[ERROR]: Query not found, but inclusive switch is on") +if args.page_size: + if args.page_size <= 0: + pp.p_error("Please provide some +ve value for `dork results`.") + quit() + fox.set_page_size(gsrch.correct_page_size(args.page_size)) + +if args.no_of_pages: + if args.no_of_pages <= 0: + pp.p_error("Please provide some +ve value for `pages to request`.") + quit() + fox.set_no_of_pages(int(args.no_of_pages)) + +if args.max_results: + if args.max_results <= 0: + pp.p_error("Please provide some +ve value for `max results`.") + quit() + fox.set_max_results(args.max_results) + +if ((args.out_fmt_json or args.out_fmt_xml or args.out_report) and + not args.out_dir): + pp.p_error("Output format is defined without specifying output directory.") quit() -if verbose and inclusive: - print ("[DEBUG]: Including dorks results along with query results") +if args.out_silent and not args.out_dir: + pp.p_warn("Can't silent links output, as no output directory defined " + + "to save them.") -category = '' -if args.category: - category = args.category.strip() +if args.out_silent: + fox.set_output_silent(True) -if verbose and category: - print ("[DEBUG]: Category recieved ==> {}".format(category)) +if args.out_dir: + fox.set_outdir(args.out_dir.strip()) -severity = 5 -if args.severity: - severity = args.severity - -if verbose and args.severity: - print ("[DEBUG]: Using severity ==> {}".format(severity)) - -severity_flag = 0 -# 0 for >= severity -# 1 for = severity -# 2 for < severity -if args.s_only: - severity_flag = 1 -if args.s_upper: - severity_flag = 2 - -if args.s_all: - severity = 0 - severity_flag = 0 - -if verbose and args.s_all: - if args.severity: - print ("[DEBUG]: Severity is overridden by `--all` switch") - print ("[DEBUG]: Using severity ==> {}".format(severity)) - -if args.s_qual: - severity = 8 - severity_flag = 0 - -if verbose and args.s_qual: - if args.severity or args.s_all: - print ("[DEBUG]: Severity is overridden by `--quality` switch") - print ("[DEBUG]: Using severity ==> {}".format(severity)) - -if verbose: - print ("[DEBUG]: Severity flag ==> {}".format(severity_flag)) - -page_size = 30 -if args.page_size: - page_size = args.page_size / 10 - - if page_size >= 10: - page_size = 100 - elif page_size >=5: - page_size = 50 - elif page_size > 0 : - page_size = page_size * 10 +if args.out_fmt_json: + fox.set_outmode_json() + +if args.out_fmt_xml: + fox.set_outmode_xml() + +if args.out_report: + fox.set_outmode_report() + +if args.no_ssl_check: + fox.set_ssl_check(False) + +if args.time_out: + if args.time_out <= 0: + pp.p_error("Please provide some +ve value for `request timeout`.") + quit() + fox.set_request_timeout(args.time_out) + +if args.proxy_conn: + if args.proxy_conn > 0: + fox.set_connection_per_proxy(args.proxy_conn) else: - page_size = 30 - -page_size = int(page_size) - -if verbose: - print ("[DEBUG]: Page size ==> {}".format(page_size)) - -dork_size = 150 -if args.dork_size and args.dork_size >= page_size: - dork_size = args.dork_size - -if verbose: - print ("[DEBUG]: Max results per dork ==> {}".format(dork_size)) - -max_results = dork_size * 100 -if args.max_results and args.max_results >= 0: - max_results = args.max_results - -if verbose: - print ("[DEBUG]: Total results limit to ==> {}".format(max_results)) - -delay = 2 -var_delay = 5 -# total delay will be calculated as delay + (var_delay * random_no(0, 1)) - -if args.delay and args.delay >= 0: - delay = args.delay - var_delay = 0 -else: - if args.min and args.max: - if args.min > 0 and args.max >= args.min: - delay = args.min - var_delay = args.max - args.min - elif args.min > 0: - delay = args.min - elif args.min: - if args.min > 0: - delay = args.min - elif args.max: - if args.max >= var_delay : - delay = args.max - var_delay - elif args.max >= 0: - delay = 0 - var_delay = args.max - -if verbose: - print ("[DEBUG]: Delay between each request ==> [{}, {}] sec".format(delay, delay + var_delay)) - -parallel_req = 5 -if args.parallel and args.parallel > 0: - parallel_req = args.parallel - -if verbose: - print ("[DEBUG]: Parallel requests set to ==> {}".format(parallel_req)) - - -useragents = [] -if args.UA: - useragents = args.UA.strip().split(',') -else: -# with open(get_value('useragents'), 'r') as uas: - # useragents = [ua.strip('\n') for ua in uas] - useragents = getUserAgents() - -if verbose: - print ("[DEBUG]: Using User-Agents ==> {}".format(useragents)) - -out_dir = '' - -if args.output: - out_dir = getNewDir(args.output) -else: - print ("[ERROR]: Output directory is not specified") - quit() - -buildReport = True -if args.json and not args.report: - buildReport = False + pp.p_error("Please provide some +ve value for connection per proxy.") -if verbose: - print ("[DEBUG]: Using output directory ==> {}".format(out_dir)) - if not buildReport: - print ("[DEBUG]: Output will be saved in JSON format") - else: - print ("[DEBUG]: Reporting is enabled, along with JSON format") +if args.proxy_open and (args.proxy or args.proxy_file): + pp.p_error("Please use only one option from `open proxies` or " + + "(proxy and proxy_file).") + quit() + +if args.proxy and args.proxy_file: + pp.p_error("Please use only one option from proxy or proxy_file.") + quit() -#---------------------------------------------------------------------------------- +if args.proxy: + fox.add_proxy(args.proxy.strip()) -print("[*] Building db.") -dbBuilder() -print("[*] Finished building db.") +if args.proxy_file: + for i in futil.get_file_aslist(args.proxy_file): + fox.add_proxy(i) -print("[*] Building JSON.") -jsonBuilder(getDir(out_dir, 'dorks')) -print("[*] Finished building JSON.") +if args.proxy_count: + if args.proxy_count <= 0: + pp.p_error("Please provide some +ve value for `open proxy count`.") + quit() -if buildReport: - print("[*] Building Report.") - reportBuilder(out_dir, 'dorks') - print ("[*] Report saved at: {}".format(getFileName(out_dir, 'report.html'))) + fox.set_open_proxy_count(args.proxy_count) + + if fox.is_verbose() and not args.proxy_open: + pp.p_info('Ignoring `open proxy count` as provided without ' + + 'enabling `open proxies` switch.') + +if args.proxy_open: + fox.collect_open_proxies() -print ("[*] Results saved at {}".format(out_dir)) +fox.execute() diff --git a/lib/config.py b/lib/config.py new file mode 100644 index 0000000..7b4f7de --- /dev/null +++ b/lib/config.py @@ -0,0 +1,44 @@ +__all__ = [] + +import configparser + + +class ConfigManager: + config = None + + def getConfig(): + if not ConfigManager.config: + ConfigManager.config = ConfigManager.Configuration() + return ConfigManager.config + + def load(cfile): + conf = configparser.ConfigParser() + conf.read(cfile) + + for s in conf.sections(): + for k in conf[s]: + ConfigManager.getConfig().set(k, conf[s].get(k)) + + return ConfigManager.getConfig() + + class Configuration(): + + def __init__(self): + self.properties = {} + + def get(self, key): + if key in self.properties.keys(): + return self.properties[key] + else: + raise self.NoSuchKeyException("No such key `{}` found" + .format(key)) + + def set(self, key, value): + self.properties[key] = value + + def getKeys(self): + return self.properties.keys() + + class NoSuchKeyException(Exception): + def __init__(self, message): + super().__init__(message) diff --git a/lib/google.py b/lib/google.py new file mode 100644 index 0000000..6e424bb --- /dev/null +++ b/lib/google.py @@ -0,0 +1,116 @@ +__all__ = [] + +import urllib.parse as urlparse +import requests +import threading +import re + + +class GoogleSearch: + _session_var = threading.local() + + def _init_session(): + if not hasattr(GoogleSearch._session_var, "session"): + GoogleSearch._session_var.session = requests.Session() + + def session_cleanup(): + if hasattr(GoogleSearch._session_var, "session"): + delattr(GoogleSearch._session_var, "session") + + def _get_session(): + GoogleSearch._init_session() + return GoogleSearch._session_var.session + + def __qencode__(q): + return urlparse.quote_plus(q) + + def correct_page_size(page_size): + default = 30 + if not page_size: + return default + + try: + page_size = int(page_size) + except: + return default + + page_size /= 10 + # consider lower bound + if page_size >= 10: + page_size = 100 + elif page_size >= 5: + page_size = 50 + elif page_size > 0: + page_size = page_size * 10 + else: + page_size = default + + return int(page_size) + + def prepare_URL(query, site='', params='', page_num=1, page_size=30): + if not query or query == '': + raise Exception("Query cannot be empty") + + if not site: + site = '' + if not params: + params = '' + + u = 'https://www.google.com/search?gbv=1&q=' + u += GoogleSearch.__qencode__(query) + if params != '': + u += '+' + GoogleSearch.__qencode__(params) + if site != '': + u += '+' + GoogleSearch.__qencode__(site) + u += '&btnG=Google+Search' + + page_size = GoogleSearch.correct_page_size(page_size) + + fmt = '{}&start={}&num={}' + page_num = int(page_num) + + if page_num <= 1: + return fmt.format(u, '', page_size) + else: + return fmt.format(u, (page_num - 1) * page_size, page_size) + + def fetch(url, UA='f0x.py (Linux; python/requests)', proxy=None, + time_out=None, sslcheck=True): + hdrs = { + 'Host': 'www.google.com', + 'User-Agent': UA, + 'Accept': 'text/html,application/xhtml+xml,application/xml;' + + ' q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'Accept-Encoding': 'gzip, deflate', + 'Referer': 'https://www.google.com/', + 'DNT': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Cache-Control': 'max-age=0, no-cache', + 'Pragma': 'no-cache', + 'TE': 'Trailers' + } + + return GoogleSearch._get_session().get(url, headers=hdrs, + proxies=proxy, timeout=time_out, + verify=sslcheck).text + + def has_next(text): + o = re.search('aria-label="Next page"[^>]*>((Next >)|' + + '(]*>>))', text, re.I) + if o: + return True + else: + return False + + def extract_urls(text): + urls = [] + for pat in re.findall('\s+href="/url\?q=([^"&]*)[^"]*"[^>]*>', text, + re.M | re.I): + if not re.search('^http(s)?://(www\.)?[^.]*\.google\.com', pat, + re.I): + up = urlparse.unquote(pat) + if not up.startswith('/search?q='): + urls += [up] + return urls diff --git a/lib/prettyPrint.py b/lib/prettyPrint.py new file mode 100644 index 0000000..2882b52 --- /dev/null +++ b/lib/prettyPrint.py @@ -0,0 +1,161 @@ +__all__ = [] + +from colorama import Fore as f +from colorama import Back as b +from colorama import Style as s + + +class Prettify: + + def __cont__(t, r, c): + return t.replace(r, r + c) + + # *** text *** + def red(text): + return f.RED + Prettify.__cont__(text, f.RESET, f.RED) + f.RESET + + def black(text): + return f.BLACK + Prettify.__cont__(text, f.RESET, f.BLACK) + f.RESET + + def blue(text): + return f.BLUE + Prettify.__cont__(text, f.RESET, f.BLUE) + f.RESET + + def cyan(text): + return f.CYAN + Prettify.__cont__(text, f.RESET, f.CYAN) + f.RESET + + def green(text): + return f.GREEN + Prettify.__cont__(text, f.RESET, f.GREEN) + f.RESET + + def magenta(text): + return f.MAGENTA + Prettify.__cont__(text, f.RESET, f.MAGENTA) + f.RESET + + def white(text): + return f.WHITE + Prettify.__cont__(text, f.RESET, f.WHITE) + f.RESET + + def yellow(text): + return f.YELLOW + Prettify.__cont__(text, f.RESET, f.YELLOW) + f.RESET + + def light_black(text): + return f.LIGHTBLACK_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTBLACK_EX) + f.RESET + + def light_blue(text): + return f.LIGHTBLUE_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTBLUE_EX) + f.RESET + + def light_cyan(text): + return f.LIGHTCYAN_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTCYAN_EX) + f.RESET + + def light_green(text): + return f.LIGHTGREEN_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTGREEN_EX) + f.RESET + + def light_magenta(text): + return f.LIGHTMAGENTA_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTMAGENTA_EX) + \ + f.RESET + + def light_red(text): + return f.LIGHTRED_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTRED_EX) + f.RESET + + def light_white(text): + return f.LIGHTWHITE_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTWHITE_EX) + f.RESET + + def light_yellow(text): + return f.LIGHTYELLOW_EX + Prettify.__cont__(text, f.RESET, + f.LIGHTYELLOW_EX) + f.RESET + + # *** background *** + def on_red(text): + return b.RED + Prettify.__cont__(text, b.RESET, b.RED) + b.RESET + + def on_black(text): + return b.BLACK + Prettify.__cont__(text, b.RESET, b.BLACK) + b.RESET + + def on_blue(text): + return b.BLUE + Prettify.__cont__(text, b.RESET, b.BLUE) + b.RESET + + def on_cyan(text): + return b.CYAN + Prettify.__cont__(text, b.RESET, b.CYAN) + b.RESET + + def on_green(text): + return b.GREEN + Prettify.__cont__(text, b.RESET, b.GREEN) + b.RESET + + def on_magenta(text): + return b.MAGENTA + Prettify.__cont__(text, b.RESET, b.MAGENTA) + b.RESET + + def on_white(text): + return b.WHITE + Prettify.__cont__(text, b.RESET, b.WHITE) + b.RESET + + def on_yellow(text): + return b.YELLOW + Prettify.__cont__(text, b.RESET, b.YELLOW) + b.RESET + + def on_light_black(text): + return b.LIGHTBLACK_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTBLACK_EX) + b.RESET + + def on_light_blue(text): + return b.LIGHTBLUE_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTBLUE_EX) + b.RESET + + def on_light_cyan(text): + return b.LIGHTCYAN_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTCYAN_EX) + b.RESET + + def on_light_green(text): + return b.LIGHTGREEN_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTGREEN_EX) + b.RESET + + def on_light_magenta(text): + return b.LIGHTMAGENTA_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTMAGENTA_EX) + \ + b.RESET + + def on_light_red(text): + return b.LIGHTRED_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTRED_EX) + b.RESET + + def on_light_white(text): + return b.LIGHTWHITE_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTWHITE_EX) + b.RESET + + def on_light_yellow(text): + return b.LIGHTYELLOW_EX + Prettify.__cont__(text, b.RESET, + b.LIGHTYELLOW_EX) + b.RESET + + # *** sytle *** + + def as_bold(text): + return s.BRIGHT + Prettify.__cont__(text, s.RESET_ALL, + s.BRIGHT) + s.RESET_ALL + + def as_dim(text): + return s.DIM + Prettify.__cont__(text, s.RESET_ALL, + s.BRIGHT) + s.RESET_ALL + + def as_normal(text): + return s.NORMAL + Prettify.__cont__(text, s.RESET_ALL, + s.BRIGHT) + s.RESET_ALL + + # *** logger *** + def p_error(text): + print("[{}]: {}".format(Prettify.as_bold(Prettify.red('ERROR')), text)) + + def p_info(text): + print("[{}]: {}".format(Prettify.cyan('INFO'), text)) + + def p_warn(text): + print("[{}]: {}".format(Prettify.red('WARN'), text)) + + def p_debug(text): + print("[{}]: {}".format(Prettify.as_bold(Prettify.yellow('DEBUG')), + text)) + + def p_log(text, header='*'): + print("[{}]: {}".format(Prettify.as_bold(Prettify.blue(header)), text)) + + def p_clog(text, header): + print("[{}]: {}".format(header, text)) diff --git a/lib/useragents.py b/lib/useragents.py new file mode 100644 index 0000000..78ae637 --- /dev/null +++ b/lib/useragents.py @@ -0,0 +1,151 @@ +__all__ = [] + +import random + + +class UA: + __ualist__ = [ + 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.' + + '84 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + + 'Chrome/60.0.3112.107 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 7.0; SM-G930VC Build/NRD90M; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + + 'Chrome/58.0.3029.83 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; SM-G935S Build/MMB29K; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + + 'Chrome/55.0.2883.91 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; SM-G920V Build/MMB29K) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.' + + '98 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 5.1.1; SM-G928X Build/LMY47X) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.' + + '83 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 6P Build/MMB29P) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.' + + '83 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 7.1.1; G8231 Build/41.2.A.0.219; wv)' + + ' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + + 'Chrome/59.0.3071.125 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; E6653 Build/32.2.A.0.253) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.' + + '98 Mobile Safari/537.36', + 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; ' + + 'Trident/5.0)', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 ' + + 'Firefox/15.0.1', + 'Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.' + + '98 Mobile Safari/537.3', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) ' + + 'AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 ' + + 'Mobile/15E148 Safari/604.1', + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FSL 7.0.7.' + + '01001)', + 'Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.02', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) ' + + 'AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/' + + '69.0.3497.105 Mobile/15E148 Safari/605.1', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) ' + + 'AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/13.' + + '2b11866 Mobile/16A366 Safari/605.1.15', + 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; ' + + '.NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET ' + + 'CLR 3.5.30729)', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) ' + + 'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/' + + '11.0 Mobile/15A372 Safari/604.1', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) ' + + 'AppleWebKit/604.1.34 (KHTML, like Gecko) Version/' + + '11.0 Mobile/15A5341f Safari/604.1', + 'Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; ' + + 'RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/' + + '52.0.2743.116 Mobile Safari/537.36 Edge/15.15254', + 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; ' + + 'RM-1127_16056) AppleWebKit/537.36(KHTML, like Gecko) ' + + 'Chrome/42.0.2311.135 Mobile Safari/537.36 Edge/12.10536', + 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like ' + + 'Gecko) Chrome/13.0.782.112 Safari/535.1', + 'Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; ' + + 'Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/13.1058', + 'Mozilla/5.0 (Linux; Android 7.0; SM-T827R4 Build/NRD90M) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.' + + '3112.116 Safari/537.36', + 'Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T550 Build/LRX22G)' + + ' AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser' + + '/3.3 Chrome/38.0.2125.102 Safari/537.36', + 'Mozilla/5.0 (Linux; Android 4.4.3; KFTHWI Build/KTU84M) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Silk/47.1.79 like' + + ' Chrome/47.0.2526.80 Safari/537.36', + 'Mozilla/5.0 (Linux; Android 5.0.2; LG-V410/V41020c Build/LRX22G)' + + ' AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + + 'Chrome/34.0.1847.118 Safari/537.36', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 ' + + 'Edge/12.246', + 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36', + 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET ' + + 'CLR 2.0.50727)', + 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 ' + + 'Firefox/12.0', + 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) ' + + 'AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 ' + + 'Mobile/15A5370a Safari/604.1', + 'Mozilla/5.0 (iPhone9,3; U; CPU iPhone OS 10_0_1 like Mac OS X) ' + + 'AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 ' + + 'Mobile/14A403 Safari/602.1', + 'Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/' + + '13.0.1', + 'Mozilla/5.0 (iPhone9,4; U; CPU iPhone OS 10_0_1 like Mac OS X) ' + + 'AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 ' + + 'Mobile/14A403 Safari/602.1', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/' + + '601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, ' + + 'like Gecko) Chrome/47.0.2526.111 Safari/537.36', + 'Mozilla/5.0 (Apple-iPhone7C2/1202.466; U; CPU like Mac OS X; en)' + + ' AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 ' + + 'Mobile/1A543 Safari/419.3', + 'Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36', + 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, ' + + 'like Gecko) Chrome/47.0.2526.111 Safari/537.36', + 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 ' + + 'Firefox/15.0.1', + 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/' + + 'bingbot.htm)', + 'Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/' + + 'help/us/ysearch/slurp)', + 'Mozilla/5.0 (Linux; Android 6.0.1; SGP771 Build/32.2.A.0.253; ' + + 'wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/' + + '52.0.2743.98 Safari/537.36', + 'Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) ' + + 'Opera 7.02 Bork-edition [en]', + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/' + + '601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9', + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' + + '(KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 ' + + 'Edge/12.246', + 'Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) ' + + 'AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 ' + + 'Chrome/52.0.2743.98 Safari/537.36', + 'Mozilla/5.0 (Linux; Android 6.0.1; SHIELD Tablet K1 Build/MRA58K' + + '; wv) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Version/4.0 Chrome/55.0.2883.91 Safari/537.36', + 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET ' + + 'CLR 1.0.3705)', + 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/' + + 'search/spider.html)', + 'Opera/9.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.01' + ] + + def get_ua_list(): + return UA.__ualist__ + + def get_random_ua(): + return random.choice(UA.__ualist__) + \ No newline at end of file diff --git a/lib/utils.py b/lib/utils.py new file mode 100644 index 0000000..84ea661 --- /dev/null +++ b/lib/utils.py @@ -0,0 +1,136 @@ +__all__ = [] + +import os +import time +import errno +import shutil +import random + + +class DirUtil: + + def join_names(parent, child): + o = parent + if o.endswith('/'): + o += child + else: + o += '/' + child + return o + + def create_dir(parent, dname=''): + o = DirUtil.join_names(parent, dname) + if not os.path.exists(o): + try: + os.makedirs(o) + except OSError as err: + if err.errno != errno.EEXIST: + raise + return o + + def get_dir(parent, child=''): + d = DirUtil.join_names(parent, child) + if not os.path.isdir(d): + raise OSError('Directory not exists.') + return d + + def create_random_dir(parent, prefix=''): + return DirUtil.create_dir(parent, prefix + str(int(time.time() * 1000))) + + def create_temp_dir(parent='', prefix=''): + return DirUtil.create_random_dir(DirUtil.create_dir('/tmp/', parent), + prefix) + + def list_dir(parent): + l = [] + for j in os.listdir(parent): + t = DirUtil.join_names(parent, j) + l += [t] + return l + + def get_dir_list(parent, recurse=False): + l = [] + for i in DirUtil.list_dir(parent): + if os.path.isdir(i): + l += [i] + if recurse: + l += DirUtil.get_dir_list(i, recurse) + return l + + def get_files_list(parent, recurse=False): + l = [] + for i in DirUtil.list_dir(parent): + if os.path.isfile(i): + l += [i] + else: + if recurse: + l += DirUtil.get_files_list(i, recurse) + return l + + def merge_dirs(source, dest): + DirUtil.create_dir(dest) + for i in os.listdir(source): + if os.path.isfile(DirUtil.join_names(source, i)): + shutil.move(DirUtil.join_names(source, i), + DirUtil.join_names(dest, i)) + else: + DirUtil.merge_dirs(DirUtil.join_names(source, i), + DirUtil.create_dir(dest, i)) + + def rmdir(dname): + shutil.rmtree(dname) + + +class FileUtil: + + def join_names(parent, fname): + o = parent + if o.endswith('/'): + o += fname + else: + o += '/' + fname + return o + + def get_file(parent, fname): + f = FileUtil.join_names(parent, fname) + if not os.path.isfile(f): + raise OSError('File not exists') + return f + + def create_random_file(parent, prefix=''): + return FileUtil.join_names(parent, prefix + str(int(time.time() * + 1000))) + + def create_temp_file(dname='', prefix=''): + return FileUtil.create_random_file(FileUtil.join_names('/tmp/', dname), + prefix) + + def dump_list(ofile, l, append=True): + if l: + m = 'a' + if not append: + m = 'w' + with open(ofile, m) as f: + for i in l: + f.write("{}\n".format(i)) + + def get_file_aslist(file): + list = [] + + with open(file, 'r') as f: + list = [l.strip() for l in f if l.strip()] + + return list + + +class Random: + + def rand_between(start, end): + if start < 0: + raise Exception('Require positive number') + if end < 0: + raise Exception('Require positive number') + + return start + ((end - start) * random.random()) + + def rand_no(max_no): + return Random.rand_between(0, max_no) diff --git a/user-agents b/user-agents deleted file mode 100644 index 033e5a1..0000000 --- a/user-agents +++ /dev/null @@ -1,53 +0,0 @@ -Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36 -Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/60.0.3112.107 Mobile Safari/537.36 -Mozilla/5.0 (Linux; Android 7.0; SM-G930VC Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/58.0.3029.83 Mobile Safari/537.36 -Mozilla/5.0 (Linux; Android 6.0.1; SM-G935S Build/MMB29K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Mobile Safari/537.36 -Mozilla/5.0 (Linux; Android 6.0.1; SM-G920V Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36 -Mozilla/5.0 (Linux; Android 5.1.1; SM-G928X Build/LMY47X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83 Mobile Safari/537.36 -Mozilla/5.0 (Linux; Android 6.0.1; Nexus 6P Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.83 Mobile Safari/537.36 -Mozilla/5.0 (Linux; Android 7.1.1; G8231 Build/41.2.A.0.219; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/59.0.3071.125 Mobile Safari/537.36 -Mozilla/5.0 (Linux; Android 6.0.1; E6653 Build/32.2.A.0.253) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36 -Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0) -Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1 -Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.3 -Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 -Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; FSL 7.0.7.01001) -Mozilla/5.0 (Windows NT 6.1; rv:5.0) Gecko/20100101 Firefox/5.02 -Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/69.0.3497.105 Mobile/15E148 Safari/605.1 -Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/13.2b11866 Mobile/16A366 Safari/605.1.15 -Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729) -Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 -Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1 -Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254 -Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; RM-1127_16056) AppleWebKit/537.36(KHTML, like Gecko) Chrome/42.0.2311.135 Mobile Safari/537.36 Edge/12.10536 -Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.112 Safari/535.1 -Mozilla/5.0 (Windows Phone 10.0; Android 4.2.1; Microsoft; Lumia 950) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Mobile Safari/537.36 Edge/13.1058 -Mozilla/5.0 (Linux; Android 7.0; SM-T827R4 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.116 Safari/537.36 -Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T550 Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.3 Chrome/38.0.2125.102 Safari/537.36 -Mozilla/5.0 (Linux; Android 4.4.3; KFTHWI Build/KTU84M) AppleWebKit/537.36 (KHTML, like Gecko) Silk/47.1.79 like Chrome/47.0.2526.80 Safari/537.36 -Mozilla/5.0 (Linux; Android 5.0.2; LG-V410/V41020c Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/34.0.1847.118 Safari/537.36 -Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 -Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36 -Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.50727) -Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0 -Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5370a Safari/604.1 -Mozilla/5.0 (iPhone9,3; U; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A403 Safari/602.1 -Mozilla/5.0 (Windows NT 5.1; rv:13.0) Gecko/20100101 Firefox/13.0.1 -Mozilla/5.0 (iPhone9,4; U; CPU iPhone OS 10_0_1 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A403 Safari/602.1 -Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9 -Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36 -Mozilla/5.0 (Apple-iPhone7C2/1202.466; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1A543 Safari/419.3 -Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36 -Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36 -Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1 -Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) -Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) -Mozilla/5.0 (Linux; Android 6.0.1; SGP771 Build/32.2.A.0.253; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36 -Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.0) Opera 7.02 Bork-edition [en] -Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9 -Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 -Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36 -Mozilla/5.0 (Linux; Android 6.0.1; SHIELD Tablet K1 Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/55.0.2883.91 Safari/537.36 -Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.0.3705) -Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html) -Opera/9.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.01