From be06782d7792661481a5c990be95d6c95f4e4141 Mon Sep 17 00:00:00 2001 From: SuperSonic Date: Sun, 1 Dec 2019 02:23:04 +0800 Subject: [PATCH 1/8] Upgrade for Python3 --- README.md | 28 ++-- setup.py | 27 ++-- webkit2png/scripts.py | 68 +++++----- webkit2png/webkit2png.py | 277 +++++++++++++++++++++++---------------- 4 files changed, 229 insertions(+), 171 deletions(-) diff --git a/README.md b/README.md index dc88b84..606a8fb 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,34 @@ -**webkit2png** -============== +# webkit2png -About ------- +## About Python script that takes screenshots (browsershots) using webkit -##Installation -Ubuntu ------- +## Requirement + python >= 3.7 + PyQt >= 5.13 + +## Installation + +### Ubuntu - Add following packages: ``apt-get install python-qt4 libqt4-webkit xvfb`` - Install the flash plugin to screenshot Adobe Flash files: ``apt-get install flashplugin-installer`` -Automated installation via ```pip``` -------------------------------------- +#### Automated installation via ```pip``` - Install pip: ```apt-get install python-pip``` - Install webkit2png: ```pip install webkit2png``` -Manual installation via Git ------------------------------ +#### Manual installation via Git - Install git: ``apt-get install git-core`` - Create directory: ``mkdir python-webkit2png`` - Clone the project: ``git clone https://github.com/adamn/python-webkit2png.git python-webkit2png`` - Install with: ``python python-webkit2png/setup.py install`` -FreeBSD -------- +### FreeBSD - install qt4 webkit: ```www/py-qt4-webkit, www/qt4-webkit, devel/py-qt4``` - install pip: ``devel/py-pip`` - install via: ``pip install webkit2png`` -Usage -===== +## Usage - For help run: ``python scripts/webkit2png -h`` ![Alt Text](http://24.media.tumblr.com/tumblr_m9trixXFHn1rxlmf0o1_400.gif) diff --git a/setup.py b/setup.py index b145841..17f0666 100755 --- a/setup.py +++ b/setup.py @@ -4,22 +4,22 @@ version = '0.8.3' -description = "Takes snapshot of webpages using Webkit and Qt4" +description = "Takes snapshot of web pages using Webkit and Qt4" long_description = description setup( - name = "webkit2png", - version = version, - url = 'http://github.com/AdamN/python-webkit2png', - license = 'LGPL', - description = description, - long_description = long_description, - author = 'Roland Tapken', - author_email = 'roland at dau-sicher de', - packages = ['webkit2png'], + name="webkit2png", + version=version, + url='http://github.com/AdamN/python-webkit2png', + license='GNU Lesser General Public License', + description=description, + long_description=long_description, + author='Roland Tapken', + author_email='roland at dau-sicher de', + packages=['webkit2png'], zip_safe=True, include_package_data=True, - package_dir = [], + package_dir=[], classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Web Environment', @@ -32,10 +32,9 @@ 'Topic :: Multimedia :: Graphics :: Capture :: Screen Capture', 'Topic :: Utilities' ], - entry_points = { + entry_points={ 'console_scripts': [ 'webkit2png = webkit2png.scripts:main', ] - }, + }, install_requires=['PyQt5'] ) - diff --git a/webkit2png/scripts.py b/webkit2png/scripts.py index 7a4bfb4..9f486d0 100755 --- a/webkit2png/scripts.py +++ b/webkit2png/scripts.py @@ -24,24 +24,26 @@ # - Add QTcpSocket support to create a "screenshot daemon" that # can handle multiple requests at the same time. +from PyQt5.QtWidgets import QApplication + from webkit2png import WebkitRenderer import sys import signal import os -import urlparse +import urllib.parse import logging from optparse import OptionParser -from PyQt4.QtCore import * -from PyQt4.QtGui import * -from PyQt4.QtWebKit import * -from PyQt4.QtNetwork import * +from PyQt5.QtCore import * +from PyQt5.QtNetwork import * +from PyQt5.QtWebEngineWidgets import QWebEngineSettings -VERSION="20091224" +VERSION = "20191201" LOG_FILENAME = 'webkit2png.log' logger = logging.getLogger('webkit2png') + def init_qtgui(display=None, style=None, qtargs=None): """Initiates the QApplication environment using the given args.""" if QApplication.instance(): @@ -72,7 +74,7 @@ def main(): # Enable HTTP proxy if 'http_proxy' in os.environ: - proxy_url = urlparse.urlparse(os.environ.get('http_proxy')) + proxy_url = urllib.parse.urljoin(os.environ.get('http_proxy')) proxy = QNetworkProxy(QNetworkProxy.HttpProxy, proxy_url.hostname, proxy_url.port) QNetworkProxy.setApplicationProxy(proxy) @@ -81,9 +83,9 @@ def main(): # $0 [--xvfb|--display=DISPLAY] [--debug] [--output=FILENAME] description = "Creates a screenshot of a website using QtWebkit." \ - + "This program comes with ABSOLUTELY NO WARRANTY. " \ - + "This is free software, and you are welcome to redistribute " \ - + "it under the terms of the GNU General Public License v2." + + "This program comes with ABSOLUTELY NO WARRANTY. " \ + + "This is free software, and you are welcome to redistribute " \ + + "it under the terms of the GNU General Public License v2." parser = OptionParser(usage="usage: %prog [options] ", version="%prog " + VERSION + ", Copyright (c) Roland Tapken", @@ -91,7 +93,8 @@ def main(): parser.add_option("-x", "--xvfb", nargs=2, type="int", dest="xvfb", help="Start an 'xvfb' instance with the given desktop size.", metavar="WIDTH HEIGHT") parser.add_option("-g", "--geometry", dest="geometry", nargs=2, default=(0, 0), type="int", - help="Geometry of the virtual browser window (0 means 'autodetect') [default: %default].", metavar="WIDTH HEIGHT") + help="Geometry of the virtual browser window (0 means 'autodetect') [default: %default].", + metavar="WIDTH HEIGHT") parser.add_option("-o", "--output", dest="output", help="Write output to FILE instead of STDOUT.", metavar="FILE") parser.add_option("-f", "--format", dest="format", default="png", @@ -104,35 +107,38 @@ def main(): choices=["javascript", "plugins"], help="Enable additional Webkit features ('javascript', 'plugins')", metavar="FEATURE") parser.add_option("-c", "--cookie", dest="cookies", action="append", - help="Add this cookie. Use multiple times for more cookies. Specification is value of a Set-Cookie HTTP response header.", metavar="COOKIE") + help="Add this cookie. Use multiple times for more cookies. Specification is value of a Set-Cookie HTTP response header.", + metavar="COOKIE") parser.add_option("-w", "--wait", dest="wait", default=0, type="int", - help="Time to wait after loading before the screenshot is taken [default: %default]", metavar="SECONDS") + help="Time to wait after loading before the screenshot is taken [default: %default]", + metavar="SECONDS") parser.add_option("-t", "--timeout", dest="timeout", default=0, type="int", help="Time before the request will be canceled [default: %default]", metavar="SECONDS") parser.add_option("-W", "--window", dest="window", action="store_true", help="Grab whole window instead of frame (may be required for plugins)", default=False) parser.add_option("-T", "--transparent", dest="transparent", action="store_true", - help="Render output on a transparent background (Be sure to have a transparent background defined in the html)", default=False) + help="Render output on a transparent background (Be sure to have a transparent background defined in the html)", + default=False) parser.add_option("", "--style", dest="style", help="Change the Qt look and feel to STYLE (e.G. 'windows').", metavar="STYLE") parser.add_option("", "--encoded-url", dest="encoded_url", action="store_true", - help="Treat URL as url-encoded", metavar="ENCODED_URL", default=False) + help="Treat URL as url-encoded", metavar="ENCODED_URL", default=False) parser.add_option("-d", "--display", dest="display", help="Connect to X server at DISPLAY.", metavar="DISPLAY") parser.add_option("--debug", action="store_true", dest="debug", help="Show debugging information.", default=False) parser.add_option("--log", action="store", dest="logfile", default=LOG_FILENAME, - help="Select the log output file",) + help="Select the log output file", ) # Parse command line arguments and validate them (as far as we can) - (options,args) = parser.parse_args() + (options, args) = parser.parse_args() if len(args) != 1: parser.error("incorrect number of arguments") if options.display and options.xvfb: parser.error("options -x and -d are mutually exclusive") options.url = args[0] - logging.basicConfig(filename=options.logfile,level=logging.WARN,) + logging.basicConfig(filename=options.logfile, level=logging.WARN, ) # Enable output of debugging information if options.debug: @@ -141,30 +147,31 @@ def main(): if options.xvfb: # Start 'xvfb' instance by replacing the current process server_num = int(os.getpid() + 1e6) - newArgs = ["xvfb-run", "--auto-servernum", "--server-num", str(server_num), "--server-args=-screen 0, %dx%dx24" % options.xvfb, sys.argv[0]] + newArgs = ["xvfb-run", "--auto-servernum", "--server-num", str(server_num), + "--server-args=-screen 0, %dx%dx24" % options.xvfb, sys.argv[0]] skipArgs = 0 for i in range(1, len(sys.argv)): if skipArgs > 0: skipArgs -= 1 elif sys.argv[i] in ["-x", "--xvfb"]: - skipArgs = 2 # following: width and height + skipArgs = 2 # following: width and height else: newArgs.append(sys.argv[i]) logger.debug("Executing %s" % " ".join(newArgs)) try: - os.execvp(newArgs[0],newArgs[1:]) + os.execvp(newArgs[0], newArgs[1:]) except OSError: logger.error("Unable to find '%s'" % newArgs[0]) - print >> sys.stderr, "Error - Unable to find '%s' for -x/--xvfb option" % newArgs[0] + sys.stderr.write("Error - Unable to find '{}' for -x/--xvfb option".format(newArgs[0])) sys.exit(1) # Prepare output ("1" means STDOUT) if options.output is None: options.output = sys.stdout else: - options.output = open(options.output, "w") + options.output = open(options.output, "wb+") - logger.debug("Version %s, Python %s, Qt %s", VERSION, sys.version, qVersion()); + logger.debug("Version %s, Python %s, Qt %s", VERSION, sys.version, qVersion()) # Technically, this is a QtGui application, because QWebPage requires it # to be. But because we will have no user interaction, and rendering can @@ -196,25 +203,26 @@ def __main_qt(): if options.features: if "javascript" in options.features: - renderer.qWebSettings[QWebSettings.JavascriptEnabled] = True + renderer.qWebSettings[QWebEngineSettings.JavascriptEnabled] = True if "plugins" in options.features: - renderer.qWebSettings[QWebSettings.PluginsEnabled] = True + renderer.qWebSettings[QWebEngineSettings.PluginsEnabled] = True renderer.render_to_file(res=options.url, file_object=options.output) options.output.close() QApplication.exit(0) - except RuntimeError, e: + except RuntimeError as e: logger.error("main: %s" % e) - print >> sys.stderr, e + sys.stderr.write(e) QApplication.exit(1) # Initialize Qt-Application, but make this script - # abortable via CTRL-C - app = init_qtgui(display = options.display, style=options.style) + # To interrupt via CTRL-C + app = init_qtgui(display=options.display, style=options.style) signal.signal(signal.SIGINT, signal.SIG_DFL) QTimer.singleShot(0, __main_qt) return app.exec_() + if __name__ == '__main__': sys.exit(main()) diff --git a/webkit2png/webkit2png.py b/webkit2png/webkit2png.py index e6a2b21..c9ddbb0 100755 --- a/webkit2png/webkit2png.py +++ b/webkit2png/webkit2png.py @@ -23,30 +23,54 @@ # - Add QTcpSocket support to create a "screenshot daemon" that # can handle multiple requests at the same time. -import time +# Third Party Update by SuperSonic to update for PB Project +# +# Modified according to GNU General Public License +# Copyright(c) 2019 SuperSonic(https://randychen.tk) +# +# ChangeLog: +# Drop support for Qt 4 +# Drop support for Python 2.x +# Support Qt 5 +# Support Python3 +# User-Agent Selection +# +# More Information: +# https://github.com/supersonictw/PBP-analytics + +import _io import os +import time -from PyQt4.QtCore import * -from PyQt4.QtGui import * -from PyQt4.QtWebKit import * -from PyQt4.QtNetwork import * +from PyQt5 import sip +from PyQt5.QtCore import QObject, QUrl, Qt, QCoreApplication, QByteArray, QBuffer, pyqtSlot +from PyQt5.QtGui import QPalette, QImage, QColor, QPainter, QGuiApplication +from PyQt5.QtNetwork import QNetworkCookieJar, QNetworkCookie, QNetworkProxy, QNetworkAccessManager, QNetworkReply +from PyQt5.QtWebEngineWidgets import QWebEngineSettings, QWebEnginePage, QWebEngineView +from PyQt5.QtWidgets import QApplication, QMainWindow, QAbstractScrollArea -# Class for Website-Rendering. Uses QWebPage, which + +# Class for Website-Rendering. Uses QtWebEngine, which # requires a running QtGui to work. + + class WebkitRenderer(QObject): """ - A class that helps to create 'screenshots' of webpages using - Qt's QWebkit. Requires PyQt4 library. - + A class that helps to create 'screenshots' of web pages using + Requires PyQt5 library. Use "render()" to get a 'QImage' object, render_to_bytes() to get the resulting image as 'str' object or render_to_file() to write the image directly into a 'file' resource. """ - def __init__(self,**kwargs): + + def __init__(self, **kwargs): """ Sets default values for the properties. """ + # UserAgents + self.user_agents = [] + # QT Initialize if not QApplication.instance(): raise RuntimeError(self.__class__.__name__ + " requires a running QApplication instance") QObject.__init__(self) @@ -61,7 +85,7 @@ def __init__(self,**kwargs): self.scaleRatio = kwargs.get('scaleRatio', 'keep') self.format = kwargs.get('format', 'png') self.logger = kwargs.get('logger', None) - + # Set this to true if you want to capture flash. # Not that your desktop must be large enough for # fitting the whole window. @@ -74,14 +98,17 @@ def __init__(self,**kwargs): self.encodedUrl = kwargs.get('encodedUrl', False) self.cookies = kwargs.get('cookies', []) - # Set some default options for QWebPage self.qWebSettings = { - QWebSettings.JavascriptEnabled : False, - QWebSettings.PluginsEnabled : False, - QWebSettings.PrivateBrowsingEnabled : True, - QWebSettings.JavascriptCanOpenWindows : False + QWebEngineSettings.JavascriptEnabled: False, + QWebEngineSettings.PluginsEnabled: False, + QWebEngineSettings.JavascriptCanOpenWindows: False } + def add_user_agent(self, user_agent_string): + self.user_agents.append(user_agent_string) + + def del_user_agent(self, user_agent_string): + self.user_agents.remove(user_agent_string) def render(self, res): """ @@ -91,7 +118,7 @@ def render(self, res): # QApplication.processEvents may be called, causing # this method to get called while it has not returned yet. helper = _WebkitRendererHelper(self) - helper._window.resize( self.width, self.height ) + helper.window.resize(self.width, self.height) image = helper.render(res) # Bind helper instance to this image to prevent the @@ -106,33 +133,36 @@ def render_to_file(self, res, file_object): Renders the image into a File resource. Returns the size of the data that has been written. """ - format = self.format # this may not be constant due to processEvents() + assert type(file_object) is _io.BufferedRandom, "Not a file object" + qt_format = self.format # this may not be constant due to processEvents() image = self.render(res) - qBuffer = QBuffer() - image.save(qBuffer, format) - file_object.write(qBuffer.buffer().data()) - return qBuffer.size() + q_buffer = QBuffer() + image.save(q_buffer, qt_format) + file_object.write(q_buffer.buffer().data()) + return q_buffer.size() def render_to_bytes(self, res): """Renders the image into an object of type 'str'""" - format = self.format # this may not be constant due to processEvents() + qt_format = self.format # this may not be constant due to processEvents() image = self.render(res) - qBuffer = QBuffer() - image.save(qBuffer, format) - return qBuffer.buffer().data() + q_buffer = QBuffer() + image.save(q_buffer, qt_format) + return q_buffer.buffer().data() + -## @brief The CookieJar class inherits QNetworkCookieJar to make a couple of functions public. +# @brief The CookieJar class inherits QNetworkCookieJar to make a couple of functions public. class CookieJar(QNetworkCookieJar): - def __init__(self, cookies, qtUrl, parent=None): - QNetworkCookieJar.__init__(self, parent) - for cookie in cookies: - QNetworkCookieJar.setCookiesFromUrl(self, QNetworkCookie.parseCookies(QByteArray(cookie)), qtUrl) + def __init__(self, cookies, qt_url, parent=None): + QNetworkCookieJar.__init__(self, parent) + for cookie in cookies: + QNetworkCookieJar.setCookiesFromUrl(self, QNetworkCookie.parseCookies(QByteArray(cookie)), qt_url) + + def allCookies(self): + return QNetworkCookieJar.allCookies(self) + + def setAllCookies(self, cookie1_list): + QNetworkCookieJar.setAllCookies(self, cookie1_list) - def allCookies(self): - return QNetworkCookieJar.allCookies(self) - - def setAllCookies(self, cookieList): - QNetworkCookieJar.setAllCookies(self, cookieList) class _WebkitRendererHelper(QObject): """ @@ -150,14 +180,14 @@ def __init__(self, parent): QObject.__init__(self) # Copy properties from parent - for key,value in parent.__dict__.items(): - setattr(self,key,value) + for key, value in parent.__dict__.items(): + setattr(self, key, value) # Determine Proxy settings proxy = QNetworkProxy(QNetworkProxy.NoProxy) if 'http_proxy' in os.environ: proxy_url = QUrl(os.environ['http_proxy']) - if unicode(proxy_url.scheme()).startswith('http'): + if proxy_url.scheme().startswith('http'): protocol = QNetworkProxy.HttpProxy else: protocol = QNetworkProxy.Socks5Proxy @@ -170,37 +200,40 @@ def __init__(self, parent): proxy_url.password() ) - # Create and connect required PyQt4 objects + # Create and connect required PyQt5 objects self._page = CustomWebPage(logger=self.logger, ignore_alert=self.ignoreAlert, - ignore_confirm=self.ignoreConfirm, ignore_prompt=self.ignorePrompt, - interrupt_js=self.interruptJavaScript) - self._page.networkAccessManager().setProxy(proxy) - self._view = QWebView() + ignore_confirm=self.ignoreConfirm, ignore_prompt=self.ignorePrompt, + interrupt_js=self.interruptJavaScript) + self._qt_proxy = QNetworkProxy() + self._qt_proxy.setApplicationProxy(proxy) + self._view = QWebEngineView() self._view.setPage(self._page) self._window = QMainWindow() self._window.setCentralWidget(self._view) + self._qt_network_access_manager = QNetworkAccessManager() # Import QWebSettings - for key, value in self.qWebSettings.iteritems(): + for key, value in self.qWebSettings.items(): self._page.settings().setAttribute(key, value) # Connect required event listeners - self.connect(self._page, SIGNAL("loadFinished(bool)"), self._on_load_finished) - self.connect(self._page, SIGNAL("loadStarted()"), self._on_load_started) - self.connect(self._page.networkAccessManager(), SIGNAL("sslErrors(QNetworkReply *,const QList&)"), self._on_ssl_errors) - self.connect(self._page.networkAccessManager(), SIGNAL("finished(QNetworkReply *)"), self._on_each_reply) + # assert False, "Not finish" + self._page.loadFinished.connect(self._on_load_finished) + self._page.loadStarted.connect(self._on_load_started) + self._qt_network_access_manager.sslErrors.connect(self._on_ssl_errors) + self._qt_network_access_manager.finished.connect(self._on_each_reply) - # The way we will use this, it seems to be unesseccary to have Scrollbars enabled - self._page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff) - self._page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff) - self._page.settings().setUserStyleSheetUrl(QUrl("data:text/css,html,body{overflow-y:hidden !important;}")) + # The way we will use this, it seems to be unnecessary to have Scrollbars enabled + self._scroll_area = QAbstractScrollArea() + self._scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) + self._scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff) # Show this widget self._window.show() def __del__(self): """ - Clean up Qt4 objects. + Clean up Qt5 objects. """ self._window.close() del self._window @@ -213,23 +246,24 @@ def render(self, res): the end of the given 'delay'. While it is waiting outstanding QApplication events are processed. After the given delay, the Window or Widget (depends - on the value of 'grabWholeWindow' is drawn into a QPixmap - and postprocessed (_post_process_image). + on the value of 'grabWholeWindow' is drawn into a QScreen + and post processed (_post_process_image). """ self._load_page(res, self.width, self.height, self.timeout) # Wait for end of timer. In this time, process # other outstanding Qt events. if self.wait > 0: - if self.logger: self.logger.debug("Waiting %d seconds " % self.wait) - waitToTime = time.time() + self.wait - while time.time() < waitToTime: + if self.logger: + self.logger.debug("Waiting {} seconds ".format(self.wait)) + wait_to_time = time.time() + self.wait + while time.time() < wait_to_time: if QApplication.hasPendingEvents(): QApplication.processEvents() if self.renderTransparentBackground: # Another possible drawing solution image = QImage(self._page.viewportSize(), QImage.Format_ARGB32) - image.fill(QColor(255,0,0,0).rgba()) + image.fill(QColor(255, 0, 0, 0).rgba()) # http://ariya.blogspot.com/2009/04/transparent-qwebview-and-qwebpage.html palette = self._view.palette() @@ -239,7 +273,7 @@ def render(self, res): painter = QPainter(image) painter.setBackgroundMode(Qt.TransparentMode) - self._page.mainFrame().render(painter) + self._window.render(painter) painter.end() else: if self.grabWholeWindow: @@ -247,9 +281,10 @@ def render(self, res): # window still has the focus when the screen is # grabbed. This might result in a race condition. self._view.activateWindow() - image = QPixmap.grabWindow(self._window.winId()) + qt_screen = QGuiApplication.primaryScreen() + image = qt_screen.grabWindow(sip.voidptr(0)) else: - image = QPixmap.grabWidget(self._window) + image = self._window.grab() return self._post_process_image(image) @@ -261,57 +296,60 @@ def _load_page(self, res, width, height, timeout): # This is an event-based application. So we have to wait until # "loadFinished(bool)" raised. - cancelAt = time.time() + timeout + cancel_at = time.time() + timeout self.__loading = True - self.__loadingResult = False # Default + self.__loadingResult = False # Default # When "res" is of type tuple, it has two elements where the first # element is the HTML code to render and the second element is a string # setting the base URL for the interpreted HTML code. # When resource is of type str or unicode, it is handled as URL which - # shal be loaded + # shall be loaded if type(res) == tuple: url = res[1] else: url = res if self.encodedUrl: - qtUrl = QUrl.fromEncoded(url) + qt_url = QUrl.fromEncoded(url) else: - qtUrl = QUrl(url) + qt_url = QUrl(url) # Set the required cookies, if any - self.cookieJar = CookieJar(self.cookies, qtUrl) - self._page.networkAccessManager().setCookieJar(self.cookieJar) + self.cookieJar = CookieJar(self.cookies, qt_url) + self._qt_network_access_manager.setCookieJar(self.cookieJar) # Load the page if type(res) == tuple: - self._page.mainFrame().setHtml(res[0], qtUrl) # HTML, baseUrl + self._page.setHtml(res[0], qt_url) # HTML, baseUrl else: - self._page.mainFrame().load(qtUrl) + self._page.load(qt_url) while self.__loading: - if timeout > 0 and time.time() >= cancelAt: - raise RuntimeError("Request timed out on %s" % res) + if timeout > 0 and time.time() >= cancel_at: + raise RuntimeError("Request timed out on {}".format(res)) while QApplication.hasPendingEvents() and self.__loading: QCoreApplication.processEvents() - if self.logger: self.logger.debug("Processing result") + if self.logger: + self.logger.debug("Processing result") - if self.__loading_result == False: - if self.logger: self.logger.warning("Failed to load %s" % res) + if not self.__loading_result: + if self.logger: + self.logger.warning("Failed to load {}".format(res)) # Set initial viewport (the size of the "window") - size = self._page.mainFrame().contentsSize() - if self.logger: self.logger.debug("contentsSize: %s", size) + size = self._page.contentsSize() + if self.logger: + self.logger.debug("contentsSize: %s", size) if width > 0: size.setWidth(width) if height > 0: size.setHeight(height) - self._window.resize(size) + self._window.resize(size.toSize()) - def _post_process_image(self, qImage): + def _post_process_image(self, q_image): """ If 'scaleToWidth' or 'scaleToHeight' are set to a value greater than zero this method will scale the image @@ -323,52 +361,64 @@ def _post_process_image(self, qImage): ratio = Qt.KeepAspectRatio elif self.scaleRatio in ['expand', 'crop']: ratio = Qt.KeepAspectRatioByExpanding - else: # 'ignore' + else: # 'ignore' ratio = Qt.IgnoreAspectRatio - qImage = qImage.scaled(self.scaleToWidth, self.scaleToHeight, ratio, Qt.SmoothTransformation) + q_image = q_image.scaled(self.scaleToWidth, self.scaleToHeight, ratio, Qt.SmoothTransformation) if self.scaleRatio == 'crop': - qImage = qImage.copy(0, 0, self.scaleToWidth, self.scaleToHeight) - return qImage + q_image = q_image.copy(0, 0, self.scaleToWidth, self.scaleToHeight) + return q_image - def _on_each_reply(self,reply): - """ - Logs each requested uri - """ - self.logger.debug("Received %s" % (reply.url().toString())) + @pyqtSlot(QNetworkReply, name='finished') + def _on_each_reply(self, reply): + """ + Logs each requested uri + """ + self.logger.debug("Received {}".format(reply.url().toString())) - # Eventhandler for "loadStarted()" signal + # Event for "loadStarted()" signal + @pyqtSlot(name='loadStarted') def _on_load_started(self): """ Slot that sets the '__loading' property to true """ - if self.logger: self.logger.debug("loading started") + if self.logger: + self.logger.debug("loading started") self.__loading = True - # Eventhandler for "loadFinished(bool)" signal + # Event for "loadFinished(bool)" signal + @pyqtSlot(bool, name='loadFinished') def _on_load_finished(self, result): - """Slot that sets the '__loading' property to false and stores + """ + Slot that sets the '__loading' property to false and stores the result code in '__loading_result'. """ - if self.logger: self.logger.debug("loading finished with result %s", result) + if self.logger: + self.logger.debug("loading finished with result %s", result) self.__loading = False self.__loading_result = result - # Eventhandler for "sslErrors(QNetworkReply *,const QList&)" signal + # Event for "sslErrors(QNetworkReply *,const QList&)" signal + @pyqtSlot('QNetworkReply*', 'QList', name='sslErrors') def _on_ssl_errors(self, reply, errors): """ Slot that writes SSL warnings into the log but ignores them. """ for e in errors: - if self.logger: self.logger.warn("SSL: " + e.errorString()) + if self.logger: + self.logger.warn("SSL: " + e.errorString()) reply.ignoreSslErrors() + @property + def window(self): + return self._window + -class CustomWebPage(QWebPage): +class CustomWebPage(QWebEnginePage): def __init__(self, **kwargs): - """ - Class Initializer - """ - super(CustomWebPage, self).__init__() + """ + Class Initializer + """ + super().__init__() self.logger = kwargs.get('logger', None) self.ignore_alert = kwargs.get('ignore_alert', True) self.ignore_confirm = kwargs.get('ignore_confirm', True) @@ -376,38 +426,41 @@ def __init__(self, **kwargs): self.interrupt_js = kwargs.get('interrupt_js', True) def javaScriptAlert(self, frame, message): - if self.logger: self.logger.debug('Alert: %s', message) + if self.logger: + self.logger.debug('Alert: {}'.format(message)) if not self.ignore_alert: - return super(CustomWebPage, self).javaScriptAlert(frame, message) + return super().javaScriptAlert(frame, message) def javaScriptConfirm(self, frame, message): - if self.logger: self.logger.debug('Confirm: %s', message) + if self.logger: + self.logger.debug('Confirm: {}'.format(message)) if not self.ignore_confirm: - return super(CustomWebPage, self).javaScriptConfirm(frame, message) + return super().javaScriptConfirm(frame, message) else: return False - def javaScriptPrompt(self, frame, message, default, result): + def javaScriptPrompt(self, frame, message, result): """ This function is called whenever a JavaScript program running inside frame tries to prompt the user for input. The program may provide an optional message, msg, as well as a default value for the input in defaultValue. - If the prompt was cancelled by the user the implementation should return false; otherwise the result should be written to result and true should be returned. If the prompt was not cancelled by the user, the implementation should return true and the result string must not be null. """ - if self.logger: self.logger.debug('Prompt: %s (%s)' % (message, default)) + if self.logger: + self.logger.debug('Prompt: {}'.format(message)) if not self.ignore_prompt: - return super(CustomWebPage, self).javaScriptPrompt(frame, message, default, result) + return super().javaScriptPrompt(frame, message, result) else: return False - def shouldInterruptJavaScript(self): + def should_interrupt_javascript(self): """ This function is called when a JavaScript program is running for a long period of time. If the user wanted to stop the JavaScript the implementation should return true; otherwise false. """ - if self.logger: self.logger.debug("WebKit ask to interrupt JavaScript") + if self.logger: + self.logger.debug("WebKit ask to interrupt JavaScript") return self.interrupt_js From b2e13580091aacfef81218c747c61fa160bae8fd Mon Sep 17 00:00:00 2001 From: SuperSonic Date: Sun, 1 Dec 2019 02:47:57 +0800 Subject: [PATCH 2/8] Update and fix bug --- setup.py | 8 ++++---- webkit2png/__init__.py | 4 ++-- webkit2png/scripts.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 17f0666..3607fac 100755 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ #!/usr/bin/env python -from setuptools import setup, find_packages +from setuptools import setup -version = '0.8.3' +version = '0.9.0' -description = "Takes snapshot of web pages using Webkit and Qt4" +description = "Takes snapshot of web pages using Webkit and Qt5" long_description = description setup( @@ -36,5 +36,5 @@ 'console_scripts': [ 'webkit2png = webkit2png.scripts:main', ] - }, install_requires=['PyQt5'] + }, install_requires=['PyQt5', 'sip'] ) diff --git a/webkit2png/__init__.py b/webkit2png/__init__.py index eec1255..a235fb6 100644 --- a/webkit2png/__init__.py +++ b/webkit2png/__init__.py @@ -1,2 +1,2 @@ -from webkit2png import WebkitRenderer -__all__ = ['WebkitRenderer'] \ No newline at end of file +from .webkit2png import WebkitRenderer +__all__ = ['WebkitRenderer'] diff --git a/webkit2png/scripts.py b/webkit2png/scripts.py index 9f486d0..8cb7303 100755 --- a/webkit2png/scripts.py +++ b/webkit2png/scripts.py @@ -74,7 +74,7 @@ def main(): # Enable HTTP proxy if 'http_proxy' in os.environ: - proxy_url = urllib.parse.urljoin(os.environ.get('http_proxy')) + proxy_url = urllib.parse.urlparse(os.environ.get('http_proxy')) proxy = QNetworkProxy(QNetworkProxy.HttpProxy, proxy_url.hostname, proxy_url.port) QNetworkProxy.setApplicationProxy(proxy) From 2afc0fc4d6601c39a7b66aa51d2804e46cabc8b2 Mon Sep 17 00:00:00 2001 From: SuperSonic Date: Sun, 1 Dec 2019 02:49:27 +0800 Subject: [PATCH 3/8] Contribute to origin --- webkit2png/webkit2png.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/webkit2png/webkit2png.py b/webkit2png/webkit2png.py index c9ddbb0..240a91d 100755 --- a/webkit2png/webkit2png.py +++ b/webkit2png/webkit2png.py @@ -23,21 +23,6 @@ # - Add QTcpSocket support to create a "screenshot daemon" that # can handle multiple requests at the same time. -# Third Party Update by SuperSonic to update for PB Project -# -# Modified according to GNU General Public License -# Copyright(c) 2019 SuperSonic(https://randychen.tk) -# -# ChangeLog: -# Drop support for Qt 4 -# Drop support for Python 2.x -# Support Qt 5 -# Support Python3 -# User-Agent Selection -# -# More Information: -# https://github.com/supersonictw/PBP-analytics - import _io import os import time From 156d2baa64896f27b4e386b80121b09aca251a81 Mon Sep 17 00:00:00 2001 From: SuperSonic Date: Sun, 1 Dec 2019 02:56:18 +0800 Subject: [PATCH 4/8] Remove the part not use in original version --- webkit2png/webkit2png.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/webkit2png/webkit2png.py b/webkit2png/webkit2png.py index 240a91d..37fd2e3 100755 --- a/webkit2png/webkit2png.py +++ b/webkit2png/webkit2png.py @@ -52,8 +52,6 @@ def __init__(self, **kwargs): """ Sets default values for the properties. """ - # UserAgents - self.user_agents = [] # QT Initialize if not QApplication.instance(): @@ -89,12 +87,6 @@ def __init__(self, **kwargs): QWebEngineSettings.JavascriptCanOpenWindows: False } - def add_user_agent(self, user_agent_string): - self.user_agents.append(user_agent_string) - - def del_user_agent(self, user_agent_string): - self.user_agents.remove(user_agent_string) - def render(self, res): """ Renders the given URL into a QImage object From 9736861fc32d0ba2e82151c6a536364abfd96333 Mon Sep 17 00:00:00 2001 From: SuperSonic Date: Sat, 30 Nov 2019 19:56:25 +0000 Subject: [PATCH 5/8] Update README --- README.md | 17 +++++++---------- setup.py | 2 +- webkit2png/webkit2png.py | 17 +++++++++++------ 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 606a8fb..18de776 100644 --- a/README.md +++ b/README.md @@ -9,26 +9,23 @@ Python script that takes screenshots (browsershots) using webkit ## Installation -### Ubuntu -- Add following packages: ``apt-get install python-qt4 libqt4-webkit xvfb`` -- Install the flash plugin to screenshot Adobe Flash files: ``apt-get install flashplugin-installer`` +### Debian/Ubuntu +- Add following packages: ``apt-get install libqt5core5a python3-pip`` #### Automated installation via ```pip``` -- Install pip: ```apt-get install python-pip``` -- Install webkit2png: ```pip install webkit2png``` +- Install webkit2png: ```pip3 install webkit2png``` #### Manual installation via Git -- Install git: ``apt-get install git-core`` -- Create directory: ``mkdir python-webkit2png`` +- Install git: ``apt-get install git`` - Clone the project: ``git clone https://github.com/adamn/python-webkit2png.git python-webkit2png`` -- Install with: ``python python-webkit2png/setup.py install`` +- Install with: ``python3 python-webkit2png/setup.py install`` ### FreeBSD -- install qt4 webkit: ```www/py-qt4-webkit, www/qt4-webkit, devel/py-qt4``` +- install qt4 webkit: ```www/py-qt5-webkit, www/qt5-webkit, devel/py-qt5``` - install pip: ``devel/py-pip`` - install via: ``pip install webkit2png`` ## Usage -- For help run: ``python scripts/webkit2png -h`` +- For help run: ``python3 webkit2png -h`` ![Alt Text](http://24.media.tumblr.com/tumblr_m9trixXFHn1rxlmf0o1_400.gif) diff --git a/setup.py b/setup.py index 3607fac..b26d90e 100755 --- a/setup.py +++ b/setup.py @@ -36,5 +36,5 @@ 'console_scripts': [ 'webkit2png = webkit2png.scripts:main', ] - }, install_requires=['PyQt5', 'sip'] + }, install_requires=['PyQt5'] ) diff --git a/webkit2png/webkit2png.py b/webkit2png/webkit2png.py index 37fd2e3..6f95871 100755 --- a/webkit2png/webkit2png.py +++ b/webkit2png/webkit2png.py @@ -27,7 +27,7 @@ import os import time -from PyQt5 import sip +import PyQt5.sip as sip from PyQt5.QtCore import QObject, QUrl, Qt, QCoreApplication, QByteArray, QBuffer, pyqtSlot from PyQt5.QtGui import QPalette, QImage, QColor, QPainter, QGuiApplication from PyQt5.QtNetwork import QNetworkCookieJar, QNetworkCookie, QNetworkProxy, QNetworkAccessManager, QNetworkReply @@ -55,7 +55,8 @@ def __init__(self, **kwargs): # QT Initialize if not QApplication.instance(): - raise RuntimeError(self.__class__.__name__ + " requires a running QApplication instance") + raise RuntimeError(self.__class__.__name__ + + " requires a running QApplication instance") QObject.__init__(self) # Initialize default properties @@ -73,7 +74,8 @@ def __init__(self, **kwargs): # Not that your desktop must be large enough for # fitting the whole window. self.grabWholeWindow = kwargs.get('grabWholeWindow', False) - self.renderTransparentBackground = kwargs.get('renderTransparentBackground', False) + self.renderTransparentBackground = kwargs.get( + 'renderTransparentBackground', False) self.ignoreAlert = kwargs.get('ignoreAlert', True) self.ignoreConfirm = kwargs.get('ignoreConfirm', True) self.ignorePrompt = kwargs.get('ignorePrompt', True) @@ -132,7 +134,8 @@ class CookieJar(QNetworkCookieJar): def __init__(self, cookies, qt_url, parent=None): QNetworkCookieJar.__init__(self, parent) for cookie in cookies: - QNetworkCookieJar.setCookiesFromUrl(self, QNetworkCookie.parseCookies(QByteArray(cookie)), qt_url) + QNetworkCookieJar.setCookiesFromUrl( + self, QNetworkCookie.parseCookies(QByteArray(cookie)), qt_url) def allCookies(self): return QNetworkCookieJar.allCookies(self) @@ -340,9 +343,11 @@ def _post_process_image(self, q_image): ratio = Qt.KeepAspectRatioByExpanding else: # 'ignore' ratio = Qt.IgnoreAspectRatio - q_image = q_image.scaled(self.scaleToWidth, self.scaleToHeight, ratio, Qt.SmoothTransformation) + q_image = q_image.scaled( + self.scaleToWidth, self.scaleToHeight, ratio, Qt.SmoothTransformation) if self.scaleRatio == 'crop': - q_image = q_image.copy(0, 0, self.scaleToWidth, self.scaleToHeight) + q_image = q_image.copy( + 0, 0, self.scaleToWidth, self.scaleToHeight) return q_image @pyqtSlot(QNetworkReply, name='finished') From 336f5c7e3b65a5dbfc60650297c58331b578224e Mon Sep 17 00:00:00 2001 From: SuperSonic Date: Sat, 30 Nov 2019 19:58:46 +0000 Subject: [PATCH 6/8] Last check --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 18de776..93573de 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,11 @@ Python script that takes screenshots (browsershots) using webkit - Install with: ``python3 python-webkit2png/setup.py install`` ### FreeBSD -- install qt4 webkit: ```www/py-qt5-webkit, www/qt5-webkit, devel/py-qt5``` +- install qt5 webkit: ```www/py-qt5-webkit, www/qt5-webkit, devel/py-qt5``` - install pip: ``devel/py-pip`` - install via: ``pip install webkit2png`` ## Usage -- For help run: ``python3 webkit2png -h`` +- For help run: ``python3 -m webkit2png -h`` ![Alt Text](http://24.media.tumblr.com/tumblr_m9trixXFHn1rxlmf0o1_400.gif) From 006170f2bebbbf29a814f064f116ba1193f34379 Mon Sep 17 00:00:00 2001 From: SuperSonic Date: Sun, 1 Dec 2019 08:47:55 +0000 Subject: [PATCH 7/8] Update setup.py and add requirements.txt --- README.md | 4 ++++ requirements.txt | 8 ++++++++ setup.py | 2 +- webkit2png/scripts.py | 2 +- webkit2png/webkit2png.py | 2 +- 5 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 requirements.txt diff --git a/README.md b/README.md index 93573de..49c05e0 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ Python script that takes screenshots (browsershots) using webkit ## Requirement python >= 3.7 PyQt >= 5.13 + PyQtWebEngine >= 5.13 + +> Notice: Since Qt5, QtWebkit has been deprecated by QtWebEngine using Blink Engine from Chromium Project. ## Installation @@ -19,6 +22,7 @@ Python script that takes screenshots (browsershots) using webkit - Install git: ``apt-get install git`` - Clone the project: ``git clone https://github.com/adamn/python-webkit2png.git python-webkit2png`` - Install with: ``python3 python-webkit2png/setup.py install`` +- If the requirement install failed, satified with: ``pip3 install -r requirements.txt`` ### FreeBSD - install qt5 webkit: ```www/py-qt5-webkit, www/qt5-webkit, devel/py-qt5``` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5225a17 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +packaging==19.2 +pyparsing==2.4.5 +PyQt5==5.13.2 +PyQt5-sip==12.7.0 +PyQtWebEngine==5.13.2 +sip==5.0.0 +toml==0.10.0 +webkit2png==0.9.0 diff --git a/setup.py b/setup.py index b26d90e..7b5d49b 100755 --- a/setup.py +++ b/setup.py @@ -36,5 +36,5 @@ 'console_scripts': [ 'webkit2png = webkit2png.scripts:main', ] - }, install_requires=['PyQt5'] + }, install_requires=['PyQt5', 'PyQtWebEngine'] ) diff --git a/webkit2png/scripts.py b/webkit2png/scripts.py index 8cb7303..9a54737 100755 --- a/webkit2png/scripts.py +++ b/webkit2png/scripts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # webkit2png.py # diff --git a/webkit2png/webkit2png.py b/webkit2png/webkit2png.py index 6f95871..77041ef 100755 --- a/webkit2png/webkit2png.py +++ b/webkit2png/webkit2png.py @@ -27,7 +27,7 @@ import os import time -import PyQt5.sip as sip +from PyQt5 import sip from PyQt5.QtCore import QObject, QUrl, Qt, QCoreApplication, QByteArray, QBuffer, pyqtSlot from PyQt5.QtGui import QPalette, QImage, QColor, QPainter, QGuiApplication from PyQt5.QtNetwork import QNetworkCookieJar, QNetworkCookie, QNetworkProxy, QNetworkAccessManager, QNetworkReply From 8f005187118698c0b7be376c1a32db2b2787298e Mon Sep 17 00:00:00 2001 From: SuperSonic Date: Sun, 1 Dec 2019 22:40:14 +0800 Subject: [PATCH 8/8] Update requirements.txt --- requirements.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5225a17..f659c93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,4 @@ -packaging==19.2 -pyparsing==2.4.5 PyQt5==5.13.2 PyQt5-sip==12.7.0 PyQtWebEngine==5.13.2 sip==5.0.0 -toml==0.10.0 -webkit2png==0.9.0