diff --git a/.github/workflows/galacteek-deploy.yml b/.github/workflows/galacteek-deploy.yml index 739d3e60..d21739da 100644 --- a/.github/workflows/galacteek-deploy.yml +++ b/.github/workflows/galacteek-deploy.yml @@ -33,6 +33,12 @@ jobs: sudo apt-get install -y dzen2 xvfb herbstluftwm sudo apt-get install -y libxcb-xkb1 libxkbcommon-x11-0 sudo apt-get install -y libzbar0 + sudo apt-get install -y tor + + - name: Install packages (macos) + if: startsWith(matrix.os, 'mac') + run: | + brew install tor - name: Install unzip (Windows) if: startsWith(matrix.os, 'windows') @@ -58,9 +64,15 @@ jobs: with: args: install nsis + - name: Install Tor (Windows) + if: startsWith(matrix.os, 'windows') + uses: crazy-max/ghaction-chocolatey@v1 + with: + args: install tor + - name: Configure environment (1) run: | - echo "G_VERSION=$(cat galacteek/version)" >> $GITHUB_ENV + echo "G_VERSION=$(cat galacteek/VERSION)" >> $GITHUB_ENV echo "COMMIT_SHORT=$(echo $GITHUB_SHA|cut -c 1-8)" >> $GITHUB_ENV echo "GIT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV @@ -256,7 +268,7 @@ jobs: - name: Release config id: release_config run: | - echo "G_VERSION=$(cat galacteek/version)" >> $GITHUB_ENV + echo "G_VERSION=$(cat galacteek/VERSION)" >> $GITHUB_ENV echo "COMMIT_SHORT=$(echo $GITHUB_SHA|cut -c 1-8)" >> $GITHUB_ENV echo "GIT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV echo "TAGNAME=continuous-${GITHUB_REF##*/}" >> $GITHUB_ENV diff --git a/AppImage/appimage-build b/AppImage/appimage-build index 2922d3d4..b7066948 100755 --- a/AppImage/appimage-build +++ b/AppImage/appimage-build @@ -97,6 +97,9 @@ find /usr/lib -iname 'libzbar.so*' -exec cp -av {} $APPDIR/usr/lib \; cp $GITHUB_WORKSPACE/go-ipfs/ipfs-${GO_IPFS_VERSION} $APPDIR/usr/bin cp $GITHUB_WORKSPACE/fs-repo-migrations/fs-repo-migrations $APPDIR/usr/bin +# Copy tor +cp /usr/bin/tor $APPDIR/usr/bin + pushd "$APPDIR"/usr/bin ln -s ipfs-${GO_IPFS_VERSION} ipfs popd diff --git a/CHANGELOG.md b/CHANGELOG.md index f4aa9db7..bc5c5474 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,30 @@ the changes in the CHANGELOG formatting. ## [Unreleased] +## [0.4.41] - 2020-12-02 +### Added +- Tor support + - Automatic use of Tor when bootstrapping succeeds + - Tor proxying for all ipfs-search and cyber requests + +- Add a new *anonymous* web profile +- Automatically fetch favicons when hashmarking an http(s) website +- Handle SSL certificate errors + +### Changed +- Bookmarking of any type of URLs +- Browser UI + - Use a block-style cursor for the address bar + - Typing an .eth domain name automatically loads it through *ens://* + - Run IPFS searches or searches with popular engines (duckduckgo, ..) + from the address bar + - Nicer history lookup interface + +- The @Earth workspace is now the default workspace in the stack + +### Fixed +- Bookmarking of clearnet URLs + ## [0.4.40] - 2020-11-29 ### Added - Lightweight BT client integration (asyncio-based) diff --git a/ci/build-dmg.sh b/ci/build-dmg.sh index 75478f81..58837162 100755 --- a/ci/build-dmg.sh +++ b/ci/build-dmg.sh @@ -81,6 +81,9 @@ mkdir -p galacteek.app/Contents/Resources/bin cp $GITHUB_WORKSPACE/go-ipfs/ipfs-${GO_IPFS_VERSION} galacteek.app/Contents/Resources/bin cp $GITHUB_WORKSPACE/fs-repo-migrations/fs-repo-migrations galacteek.app/Contents/Resources/bin +# copy tor +cp /usr/local/bin/tor galacteek.app/Contents/Resources/bin + pushd galacteek.app/Contents/Resources/bin ln -s ipfs-${GO_IPFS_VERSION} ipfs popd diff --git a/ci/build-pyinstaller-nsi.sh b/ci/build-pyinstaller-nsi.sh index ba78f2a4..64371ef4 100644 --- a/ci/build-pyinstaller-nsi.sh +++ b/ci/build-pyinstaller-nsi.sh @@ -3,17 +3,6 @@ set -x set -e -cat < galacteek_win.py -import sys -import faulthandler -print("Starting galacteek ..") - -faulthandler.enable(sys.stdout) - -from galacteek.guientrypoint import start -start() -EOF - export PYTHONPATH=$GITHUB_WORKSPACE unset VIRTUAL_ENV @@ -25,9 +14,14 @@ pip install pywin32 cp packaging/windows/pyimod03_importers.py \ c:\\hostedtoolcache\\windows\\python\\3.7.9\\x64\\lib\\site-packages\\PyInstaller\\loader -echo "Running pyinstaller" +# Copy tor and the dlls +cp /c/ProgramData/chocolatey/lib/tor/tools/Tor/* packaging/windows/tor +cp packaging/windows/galacteek_win.py . cp packaging/windows/galacteek_folder.spec . + +echo "Running pyinstaller" + pyinstaller galacteek_folder.spec echo "Success, packaging folder" diff --git a/galacteek/VERSION b/galacteek/VERSION new file mode 100644 index 00000000..4fc37cdb --- /dev/null +++ b/galacteek/VERSION @@ -0,0 +1 @@ +0.4.41 diff --git a/galacteek/__version__.py b/galacteek/__version__.py index 9e618831..29cb3a43 100644 --- a/galacteek/__version__.py +++ b/galacteek/__version__.py @@ -1 +1 @@ -__version__ = '0.4.40' +__version__ = '0.4.41' diff --git a/galacteek/application.py b/galacteek/application.py index 68d242c6..4c1e8a12 100644 --- a/galacteek/application.py +++ b/galacteek/application.py @@ -15,6 +15,7 @@ import aiojobs import shutil import signal +import psutil from pathlib import Path from filelock import FileLock @@ -38,6 +39,8 @@ from PyQt5.QtCore import QDir from PyQt5.QtCore import QMimeDatabase +from PyQt5.QtNetwork import QNetworkProxy + from PyQt5.QtGui import QCursor from galacteek import log @@ -55,6 +58,8 @@ from galacteek.core.db import SqliteDatabase from galacteek.core import pkgResourcesListDir from galacteek.core import pkgResourcesRscFilename +from galacteek.core.tor import TorLauncher +from galacteek.core.webproxy import NullProxy from galacteek import database from galacteek.database import models @@ -96,6 +101,7 @@ from galacteek.core.webprofiles import IPFSProfile from galacteek.core.webprofiles import Web3Profile from galacteek.core.webprofiles import MinimalProfile +from galacteek.core.webprofiles import AnonymousProfile from galacteek.dweb.webscripts import ipfsClientScripts from galacteek.dweb.render import defaultJinjaEnv @@ -187,6 +193,14 @@ def __init__(self, host, apiport, gwport): scheme='http', path='') + def asDict(self): + return { + 'host': self.host, + 'apiPort': self.apiPort, + 'gatewayPort': self.gatewayPort, + 'gatewayUrl': self.gatewayUrl + } + @property def host(self): return self._host @@ -243,6 +257,7 @@ def __init__(self, debug=False, profile='main', sslverify=True, self._urlSchemes = {} self._shuttingDown = False self._freshInstall = False + self._process = psutil.Process(os.getpid()) self._icons = {} self._ipfsIconsCache = {} @@ -251,6 +266,7 @@ def __init__(self, debug=False, profile='main', sslverify=True, self.enableOrbital = enableOrbital self.orbitConnector = None + self.netProxy = None self.translator = None self.mainWindow = None @@ -443,6 +459,17 @@ def debug(self, msg): if self.debugEnabled: log.debug(msg) + def networkProxy(self): + # return QNetworkProxy.applicationProxy() + return self.netProxy + + def networkProxySet(self, proxy): + QNetworkProxy.setApplicationProxy(proxy) + self.netProxy = proxy + + def networkProxySetNull(self): + self.networkProxySet(NullProxy()) + def initSystemTray(self): self.systemTray = QSystemTrayIcon(self) self.systemTray.setIcon(getIcon('galacteek-incandescent.png')) @@ -464,6 +491,9 @@ def initSystemTray(self): self.systemTray.setContextMenu(systemTrayMenu) def initMisc(self): + # Start with no proxy + self.networkProxySet(NullProxy()) + self.multihashDb = IPFSObjectMetadataDatabase(self._mHashDbLocation, loop=self.loop) @@ -481,6 +511,7 @@ def initMisc(self): self.tempDirWeb = self.tempDirCreate( self.tempDir.path(), 'webdownloads') + self.tor = TorLauncher(self._torConfigLocation) self._goIpfsBinPath = self.suitableGoIpfsBinary() def tempDirCreate(self, basedir, name=None): @@ -794,6 +825,8 @@ async def onDbConfigured(self, configured): self.createMainWindow() self.clipboardInit() + await self.tor.start() + await self.setupIpfsConnection() async def fetchGoIpfs(self): @@ -888,6 +921,10 @@ def repolishWidget(self, widget): self.style().unpolish(widget) self.style().polish(widget) + def webClientSession(self): + from galacteek.core.asynclib import clientSessionWithProxy + return clientSessionWithProxy(self.netProxy.url()) + def setupAsyncLoop(self): """ Install the asyncqt event loop and enable debugging @@ -970,6 +1007,7 @@ def setupPaths(self): self._orbitDataLocation = os.path.join(self.dataLocation, 'orbitdb') self._mHashDbLocation = os.path.join(self.dataLocation, 'mhashmetadb') self._sqliteDbLocation = os.path.join(self.dataLocation, 'db.sqlite') + self._torConfigLocation = os.path.join(self.dataLocation, 'torrc') self._pLockLocation = os.path.join(self.dataLocation, 'profile.lock') self._mainDbLocation = os.path.join( self.dataLocation, 'db_main.sqlite3') @@ -1315,9 +1353,16 @@ def initWebProfiles(self): self.webProfiles = { 'minimal': MinimalProfile(parent=self), 'ipfs': IPFSProfile(parent=self), - 'web3': Web3Profile(parent=self) + 'web3': Web3Profile(parent=self), + 'anonymous': AnonymousProfile(parent=self) } + def allWebProfilesSetAttribute(self, attribute, val): + for pName, profile in self.webProfiles.items(): + log.debug(f'Web profile {pName}: setting attr ' + f'{attribute}:{val}') + profile.webSettings.setAttribute(attribute, val) + def availableWebProfilesNames(self): return [p.profileName for n, p in self.webProfiles.items()] @@ -1405,6 +1450,9 @@ async def exitApp(self, detachIpfsd=False): self.lock.release() + if self.tor: + self.tor.stop() + self.mainWindow.stopTimers() await self.mainWindow.stack.shutdown() @@ -1447,7 +1495,8 @@ async def exitApp(self, detachIpfsd=False): self.quit() if self.windowsSystem: - sys.exit(0) + # temporary, need to fix #33 + self._process.kill() class ManualsManager(QObject): diff --git a/galacteek/core/asynclib.py b/galacteek/core/asynclib.py index 5f776ed6..667ad0f2 100644 --- a/galacteek/core/asynclib.py +++ b/galacteek/core/asynclib.py @@ -6,6 +6,9 @@ import functools from asyncio_extras.file import open_async import aiofiles +import aiohttp +from aiohttp_socks import ProxyConnector + from PyQt5.QtWidgets import QApplication @@ -300,3 +303,11 @@ async def asyncRmTree(path): shutil.rmtree, path ) + + +def clientSessionWithProxy(proxyUrl): + if proxyUrl and proxyUrl.startswith('socks5://'): + return aiohttp.ClientSession( + connector=ProxyConnector.from_url(proxyUrl)) + else: + return aiohttp.ClientSession() diff --git a/galacteek/core/schemes.py b/galacteek/core/schemes.py index 73480be8..47b6e5b0 100644 --- a/galacteek/core/schemes.py +++ b/galacteek/core/schemes.py @@ -39,7 +39,7 @@ from galacteek.core.asynccache import cachedcoromethod -# Core schemes +# Core schemes (the URL schemes your children will soon teach you how to use) SCHEME_DWEB = 'dweb' SCHEME_DWEBGW = 'dwebgw' SCHEME_FS = 'fs' @@ -51,13 +51,21 @@ SCHEME_ENSR = 'ensr' SCHEME_E = 'e' +# Obsolete schemes :) +SCHEME_HTTP = 'http' +SCHEME_HTTPS = 'https' +SCHEME_FTP = 'ftp' + # Misc schemes SCHEME_Z = 'z' SCHEME_Q = 'q' -SCHEME_GALACTEEK = 'glk' +SCHEME_GALACTEEK = 'g' +SCHEME_DISTRIBUTED = 'd' SCHEME_PALACE = 'palace' SCHEME_MANUAL = 'manual' +SCHEME_CHROMIUM = 'chromium' + # Default flags used by declareUrlScheme() defaultSchemeFlags = QWebEngineUrlScheme.SecureScheme | \ @@ -84,6 +92,15 @@ def isSchemeRegistered(scheme): return True +def isUrlSupported(url): + return isSchemeRegistered(url.scheme()) or url.scheme() in [ + SCHEME_HTTP, + SCHEME_HTTPS, + SCHEME_FTP, + SCHEME_CHROMIUM + ] + + def declareUrlScheme(name, flags=defaultSchemeFlags, syntax=QWebEngineUrlScheme.Syntax.Host, @@ -116,6 +133,11 @@ def registerMiscSchemes(): flags=QWebEngineUrlScheme.LocalScheme, schemeSection='misc' ) + declareUrlScheme(SCHEME_GALACTEEK, + syntax=QWebEngineUrlScheme.Syntax.Path, + flags=QWebEngineUrlScheme.LocalScheme, + schemeSection='core' + ) declareUrlScheme(SCHEME_Q, syntax=QWebEngineUrlScheme.Syntax.Host, flags=QWebEngineUrlScheme.LocalScheme, @@ -189,6 +211,11 @@ def isEnsUrl(url): return url.scheme() in [SCHEME_ENS, SCHEME_ENSR] +def isHttpUrl(url): + if url.isValid(): + return url.scheme() in [SCHEME_HTTP, SCHEME_HTTPS] + + class BaseURLSchemeHandler(QWebEngineUrlSchemeHandler): webProfileNeeded = None @@ -225,6 +252,15 @@ def allocReqId(self, req): return uid + async def handleRequest(self, request, uid): + return self.urlInvalid(request) + + def requestStarted(self, request): + uid = str(uuid.uuid4()) + self.requests[uid] = request + request.destroyed.connect(lambda: self.onRequestDestroyed(uid)) + ensure(self.handleRequest(self.requests[uid])) + class IPFSObjectProxyScheme: """ diff --git a/galacteek/core/tor.py b/galacteek/core/tor.py new file mode 100644 index 00000000..99dd6525 --- /dev/null +++ b/galacteek/core/tor.py @@ -0,0 +1,212 @@ +import asyncio +import re +import signal +import psutil +import platform +import subprocess +import tempfile + +from galacteek import log +from galacteek import ensure +from galacteek import AsyncSignal +from galacteek.core.asynclib import asyncWriteFile +from galacteek.core import unusedTcpPort + + +class TorProtocol(asyncio.SubprocessProtocol): + def __init__(self, loop, exitFuture, startedFuture, debug=False): + self._loop = loop + self.debug = debug + self.eventStarted = asyncio.Event() + self.exitFuture = exitFuture + self.startedFuture = startedFuture + self.errAlreadyRunning = False + + self.sTorBootstrapStatus = AsyncSignal(int, str) + + @property + def loop(self): + return self._loop + + def pipe_data_received(self, fd, data): + try: + msg = data.decode().strip() + except BaseException: + return + + for line in msg.split('\n'): + ma = re.search( + r'\w*\s\d*\s\d+:\d+:\d+\.\d* \[.*\] ' + r'Bootstrapped (\d+)\%\s\(([\w\_]*)\)', + line) + if ma: + try: + pc = ma.group(1) + status = ma.group(2) + log.debug(f'TOR bootstrapped at {pc} percent: {status}') + ensure(self.sTorBootstrapStatus.emit(int(pc), status)) + except Exception: + continue + + if self.debug: + log.debug(f'TOR: {line}') + + def process_exited(self): + if not self.exitFuture.done(): + self.exitFuture.set_result(True) + + +torConfigTemplate = ''' +SOCKSPort {socksPort} +ControlPort {controlPort} +SOCKSPolicy accept 127.0.0.1 +SOCKSPolicy reject * +DNSPort {dnsPort} +AutomapHostsOnResolve 1 +AutomapHostsSuffixes .exit,.onion +DataDirectory {dataDir} +''' + + +class TorConfigBuilder: + def __init__(self): + self._socksPort = None + self._controlPort = None + self._dnsPort = None + self._hostname = '127.0.0.1' + self._dataDir = tempfile.mkdtemp(prefix='gtor') + + @property + def socksPort(self): + return self._socksPort + + @property + def controlPort(self): + return self._controlPort + + @property + def dnsPort(self): + return self._dnsPort + + @property + def hostname(self): + return self._hostname + + @socksPort.setter + def socksPort(self, v): + self._socksPort = v + self._controlPort = v + 1 + self._dnsPort = v + 2 + + def __str__(self): + return torConfigTemplate.format( + socksPort=self.socksPort, + controlPort=self.controlPort, + dnsPort=self.dnsPort, + dataDir=self._dataDir + ) + + +class TorLauncher: + def __init__(self, configPath, torPath='tor', debug=True, loop=None): + + self.loop = loop if loop else asyncio.get_event_loop() + self.exitFuture = asyncio.Future(loop=self.loop) + self.startedFuture = asyncio.Future(loop=self.loop) + + self._procPid = None + self._process = None + self.torPath = torPath + self.configPath = configPath + self.debug = debug + self.transport, self.proto = None, None + self.torCfg = TorConfigBuilder() + self.torProto = TorProtocol(self.loop, self.exitFuture, + self.startedFuture, + debug=self.debug) + + @property + def process(self): + return self._process + + @process.setter + def process(self, p): + if p: + log.debug(f'Tor process changed, PID: {p.pid}') + else: + log.debug('Tor process reset') + + self._process = p + + @property + def pid(self): + return self._procPid + + @property + def running(self): + return self.pid is not None + + def message(self, msg): + log.debug(msg) + + async def start(self): + pCreationFlags = 0 + + startupInfo = None + if platform.system() == 'Windows': + startupInfo = subprocess.STARTUPINFO() + startupInfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + startupInfo.wShowWindow = subprocess.SW_HIDE + + # for socksPort in range(9052, 9080): + for x in range(0, 12): + socksPort = unusedTcpPort() + + self.torCfg.socksPort = socksPort + await asyncWriteFile(self.configPath, str(self.torCfg), 'w+t') + + args = [ + self.torPath, + '-f', self.configPath + ] + log.debug(f'Starting TOR with args: {args}') + + try: + f = self.loop.subprocess_exec( + lambda: self.torProto, + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + startupinfo=startupInfo, + creationflags=pCreationFlags) + + self.transport, self.proto = await f + + self._procPid = self.transport.get_pid() + self.process = psutil.Process(self._procPid) + except Exception: + log.debug(f'Starting TOR failed on port {socksPort}') + continue + else: + log.debug(f'Starting TOR OK on port {socksPort}') + break + + def stop(self): + self.message('Stopping Tor') + try: + if not self.process: + raise Exception('Process not found') + + if platform.system() == 'Windows': + self.process.kill() + else: + self.process.send_signal(signal.SIGINT) + self.process.send_signal(signal.SIGHUP) + + self._procPid = None + return True + except Exception as err: + self.message(f'Error shutting down daemon: {err}') + self._procPid = None + self.terminateException = err + return False diff --git a/galacteek/core/webprofiles.py b/galacteek/core/webprofiles.py index 609f5c6a..2726d34f 100644 --- a/galacteek/core/webprofiles.py +++ b/galacteek/core/webprofiles.py @@ -15,6 +15,7 @@ from galacteek.core.schemes import isIpfsUrl +WP_NAME_ANON = 'anonymous' WP_NAME_MINIMAL = 'minimal' WP_NAME_IPFS = 'ipfs' WP_NAME_WEB3 = 'web3' @@ -95,6 +96,23 @@ def __init__(self, storageName=WP_NAME_MINIMAL, parent=None): super(MinimalProfile, self).__init__(storageName, parent) +class AnonymousProfile(BaseProfile): + """ + Anonymous web profile. No JS, no cache, no cookies. + """ + def __init__(self, storageName=WP_NAME_ANON, parent=None): + super(AnonymousProfile, self).__init__(storageName, parent) + + def setSettings(self): + super().setSettings() + self.webSettings.setAttribute(QWebEngineSettings.JavascriptEnabled, + False) + self.webSettings.setAttribute(QWebEngineSettings.XSSAuditingEnabled, + True) + self.setPersistentCookiesPolicy(QWebEngineProfile.NoPersistentCookies) + self.setHttpCacheType(QWebEngineProfile.NoCache) + + class IPFSProfile(BaseProfile): """ IPFS web profile diff --git a/galacteek/core/webproxy.py b/galacteek/core/webproxy.py new file mode 100644 index 00000000..09e157e7 --- /dev/null +++ b/galacteek/core/webproxy.py @@ -0,0 +1,31 @@ +from PyQt5.QtNetwork import QNetworkProxy + + +class Proxy(QNetworkProxy): + def apply(self): + QNetworkProxy.setApplicationProxy(self) + + def url(self): + return None + + +class TorNetworkProxy(QNetworkProxy): + def __init__(self, torCfg): + super().__init__() + self.torCfg = torCfg + + self.setType(QNetworkProxy.Socks5Proxy) + self.setHostName(self.torCfg.hostname) + self.setPort(self.torCfg.socksPort) + + def url(self): + return f'socks5://{self.torCfg.hostname}:{self.torCfg.socksPort}' + + +class NullProxy(QNetworkProxy): + def __init__(self): + super().__init__() + self.setType(QNetworkProxy.NoProxy) + + def url(self): + return None diff --git a/galacteek/database/__init__.py b/galacteek/database/__init__.py index cdfa7163..eb641dcc 100644 --- a/galacteek/database/__init__.py +++ b/galacteek/database/__init__.py @@ -168,7 +168,6 @@ async def hashmarkAdd(path: str, datecreated=None, source=None, **kw): - mark = await hashmarksByPath(path) if mark: return mark @@ -202,7 +201,8 @@ async def hashmarkAdd(path: str, extra['url'] = ipfsPath.ipfsUrl else: url = QUrl(path) - if url.isValid() and url.scheme() in ['ens', 'ensr']: + if url.isValid() and url.scheme() in [ + 'ens', 'ensr', 'http', 'https', 'ftp']: extra['url'] = url.toString() if isinstance(icon, str): diff --git a/galacteek/docs/manual/en/browsing.rst b/galacteek/docs/manual/en/browsing.rst index d1d03302..05a6bc51 100644 --- a/galacteek/docs/manual/en/browsing.rst +++ b/galacteek/docs/manual/en/browsing.rst @@ -7,13 +7,24 @@ URL bar In the address bar you can type (or paste) a full URL, an IPFS :term:`CID` or a full :term:`IPFS path` (they will be -loaded with the appropriate scheme). +loaded with the appropriate scheme). If you type +an ENS domain name or a regular domain name it will be +loaded automatically with the right scheme. You can also type words that you want to search for in the hashmarks -database and the visited URLs history (search results will -pop up after a short amount of time). Hitting the *Escape* key +database and the visited URLs history. Search results will +pop up after a short amount of time. Hitting the *Escape* key will hide the results. +You can also use specific syntax to search with certain +search engines: + +- Use the **d** command to search with the + [DuckDuckGo](https://duckduckgo.com/) web search engine. + Example: **d distributed web** +- Use the **i** or **ip** command to run a search on the IPFS + network. Example: **i distributed web** + CID status icon ^^^^^^^^^^^^^^^ @@ -168,6 +179,16 @@ You can change the default web profile that will be used when opening a browser tab by changing the *Default web profile* setting in the *UI* section of the application settings. +Anomymus profile +^^^^^^^^^^^^^^^^ + +Anonymous profile: + +- Javascript is disabled +- Caching is disabled +- No persistent cookies +- XSS auditing + Minimal profile ^^^^^^^^^^^^^^^ diff --git a/galacteek/dweb/cyber.py b/galacteek/dweb/cyber.py index 52e51bd5..c32394a4 100644 --- a/galacteek/dweb/cyber.py +++ b/galacteek/dweb/cyber.py @@ -1,7 +1,7 @@ from galacteek import log from galacteek.ipfs import ipfsOpFn +from galacteek.core.asynclib import clientSessionWithProxy -import aiohttp import async_timeout @@ -27,6 +27,7 @@ def hitsCount(self): @ipfsOpFn async def cyberSearch(ipfsop, query: str, page=0, perPage=10, sslverify=True, + proxyUrl=None, timeout=8): entry = await ipfsop.hashComputeString(query, cidversion=0) @@ -38,7 +39,7 @@ async def cyberSearch(ipfsop, query: str, page=0, perPage=10, sslverify=True, try: with async_timeout.timeout(timeout): - async with aiohttp.ClientSession() as session: + async with clientSessionWithProxy(proxyUrl) as session: async with session.get('https://{host}/api/search'.format( host='titan.cybernode.ai'), params=params, diff --git a/galacteek/dweb/render.py b/galacteek/dweb/render.py index 72468411..f391f183 100644 --- a/galacteek/dweb/render.py +++ b/galacteek/dweb/render.py @@ -5,6 +5,8 @@ from datetime import datetime from dateutil import parser as dateparser from tempfile import TemporaryDirectory +from cachetools import TTLCache +from cachetools import keys from galacteek import log from galacteek.core import isoformat @@ -83,14 +85,42 @@ def renderWrapper(tmpl, **kw): return data -async def renderTemplate(tmplname, loop=None, env=None, **kw): - env = env if env else defaultJinjaEnv() - tmpl = env.get_template(tmplname) - if not tmpl: - raise Exception('template not found') +templatesCache = TTLCache(64, 60) + + +async def renderTemplate(tmplname, loop=None, env=None, + _cache=False, + _cacheKeyAttrs=None, + **kw): + global templatesCache + + key = None + if _cache: + if isinstance(_cacheKeyAttrs, list): + # use certain kw attributes to form the + # entry's key in the cache + attrs = {} + for attr in _cacheKeyAttrs: + attrs[attr] = kw.get(attr) + key = keys.hashkey(tmplname, **attrs) + else: + # use all kw attributes + key = keys.hashkey(tmplname, **kw) + + try: + if key is None: + raise KeyError('Not using cache') - data = await tmpl.render_async(**kw) - return data + return templatesCache[key] + except KeyError: + env = env if env else defaultJinjaEnv() + tmpl = env.get_template(tmplname) + if not tmpl: + raise Exception('template not found') + + data = await tmpl.render_async(**kw) + templatesCache[key] = data + return data @ipfsOpFn diff --git a/galacteek/guientrypoint.py b/galacteek/guientrypoint.py index d389ea47..411319ce 100644 --- a/galacteek/guientrypoint.py +++ b/galacteek/guientrypoint.py @@ -202,7 +202,7 @@ def hideConsoleWindow(): def start(): global appStarter - if platform.system() == 'Windows' and inPyInstaller(): + if platform.system() == 'Windows' and inPyInstaller() and 0: # Hide the console window when running with pyinstaller hideConsoleWindow() diff --git a/galacteek/ipfs/ipfssearch.py b/galacteek/ipfs/ipfssearch.py index 71b34123..3e8a9f12 100755 --- a/galacteek/ipfs/ipfssearch.py +++ b/galacteek/ipfs/ipfssearch.py @@ -1,9 +1,11 @@ from galacteek import log +from galacteek.core.asynclib import clientSessionWithProxy import aiohttp import async_timeout + ipfsSearchApiHost = 'api.ipfs-search.com' @@ -35,7 +37,8 @@ def hitsCount(self): emptyResults = IPFSSearchResults(0, {}) -async def searchPage(query, page, filters={}, sslverify=True): +async def searchPage(query, page, filters={}, sslverify=True, + proxyUrl=None): params = { 'q': query, 'page': page @@ -45,7 +48,7 @@ async def searchPage(query, page, filters={}, sslverify=True): params['q'] += ' {fkey}:{fvalue}'.format( fkey=fkey, fvalue=fvalue) - async with aiohttp.ClientSession() as session: + async with clientSessionWithProxy(proxyUrl) as session: async with session.get('https://{host}/v1/search'.format( host=ipfsSearchApiHost), params=params, @@ -54,10 +57,12 @@ async def searchPage(query, page, filters={}, sslverify=True): async def getPageResults(query, page, filters={}, sslverify=True, + proxyUrl=None, timeout=12): try: with async_timeout.timeout(timeout): results = await searchPage(query, page, filters=filters, + proxyUrl=proxyUrl, sslverify=sslverify) return IPFSSearchResults(page, results) except Exception as err: diff --git a/galacteek/ipfs/search.py b/galacteek/ipfs/search.py index fc2a0969..3214ce21 100644 --- a/galacteek/ipfs/search.py +++ b/galacteek/ipfs/search.py @@ -13,7 +13,8 @@ def alternate(s1, s2): async def multiSearch(query, page=0, - filters=None, sslverify=True): + filters=None, sslverify=True, + proxyUrl=None): """ Perform a search query on multiple search engines (ipfs-search and cyber). This is an @@ -24,8 +25,10 @@ async def multiSearch(query, page=0, results = await asyncio.gather( ipfssearch.getPageResults(query, page, filters=filters if filters else {}, - sslverify=sslverify), - cyberSearch(query, page=page, perPage=20, sslverify=sslverify) + sslverify=sslverify, + proxyUrl=proxyUrl), + cyberSearch(query, page=page, perPage=20, sslverify=sslverify, + proxyUrl=proxyUrl) ) if len(results) == 2: diff --git a/galacteek/templates/ipfssearch.html b/galacteek/templates/ipfssearch.html index 3636dfa0..453be808 100644 --- a/galacteek/templates/ipfssearch.html +++ b/galacteek/templates/ipfssearch.html @@ -4,7 +4,7 @@