Add support for Tor proxying

* Tor proxying on all platforms

* Tor proxying of ipfs-search/cyber requests

* Templates caching

* Nicer history completion interface

* IPFS search from any browser tab

.github/workflows/galacteek-deploy.yml

@@ -33,6 +33,12 @@ jobs:
         sudo apt-get install -y dzen2 xvfb herbstluftwm
         sudo apt-get install -y libxcb-xkb1 libxkbcommon-x11-0
         sudo apt-get install -y libzbar0
+        sudo apt-get install -y tor
+
+    - name: Install packages (macos)
+      if: startsWith(matrix.os, 'mac')
+      run: |
+        brew install tor
 
     - name: Install unzip (Windows)
       if: startsWith(matrix.os, 'windows')
@@ -58,9 +64,15 @@ jobs:
       with:
         args: install nsis
 
+    - name: Install Tor (Windows)
+      if: startsWith(matrix.os, 'windows')
+      uses: crazy-max/ghaction-chocolatey@v1
+      with:
+        args: install tor
+
     - name: Configure environment (1)
       run: |
-        echo "G_VERSION=$(cat galacteek/version)" >> $GITHUB_ENV
+        echo "G_VERSION=$(cat galacteek/VERSION)" >> $GITHUB_ENV
 
         echo "COMMIT_SHORT=$(echo $GITHUB_SHA|cut -c 1-8)" >> $GITHUB_ENV
         echo "GIT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
@@ -256,7 +268,7 @@ jobs:
       - name: Release config
         id: release_config
         run: |
-          echo "G_VERSION=$(cat galacteek/version)" >> $GITHUB_ENV
+          echo "G_VERSION=$(cat galacteek/VERSION)" >> $GITHUB_ENV
           echo "COMMIT_SHORT=$(echo $GITHUB_SHA|cut -c 1-8)" >> $GITHUB_ENV
           echo "GIT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
           echo "TAGNAME=continuous-${GITHUB_REF##*/}" >> $GITHUB_ENV

AppImage/appimage-build

@@ -97,6 +97,9 @@ find /usr/lib -iname 'libzbar.so*' -exec cp -av {} $APPDIR/usr/lib \;
 cp $GITHUB_WORKSPACE/go-ipfs/ipfs-${GO_IPFS_VERSION} $APPDIR/usr/bin
 cp $GITHUB_WORKSPACE/fs-repo-migrations/fs-repo-migrations $APPDIR/usr/bin
 
+# Copy tor
+cp /usr/bin/tor $APPDIR/usr/bin
+
 pushd "$APPDIR"/usr/bin
 ln -s ipfs-${GO_IPFS_VERSION} ipfs
 popd

CHANGELOG.md

@@ -8,6 +8,30 @@ the changes in the CHANGELOG formatting.
 
 ## [Unreleased]
 
+## [0.4.41] - 2020-12-02
+### Added
+- Tor support
+  - Automatic use of Tor when bootstrapping succeeds
+  - Tor proxying for all ipfs-search and cyber requests
+
+- Add a new *anonymous* web profile
+- Automatically fetch favicons when hashmarking an http(s) website
+- Handle SSL certificate errors
+
+### Changed
+- Bookmarking of any type of URLs
+- Browser UI
+  - Use a block-style cursor for the address bar
+  - Typing an .eth domain name automatically loads it through *ens://*
+  - Run IPFS searches or searches with popular engines (duckduckgo, ..)
+    from the address bar
+  - Nicer history lookup interface
+
+- The @Earth workspace is now the default workspace in the stack
+
+### Fixed
+- Bookmarking of clearnet URLs
+
 ## [0.4.40] - 2020-11-29
 ### Added
 - Lightweight BT client integration (asyncio-based)

ci/build-dmg.sh

@@ -81,6 +81,9 @@ mkdir -p galacteek.app/Contents/Resources/bin
 cp $GITHUB_WORKSPACE/go-ipfs/ipfs-${GO_IPFS_VERSION} galacteek.app/Contents/Resources/bin
 cp $GITHUB_WORKSPACE/fs-repo-migrations/fs-repo-migrations galacteek.app/Contents/Resources/bin
 
+# copy tor
+cp /usr/local/bin/tor galacteek.app/Contents/Resources/bin
+
 pushd galacteek.app/Contents/Resources/bin
 ln -s ipfs-${GO_IPFS_VERSION} ipfs
 popd

ci/build-pyinstaller-nsi.sh

@@ -3,17 +3,6 @@
 set -x
 set -e
 
-cat <<EOF > galacteek_win.py
-import sys
-import faulthandler
-print("Starting galacteek ..")
-
-faulthandler.enable(sys.stdout)
-
-from galacteek.guientrypoint import start
-start()
-EOF
-
 export PYTHONPATH=$GITHUB_WORKSPACE
 
 unset VIRTUAL_ENV
@@ -25,9 +14,14 @@ pip install pywin32
 cp packaging/windows/pyimod03_importers.py \
     c:\\hostedtoolcache\\windows\\python\\3.7.9\\x64\\lib\\site-packages\\PyInstaller\\loader
 
-echo "Running pyinstaller"
+# Copy tor and the dlls
+cp /c/ProgramData/chocolatey/lib/tor/tools/Tor/*  packaging/windows/tor
 
+cp packaging/windows/galacteek_win.py .
 cp packaging/windows/galacteek_folder.spec .
+
+echo "Running pyinstaller"
+
 pyinstaller galacteek_folder.spec
 
 echo "Success, packaging folder"

galacteek/VERSION

@@ -0,0 +1 @@
+0.4.41

galacteek/__version__.py

@@ -1 +1 @@
-__version__ = '0.4.40'
+__version__ = '0.4.41'

galacteek/application.py

@@ -15,6 +15,7 @@
 import aiojobs
 import shutil
 import signal
+import psutil
 from pathlib import Path
 from filelock import FileLock
 
@@ -38,6 +39,8 @@
 from PyQt5.QtCore import QDir
 from PyQt5.QtCore import QMimeDatabase
 
+from PyQt5.QtNetwork import QNetworkProxy
+
 from PyQt5.QtGui import QCursor
 
 from galacteek import log
@@ -55,6 +58,8 @@
 from galacteek.core.db import SqliteDatabase
 from galacteek.core import pkgResourcesListDir
 from galacteek.core import pkgResourcesRscFilename
+from galacteek.core.tor import TorLauncher
+from galacteek.core.webproxy import NullProxy
 
 from galacteek import database
 from galacteek.database import models
@@ -96,6 +101,7 @@
 from galacteek.core.webprofiles import IPFSProfile
 from galacteek.core.webprofiles import Web3Profile
 from galacteek.core.webprofiles import MinimalProfile
+from galacteek.core.webprofiles import AnonymousProfile
 
 from galacteek.dweb.webscripts import ipfsClientScripts
 from galacteek.dweb.render import defaultJinjaEnv
@@ -187,6 +193,14 @@ def __init__(self, host, apiport, gwport):
             scheme='http',
             path='')
 
+    def asDict(self):
+        return {
+            'host': self.host,
+            'apiPort': self.apiPort,
+            'gatewayPort': self.gatewayPort,
+            'gatewayUrl': self.gatewayUrl
+        }
+
     @property
     def host(self):
         return self._host
@@ -243,6 +257,7 @@ def __init__(self, debug=False, profile='main', sslverify=True,
         self._urlSchemes = {}
         self._shuttingDown = False
         self._freshInstall = False
+        self._process = psutil.Process(os.getpid())
 
         self._icons = {}
         self._ipfsIconsCache = {}
@@ -251,6 +266,7 @@ def __init__(self, debug=False, profile='main', sslverify=True,
 
         self.enableOrbital = enableOrbital
         self.orbitConnector = None
+        self.netProxy = None
 
         self.translator = None
         self.mainWindow = None
@@ -443,6 +459,17 @@ def debug(self, msg):
         if self.debugEnabled:
             log.debug(msg)
 
+    def networkProxy(self):
+        # return QNetworkProxy.applicationProxy()
+        return self.netProxy
+
+    def networkProxySet(self, proxy):
+        QNetworkProxy.setApplicationProxy(proxy)
+        self.netProxy = proxy
+
+    def networkProxySetNull(self):
+        self.networkProxySet(NullProxy())
+
     def initSystemTray(self):
         self.systemTray = QSystemTrayIcon(self)
         self.systemTray.setIcon(getIcon('galacteek-incandescent.png'))
@@ -464,6 +491,9 @@ def initSystemTray(self):
         self.systemTray.setContextMenu(systemTrayMenu)
 
     def initMisc(self):
+        # Start with no proxy
+        self.networkProxySet(NullProxy())
+
         self.multihashDb = IPFSObjectMetadataDatabase(self._mHashDbLocation,
                                                       loop=self.loop)
 
@@ -481,6 +511,7 @@ def initMisc(self):
         self.tempDirWeb = self.tempDirCreate(
             self.tempDir.path(), 'webdownloads')
 
+        self.tor = TorLauncher(self._torConfigLocation)
         self._goIpfsBinPath = self.suitableGoIpfsBinary()
 
     def tempDirCreate(self, basedir, name=None):
@@ -794,6 +825,8 @@ async def onDbConfigured(self, configured):
         self.createMainWindow()
         self.clipboardInit()
 
+        await self.tor.start()
+
         await self.setupIpfsConnection()
 
     async def fetchGoIpfs(self):
@@ -888,6 +921,10 @@ def repolishWidget(self, widget):
         self.style().unpolish(widget)
         self.style().polish(widget)
 
+    def webClientSession(self):
+        from galacteek.core.asynclib import clientSessionWithProxy
+        return clientSessionWithProxy(self.netProxy.url())
+
     def setupAsyncLoop(self):
         """
         Install the asyncqt event loop and enable debugging
@@ -970,6 +1007,7 @@ def setupPaths(self):
         self._orbitDataLocation = os.path.join(self.dataLocation, 'orbitdb')
         self._mHashDbLocation = os.path.join(self.dataLocation, 'mhashmetadb')
         self._sqliteDbLocation = os.path.join(self.dataLocation, 'db.sqlite')
+        self._torConfigLocation = os.path.join(self.dataLocation, 'torrc')
         self._pLockLocation = os.path.join(self.dataLocation, 'profile.lock')
         self._mainDbLocation = os.path.join(
             self.dataLocation, 'db_main.sqlite3')
@@ -1315,9 +1353,16 @@ def initWebProfiles(self):
         self.webProfiles = {
             'minimal': MinimalProfile(parent=self),
             'ipfs': IPFSProfile(parent=self),
-            'web3': Web3Profile(parent=self)
+            'web3': Web3Profile(parent=self),
+            'anonymous': AnonymousProfile(parent=self)
         }
 
+    def allWebProfilesSetAttribute(self, attribute, val):
+        for pName, profile in self.webProfiles.items():
+            log.debug(f'Web profile {pName}: setting attr '
+                      f'{attribute}:{val}')
+            profile.webSettings.setAttribute(attribute, val)
+
     def availableWebProfilesNames(self):
         return [p.profileName for n, p in self.webProfiles.items()]
 
@@ -1405,6 +1450,9 @@ async def exitApp(self, detachIpfsd=False):
 
         self.lock.release()
 
+        if self.tor:
+            self.tor.stop()
+
         self.mainWindow.stopTimers()
         await self.mainWindow.stack.shutdown()
 
@@ -1447,7 +1495,8 @@ async def exitApp(self, detachIpfsd=False):
         self.quit()
 
         if self.windowsSystem:
-            sys.exit(0)
+            # temporary, need to fix #33
+            self._process.kill()
 
 
 class ManualsManager(QObject):

galacteek/core/asynclib.py

@@ -6,6 +6,9 @@
 import functools
 from asyncio_extras.file import open_async
 import aiofiles
+import aiohttp
+from aiohttp_socks import ProxyConnector
+
 
 from PyQt5.QtWidgets import QApplication
 
@@ -300,3 +303,11 @@ async def asyncRmTree(path):
         shutil.rmtree,
         path
     )
+
+
+def clientSessionWithProxy(proxyUrl):
+    if proxyUrl and proxyUrl.startswith('socks5://'):
+        return aiohttp.ClientSession(
+            connector=ProxyConnector.from_url(proxyUrl))
+    else:
+        return aiohttp.ClientSession()