Add MFS metadata when linking in the filemanager

Use tox in the CI process

Cyber search

README.rst

@@ -9,9 +9,6 @@
 .. image:: https://travis-ci.org/pinnaculum/galacteek.svg?branch=master
     :target: https://travis-ci.org/pinnaculum/galacteek
 
-.. image:: https://badges.gitter.im/galacteek/community.svg
-   :target: https://gitter.im/galacteek/galacteek?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge
-
 **galacteek** is a multi-platform Qt5-based browser/toolbox
 for the IPFS_ peer-to-peer network.
 
@@ -82,11 +79,12 @@ should enable pubsub and p2p streams, or some features won't be available.
   (Verifiable Credentials with RSA-PSS)
 - Browsing sessions with automatic pinning (pins every page you browse)
 - Distributed chat with pubsub (chat channels syncronized with CRDT+DAG)
-- File manager with drag-and-drop support
+- File manager with drag-and-drop support and timeframes (per-day view
+  of files in the MFS)
 - Webcam to IPFS capture (image and videos)
 - Run WASM binaries with wasmer_ (use *Open* on a WASM object from the
   clipboard manager)
-- Search content with the ipfs-search_ search engine
+- Search content with the ipfs-search_ search engine as well as with cyber_
 - Use the IPFS filestore to avoid file duplication
 - Atom feeds (subscribe to feeds on the dweb)
 - ENS_ (Ethereum Name Service) resolving (access to ENS+IPFS websites)
@@ -191,3 +189,4 @@ from the ipfs-logo_ project's repository is included, unchanged.
 .. _AppImage: https://appimage.org/
 .. _IPID: https://github.com/jonnycrunch/ipid
 .. _wasmer: https://wasmer.io/
+.. _cyber: https://cybercongress.ai

galacteek/__init__.py

@@ -46,7 +46,7 @@
 # ..............................................................................
 #
 
-__version__ = '0.4.28'
+__version__ = '0.4.29'
 
 from galacteek.core.asynclib import *  # noqa
 from galacteek.core import glogger

galacteek/application.py

@@ -575,6 +575,7 @@ async def setupRepository(self, op):
             self.feedFollower.process())
 
         await self.ipfsCtx.ipfsRepositoryReady.emit()
+        self.ipfsCtx._ipfsRepositoryReady.emit()
 
         #
         # If the application's binary name is a valid CID, pin it!

galacteek/core/analyzer.py

@@ -4,12 +4,17 @@
 from PyQt5.QtCore import QObject
 
 from galacteek import log
+from galacteek import ensure
 
 from galacteek.ipfs import ipfsOp
 
 from galacteek.ipfs.cidhelpers import IPFSPath
+from galacteek.ipfs.cidhelpers import getCID
+from galacteek.ipfs.cidhelpers import cidDowngrade
 from galacteek.ipfs.mimetype import MIMEType
 from galacteek.ipfs.mimetype import detectMimeType
+from galacteek.ipfs.stat import StatInfo
+from galacteek.ipfs.ipfssearch import objectMetadata
 
 from galacteek.crypto.qrcode import IPFSQrDecoder
 
@@ -62,11 +67,31 @@ async def __call__(self, ipfsop, pathRef):
                 stat=statInfo
             )
 
+            # Fetch additional metadata in another task
+            ensure(self.fetchMetadata(path, statInfo))
+
         if mimetype and mimetype.valid:
             return mimetype, statInfo
 
         return None, None
 
+    async def fetchMetadata(self, path, stat):
+        sInfo = StatInfo(stat)
+
+        cidobj = getCID(sInfo.cid)
+        cid = cidDowngrade(cidobj)
+
+        if not cid:
+            return
+
+        metadata = await objectMetadata(str(cid))
+
+        if metadata:
+            await self.app.multihashDb.store(
+                path,
+                objmetadata=metadata
+            )
+
     @ipfsOp
     async def decodeQrCodes(self, ipfsop, path):
         try:

galacteek/core/ctx.py

@@ -214,7 +214,7 @@ def peersHandles(self):
         return [pCtx.ident.iphandle for peerId, pCtx in self.byPeerId.items()]
 
     async def unregister(self, peerId):
-        with await self.lock:
+        async with self.lock:
             if peerId in self.byPeerId:
                 del self.byPeerId[peerId]
             await self.peerLogout.emit(peerId)
@@ -271,7 +271,7 @@ async def registerFromIdent(self, op, iMsg):
                 log.debug('Cannot load DID: {}'.format(personDid))
                 return
 
-            with await self.lock:
+            async with self.lock:
                 pCtx = PeerCtx(self.ctx, iMsg.peer, iMsg, ipid,
                                pingavg=avgPing if avgPing else 0,
                                pinglast=now if avgPing else 0,
@@ -291,7 +291,7 @@ async def registerFromIdent(self, op, iMsg):
             # This peer is already registered
             # What we ought to do here is just to refresh the DID document
 
-            with await self.lock:
+            async with self.lock:
                 pCtx = self.getByPeerId(iMsg.peer)
                 if pCtx:
                     log.debug('Updating ident for peer {}'.format(iMsg.peer))
@@ -509,6 +509,8 @@ class IPFSContext(QObject):
     ipfsConnectionReady = AsyncSignal()
     ipfsRepositoryReady = AsyncSignal()
 
+    _ipfsRepositoryReady = pyqtSignal()  # for pytest
+
     # profiles
     profilesAvailable = pyqtSignal(list)
     profileChanged = pyqtSignal(str, UserProfile)

galacteek/core/models/mfs.py

@@ -1,4 +1,6 @@
 import os.path
+from datetime import datetime
+from datetime import timedelta
 
 from PyQt5.QtWidgets import QApplication
 
@@ -51,11 +53,16 @@ def parentHash(self):
     def setParentHash(self, pHash):
         self._parentHash = pHash
 
-    def childrenItems(self):
+    def childrenItems(self, type=None):
         for row in range(0, self.rowCount()):
             child = self.child(row, 0)
+
             if child:
-                yield child
+                if type:
+                    if isinstance(child, type):
+                        yield child
+                else:
+                    yield child
 
     def findChildByMultihash(self, multihash):
         for item in self.childrenItems():
@@ -66,10 +73,12 @@ def findChildByMultihash(self, multihash):
 
     def findChildByName(self, name):
         for item in self.childrenItems():
-            if not isinstance(item, MFSNameItem):
-                continue
-            if item.entry['Name'] == name:
-                return item
+            if isinstance(item, MFSNameItem):
+                if item.entry['Name'] == name:
+                    return item
+            elif isinstance(item, MFSTimeFrameItem):
+                if item.text() == name:
+                    return item
 
 
 class MFSRootItem(MFSItem):
@@ -93,6 +102,41 @@ def __init__(self, text, path=None, parenthash=None, icon=None,
         self.cidString = None
 
 
+class MFSTimeFrameItem(MFSItem):
+    """
+    Model item that holds MFS files which were written
+    at a date corresponding to this item's time frame
+    (right now it's a 1-day time frame but we can extend
+    to 3-days or 1 week if it proves to be cumbersome in
+    the UI)
+    """
+
+    def __init__(self, date, text, icon=None):
+        super(MFSTimeFrameItem, self).__init__(text, icon=icon)
+
+        self.date = date
+        self.cidString = None
+
+    def inRange(self, dateFrom, dateTo):
+        return (self.date >= dateFrom and self.date <= dateTo)
+
+    def isToday(self):
+        return self.isDay(datetime.today())
+
+    def isPast3Days(self):
+        return any(
+            self.isPastDay(d) is True for d in range(1, 4)
+        )
+
+    def isPastDay(self, bdays):
+        return self.isDay(datetime.today() - timedelta(days=bdays))
+
+    def isDay(self, day):
+        return (day.year == self.date.year and
+                day.month == self.date.month and
+                day.day == self.date.day)
+
+
 class MFSNameItem(MFSItem):
     def __init__(self, entry, text, icon, cidString):
         super().__init__(text, icon=icon)
@@ -166,6 +210,7 @@ class MFSItemModel(QStandardItemModel):
 
     # Specific roles
     CidRole = 0x0108
+    TimeFrameRole = 0x0109
 
     def __init__(self):
         QStandardItemModel.__init__(self)
@@ -264,9 +309,10 @@ async def setupQrCodesFolder(self, ipfsop, profile, model, files):
                     await ipfsop.sleep()
 
                     if not exists:
-                        await files.linkEntry(ipfsop, entry,
-                                              model.itemQrCodes.path,
-                                              entry['Name'])
+                        await files.linkEntryNoMetadata(
+                            entry,
+                            model.itemQrCodes.path,
+                            entry['Name'])
 
                     tmpFile.remove()
 
@@ -315,7 +361,7 @@ def dropMimeData(self, data, action, row, column, parent):
 
     def getHashFromIdx(self, idx):
         item = self.getNameItemFromIdx(idx)
-        if item:
+        if item and hasattr(item, 'cidString'):
             return item.cidString
 
     def getNameFromIdx(self, idx):
@@ -326,7 +372,9 @@ def getNameFromIdx(self, idx):
     def getNameItemFromIdx(self, idx):
         idxName = self.index(idx.row(), 0, idx.parent())
         if idxName.isValid():
-            return self.itemFromIndex(idxName)
+            item = self.itemFromIndex(idxName)
+            if isinstance(item, MFSNameItem):
+                return item
 
     def data(self, index, role):
         if not index.isValid():
@@ -337,6 +385,8 @@ def data(self, index, role):
         if role == self.CidRole and isinstance(item, MFSNameItem):
             # For the CidRole we return the CID associated with the item
             return item.cidString
+        elif role == self.TimeFrameRole and isinstance(item, MFSTimeFrameItem):
+            return item.text()
         else:
             return super().data(index, role)
 

galacteek/core/userdag.py

@@ -405,7 +405,7 @@ async def update(self, op):
                 for idx, request in enumerate(requests):
                     path = '{0}/{1}'.format(PINREQS_NODEKEY, idx)
                     if not await dag.get(
-                            '{path}/view'.format(path=path, idx=idx)):
+                            '{path}/view'.format(path=path)):
                         self.dagRequests[idx]['view'] = \
                             await self.renderLink(
                                 'usersite/pinrequest.html',

galacteek/dweb/cyber.py

@@ -0,0 +1,83 @@
+from galacteek import log
+from galacteek.ipfs import ipfsOpFn
+
+import aiohttp
+
+
+class CyberSearchResults:
+    def __init__(self, pageCount, page, results):
+        self.pageCount = pageCount
+        self.page = page
+        self.results = results
+
+    @property
+    def hits(self):
+        return [{
+            'hit': hit,
+            'pageCount': self.pageCount,
+            'page': self.page,
+            'engine': 'cyber'
+        } for hit in self.results['hits']]
+
+    @property
+    def hitsCount(self):
+        return len(self.hits)
+
+
+@ipfsOpFn
+async def cyberSearch(ipfsop, query: str, page=0, perPage=10, sslverify=True):
+    entry = await ipfsop.hashComputeString(query, cidversion=0)
+
+    params = {
+        'cid': '"{qcid}"'.format(qcid=entry['Hash']),
+        'page': page,
+        'perPage': perPage
+    }
+
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get('https://{host}/api/search'.format(
+                    host='titan.cybernode.ai'),
+                    params=params,
+                    verify_ssl=sslverify) as resp:
+                resp = await resp.json()
+                result = resp['result']
+                total = int(result['total'])
+                cids = result['cids']
+                pageCount = int(total / perPage)
+    except Exception as err:
+        log.debug(str(err))
+        return CyberSearchResults(
+            0,
+            page,
+            {
+                'total': 0,
+                'page_size': perPage,
+                'page_count': 0,
+                'hits': []
+            }
+        )
+    else:
+        _sorted = sorted(
+            cids,
+            reverse=True,
+            key=lambda it: it['rank']
+        )
+
+        def _format(r):
+            return {
+                'hash': r['cid'],
+                'title': r['cid'],
+                'score': r['rank']
+            }
+
+        return CyberSearchResults(
+            pageCount,
+            page,
+            {
+                'total': total,
+                'page_size': perPage,
+                'page_count': pageCount,
+                'hits': [_format(r) for r in _sorted]
+            }
+        )