Merge branch 'iosoffline'

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "stalkerware-indicators"]
+	path = stalkerware-indicators
+	url = [email protected]:AssoEchap/stalkerware-indicators.git

Brewfile

@@ -4,8 +4,4 @@ cask "android-platform-tools"
 brew "expect"
 
 brew "libimobiledevice"
-brew "ideviceinstaller"
-
-brew "autoconf"
-brew "automake"
-brew "libtool"
+brew "ideviceinstaller"

config.py

@@ -1,10 +1,32 @@
 import hashlib
 import hmac
 import os
+import secrets
 import shlex
 from pathlib import Path
 from sys import platform
 
+import logging
+import logging.handlers as handlers
+
+def setup_logger():
+    """
+    Set up a logger with a rotating file handler.
+
+    The logger will write in a file named 'app.log' in the 'logs' directory.
+    The log file will rotate when it reaches 100,000 bytes, keeps a maximum of 30 files.
+
+    Returns:
+        logging.Logger: The configured logger object.
+    """
+    handler = handlers.RotatingFileHandler(
+        'logs/app.log', maxBytes=100000, 
+        backupCount=30)
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.INFO)
+    logger.addHandler(handler)
+    return logger
+
 DEV_SUPPRTED = ["android", "ios"]  # 'windows', 'mobileos', later
 THIS_DIR = Path(__file__).absolute().parent
 
@@ -16,6 +38,7 @@
 }
 spyware_list_file = "static_data/spyware.csv"  # hand picked
 
+
 # ---------------------------------------------------------
 DEBUG = bool(int(os.getenv("DEBUG", "0")))
 TEST = bool(int(os.getenv("TEST", "0")))
@@ -41,40 +64,38 @@
 
 TITLE = {"title": "IPV Spyware Discovery (ISDi){}".format(" (test)" if TEST else "")}
 
+
 APP_FLAGS_FILE = "static_data/app-flags.csv"
 APP_INFO_SQLITE_FILE = "sqlite:///static_data/app-info.db"
 
 # IOC stalkware indicators
-IOC_PATH = "data/stalkerware-indicators/"
-IOC_FILE = IOC_PATH + "ioc.yaml"
-
-# IOC stalkware indicators
-IOC_PATH = "data/stalkerware-indicators/"
-IOC_FILE = IOC_PATH + "ioc.yaml"
+IOC_PATH = "stalkerware-indicators"
+IOC_FILE = os.path.join(IOC_PATH, "ioc.yaml")
 
 # we will resolve the database path using an absolute path to __FILE__ because
 # there are a couple of sources of truth that may disagree with their "path
 # relavitity". Needless to say, FIXME
-SQL_DB_PATH = "sqlite:///{}".format(str(THIS_DIR / "data/fieldstudy.db"))
-# SQL_DB_CONSULT_PATH = 'sqlite:///data/consultnotes.db' + ("~test" if TEST else "")
-
+SQL_DB_PATH = f"sqlite:///{str(THIS_DIR / 'data/fieldstudy.db')}"
+#SQL_DB_CONSULT_PATH = 'sqlite:///data/consultnotes.db' + ("~test" if TEST else "")
 
 def set_test_mode(test):
-    global TEST, APP_FLAGS_FILE, SQL_DB_PATH
-    TEST = test
-    if TEST:
-        if not APP_FLAGS_FILE.endswith("~test"):
-            APP_FLAGS_FILE = APP_FLAGS_FILE + "~test"
-        if not SQL_DB_PATH.endswith("~test"):
-            SQL_DB_PATH = SQL_DB_PATH + "~test"
+    """
+    Sets the test mode to the given value and returns the new values of APP_FLAGS_FILE and SQL_DB_PATH.
+    """
+    app_flags_file, sql_db_path = APP_FLAGS_FILE, SQL_DB_PATH
+    if test:
+        if not app_flags_file.endswith('~test'):
+            app_flags_file = APP_FLAGS_FILE + "~test"
+        if not sql_db_path.endswith('~test'):
+            sql_db_path = sql_db_path + "~test"
     else:
-        if APP_FLAGS_FILE.endswith("~test"):
-            APP_FLAGS_FILE = APP_FLAGS_FILE.replace("~test", "")
-        if SQL_DB_PATH.endswith("~test"):
-            SQL_DB_PATH = SQL_DB_PATH.replace("~test", "")
-
+        if app_flags_file.endswith('~test'):
+            app_flags_file = app_flags_file.replace("~test", '')
+        if sql_db_path.endswith('~test'):
+            sql_db_path = sql_db_path.replace("~test", '')
+    return app_flags_file, sql_db_path
 
-set_test_mode(TEST)
+APP_FLAGS_FILE, SQL_DB_PATH = set_test_mode(TEST)
 
 
 STATIC_DATA = THIS_DIR / "static_data"
@@ -117,10 +138,18 @@ def set_test_mode(test):
 
 
 def open_or_create_random_key(fpath, keylen=32):
-    def create():
-        import secrets
+    """
+    Opens the file at the given path or creates a new file with a random key of the specified length.
+
+    Args:
+        fpath (str): The path to the file.
+        keylen (int, optional): The length of the random key. Defaults to 32.
 
-        with fpath.open("wb") as f:
+    Returns:
+        bytes: The contents of the file as bytes.
+    """
+    def create():
+        with fpath.open('wb') as f:
             f.write(secrets.token_bytes(keylen))
 
     if not fpath.exists():
@@ -162,5 +191,5 @@ def error():
     if len(ERROR_LOG) > 0:
         e, ERROR_LOG = ERROR_LOG[0], ERROR_LOG[1:]
 
-        print("ERROR: {}".format(e))
+        print(f"ERROR: {e}")
     return e.replace("\n", "<br/>")

data_process.py

@@ -37,9 +37,9 @@ def create_app_flags_file():
     sys.stderr.write("Concatenating...")
     fulld = pd.concat(dlist)
     sys.stderr.write("done\n")
-    spyware = pd.read_csv(config.spyware_list_file, index_col='appId')
+    spyware = pd.read_csv(config.SPYWARE_LIST_FILE, index_col='appId')
     fulld.loc[spyware.index, 'flag'] = 'spyware'
-    print("Writing to the file: {}".format(config.APP_FLAGS_FILE))
+    print("Writing to the file: {config.APP_FLAGS_FILE}")
     fulld.to_csv(config.APP_FLAGS_FILE)
 
 

isdi

@@ -1,31 +1,34 @@
 #!/usr/bin/env python3
-import logging
-import logging.handlers as handlers
+"""
+The mainfile of ISDi repo. 
+"""
 import webbrowser
 from threading import Timer
 
 import config
 from db import init_db
 from web import app, sa
 
+PORT = 6200 if not (config.TEST or config.DEBUG) else 6202
+HOST = "127.0.0.1" if config.DEBUG else "0.0.0.0"
+
 def open_browser():
-    webbrowser.open('http://127.0.0.1:' + str(PORT), new=0, autoraise=True)
+    """Opens a browser to make it easy to navigate to ISDi
+    """
+    if not config.TEST:
+        webbrowser.open('http://127.0.0.1:' + str(PORT), new=0, autoraise=True)
 
 if __name__ == "__main__":
     import sys
     if 'TEST' in sys.argv[1:] or 'test' in sys.argv[1:]:
         print("Running in test mode.")
         config.set_test_mode(True)
-        print("Checking mode = {}\nApp flags: {}\nSQL_DB: {}"
-              .format(config.TEST, config.APP_FLAGS_FILE,
-                      config.SQL_DB_PATH))
-    print("TEST={}".format(config.TEST))
+        print(f"Checking mode = {config.TEST}\n"
+              "App flags: {config.APP_FLAGS_FILE}\n"
+              "SQL_DB: {config.SQL_DB_PATH}")
+
+    print(f"TEST={config.TEST}")
     init_db(app, sa, force=config.TEST)
-    handler = handlers.RotatingFileHandler('logs/app.log', maxBytes=100000, 
-                                           backupCount=30)
-    logger = logging.getLogger(__name__)
-    logger.setLevel(logging.ERROR)
-    logger.addHandler(handler)
-    PORT = 6200 if not config.TEST else 6202
+    config.setup_logger()
     Timer(1, open_browser).start()
-    app.run(host="0.0.0.0", port=PORT, debug=config.DEBUG, use_reloader= False)
+    app.run(host=HOST, port=PORT, debug=config.DEBUG, use_reloader=config.DEBUG)

parse_dump.py

@@ -320,10 +320,14 @@ def get_data_usage(d, process_uid):
             return {"foreground": "unknown", "background": "unknown"}
         # FIXME: pandas.errors.ParserError: Error tokenizing data. C error: Expected 21 fields in line 556, saw 22
         # parser error (tested on SM-G965U,Samsung,8.0.0)
+        try:
+            net_stats = pd.read_csv(io.StringIO(
+                '\n'.join(d['net_stats'])
+            ), on_bad_lines='warn')
+        except pd.errors.EmptyDataError:
+            config.logging.warning(f"No net_stats for {d['appId']} is empty and has been skipped.")
+            net_stats = pd.DataFrame()
 
-        net_stats = pd.read_csv(
-            io.StringIO("\n".join(d["net_stats"])), on_bad_lines="warn"
-        )
         d = net_stats.query('uid_tag_int == "{}"'.format(process_uid))[
             ["uid_tag_int", "cnt_set", "rx_bytes", "tx_bytes"]
         ].astype(int)
@@ -478,7 +482,6 @@ def load_file(self):
                 apps_plist = load(app_data)
             d = pd.DataFrame(apps_plist)
             d['appId'] = d['CFBundleIdentifier']
-            # d.set_index('appId', inplace=True)
             return d
         except Exception as ex:
             print(ex)
@@ -575,8 +578,9 @@ def info(self, appid):
         # app = self.df.iloc[appidx,:].dropna()
         app = self.df[self.df["CFBundleIdentifier"] == appid].squeeze().dropna()
         party = app.ApplicationType.lower()
-        if party in ["system", "user"]:
-            print(f"{app["CFBundleName"]} ({app["CFBundleIdentifier"]}) is a {party} app and has permissions:")
+        permissions = []
+        if party in ["system", "user", "hidden"]:
+            print(f"{app['CFBundleName']} ({app['CFBundleIdentifier']}) is a {party} app and has permissions:")
             # permissions are an array that returns the permission id and an explanation.
             permissions = self.get_permissions(app)
         res["permissions"] = [(p.capitalize(), r) for p, r in permissions]
@@ -622,7 +626,8 @@ def system_apps(self):
 
     def installed_apps_titles(self) -> pd.DataFrame:
         if self:
-            return self.df.rename(index=str, columns={"CFBundleExecutable": "title"})
+            return self.df.rename(index=str,
+                                  columns={'CFBundleExecutable': 'title'}).set_index('appId')
 
     def installed_apps(self):
         # return self.df.index
@@ -646,5 +651,5 @@ def installed_apps(self):
         print(json.dumps(ddump.info("ru.kidcontrol.gpstracker"), indent=2))
     elif sys.argv[2] == "ios":
         ddump = IosDump(fname)
-        # print(ddump.installed_apps())
+        print(ddump.installed_apps())
         print(ddump.installed_apps_titles().to_csv())

phone_scanner.py

@@ -86,7 +86,7 @@ def dump_path(self, serial, fkind="json"):
             config.DUMP_DIR, "{}_{}.{}".format(hmac_serial, self.device_type, fkind)
         )
 
-    def app_details(self, serialno, appid):
+    def app_details(self, serialno, appid) -> (dict, dict):
         try:
             d = pd.read_sql(
                 "select * from apps where appid=?", self.app_info_conn, params=(appid,)
@@ -107,21 +107,20 @@ def app_details(self, serialno, appid):
 
             info = ddump.info(appid)
 
-            print("BEGIN APP INFO")
-            print("info={}".format(info))
-            print("END APP INFO")
+            config.logging.info('BEGIN APP INFO')
+            config.logging.info("info={}".format(info))
+            config.logging.info('END APP INFO')
             # FIXME: sloppy iOS hack but should fix later, just add these to DF
             # directly.
             if self.device_type == "ios":
                 # TODO: add extra info about iOS? Like idevicediagnostics
                 # ioregentry AppleARMPMUCharger or IOPMPowerSource or
                 # AppleSmartBattery.
-                d["permissions"] = pd.Series(info.get("permissions", ""))
-                # d['permissions'] = [info.get('permissions','')]
-                d["title"] = pd.Series(info.get("title", ""))
-                # del info['permissions']
-            print("AppInfo: ", info, appid, dfname, ddump)
-            return d.fillna(""), info
+                d['permissions'] = pd.Series(info.get('permissions',''), dtype=object)
+                d['title'] = pd.Series(info.get('title',''))
+                del info['permissions']
+            d = d.fillna('').to_dict(orient='index').get(0, {})
+            return d, info
         except KeyError as ex:
             print(">>> Exception:::", ex, file=sys.stderr)
             return pd.DataFrame([]), dict()
@@ -264,14 +263,14 @@ def get_apps(self, serialno: str, from_dump: bool=True) -> list:
         self.installed_apps = installed_apps
         return installed_apps
 
-    def get_system_apps(self, serialno, from_dump=False):
+    def get_system_apps(self, serialno, from_dump=False) -> list:
         if (not from_dump):
             apps = self._get_apps_from_device(serialno, '-s')
         else:
             apps = []  # TODO: fix this later, not sure how to get from dump
         return apps
 
-    def get_offstore_apps(self, serialno, from_dump=False):
+    def get_offstore_apps(self, serialno, from_dump=False) -> list:
         if from_dump:
             return []  # TODO: fix this later, not sure how to get from dump
         offstore = []
@@ -364,18 +363,12 @@ def uninstall(self, serial, appid):
         )
         return s != -1
 
-    def app_details(self, serialno, appid):
+    def app_details(self, serialno, appid) -> (dict, dict):
         d, info = super(AndroidScan, self).app_details(serialno, appid)
         # part that requires android to be connected / store this somehow.
         hf_recent, non_hf_recent, non_hf, stats = all_permissions(
             self.dump_path(serialno), appid
         )
-        # print(f"Permissions:\n"\
-        #       f"hf_recent=\n{hf_recent}\n"\
-        #       f"non_hf_recent=\n{non_hf_recent}\n"\
-        #       f"no_hf=\n{non_hf}\n"\
-        #       f"stats=\n{stats}\n")
-
         # FIXME: some appopps in non_hf_recent are not included in the
         # output.  maybe concat hf_recent with them?
         info["Date of Scan"] = datetime.now().strftime(config.DATE_STR)
@@ -406,13 +399,15 @@ def app_details(self, serialno, appid):
                 ),
                 axis=1,
             )
-
         # print("hf_recent['label']=", hf_recent['label'].tolist())
-        # print(~hf_recent['timestamp'].str.contains('unknown'))
-        d.at[0, "permissions"] = hf_recent["label"].tolist()
-        non_hf_recent.drop("appId", axis=1, inplace=True)
-        d.at[0, "non_hf_permissions_html"] = non_hf_recent.to_html()
-
+        #print(~hf_recent['timestamp'].str.contains('unknown'))
+        non_hf_recent.drop('appId', axis=1, inplace=True)
+        d = d.fillna('').to_dict(orient='index').get(0, {})
+        print(d)
+        # d.at[0, 'permissions'] = hf_recent['label'].tolist()
+        # d.at[0, 'non_hf_permissions_html'] = non_hf_recent.to_html()
+        d['permissions'] = hf_recent['label'].tolist()
+        d['non_hf_permissions_html'] = non_hf_recent.to_html()
         print("App info dict:", d)
 
         # hf_recent['label'] = hf_recent['label'].map(str) + " (last used by app: "+\
@@ -589,12 +584,13 @@ def _is_device(x):
         cmd = "{}idevice_id -l | tail -n 1".format(self.cli)
         self.serialno = None
         s = catch_err(run_command(cmd), cmd=cmd, msg="")
+
         d = [
             line.strip()
             for line in s.split("\n")
             if line.strip() and _is_device(line.strip())
         ]
-        print("Devices found:", d)
+        config.logging.info("Devices found:", d)
         return d
 
     def device_info(self, serial):