Merge pull request #13 from prise6/dev

Version 0.1.3: fix scoring logic

.gitignore

@@ -90,6 +90,7 @@ target/
 # VS Code
 .vscode/
 .devcontainer.json
+devcontainer.json
 
 # Spyder
 .spyproject/

Dockerfile

@@ -5,7 +5,7 @@ ENV USERNAME=mediastrends
 ARG PUID=1001
 ARG PGID=1001
 ARG WORKDIR=/app
-ARG VERSION=0.1.2
+ARG VERSION=0.1.3
 ARG TARGETOS
 ARG TARGETARCH
 ARG TARGETVARIANT
@@ -48,7 +48,7 @@ RUN groupadd --gid $PGID $USERNAME \
 
 RUN mkdir -p $WORKDIR/scripts $WORKDIR/logs
 
-COPY scripts/add_movie_torrents.py scripts/stats_movie_torrents.py scripts/
+COPY scripts/add_movie_torrents.py scripts/stats_movie_torrents.py scripts/migrate_db_v0.1.1_to_v0.1.2.py scripts/
 COPY Makefile .
 
 ENTRYPOINT ["tini", "--", "/entrypoint.sh"]

README.md

@@ -68,7 +68,7 @@ Content of mediastrends.ini (default config)
 [directory]
 base=
 data=
-log=${base}/logs
+logs=${base}/logs
 sqlite=
 
 [db]
@@ -92,11 +92,24 @@ batch_size=50
 tries=10
 delay=5
 
-[stats_manager]
-new_delay=3
-
-[trends]
-tau=0.2
+[torrents_manager]
+; new_delay=3
+; delta_hours=1
+; min_date=YYYY-mm-dd HH:MM
+; max_date=YYYY-mm-dd HH:MM
+; candidates_status=new,follow
+
+[trends_manager]
+; min_date=YYYY-mm-dd HH:MM
+; max_date=YYYY-mm-dd HH:MM
+; delta_days=31
+; torrents_status=new,follow
+; tau=0.2
+; max_trendings=50
+weight_seeders=0.4
+weight_completed=0.4
+weight_leechers=0.2
+lambda=0.8
 
 [jackettapi]
 scheme=http
@@ -235,6 +248,11 @@ _To-do_
 
 ## Changelog
 
+### version 0.1.3
+
+* resolve issue #12 (wrong score order)
+* new score logic with parameter to tune in config
+
 ### version 0.1.2
 
 * resolve issue #9

docker-compose-dev.yaml

@@ -7,7 +7,7 @@ services:
       args:
         USER_UID: 1000
         WORKDIR: /package
-    image: mediastrends-core-dev:0.1.2
+    image: mediastrends-core-dev:0.1.3
     container_name: mediastrends-core-dev
     user: vscode
     volumes:

docker-compose.yaml

@@ -4,7 +4,7 @@ services:
     build:
       context: .
       dockerfile: Dockerfile
-    image: mediastrends-core-prod:0.1.2
+    image: mediastrends-core-prod:0.1.3
     container_name: mediastrends-core-prod
     user: root
     command: get_movie_trends

docker/entrypoint.sh

@@ -28,6 +28,12 @@ else
     JACKETT=${JACKETT}
 fi
 
+if [ -z "${TIMEOUT}" ]; then
+    TIMEOUT="15"
+else 
+    TIMEOUT=${TIMEOUT}
+fi
+
 if [ -z "${MEDIASTRENDS_CACHE_TIME}" ]; then
     CACHE_TIME=3600*6
 else
@@ -52,7 +58,7 @@ if [ "$1" = 'stats_movie_torrents' ] \
     || [ "$1" = 'add_movie_torrents' ] \
     || [ "$1" = 'compute_movie_trends' ] \
     || [ "$1" = 'get_movie_trends' ]; then
-    CMD="./wait-for-it.sh $JACKETT -- mediastrends database create && make $@"
+    CMD="./wait-for-it.sh $JACKETT -t $TIMEOUT -- mediastrends database create && make $@"
 else
     CMD="$@"
 fi

mediastrends/cli.py

@@ -78,6 +78,10 @@ def _argument_maxdate(parser):
     parser.add_argument("-dx", "--maxdate", help="Max datetime: YYYYMMDDHHMM", type=lambda s: datetime.datetime.strptime(s, '%Y%m%d%H%M'))
 
 
+def _argument_deltahours(parser):
+    parser.add_argument("-dh", "--delta-hours", help="Delta hours to define mindate: mindate := maxdate - delta_hours", type=int, default=1)
+
+
 def _argument_tracker(parser):
     parser.add_argument("-t", "--tracker-name", help="Tracker name", type=str, choices=trackers_config.keys(), required=True)
 
@@ -155,6 +159,7 @@ def build(self):
         _argument_category(self.parser)
         _argument_mindate(self.parser)
         _argument_maxdate(self.parser)
+        _argument_deltahours(self.parser)
         _argument_test(self.parser)
 
     def task(self, **kwargs):
@@ -165,8 +170,6 @@ class TorrentsTrendsComputeParser(AbstractParser):
 
     def build(self):
         _argument_category(self.parser)
-        _argument_mindate(self.parser)
-        _argument_maxdate(self.parser)
         _argument_test(self.parser)
 
     def task(self, **kwargs):
@@ -259,6 +262,7 @@ class MoviesTrendsGetParser(AbstractParser):
     def build(self):
         _argument_mindate(self.parser)
         _argument_maxdate(self.parser)
+        _argument_deltahours(self.parser)
         _argument_test(self.parser)
 
     def task(self, **kwargs):
@@ -268,8 +272,6 @@ def task(self, **kwargs):
 class MoviesTrendsComputeParser(AbstractParser):
 
     def build(self):
-        _argument_mindate(self.parser)
-        _argument_maxdate(self.parser)
         _argument_test(self.parser)
 
     def task(self, **kwargs):

mediastrends/database/DbManager.py

@@ -117,7 +117,7 @@ def get_torrents_by_tracker(tracker: Tracker, status: list, category: list = Non
         return
 
     @abstractstaticmethod
-    def get_trending_torrents_by_category(category: list = None, min_date=None, max_date=None):
+    def get_trending_torrents_by_category(category: list = None, min_date=None, max_date=None, delta_hours=1):
         return
 
     @abstractstaticmethod

mediastrends/database/peewee/PDbManager.py

@@ -338,13 +338,15 @@ def get_torrents_by_tracker(tracker: Tracker, status: list = None, category: lis
         result = result.where(expression)
         return [PDbManager.db_to_torrent(db_torrent) for db_torrent in result]
 
-    def get_trending_torrents_by_category(category: list = None, min_date=None, max_date=None):
+    def get_trending_torrents_by_category(category: list = None, min_date=None, max_date=None, delta_hours=1):
 
-        if not min_date and not max_date:
+        if not max_date:
             max_date = PDbManager.get_max_trend_date_by_category(category)
             if max_date is None:
                 raise ValueError("Maximum trend date is null")
-            min_date = max_date - datetime.timedelta(hours=1)
+
+        if not min_date:
+            min_date = max_date - datetime.timedelta(hours=delta_hours)
 
         sub_q = PTrends.select(PTrends.id, fn.row_number().over(
             partition_by=[PTrends.torrent],
@@ -365,13 +367,15 @@ def get_trending_torrents_by_category(category: list = None, min_date=None, max_
 
         return [(PDbManager.db_to_torrent(db_trends.torrent), db_trends.score, db_trends.valid_date) for db_trends in result]
 
-    def get_trending_movies(min_date=None, max_date=None):
+    def get_trending_movies(min_date=None, max_date=None, delta_hours=1):
 
-        if not min_date and not max_date:
+        if not max_date:
             max_date = PDbManager.get_max_trend_date_by_category(Torrent._CAT_MOVIE)
             if max_date is None:
                 raise ValueError("Maximum trend date is null")
-            min_date = max_date - datetime.timedelta(hours=1)
+
+        if not min_date:
+            min_date = max_date - datetime.timedelta(hours=delta_hours)
 
         sub_q = PTrends.select(PTrends.torrent_id, fn.row_number().over(
             partition_by=[PTrends.torrent],

mediastrends/mediastrends.ini

@@ -25,11 +25,24 @@ batch_size=50
 tries=10
 delay=5
 
-[stats_manager]
-new_delay=3
-
-[trends]
-tau=0.2
+[torrents_manager]
+; new_delay=3
+; delta_hours=1
+; min_date=YYYY-mm-dd HH:MM
+; max_date=YYYY-mm-dd HH:MM
+; candidates_status=new,follow
+
+[trends_manager]
+; min_date=YYYY-mm-dd HH:MM
+; max_date=YYYY-mm-dd HH:MM
+; delta_days=31
+; torrents_status=new,follow
+; tau=0.2
+; max_trendings=50
+weight_seeders=0.4
+weight_completed=0.4
+weight_leechers=0.2
+lambda=0.8
 
 [jackettapi]
 scheme=http

mediastrends/tasks/movies.py

@@ -1,24 +1,26 @@
 import logging
-from mediastrends import db_factory
+from mediastrends import db_factory, config, CATEGORY_NAME
 from mediastrends.database.peewee.PDbManager import PDbManager
 from mediastrends.torrent.IMDBObject import movies_from_torrents
-from mediastrends.torrent.Torrent import Torrent
+from mediastrends.torrent.TorrentsManager import TorrentsManager
 
 logger = logging.getLogger(__name__)
 
 
 # region
-def compute_trending(test, mindate=None, maxdate=None, **kwargs):
+def compute_trending(test, **kwargs):
     if test:
         logger.debug("get_trending task")
         return
 
     trendings_movies = []
 
+    torrents_manager = TorrentsManager(config, PDbManager, CATEGORY_NAME['movies'])
+
     try:
         with db_factory.get_instance():
-            trendings_torrents = PDbManager.get_trending_torrents_by_category(Torrent._CAT_MOVIE, mindate, maxdate)
-
+            # trendings_torrents = PDbManager.get_trending_torrents_by_category(Torrent._CAT_MOVIE, mindate, maxdate, delta_hours)
+            trendings_torrents = torrents_manager.torrents_trending
         trendings_movies = movies_from_torrents([t for t, _, _ in trendings_torrents])
     except ValueError as err:
         logger.warning(err)
@@ -34,7 +36,7 @@ def compute_trending(test, mindate=None, maxdate=None, **kwargs):
 
 
 # region
-def get_trending(test, mindate=None, maxdate=None, **kwargs):
+def get_trending(test, mindate=None, maxdate=None, delta_hours=1, **kwargs):
 
     results = None
 
@@ -43,7 +45,7 @@ def get_trending(test, mindate=None, maxdate=None, **kwargs):
         return
     try:
         with db_factory.get_instance():
-            results = PDbManager.get_trending_movies()
+            results = PDbManager.get_trending_movies(mindate, maxdate, delta_hours)
             for item in results:
                 print(item)
     except ValueError as err:

mediastrends/tasks/torrents.py

@@ -1,4 +1,3 @@
-import datetime
 import logging
 from mediastrends import config, db_factory, indexers_config, CATEGORY_NAME, trackers_config
 from mediastrends.database.peewee.PDbManager import PDbManager
@@ -163,15 +162,15 @@ def torrents_stats_with_tracker(tracker: Tracker, category: list = None):
 
 
 # region
-def compute_trending(test, category: list = None, mindate=None, maxdate=datetime.datetime.now(), **kwargs):
+def compute_trending(test, category: list = None, **kwargs):
     if category is not None:
         category = [CATEGORY_NAME.get(c) for c in category]
     if test:
         logger.debug("compute_trending task")
         return
     with db_factory.get_instance():
-        trends_manager = TrendsManager(config, PDbManager, category, mindate, maxdate)
-        trends_manager.evaluate(NormalizedTrendsEngine())
+        trends_manager = TrendsManager(config, PDbManager, category)
+        trends_manager.evaluate(NormalizedTrendsEngine(config))
         trends_manager.save_trends()
 # endregion
 
@@ -191,7 +190,7 @@ def update_status(test, category: list = None, **kwargs):
 
 
 # region
-def get_trending(test, category: list = None, mindate=None, maxdate=None, **kwargs):
+def get_trending(test, category: list = None, mindate=None, maxdate=None, delta_hours=1, **kwargs):
 
     if category is not None:
         category = [CATEGORY_NAME.get(c) for c in category]
@@ -203,7 +202,7 @@ def get_trending(test, category: list = None, mindate=None, maxdate=None, **kwar
         return
     try:
         with db_factory.get_instance():
-            results = PDbManager.get_trending_torrents_by_category(category, mindate, maxdate)
+            results = PDbManager.get_trending_torrents_by_category(category, mindate, maxdate, delta_hours)
 
         for t, score, valid_date in results:
             print("%s / %s / %s" % (t, score, valid_date))

mediastrends/tools/config.py

@@ -1,5 +1,6 @@
 import os
 import yaml
+import datetime
 import configparser
 from configparser import ExtendedInterpolation
 
@@ -16,7 +17,15 @@
 
 
 def init_config():
-    config = configparser.ConfigParser(interpolation=ExtendedInterpolation())
+    config = configparser.ConfigParser(
+        interpolation=ExtendedInterpolation(),
+        converters={
+            'datetime': lambda s: datetime.datetime.strptime(s, '%Y-%m-%d %H:%M'),
+            'list': lambda s: s.split(',')
+        },
+        allow_no_value=True
+        # converters={'datetime': datetime.datetime}
+    )
 
     return config
 

mediastrends/tools/movies.py

@@ -32,7 +32,13 @@ def group_torrents_by_name(torrents: List[Torrent]) -> dict:
     groups = {}
     torrents_name = [t.name for t in torrents]
     torrents_name_parsed = [PTN.parse(name) for name in torrents_name]
-    torrents_name_parsed_title = [el.get('title', '').lower() for el in torrents_name_parsed]
+    torrents_name_parsed_title = []
+    for idx, el in enumerate(torrents_name_parsed):
+        title = el.get('title', None)
+        if not (title and title.strip() != ''):
+            title = torrents_name[idx]
+        torrents_name_parsed_title.append(title.lower())
+
     nb_torrents = len(torrents_name_parsed)
     similarity_matrix = np.zeros((nb_torrents, nb_torrents))
     np.fill_diagonal(similarity_matrix, 0)