Merge pull request pkreissel#6 from pkreissel/recommender

Recommender

.github/workflows/CI.yaml

@@ -8,12 +8,15 @@ on:
 jobs:
   build-and-test:
     runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
             ref: ${{ github.head_ref }}
-            token: ${{ secrets.GH_TOKEN }}
 
       - name: Set up Node.js
         uses: actions/setup-node@v3
@@ -22,6 +25,9 @@ jobs:
 
       - name: Install dependencies
         run: npm ci
+
+      - name: Lint TypeScript
+        run: npm run lint
 
       - name: Compile TypeScript
         run: npm run build

.gitignore

@@ -1,3 +1,5 @@
 node_modules
 coverage
-.env
+.env
+
+.DS_Store

dist/Paginator.d.ts

@@ -6,7 +6,7 @@ export default class ForYouPaginator implements mastodon.Paginator<mastodon.v1.S
     constructor(data: mastodon.v1.Status[]);
     return(value: PromiseLike<undefined> | undefined): Promise<IteratorResult<mastodon.v1.Status[], undefined>>;
     [Symbol.asyncIterator](): AsyncIterator<mastodon.v1.Status[], undefined, string | undefined>;
-    then<TResult1 = mastodon.v1.Status[], TResult2 = never>(onfulfilled?: ((value: mastodon.v1.Status[]) => TResult1 | PromiseLike<TResult1>) | null | undefined, onrejected?: ((reason: any) => TResult2 | PromiseLike<TResult2>) | null | undefined): PromiseLike<TResult1 | TResult2>;
+    then<TResult1 = mastodon.v1.Status[], TResult2 = never>(onfulfilled?: ((value: mastodon.v1.Status[]) => TResult1 | PromiseLike<TResult1>) | null | undefined, onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | null | undefined): PromiseLike<TResult1 | TResult2>;
     next(): Promise<IteratorResult<mastodon.v1.Status[], undefined>>;
     getDirection(): "next" | "prev";
     setDirection(direction: "next" | "prev"): mastodon.Paginator<mastodon.v1.Status[], undefined>;

dist/Paginator.js

@@ -10,13 +10,13 @@ class ForYouPaginator {
         this.direction = "next";
     }
     return(value) {
-        throw new Error("Method not implemented.");
+        throw new Error(`Method not implemented. ${value}`);
     }
     [Symbol.asyncIterator]() {
         throw new Error("Method not implemented.");
     }
     then(onfulfilled, onrejected) {
-        throw new Error("Method not implemented.");
+        throw new Error(`Method not implemented. ${onfulfilled} ${onrejected}`);
     }
     async next() {
         if (this.currentIndex < this.data.length) {
@@ -39,6 +39,7 @@ class ForYouPaginator {
         return clonedPaginator;
     }
     async throw(e) {
+        console.error(e);
         return { value: undefined, done: true };
     }
     async *values() {

dist/Storage.d.ts

@@ -13,7 +13,7 @@ export declare enum Key {
 export default class Storage {
     protected static get(key: Key, groupedByUser?: boolean, suffix?: string): Promise<StorageValue>;
     protected static set(key: Key, value: StorageValue, groupedByUser?: boolean, suffix?: string): Promise<void>;
-    static suffix(key: Key, suffix: any): string;
+    static suffix(key: Key, suffix: string): string;
     protected static remove(key: Key, groupedByUser?: boolean, suffix?: string): Promise<void>;
     protected static prefix(key: string): Promise<string>;
     static logOpening(): Promise<void>;

dist/features/FeatureStore.js

@@ -71,7 +71,7 @@ class FeatureStorage extends Storage_1.default {
     static async getCoreServer(api) {
         const coreServer = await this.get(Storage_1.Key.CORE_SERVER);
         console.log(coreServer);
-        if (coreServer != null && await this.getOpenings() % 10 < 9) {
+        if (coreServer != null && await this.getOpenings() % 10 != 9) {
             return coreServer;
         }
         else {

dist/features/coreServerFeature.js

@@ -1,5 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const helpers_1 = require("../helpers");
+async function getMonthlyUsers(server) {
+    try {
+        const instance = await (0, helpers_1.mastodonFetch)(server, "api/v2/instance");
+        console.log(instance);
+        return instance ? instance.usage.users.activeMonth : 0;
+    }
+    catch (error) {
+        console.error(`Error fetching data for server ${server}:`, error);
+        return 0; // Return 0 if we can't get the data
+    }
+}
 async function coreServerFeature(api, user) {
     let results = [];
     let pages = 10;
@@ -13,6 +25,7 @@ async function coreServerFeature(api, user) {
         }
     }
     catch (e) {
+        console.error(e);
         return {};
     }
     const serverFrequ = results.reduce((accumulator, follower) => {
@@ -25,6 +38,19 @@ async function coreServerFeature(api, user) {
         }
         return accumulator;
     }, {});
-    return serverFrequ;
+    console.log(serverFrequ);
+    // for top 30 servers
+    const top30 = Object.keys(serverFrequ).sort((a, b) => serverFrequ[b] - serverFrequ[a]).slice(0, 30);
+    console.log("Top 30 servers: ", top30);
+    const monthlyUsers = await Promise.all(top30.map(server => getMonthlyUsers(server)));
+    console.log("Monthly Users: ", monthlyUsers);
+    const overrepresentedServerFrequ = top30.reduce((acc, server, index) => {
+        const activeUsers = monthlyUsers[index];
+        if (activeUsers < 1)
+            return acc;
+        const ratio = serverFrequ[server] / activeUsers;
+        return { ...acc, [server]: ratio };
+    }, {});
+    return overrepresentedServerFrequ;
 }
 exports.default = coreServerFeature;

dist/features/favsFeature.js

@@ -13,6 +13,7 @@ async function favFeature(api) {
         }
     }
     catch (e) {
+        console.error(e);
         return {};
     }
     const favFrequ = results.reduce((accumulator, status) => {

dist/features/interactsFeature.js

@@ -13,6 +13,7 @@ async function interactFeature(api) {
         }
     }
     catch (e) {
+        console.error(e);
         return {};
     }
     const interactFrequ = results.reduce((accumulator, status) => {

dist/features/reblogsFeature.d.ts

@@ -1,2 +1,2 @@
 import { mastodon } from "masto";
-export default function getReblogsFeature(api: mastodon.rest.Client): Promise<any>;
+export default function getReblogsFeature(api: mastodon.rest.Client): Promise<Record<string, number>>;

dist/features/reblogsFeature.js

@@ -13,6 +13,7 @@ async function getReblogsFeature(api) {
         }
     }
     catch (e) {
+        console.error(e);
         return {};
     }
     const reblogFrequ = results.reduce((accumulator, status) => {

dist/feeds/homeFeed.d.ts

@@ -1,2 +1,3 @@
 import { mastodon } from "masto";
-export default function getHomeFeed(api: mastodon.rest.Client, user: mastodon.v1.Account): Promise<any[]>;
+import { StatusType } from "../types";
+export default function getHomeFeed(api: mastodon.rest.Client, _user: mastodon.v1.Account): Promise<StatusType[]>;

dist/feeds/homeFeed.js

@@ -4,10 +4,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const Storage_1 = __importDefault(require("../Storage"));
-async function getHomeFeed(api, user) {
+async function getHomeFeed(api, _user) {
     let results = [];
     let pages = 10;
-    const lastOpened = new Date(await Storage_1.default.getLastOpened() - 600) ?? new Date(0);
+    const lastOpened = new Date((await Storage_1.default.getLastOpened() ?? 0) - 600);
     const defaultCutoff = new Date(Date.now() - 43200000);
     const dateCutoff = lastOpened < defaultCutoff ? defaultCutoff : lastOpened;
     console.log("Date Cutoff: ", dateCutoff);

dist/feeds/recommenderFeed.d.ts

@@ -0,0 +1,3 @@
+import { mastodon } from "masto";
+import { StatusType } from "../types";
+export default function getRecommenderFeed(_api: mastodon.rest.Client, _user: mastodon.v1.Account): Promise<StatusType[]>;

dist/feeds/recommenderFeed.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const change_case_1 = require("change-case");
+const helpers_1 = require("../helpers");
+async function getRecommenderFeed(_api, _user) {
+    let data, res;
+    try {
+        res = await fetch("http://127.0.0.1:5000");
+        data = await res.json();
+    }
+    catch (e) {
+        console.log(e);
+        return [];
+    }
+    if (!res.ok) {
+        return [];
+    }
+    const statuses = data.statuses.map((status) => {
+        status.recommended = true;
+        return status;
+    });
+    return (0, helpers_1._transformKeys)(statuses, change_case_1.camelCase);
+}
+exports.default = getRecommenderFeed;

dist/feeds/topPostsFeed.d.ts

@@ -1,2 +1,3 @@
 import { mastodon } from "masto";
-export default function getTopPostFeed(api: mastodon.rest.Client): Promise<mastodon.v1.Status[]>;
+import { StatusType } from "../types";
+export default function getTopPostFeed(api: mastodon.rest.Client): Promise<StatusType[]>;

dist/feeds/topPostsFeed.js

@@ -4,25 +4,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 const FeatureStore_1 = __importDefault(require("../features/FeatureStore"));
-const change_case_1 = require("change-case");
 const Storage_1 = __importDefault(require("../Storage"));
+const helpers_1 = require("../helpers");
 async function getTopPostFeed(api) {
     const core_servers = await FeatureStore_1.default.getCoreServer(api);
     let results = [];
-    //Masto does not support top posts from foreign servers, so we have to do it manually
-    const isRecord = (x) => typeof x === "object" && x !== null && x.constructor.name === "Object";
-    const _transformKeys = (data, transform) => {
-        if (Array.isArray(data)) {
-            return data.map((value) => _transformKeys(value, transform));
-        }
-        if (isRecord(data)) {
-            return Object.fromEntries(Object.entries(data).map(([key, value]) => [
-                transform(key),
-                _transformKeys(value, transform),
-            ]));
-        }
-        return data;
-    };
     //Get Top Servers
     const servers = Object.keys(core_servers).sort((a, b) => {
         return core_servers[b] - core_servers[a];
@@ -33,29 +19,14 @@ async function getTopPostFeed(api) {
     results = await Promise.all(servers.map(async (server) => {
         if (server === "undefined" || typeof server == "undefined" || server === "")
             return [];
-        let res, json;
-        try {
-            res = await fetch("https://" + server + "/api/v1/trends/statuses");
-            json = await res.json();
-        }
-        catch (e) {
-            console.log(e);
-            return [];
-        }
-        if (!res.ok) {
-            return [];
-        }
-        const data = _transformKeys(json, change_case_1.camelCase);
-        if (data === undefined) {
-            return [];
-        }
-        return data.map((status) => {
+        const data = await (0, helpers_1.mastodonFetch)(server, "api/v1/timelines/public");
+        return data?.map((status) => {
             status.topPost = true;
             return status;
-        }).slice(0, 10);
+        }).slice(0, 10) ?? [];
     }));
     console.log(results);
-    const lastOpened = new Date(await Storage_1.default.getLastOpened() - 28800000) ?? new Date(0);
+    const lastOpened = new Date((await Storage_1.default.getLastOpened() ?? 0) - 28800000);
     return results.flat().filter((status) => new Date(status.createdAt) > lastOpened);
 }
 exports.default = getTopPostFeed;

dist/helpers.d.ts

@@ -0,0 +1,3 @@
+export declare const isRecord: (x: unknown) => x is Record<string, unknown>;
+export declare const _transformKeys: <T>(data: T, transform: (key: string) => string) => T;
+export declare const mastodonFetch: <T>(server: string, endpoint: string) => Promise<T | undefined>;

dist/helpers.js

@@ -0,0 +1,34 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mastodonFetch = exports._transformKeys = exports.isRecord = void 0;
+const axios_1 = __importDefault(require("axios"));
+const change_case_1 = require("change-case");
+//Masto does not support top posts from foreign servers, so we have to do it manually
+const isRecord = (x) => typeof x === "object" && x !== null && x.constructor.name === "Object";
+exports.isRecord = isRecord;
+const _transformKeys = (data, transform) => {
+    if (Array.isArray(data)) {
+        return data.map((value) => (0, exports._transformKeys)(value, transform));
+    }
+    if ((0, exports.isRecord)(data)) {
+        return Object.fromEntries(Object.entries(data).map(([key, value]) => [
+            transform(key),
+            (0, exports._transformKeys)(value, transform),
+        ]));
+    }
+    return data;
+};
+exports._transformKeys = _transformKeys;
+const mastodonFetch = async (server, endpoint) => {
+    const json = await axios_1.default.get(`https://${server}${endpoint}`);
+    if (!(json.status === 200) || !json.data) {
+        console.error(`Error fetching data for server ${server}:`, json);
+        return;
+    }
+    const data = (0, exports._transformKeys)(json.data, change_case_1.camelCase);
+    return data;
+};
+exports.mastodonFetch = mastodonFetch;

dist/index.js

@@ -10,10 +10,17 @@ const topPostsFeed_1 = __importDefault(require("./feeds/topPostsFeed"));
 const Storage_1 = __importDefault(require("./Storage"));
 const Paginator_1 = __importDefault(require("./Paginator"));
 const chaosFeatureScorer_1 = __importDefault(require("./scorer/feature/chaosFeatureScorer"));
+const recommenderFeed_1 = __importDefault(require("./feeds/recommenderFeed"));
 class TheAlgorithm {
     user;
-    fetchers = [homeFeed_1.default, topPostsFeed_1.default];
-    featureScorer = [new scorer_1.favsFeatureScorer(), new scorer_1.reblogsFeatureScorer(), new scorer_1.interactsFeatureScorer(), new scorer_1.topPostFeatureScorer(), new chaosFeatureScorer_1.default()];
+    fetchers = [homeFeed_1.default, topPostsFeed_1.default, recommenderFeed_1.default];
+    featureScorer = [
+        new scorer_1.favsFeatureScorer(),
+        new scorer_1.reblogsFeatureScorer(),
+        new scorer_1.interactsFeatureScorer(),
+        new scorer_1.topPostFeatureScorer(),
+        new chaosFeatureScorer_1.default(),
+    ];
     feedScorer = [new scorer_1.reblogsFeedScorer(), new scorer_1.diversityFeedScorer()];
     feed = [];
     api;
@@ -64,8 +71,8 @@ class TheAlgorithm {
             .filter((item) => item.inReplyToId === null)
             .filter((item) => item.content.includes("RT @") === false)
             .filter((item) => !(item?.reblog?.reblogged ?? false))
-            .filter((item) => (!item?.reblog?.muted ?? true))
-            .filter((item) => (!item?.muted ?? true));
+            .filter((item) => !(item?.reblog?.muted ?? false))
+            .filter((item) => !(item?.muted ?? false));
         // Add Time Penalty
         scoredFeed = scoredFeed.map((item) => {
             const seconds = Math.floor((new Date().getTime() - new Date(item.createdAt).getTime()) / 1000);
@@ -89,7 +96,7 @@ class TheAlgorithm {
     async _getValueFromScores(scores) {
         const weights = await weightsStore_1.default.getWeightsMulti(Object.keys(scores));
         const weightedScores = Object.keys(scores).reduce((obj, cur) => {
-            obj = obj + (scores[cur] * weights[cur] ?? 0);
+            obj = obj + (scores[cur] ?? 0) * (weights[cur] ?? 0);
             return obj;
         }, 0);
         return weightedScores;
@@ -147,8 +154,8 @@ class TheAlgorithm {
         const mean = Object.values(statusWeights).filter((value) => !isNaN(value)).reduce((accumulator, currentValue) => accumulator + Math.abs(currentValue), 0) / Object.values(statusWeights).length;
         const currentWeight = await this.getWeights();
         const currentMean = Object.values(currentWeight).filter((value) => !isNaN(value)).reduce((accumulator, currentValue) => accumulator + currentValue, 0) / Object.values(currentWeight).length;
-        for (let key in currentWeight) {
-            let reweight = 1 - (Math.abs(statusWeights[key]) / mean) / (currentWeight[key] / currentMean);
+        for (const key in currentWeight) {
+            const reweight = 1 - (Math.abs(statusWeights[key]) / mean) / (currentWeight[key] / currentMean);
             currentWeight[key] = currentWeight[key] - step * currentWeight[key] * reweight;
         }
         await this.setWeights(currentWeight);

dist/scorer/FeatureScorer.d.ts

@@ -1,5 +1,5 @@
 import { mastodon } from "masto";
-import { StatusType, accFeatureType } from "../types";
+import { accFeatureType, StatusType } from "../types";
 interface RankParams {
     featureGetter: (api: mastodon.rest.Client) => Promise<accFeatureType>;
     verboseName: string;
@@ -15,7 +15,7 @@ export default class FeatureScorer {
     private _defaultWeight;
     constructor(params: RankParams);
     getFeature(api: mastodon.rest.Client): Promise<void>;
-    score(api: mastodon.rest.Client, status: StatusType): Promise<number>;
+    score(_api: mastodon.rest.Client, _status: StatusType): Promise<number>;
     getVerboseName(): string;
     getDescription(): string;
     getDefaultWeight(): number;

dist/scorer/FeatureScorer.js

@@ -17,11 +17,7 @@ class FeatureScorer {
         this._isReady = true;
         this.feature = await this.featureGetter(api);
     }
-    async score(api, status) {
-        if (!this._isReady) {
-            await this.getFeature(api);
-            this._isReady = true;
-        }
+    async score(_api, _status) {
         return 0;
     }
     getVerboseName() {