Merge branch 'next' into SER-2099-do-a-small-product-frontend-improve…

…ment-for-alerts

CHANGELOG.md

@@ -35,6 +35,18 @@ Enterprise Features:
 - [users] UI improvements
 - [views] Added a quick transition to drill
 
+## Version 24.05.17
+Fixes:
+- [push] Improved ability to observe push related errors
+
+Enterprise fixes:
+- [cohorts] Fixed issues with nightly cleanup
+- [data-manager] Fixed UI bug where rules were not visible when editing "Merge by regex" transformations
+- [drill] Fixed wrong pie chart label  tooltip in dashboard widget
+- [flows] Fixed bug in case of null data in schema
+- [nps] Fixed bug in the editor where the "internal name" field was not mandatory
+- [ratings] Fixed UI bug where "Internal name" was not a mandatory field
+
 ## Version 24.05.16
 Fixes:
 - [core] Replaced "Users" with "Sessions" label on technology home widgets

api/api.js

@@ -313,6 +313,7 @@ plugins.connectToAllDatabases().then(function() {
             jobs.job('api:clearAutoTasks').replace().schedule('every 1 day');
             jobs.job('api:task').replace().schedule('every 5 minutes');
             jobs.job('api:userMerge').replace().schedule('every 10 minutes');
+            jobs.job("api:ttlCleanup").replace().schedule("every 1 minute");
             //jobs.job('api:appExpire').replace().schedule('every 1 day');
         }, 10000);
 

api/jobs/ttlCleanup.js

@@ -0,0 +1,48 @@
+const plugins = require("../../plugins/pluginManager.js");
+const common = require('../utils/common');
+const job = require("../parts/jobs/job.js");
+const log = require("../utils/log.js")("job:ttlCleanup");
+
+/**
+ * Class for job of cleaning expired records inside ttl collections
+ */
+class TTLCleanup extends job.Job {
+    /**
+     * Run the job
+     */
+    async run() {
+        log.d("Started running TTL clean up job");
+        for (let i = 0; i < plugins.ttlCollections.length; i++) {
+            const {
+                db = "countly",
+                collection,
+                property,
+                expireAfterSeconds = 0
+            } = plugins.ttlCollections[i];
+            let dbInstance;
+            switch (db) {
+            case "countly": dbInstance = common.db; break;
+            case "countly_drill": dbInstance = common.drillDb; break;
+            case "countly_out": dbInstance = common.outDb; break;
+            }
+            if (!dbInstance) {
+                log.e("Invalid db selection:", db);
+                continue;
+            }
+
+            log.d("Started cleaning up", collection);
+            const result = await dbInstance.collection(collection).deleteMany({
+                [property]: {
+                    $lte: new Date(Date.now() - expireAfterSeconds * 1000)
+                }
+            });
+            log.d("Finished cleaning up", result.deletedCount, "records from", collection);
+
+            // Sleep 1 second to prevent sending too many deleteMany queries
+            await new Promise(res => setTimeout(res, 1000));
+        }
+        log.d("Finished running TTL clean up job");
+    }
+}
+
+module.exports = TTLCleanup;

bin/scripts/expire-data/delete_custom_events_regex.js

@@ -6,7 +6,6 @@
  */
 
 
-const { ObjectId } = require('mongodb');
 const pluginManager = require('../../../plugins/pluginManager.js');
 const common = require('../../../api/utils/common.js');
 const drillCommon = require('../../../plugins/drill/api/common.js');
@@ -25,7 +24,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
 
     //GET APP
     try {
-        const app = await countlyDb.collection("apps").findOne({_id: ObjectId(APP_ID)}, {_id: 1, name: 1});
+        const app = await countlyDb.collection("apps").findOne({_id: countlyDb.ObjectID(APP_ID)}, {_id: 1, name: 1});
         console.log("App:", app.name);
         //GET EVENTS
         var events = [];
@@ -51,6 +50,27 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
                     }
                 ]).toArray();
                 events = events.length ? events[0].list : [];
+                const metaEvents = await drillDb.collection("drill_meta").aggregate([
+                    {
+                        $match: {
+                            'app_id': app._id + "",
+                            "type": "e",
+                            "e": { $regex: regex, $options: CASE_INSENSITIVE ? "i" : "", $nin: events }
+                        }
+                    },
+                    {
+                        $group: {
+                            _id: "$e"
+                        }
+                    },
+                    {
+                        $project: {
+                            _id: 0,
+                            e: "$_id"
+                        }
+                    }
+                ]).toArray();
+                events = events.concat(metaEvents.map(e => e.e));
             }
             catch (err) {
                 close("Invalid regex");
@@ -86,6 +106,7 @@ Promise.all([pluginManager.dbConnection("countly"), pluginManager.dbConnection("
         close(err);
     }
 
+
     async function deleteDrillEvents(appId, events) {
         for (let i = 0; i < events.length; i++) {
             var collectionName = drillCommon.getCollectionName(events[i], appId);

frontend/express/app.js

@@ -974,6 +974,7 @@ Promise.all([plugins.dbConnection(countlyConfig), plugins.dbConnection("countly_
                     timezones: timezones,
                     countlyTypeName: COUNTLY_NAMED_TYPE,
                     countlyTypeTrack: COUNTLY_TRACK_TYPE,
+                    countlyTypeCE: COUNTLY_TYPE_CE,
                     countly_tracking,
                     countly_domain,
                     frontend_app: versionInfo.frontend_app || 'e70ec21cbe19e799472dfaee0adb9223516d238f',

frontend/express/public/core/app-management/javascripts/countly.views.js

@@ -392,10 +392,10 @@
                                     label: data.name
                                 });
                                 self.$store.dispatch("countlyCommon/addToAllApps", data);
+                                self.$store.dispatch("countlyCommon/updateActiveApp", data._id + "");
                                 if (self.firstApp) {
                                     countlyCommon.ACTIVE_APP_ID = data._id + "";
                                     app.onAppManagementSwitch(data._id + "", data && data.type || "mobile");
-                                    self.$store.dispatch("countlyCommon/updateActiveApp", data._id + "");
                                     app.initSidebar();
                                 }
                                 self.firstApp = self.checkIfFirst();

frontend/express/public/core/onboarding/javascripts/countly.views.js

@@ -99,6 +99,7 @@
                 countlyPopulator.setStartTime(countlyCommon.periodObj.start / 1000);
                 countlyPopulator.setEndTime(countlyCommon.periodObj.end / 1000);
                 countlyPopulator.setSelectedTemplate(selectedAppTemplate);
+                countlyPopulator.setSelectedFeatures("all");
                 countlyPopulator.getTemplate(selectedAppTemplate, function(template) {
                     countlyPopulator.generateUsers(10, template);
                     self.populatorProgress = 0;

frontend/express/public/javascripts/countly/countly.auth.js

@@ -39,7 +39,7 @@
         }
 
         if (!member.global_admin) {
-            var isPermissionObjectExistForAccessType = (typeof member.permission[accessType] === "object" && typeof member.permission[accessType][app_id] === "object");
+            var isPermissionObjectExistForAccessType = (member.permission && typeof member.permission[accessType] === "object" && typeof member.permission[accessType][app_id] === "object");
 
             var memberHasAllFlag = member.permission && member.permission[accessType] && member.permission[accessType][app_id] && member.permission[accessType][app_id].all;
             var memberHasAllowedFlag = false;
@@ -96,7 +96,7 @@
             return false;
         }
         if (!member.global_admin) {
-            var isPermissionObjectExistForRead = (typeof member.permission.r === "object" && typeof member.permission.r[app_id] === "object");
+            var isPermissionObjectExistForRead = (member.permission && typeof member.permission.r === "object" && typeof member.permission.r[app_id] === "object");
             // TODO: make here better. create helper method for these checks
             var memberHasAllFlag = member.permission && member.permission.r && member.permission.r[app_id] && member.permission.r[app_id].all;
             var memberHasAllowedFlag = false;

frontend/express/public/javascripts/countly/vue/components/sidebar.js

@@ -742,6 +742,9 @@
                 },
                 helpCenterTarget: function() {
                     return this.enableGuides ? '_self' : "_blank";
+                },
+                isCommunityEdition: function() {
+                    return countlyGlobal.countlyTypeCE;
                 }
             },
             methods: {
@@ -894,6 +897,12 @@
 
                         return menu;
                     });
+                },
+                handleButtonClick: function() {
+                    CountlyHelpers.goTo({
+                        url: "https://flex.countly.com",
+                        isExternalLink: true
+                    });
                 }
             },
             mounted: function() {

frontend/express/public/javascripts/countly/vue/templates/sidebar/sidebar.html

@@ -42,7 +42,18 @@
             <analytics-menu ref="analytics" v-show="visibleSidebarMenu === 'analytics'"></analytics-menu>
             <management-menu ref="management" v-show="visibleSidebarMenu === 'management'" @management-menu-ready="onManagementMenuReady"></management-menu>
             <component :ref="item.name" v-for="item in components" :is="item.component" v-show="visibleSidebarMenu === item.name" :key="item.name"></component>
-            <a :href="countlySidebarVersionPath"><div class="cly-vue-sidebar__version" data-test-id="sidebar-menu-version"> {{versionInfo}} </div></a>
+            <div v-if="isCommunityEdition" class="cly-vue-sidebar__version" data-test-id="sidebar-menu-version">
+                <div class="cly-vue-sidebar__version__banner">
+                    <div class="text">{{i18n('sidebar.banner.text')}}</div>
+                    <div class="text bu-has-text-weight-medium">{{i18n('sidebar.banner.upgrade')}}</div>
+                    <el-button class="button" type="success" @click="handleButtonClick">{{i18n('sidebar.banner.upgrade-button')}}</el-button>
+                </div>
+            </div>
+            <a v-else :href="countlySidebarVersionPath">
+                <div class="cly-vue-sidebar__version" data-test-id="sidebar-menu-version">
+                    {{versionInfo}}
+                </div>
+            </a>
         </div>
     </transition>
 </div>

frontend/express/public/localization/dashboard/dashboard.properties

@@ -440,6 +440,9 @@ sidebar.dashboard-tooltip = Dashboards
 sidebar.main-menu = Main Menu
 sidebar.my-profile = My Profile
 sidebar.copy-api-key-success-message = Api Key has been copied to clipboard!
+sidebar.banner.text = You are using a free plan.
+sidebar.banner.upgrade = Upgrade and get more.
+sidebar.banner.upgrade-button = Manage your plan
 
 #dashboard
 dashboard.apply = Apply

frontend/express/public/stylesheets/styles/blocks/_sidebar.scss

@@ -378,6 +378,26 @@ $bg-color: #24292e;
         width: 224px;
         font-size: 10px;
         background-color: #24292e;
+
+        &__banner {
+            background-color: #191C20;
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+            justify-content: center;
+            border-radius: 4px;
+            padding: 16px;
+            margin-bottom: 30%;
+            cursor: default;
+            .text {
+                font-size: 14px;
+                color: #fff;
+            }
+            .button {
+                width: 100%;
+                margin-top: 12px;
+            }
+        }
     }
 
     .cly-icon {

plugins/alerts/README.md

@@ -1,6 +1,6 @@
 # Alerts plugin
 
-The Alerts plugin is designed to monitor various metrics and events within the Countly analytics platform. It provides a structured way to set up alerts for different types of data, ensuring that users are notified when specific conditions are met. Below is a detailed breakdown of the plugin's file structure and its components:
+The Alerts plugin in Countly is a reactive tool designed to keep you informed about critical changes in your application’s metrics, even when you’re not monitoring dashboards. It sends email notifications when specific conditions on important metrics are met, enabling you to respond quickly to potential issues. This feature helps ensure your app maintains high performance and provides a positive user experience by alerting you to areas that may need immediate attention.
 
 ## File structure
 File structure follows usual Countly plugin structure
@@ -41,6 +41,17 @@ alerts/
 └── tests.js                                    # plugin tests
 ```
 
+## Key Features
+
+- **Customizable Alerts:** Define specific conditions for metrics such as crashes, cohorts, data points, events, Net Promoter Score (NPS), online users, rating, revenue, sessions, surveys, users, and views. Get notified whenever these conditions are met.
+- **Real-Time Notifications:** Receive email alerts for immediate awareness of changes in your metrics.
+- **Detailed Monitoring:** Track a broad range of metrics, including user engagement, performance, user feedback, and error rates.
+- **Easy Setup:** Simple configuration allows you to set and customize alerts quickly to fit your needs.
+
+## Example Use Case
+
+Imagine you’ve released a new version of your app. Although it passed all tests, some critical bugs may still slip through. These bugs might prevent users from fully using the app. By setting up alerts for sudden spikes in crashes or decreased user activity, you can catch these issues early and work to resolve them, ensuring minimal disruption.
+
 ## Generate alerts job
 
 Job name: alerts:monitor

plugins/alerts/api/api.js

@@ -44,7 +44,11 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
         if (typeof alertID === 'string') {
             alertID = common.db.ObjectID(alertID);
         }
-        common.db.collection("jobs").remove({ 'data.alertID': alertID }, function() {
+        common.db.collection("jobs").remove({ 'data.alertID': alertID }, function(err) {
+            if (err) {
+                log.e('delete job failed, alertID:', alertID, err);
+                return;
+            }
             log.d('delete job, alertID:', alertID);
             if (callback) {
                 callback();
@@ -241,8 +245,8 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
                 );
             }
             catch (err) {
-                log.e('Parse alert failed', alertConfig);
-                common.returnMessage(params, 500, "Failed to create an alert");
+                log.e('Parse alert failed', alertConfig, err);
+                common.returnMessage(params, 500, "Failed to create an alert" + err.message);
             }
         });
         return true;
@@ -284,8 +288,8 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
                 );
             }
             catch (err) {
-                log.e('delete alert failed', alertID);
-                common.returnMessage(params, 500, "Failed to delete an alert");
+                log.e('delete alert failed', alertID, err);
+                common.returnMessage(params, 500, "Failed to delete an alert" + err.message);
             }
         });
         return true;
@@ -411,8 +415,8 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
                 });
             }
             catch (err) {
-                log.e('get alert list failed');
-                common.returnMessage(params, 500, "Failed to get alert list");
+                log.e('get alert list failed', err);
+                common.returnMessage(params, 500, "Failed to get alert list" + err.message);
             }
         });
         return true;

plugins/compliance-hub/README.md

@@ -27,4 +27,18 @@ compliance-hub/
 ├── package.json
 ├── README.md
 └── tests.js                                  # plugin tests
-```
+```
+
+## Key Features
+
+- **Collect User Consents:** Prompt first-time users to consent to data collection, detailing which types of data (e.g., sessions, crashes, views) will be collected. No data is sent unless the user opts in.
+- **Manage User Requests:** The "Consents" tab (available in Enterprise Edition) lets admins view and fulfill user requests for data export or deletion.
+- **SDK Integration:** Countly SDKs (iOS, Android, Node.js, Web) support flexible consent management, allowing opt-in/opt-out on a per-feature basis. SDKs default to opt-in for backward compatibility but can be configured to require opt-in consent at initialization.
+
+## Using the Compliance Hub
+
+Access the Compliance Hub via Main Menu > Utilities > Compliance Hub. The Compliance Hub offers the following views:
+1. **Metrics View:** Track opt-ins and opt-outs across various features (e.g., sessions, crashes) over time in a time-series graph.
+2. **Users View:** List users with consent histories. Each entry shows user ID, device info, app version, and consent types. Options include viewing consent history, exporting user data, and purging data if required.
+3. **Consent History:** A complete list of all opt-in and opt-out actions across metrics, allowing for easy tracking.
+4. **Export/Purge History:** See a record of all data export and deletion actions for compliance tracking.

plugins/data-manager/frontend/public/javascripts/countly.views.js

@@ -1148,11 +1148,12 @@
                 }
                 if (doc.actionType === 'EVENT_MERGE' && doc.isRegexMerge === true) {
                     doc.actionType = 'merge-regex';
+                    doc.eventTransformTargetRegex = doc.transformTarget[0];
                 }
                 else {
                     doc.actionType = doc.actionType.split('_')[1].toLowerCase();
                 }
-                doc.isExistingEvent = 'true';
+                doc.isExistingEvent = doc.isExistingEvent ? 'true' : 'false';
                 // doc.tab;
                 // delete doc.transformType;
                 doc.name = countlyCommon.unescapeHtml(doc.name);