diff --git a/.babelrc b/.babelrc new file mode 100644 index 0000000000..f6bcb726d8 --- /dev/null +++ b/.babelrc @@ -0,0 +1,4 @@ +{ + "presets": ["@babel/preset-flow"], + "plugins": ["babel-plugin-syntax-hermes-parser"] +} diff --git a/.flowconfig b/.flowconfig new file mode 100644 index 0000000000..95fde81def --- /dev/null +++ b/.flowconfig @@ -0,0 +1,14 @@ +[ignore] + +[include] + +[libs] + +[lints] +untyped-type-import=error +internal-type=error +deprecated-type-bool=error + +[options] + +[strict] diff --git a/.github/workflows/azure-deploy-f24.yml b/.github/workflows/azure-deploy-f24.yml index e35e0f7018..752ac4b7bf 100644 --- a/.github/workflows/azure-deploy-f24.yml +++ b/.github/workflows/azure-deploy-f24.yml @@ -19,7 +19,7 @@ jobs: ./.github/workflows/test.yaml build-and-deploy: - if: github.repository == 'cmu-313/NodeBB' + if: github.repository == 'cmu-313/nodebb-f24-hunan-hunters' needs: lint-and-test runs-on: ubuntu-latest @@ -34,7 +34,7 @@ jobs: - name: Set up NodeBB run: | - ./nodebb setup '{"url":"https://nodebb-f24.azurewebsites.net:443", + ./nodebb setup '{"url":"https://nodebb-hunanhunters.azurewebsites.net:443", "admin:username": "admin", "admin:password": "${{ secrets.ADMIN_PASSWORD }}", "admin:password:confirm": "${{ secrets.ADMIN_PASSWORD }}", @@ -48,7 +48,7 @@ jobs: id: deploy-to-webapp uses: azure/webapps-deploy@v2 with: - app-name: 'nodebb-f24' + app-name: 'nodebb-hunanhunters' slot-name: 'Production' - publish-profile: ${{ secrets.AZUREAPPSERVICE_PUBLISHPROFILE_BFAB97B1AB1441ACA7C63280F91AD3F3 }} + publish-profile: ${{ secrets.AZUREAPPSERVICE_PUBLISHPROFILE_8CE2E89DDD294B98BF913663FFF5E9D5 }} package: . diff --git a/README.md b/README.md index 6ef180f625..2fd96967ba 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +Team Members: Andrew Cheng, Emily Yu, Tin Chi Lo, Joyce Lam, Jorge Gracia + +[![Review Assignment Due Date](https://classroom.github.com/assets/deadline-readme-button-22041afd0340ce965d47ae6ef1cefeee28c7c493a6346c4f15d667ab976d596c.svg)](https://classroom.github.com/a/ithVU1OO) # ![NodeBB](public/images/sm-card.png) [![Workflow](https://github.com/CMU-313/NodeBB/actions/workflows/test.yaml/badge.svg)](https://github.com/CMU-313/NodeBB/actions/workflows/test.yaml) diff --git a/UserGuide.md b/UserGuide.md new file mode 100644 index 0000000000..3240426214 --- /dev/null +++ b/UserGuide.md @@ -0,0 +1,12 @@ +User Story #11: +As a user, I would like to be able to visualize instructor posts differently so that I can make sure that I don't miss any important announcements from the instructor. + +To test the flagging instructor posts feature, run nodebb with the frontend changes, any signed in account should work. When opening a post or a list of posts, you should be able to see "! instructor" next to all instructor posts. Unfortunately, this feature did not work as we expected, and we could not run our backend functions that check if a post was created by an instructor in the frontend. Meaning, all posts and post lists show the flag regardless of whether or not they were created by an instructor. + +We also created tests for the isInstructor function (which checks if a user is an instructor) in tests/user.js. + +User Story #14: +As a user, I would like to be able to pin posts so that I can focus on certain posts that I am interested in. + +This feature is not fully developed. We are struggling to join the front_end feature with the backend methods and implementation for toggling the pinned field within the post object. Our front end feature is fully developed and visible through + diff --git a/lib/admin/search.js b/lib/admin/search.js new file mode 100644 index 0000000000..bc4803462a --- /dev/null +++ b/lib/admin/search.js @@ -0,0 +1,96 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const sanitizeHTML = require('sanitize-html'); +const nconf = require('nconf'); +const winston = require('winston'); +const file = require('../file'); +const { + Translator +} = require('../translator'); +function filterDirectories(directories) { + return directories.map(dir => dir.replace(/^.*(admin.*?).tpl$/, '$1').split(path.sep).join('/')).filter(dir => !dir.endsWith('.js') && !dir.includes('/partials/') && /\/.*\//.test(dir) && !/manage\/(category|group|category-analytics)$/.test(dir)); +} +async function getAdminNamespaces() { + const directories = await file.walk(path.resolve(nconf.get('views_dir'), 'admin')); + return filterDirectories(directories); +} +function sanitize(html) { + return sanitizeHTML(html, { + allowedTags: [], + allowedAttributes: [] + }); +} +function simplify(translations) { + return translations.replace(/(?:\{{1,2}[^}]*?\}{1,2})/g, '').replace(/(?:[ \t]*[\n\r]+[ \t]*)+/g, '\n').replace(/[\t ]+/g, ' '); +} +function nsToTitle(namespace) { + return namespace.replace('admin/', '').split('/').map(str => str[0].toUpperCase() + str.slice(1)).join(' > ').replace(/[^a-zA-Z> ]/g, ' '); +} +const fallbackCache = {}; +async function initFallback(namespace) { + const template = await fs.promises.readFile(path.resolve(nconf.get('views_dir'), `${namespace}.tpl`), 'utf8'); + const title = nsToTitle(namespace); + let translations = sanitize(template); + translations = Translator.removePatterns(translations); + translations = simplify(translations); + translations += `\n${title}`; + return { + namespace: namespace, + translations: translations, + title: title + }; +} +async function fallback(namespace) { + if (fallbackCache[namespace]) { + return fallbackCache[namespace]; + } + const params = await initFallback(namespace); + fallbackCache[namespace] = params; + return params; +} +async function initDict(language) { + const namespaces = await getAdminNamespaces(); + return await Promise.all(namespaces.map(ns => buildNamespace(language, ns))); +} +async function buildNamespace(language, namespace) { + const translator = Translator.create(language); + try { + const translations = await translator.getTranslation(namespace); + if (!translations || !Object.keys(translations).length) { + return await fallback(namespace); + } + let str = Object.keys(translations).map(key => translations[key]).join('\n'); + str = sanitize(str); + let title = namespace; + title = title.match(/admin\/(.+?)\/(.+?)$/); + title = `[[admin/menu:section-${title[1] === 'development' ? 'advanced' : title[1]}]]${title[2] ? ` > [[admin/menu:${title[1]}/${title[2]}]]` : ''}`; + title = await translator.translate(title); + return { + namespace: namespace, + translations: `${str}\n${title}`, + title: title + }; + } catch (err) { + winston.error(err.stack); + return { + namespace: namespace, + translations: '' + }; + } +} +const cache = {}; +async function getDictionary(language) { + if (cache[language]) { + return cache[language]; + } + const params = await initDict(language); + cache[language] = params; + return params; +} +module.exports.getDictionary = getDictionary; +module.exports.filterDirectories = filterDirectories; +module.exports.simplify = simplify; +module.exports.sanitize = sanitize; +require('../promisify')(module.exports); \ No newline at end of file diff --git a/lib/admin/versions.js b/lib/admin/versions.js new file mode 100644 index 0000000000..30ce8983af --- /dev/null +++ b/lib/admin/versions.js @@ -0,0 +1,40 @@ +'use strict'; + +const request = require('../request'); +const meta = require('../meta'); +let versionCache = ''; +let versionCacheLastModified = ''; +const isPrerelease = /^v?\d+\.\d+\.\d+-.+$/; +const latestReleaseUrl = 'https://api.github.com/repos/NodeBB/NodeBB/releases/latest'; +async function getLatestVersion() { + const headers = { + Accept: 'application/vnd.github.v3+json', + 'User-Agent': encodeURIComponent(`NodeBB Admin Control Panel/${meta.config.title}`) + }; + if (versionCacheLastModified) { + headers['If-Modified-Since'] = versionCacheLastModified; + } + const { + body: latestRelease, + response + } = await request.get(latestReleaseUrl, { + headers: headers, + timeout: 2000 + }); + if (response.statusCode === 304) { + return versionCache; + } + if (response.statusCode !== 200) { + throw new Error(response.statusText); + } + if (!latestRelease || !latestRelease.tag_name) { + throw new Error('[[error:cant-get-latest-release]]'); + } + const tagName = latestRelease.tag_name.replace(/^v/, ''); + versionCache = tagName; + versionCacheLastModified = response.headers['last-modified']; + return versionCache; +} +exports.getLatestVersion = getLatestVersion; +exports.isPrerelease = isPrerelease; +require('../promisify')(exports); \ No newline at end of file diff --git a/lib/als.js b/lib/als.js new file mode 100644 index 0000000000..afe0932753 --- /dev/null +++ b/lib/als.js @@ -0,0 +1,7 @@ +'use strict'; + +const { + AsyncLocalStorage +} = require('async_hooks'); +const asyncLocalStorage = new AsyncLocalStorage(); +module.exports = asyncLocalStorage; \ No newline at end of file diff --git a/lib/analytics.js b/lib/analytics.js new file mode 100644 index 0000000000..95c8c3f420 --- /dev/null +++ b/lib/analytics.js @@ -0,0 +1,235 @@ +'use strict'; + +const cronJob = require('cron').CronJob; +const winston = require('winston'); +const nconf = require('nconf'); +const crypto = require('crypto'); +const util = require('util'); +const _ = require('lodash'); +const sleep = util.promisify(setTimeout); +const db = require('./database'); +const utils = require('./utils'); +const plugins = require('./plugins'); +const meta = require('./meta'); +const pubsub = require('./pubsub'); +const cacheCreate = require('./cache/lru'); +const Analytics = module.exports; +const secret = nconf.get('secret'); +let local = { + counters: {}, + pageViews: 0, + pageViewsRegistered: 0, + pageViewsGuest: 0, + pageViewsBot: 0, + uniqueIPCount: 0, + uniquevisitors: 0 +}; +const empty = _.cloneDeep(local); +const total = _.cloneDeep(local); +let ipCache; +const runJobs = nconf.get('runJobs'); +Analytics.init = async function () { + ipCache = cacheCreate({ + max: parseInt(meta.config['analytics:maxCache'], 10) || 500, + ttl: 0 + }); + new cronJob('*/10 * * * * *', async () => { + publishLocalAnalytics(); + if (runJobs) { + await sleep(2000); + await Analytics.writeData(); + } + }, null, true); + if (runJobs) { + pubsub.on('analytics:publish', data => { + incrementProperties(total, data.local); + }); + } +}; +function publishLocalAnalytics() { + pubsub.publish('analytics:publish', { + local: local + }); + local = _.cloneDeep(empty); +} +function incrementProperties(obj1, obj2) { + for (const [key, value] of Object.entries(obj2)) { + if (typeof value === 'object') { + incrementProperties(obj1[key], value); + } else if (utils.isNumber(value)) { + obj1[key] = obj1[key] || 0; + obj1[key] += obj2[key]; + } + } +} +Analytics.increment = function (keys, callback) { + keys = Array.isArray(keys) ? keys : [keys]; + plugins.hooks.fire('action:analytics.increment', { + keys: keys + }); + keys.forEach(key => { + local.counters[key] = local.counters[key] || 0; + local.counters[key] += 1; + }); + if (typeof callback === 'function') { + callback(); + } +}; +Analytics.getKeys = async () => db.getSortedSetRange('analyticsKeys', 0, -1); +Analytics.pageView = async function (payload) { + local.pageViews += 1; + if (payload.uid > 0) { + local.pageViewsRegistered += 1; + } else if (payload.uid < 0) { + local.pageViewsBot += 1; + } else { + local.pageViewsGuest += 1; + } + if (payload.ip) { + let hash = ipCache.get(payload.ip + secret); + if (!hash) { + hash = crypto.createHash('sha1').update(payload.ip + secret).digest('hex'); + ipCache.set(payload.ip + secret, hash); + } + const score = await db.sortedSetScore('ip:recent', hash); + if (!score) { + local.uniqueIPCount += 1; + } + const today = new Date(); + today.setHours(today.getHours(), 0, 0, 0); + if (!score || score < today.getTime()) { + local.uniquevisitors += 1; + await db.sortedSetAdd('ip:recent', Date.now(), hash); + } + } +}; +Analytics.writeData = async function () { + const today = new Date(); + const month = new Date(); + const dbQueue = []; + const incrByBulk = []; + let metrics = ['pageviews', 'pageviews:month']; + metrics.forEach(metric => { + const toAdd = ['registered', 'guest', 'bot'].map(type => `${metric}:${type}`); + metrics = [...metrics, ...toAdd]; + }); + metrics.push('uniquevisitors'); + today.setHours(today.getHours(), 0, 0, 0); + month.setMonth(month.getMonth(), 1); + month.setHours(0, 0, 0, 0); + if (total.pageViews > 0) { + incrByBulk.push(['analytics:pageviews', total.pageViews, today.getTime()]); + incrByBulk.push(['analytics:pageviews:month', total.pageViews, month.getTime()]); + total.pageViews = 0; + } + if (total.pageViewsRegistered > 0) { + incrByBulk.push(['analytics:pageviews:registered', total.pageViewsRegistered, today.getTime()]); + incrByBulk.push(['analytics:pageviews:month:registered', total.pageViewsRegistered, month.getTime()]); + total.pageViewsRegistered = 0; + } + if (total.pageViewsGuest > 0) { + incrByBulk.push(['analytics:pageviews:guest', total.pageViewsGuest, today.getTime()]); + incrByBulk.push(['analytics:pageviews:month:guest', total.pageViewsGuest, month.getTime()]); + total.pageViewsGuest = 0; + } + if (total.pageViewsBot > 0) { + incrByBulk.push(['analytics:pageviews:bot', total.pageViewsBot, today.getTime()]); + incrByBulk.push(['analytics:pageviews:month:bot', total.pageViewsBot, month.getTime()]); + total.pageViewsBot = 0; + } + if (total.uniquevisitors > 0) { + incrByBulk.push(['analytics:uniquevisitors', total.uniquevisitors, today.getTime()]); + total.uniquevisitors = 0; + } + if (total.uniqueIPCount > 0) { + dbQueue.push(db.incrObjectFieldBy('global', 'uniqueIPCount', total.uniqueIPCount)); + total.uniqueIPCount = 0; + } + for (const [key, value] of Object.entries(total.counters)) { + incrByBulk.push([`analytics:${key}`, value, today.getTime()]); + metrics.push(key); + delete total.counters[key]; + } + if (incrByBulk.length) { + dbQueue.push(db.sortedSetIncrByBulk(incrByBulk)); + } + dbQueue.push(db.sortedSetAdd('analyticsKeys', metrics.map(() => +Date.now()), metrics)); + try { + await Promise.all(dbQueue); + } catch (err) { + winston.error(`[analytics] Encountered error while writing analytics to data store\n${err.stack}`); + } +}; +Analytics.getHourlyStatsForSet = async function (set, hour, numHours) { + if (!set.startsWith('analytics:')) { + set = `analytics:${set}`; + } + const terms = {}; + const hoursArr = []; + hour = new Date(hour); + hour.setHours(hour.getHours(), 0, 0, 0); + for (let i = 0, ii = numHours; i < ii; i += 1) { + hoursArr.push(hour.getTime() - i * 3600 * 1000); + } + const counts = await db.sortedSetScores(set, hoursArr); + hoursArr.forEach((term, index) => { + terms[term] = parseInt(counts[index], 10) || 0; + }); + const termsArr = []; + hoursArr.reverse(); + hoursArr.forEach(hour => { + termsArr.push(terms[hour]); + }); + return termsArr; +}; +Analytics.getDailyStatsForSet = async function (set, day, numDays) { + if (!set.startsWith('analytics:')) { + set = `analytics:${set}`; + } + day = new Date(day); + day.setDate(day.getDate() + 1); + day.setHours(0, 0, 0, 0); + async function getHourlyStats(hour) { + const dayData = await Analytics.getHourlyStatsForSet(set, hour, 24); + return dayData.reduce((cur, next) => cur + next); + } + const hours = []; + while (numDays > 0) { + hours.push(day.getTime() - 1000 * 60 * 60 * 24 * (numDays - 1)); + numDays -= 1; + } + return await Promise.all(hours.map(getHourlyStats)); +}; +Analytics.getUnwrittenPageviews = function () { + return local.pageViews; +}; +Analytics.getSummary = async function () { + const today = new Date(); + today.setHours(0, 0, 0, 0); + const [seven, thirty] = await Promise.all([Analytics.getDailyStatsForSet('analytics:pageviews', today, 7), Analytics.getDailyStatsForSet('analytics:pageviews', today, 30)]); + return { + seven: seven.reduce((sum, cur) => sum + cur, 0), + thirty: thirty.reduce((sum, cur) => sum + cur, 0) + }; +}; +Analytics.getCategoryAnalytics = async function (cid) { + return await utils.promiseParallel({ + 'pageviews:hourly': Analytics.getHourlyStatsForSet(`analytics:pageviews:byCid:${cid}`, Date.now(), 24), + 'pageviews:daily': Analytics.getDailyStatsForSet(`analytics:pageviews:byCid:${cid}`, Date.now(), 30), + 'topics:daily': Analytics.getDailyStatsForSet(`analytics:topics:byCid:${cid}`, Date.now(), 7), + 'posts:daily': Analytics.getDailyStatsForSet(`analytics:posts:byCid:${cid}`, Date.now(), 7) + }); +}; +Analytics.getErrorAnalytics = async function () { + return await utils.promiseParallel({ + 'not-found': Analytics.getDailyStatsForSet('analytics:errors:404', Date.now(), 7), + toobusy: Analytics.getDailyStatsForSet('analytics:errors:503', Date.now(), 7) + }); +}; +Analytics.getBlacklistAnalytics = async function () { + return await utils.promiseParallel({ + daily: Analytics.getDailyStatsForSet('analytics:blacklist', Date.now(), 7), + hourly: Analytics.getHourlyStatsForSet('analytics:blacklist', Date.now(), 24) + }); +}; +require('./promisify')(Analytics); \ No newline at end of file diff --git a/lib/api/admin.js b/lib/api/admin.js new file mode 100644 index 0000000000..759691d712 --- /dev/null +++ b/lib/api/admin.js @@ -0,0 +1,45 @@ +'use strict'; + +const meta = require('../meta'); +const analytics = require('../analytics'); +const privileges = require('../privileges'); +const groups = require('../groups'); +const adminApi = module.exports; +adminApi.updateSetting = async (caller, { + setting, + value +}) => { + const ok = await privileges.admin.can('admin:settings', caller.uid); + if (!ok) { + throw new Error('[[error:no-privileges]]'); + } + await meta.configs.set(setting, value); +}; +adminApi.getAnalyticsKeys = async () => { + const keys = await analytics.getKeys(); + return keys.sort((a, b) => a < b ? -1 : 1); +}; +adminApi.getAnalyticsData = async (caller, { + set, + until, + amount, + units +}) => { + if (!amount) { + if (units === 'days') { + amount = 30; + } else { + amount = 24; + } + } + const getStats = units === 'days' ? analytics.getDailyStatsForSet : analytics.getHourlyStatsForSet; + return await getStats(`analytics:${set}`, parseInt(until, 10) || Date.now(), amount); +}; +adminApi.listGroups = async () => { + const payload = await groups.getNonPrivilegeGroups('groups:createtime', 0, -1, { + ephemeral: false + }); + return { + groups: payload + }; +}; \ No newline at end of file diff --git a/lib/api/categories.js b/lib/api/categories.js new file mode 100644 index 0000000000..4a35ad5f1e --- /dev/null +++ b/lib/api/categories.js @@ -0,0 +1,226 @@ +'use strict'; + +const meta = require('../meta'); +const categories = require('../categories'); +const topics = require('../topics'); +const events = require('../events'); +const user = require('../user'); +const groups = require('../groups'); +const privileges = require('../privileges'); +const categoriesAPI = module.exports; +const hasAdminPrivilege = async (uid, privilege = 'categories') => { + const ok = await privileges.admin.can(`admin:${privilege}`, uid); + if (!ok) { + throw new Error('[[error:no-privileges]]'); + } +}; +categoriesAPI.list = async caller => { + async function getCategories() { + const cids = await categories.getCidsByPrivilege('categories:cid', caller.uid, 'find'); + return await categories.getCategoriesData(cids); + } + const [isAdmin, categoriesData] = await Promise.all([user.isAdministrator(caller.uid), getCategories()]); + return { + categories: categoriesData.filter(category => category && (!category.disabled || isAdmin)) + }; +}; +categoriesAPI.get = async function (caller, data) { + const [userPrivileges, category] = await Promise.all([privileges.categories.get(data.cid, caller.uid), categories.getCategoryData(data.cid)]); + if (!category || !userPrivileges.read) { + return null; + } + return category; +}; +categoriesAPI.create = async function (caller, data) { + await hasAdminPrivilege(caller.uid); + const response = await categories.create(data); + const categoryObjs = await categories.getCategories([response.cid]); + return categoryObjs[0]; +}; +categoriesAPI.update = async function (caller, data) { + await hasAdminPrivilege(caller.uid); + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + const { + cid, + values + } = data; + const payload = {}; + payload[cid] = values; + await categories.update(payload); +}; +categoriesAPI.delete = async function (caller, { + cid +}) { + await hasAdminPrivilege(caller.uid); + const name = await categories.getCategoryField(cid, 'name'); + await categories.purge(cid, caller.uid); + await events.log({ + type: 'category-purge', + uid: caller.uid, + ip: caller.ip, + cid: cid, + name: name + }); +}; +categoriesAPI.getTopicCount = async (caller, { + cid +}) => { + const allowed = await privileges.categories.can('find', cid, caller.uid); + if (!allowed) { + throw new Error('[[error:no-privileges]]'); + } + const count = await categories.getCategoryField(cid, 'topic_count'); + return { + count + }; +}; +categoriesAPI.getPosts = async (caller, { + cid +}) => await categories.getRecentReplies(cid, caller.uid, 0, 4); +categoriesAPI.getChildren = async (caller, { + cid, + start +}) => { + if (!start || start < 0) { + start = 0; + } + start = parseInt(start, 10); + const allowed = await privileges.categories.can('read', cid, caller.uid); + if (!allowed) { + throw new Error('[[error:no-privileges]]'); + } + const category = await categories.getCategoryData(cid); + await categories.getChildrenTree(category, caller.uid); + const allCategories = []; + categories.flattenCategories(allCategories, category.children); + await categories.getRecentTopicReplies(allCategories, caller.uid); + const payload = category.children.slice(start, start + category.subCategoriesPerPage); + return { + categories: payload + }; +}; +categoriesAPI.getTopics = async (caller, data) => { + data.query = data.query || {}; + const [userPrivileges, settings, targetUid] = await Promise.all([privileges.categories.get(data.cid, caller.uid), user.getSettings(caller.uid), user.getUidByUserslug(data.query.author)]); + if (!userPrivileges.read) { + throw new Error('[[error:no-privileges]]'); + } + const infScrollTopicsPerPage = 20; + const sort = data.sort || data.categoryTopicSort || meta.config.categoryTopicSort || 'recently_replied'; + let start = Math.max(0, parseInt(data.after || 0, 10)); + if (parseInt(data.direction, 10) === -1) { + start -= infScrollTopicsPerPage; + } + let stop = start + infScrollTopicsPerPage - 1; + start = Math.max(0, start); + stop = Math.max(0, stop); + const result = await categories.getCategoryTopics({ + uid: caller.uid, + cid: data.cid, + start, + stop, + sort, + settings, + query: data.query, + tag: data.query.tag, + targetUid + }); + categories.modifyTopicsByPrivilege(result.topics, userPrivileges); + return { + ...result, + privileges: userPrivileges + }; +}; +categoriesAPI.setWatchState = async (caller, { + cid, + state, + uid +}) => { + let targetUid = caller.uid; + const cids = Array.isArray(cid) ? cid.map(cid => parseInt(cid, 10)) : [parseInt(cid, 10)]; + if (uid) { + targetUid = uid; + } + await user.isAdminOrGlobalModOrSelf(caller.uid, targetUid); + const allCids = await categories.getAllCidsFromSet('categories:cid'); + const categoryData = await categories.getCategoriesFields(allCids, ['cid', 'parentCid']); + let cat; + do { + cat = categoryData.find(c => !cids.includes(c.cid) && cids.includes(c.parentCid)); + if (cat) { + cids.push(cat.cid); + } + } while (cat); + await user.setCategoryWatchState(targetUid, cids, state); + await topics.pushUnreadCount(targetUid); + return { + cids + }; +}; +categoriesAPI.getPrivileges = async (caller, { + cid +}) => { + await hasAdminPrivilege(caller.uid, 'privileges'); + let responsePayload; + if (cid === 'admin') { + responsePayload = await privileges.admin.list(caller.uid); + } else if (!parseInt(cid, 10)) { + responsePayload = await privileges.global.list(); + } else { + responsePayload = await privileges.categories.list(cid); + } + return responsePayload; +}; +categoriesAPI.setPrivilege = async (caller, data) => { + await hasAdminPrivilege(caller.uid, 'privileges'); + const [userExists, groupExists] = await Promise.all([user.exists(data.member), groups.exists(data.member)]); + if (!userExists && !groupExists) { + throw new Error('[[error:no-user-or-group]]'); + } + const privs = Array.isArray(data.privilege) ? data.privilege : [data.privilege]; + const type = data.set ? 'give' : 'rescind'; + if (!privs.length) { + throw new Error('[[error:invalid-data]]'); + } + if (parseInt(data.cid, 10) === 0) { + const adminPrivList = await privileges.admin.getPrivilegeList(); + const adminPrivs = privs.filter(priv => adminPrivList.includes(priv)); + if (adminPrivs.length) { + await privileges.admin[type](adminPrivs, data.member); + } + const globalPrivList = await privileges.global.getPrivilegeList(); + const globalPrivs = privs.filter(priv => globalPrivList.includes(priv)); + if (globalPrivs.length) { + await privileges.global[type](globalPrivs, data.member); + } + } else { + const categoryPrivList = await privileges.categories.getPrivilegeList(); + const categoryPrivs = privs.filter(priv => categoryPrivList.includes(priv)); + await privileges.categories[type](categoryPrivs, data.cid, data.member); + } + await events.log({ + uid: caller.uid, + type: 'privilege-change', + ip: caller.ip, + privilege: data.privilege.toString(), + cid: data.cid, + action: data.set ? 'grant' : 'rescind', + target: data.member + }); +}; +categoriesAPI.setModerator = async (caller, { + cid, + member, + set +}) => { + await hasAdminPrivilege(caller.uid, 'admins-mods'); + const privilegeList = await privileges.categories.getUserPrivilegeList(); + await categoriesAPI.setPrivilege(caller, { + cid, + privilege: privilegeList, + member, + set + }); +}; \ No newline at end of file diff --git a/lib/api/chats.js b/lib/api/chats.js new file mode 100644 index 0000000000..26944e115c --- /dev/null +++ b/lib/api/chats.js @@ -0,0 +1,414 @@ +'use strict'; + +const validator = require('validator'); +const winston = require('winston'); +const db = require('../database'); +const user = require('../user'); +const meta = require('../meta'); +const messaging = require('../messaging'); +const notifications = require('../notifications'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const websockets = require('../socket.io'); +const socketHelpers = require('../socket.io/helpers'); +const chatsAPI = module.exports; +async function rateLimitExceeded(caller, field) { + const session = caller.request ? caller.request.session : caller.session; + const now = Date.now(); + const [isPrivileged, reputation] = await Promise.all([user.isPrivileged(caller.uid), user.getUserField(caller.uid, 'reputation')]); + const newbie = !isPrivileged && meta.config.newbieReputationThreshold > reputation; + const delay = newbie ? meta.config.newbieChatMessageDelay : meta.config.chatMessageDelay; + session[field] = session[field] || 0; + if (now - session[field] < delay) { + return true; + } + session[field] = now; + return false; +} +chatsAPI.list = async (caller, { + uid = caller.uid, + start, + stop, + page, + perPage +} = {}) => { + if ((!utils.isNumber(start) || !utils.isNumber(stop)) && !utils.isNumber(page)) { + throw new Error('[[error:invalid-data]]'); + } + if (!start && !stop && page) { + winston.warn('[api/chats] Sending `page` and `perPage` to .list() is deprecated in favour of `start` and `stop`. The deprecated parameters will be removed in v4.'); + start = Math.max(0, page - 1) * perPage; + stop = start + perPage - 1; + } + return await messaging.getRecentChats(caller.uid, uid || caller.uid, start, stop); +}; +chatsAPI.create = async function (caller, data) { + if (await rateLimitExceeded(caller, 'lastChatRoomCreateTime')) { + throw new Error('[[error:too-many-messages]]'); + } + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + const isPublic = data.type === 'public'; + const isAdmin = await user.isAdministrator(caller.uid); + if (isPublic && !isAdmin) { + throw new Error('[[error:no-privileges]]'); + } + if (!data.uids || !Array.isArray(data.uids)) { + throw new Error(`[[error:wrong-parameter-type, uids, ${typeof data.uids}, Array]]`); + } + if (!isPublic && !data.uids.length) { + throw new Error('[[error:no-users-selected]]'); + } + if (isPublic && (!Array.isArray(data.groups) || !data.groups.length)) { + throw new Error('[[error:no-groups-selected]]'); + } + data.notificationSetting = isPublic ? messaging.notificationSettings.ATMENTION : messaging.notificationSettings.ALLMESSAGES; + await Promise.all(data.uids.map(uid => messaging.canMessageUser(caller.uid, uid))); + const roomId = await messaging.newRoom(caller.uid, data); + return await messaging.getRoomData(roomId); +}; +chatsAPI.getUnread = async caller => { + const count = await messaging.getUnreadCount(caller.uid); + return { + count + }; +}; +chatsAPI.sortPublicRooms = async (caller, { + roomIds, + scores +}) => { + [roomIds, scores].forEach(arr => { + if (!Array.isArray(arr) || !arr.every(value => isFinite(value))) { + throw new Error('[[error:invalid-data]]'); + } + }); + const isAdmin = await user.isAdministrator(caller.uid); + if (!isAdmin) { + throw new Error('[[error:no-privileges]]'); + } + await db.sortedSetAdd(`chat:rooms:public:order`, scores, roomIds); + require('../cache').del(`chat:rooms:public:order:all`); +}; +chatsAPI.get = async (caller, { + uid, + roomId +}) => await messaging.loadRoom(caller.uid, { + uid, + roomId +}); +chatsAPI.post = async (caller, data) => { + if (await rateLimitExceeded(caller, 'lastChatMessageTime')) { + throw new Error('[[error:too-many-messages]]'); + } + if (!data || !data.roomId || !caller.uid) { + throw new Error('[[error:invalid-data]]'); + } + ({ + data + } = await plugins.hooks.fire('filter:messaging.send', { + data, + uid: caller.uid + })); + await messaging.canMessageRoom(caller.uid, data.roomId); + const message = await messaging.sendMessage({ + uid: caller.uid, + roomId: data.roomId, + content: data.message, + toMid: data.toMid, + timestamp: Date.now(), + ip: caller.ip + }); + messaging.notifyUsersInRoom(caller.uid, data.roomId, message); + user.updateOnlineUsers(caller.uid); + return message; +}; +chatsAPI.update = async (caller, data) => { + if (!data || !data.roomId) { + throw new Error('[[error:invalid-data]]'); + } + if (data.hasOwnProperty('name')) { + if (!data.name && data.name !== '') { + throw new Error('[[error:invalid-data]]'); + } + await messaging.renameRoom(caller.uid, data.roomId, data.name); + } + const [roomData, isAdmin] = await Promise.all([messaging.getRoomData(data.roomId), user.isAdministrator(caller.uid)]); + if (!roomData) { + throw new Error('[[error:invalid-data]]'); + } + if (data.hasOwnProperty('groups')) { + if (roomData.public && isAdmin) { + await db.setObjectField(`chat:room:${data.roomId}`, 'groups', JSON.stringify(data.groups)); + } + } + if (data.hasOwnProperty('notificationSetting') && isAdmin) { + await db.setObjectField(`chat:room:${data.roomId}`, 'notificationSetting', data.notificationSetting); + } + const loadedRoom = await messaging.loadRoom(caller.uid, { + roomId: data.roomId + }); + if (data.hasOwnProperty('name')) { + const ioRoom = require('../socket.io').in(`chat_room_${data.roomId}`); + if (ioRoom) { + ioRoom.emit('event:chats.roomRename', { + roomId: data.roomId, + newName: validator.escape(String(data.name)), + chatWithMessage: loadedRoom.chatWithMessage + }); + } + } + return loadedRoom; +}; +chatsAPI.rename = async (caller, data) => { + if (!data || !data.roomId || !data.name) { + throw new Error('[[error:invalid-data]]'); + } + return await chatsAPI.update(caller, data); +}; +chatsAPI.mark = async (caller, data) => { + if (!caller.uid || !data || !data.roomId) { + throw new Error('[[error:invalid-data]]'); + } + const { + roomId, + state + } = data; + if (state) { + await messaging.markUnread([caller.uid], roomId); + } else { + await messaging.markRead(caller.uid, roomId); + socketHelpers.emitToUids('event:chats.markedAsRead', { + roomId: roomId + }, [caller.uid]); + const nids = await user.notifications.getUnreadByField(caller.uid, 'roomId', [roomId]); + await notifications.markReadMultiple(nids, caller.uid); + user.notifications.pushCount(caller.uid); + } + socketHelpers.emitToUids('event:chats.mark', { + roomId, + state + }, [caller.uid]); + messaging.pushUnreadCount(caller.uid); +}; +chatsAPI.watch = async (caller, { + roomId, + state +}) => { + const inRoom = await messaging.isUserInRoom(caller.uid, roomId); + if (!inRoom) { + throw new Error('[[error:no-privileges]]'); + } + await messaging.setUserNotificationSetting(caller.uid, roomId, state); +}; +chatsAPI.toggleTyping = async (caller, { + roomId, + typing +}) => { + if (!utils.isNumber(roomId) || typeof typing !== 'boolean') { + throw new Error('[[error:invalid-data]]'); + } + const [isInRoom, username] = await Promise.all([messaging.isUserInRoom(caller.uid, roomId), user.getUserField(caller.uid, 'username')]); + if (!isInRoom) { + throw new Error('[[error:no-privileges]]'); + } + websockets.in(`chat_room_${roomId}`).emit('event:chats.typing', { + uid: caller.uid, + roomId, + typing, + username + }); +}; +chatsAPI.users = async (caller, data) => { + const start = data.hasOwnProperty('start') ? data.start : 0; + const stop = start + 39; + const io = require('../socket.io'); + const [isOwner, isUserInRoom, users, isAdmin, onlineUids] = await Promise.all([messaging.isRoomOwner(caller.uid, data.roomId), messaging.isUserInRoom(caller.uid, data.roomId), messaging.getUsersInRoomFromSet(`chat:room:${data.roomId}:uids:online`, data.roomId, start, stop, true), user.isAdministrator(caller.uid), io.getUidsInRoom(`chat_room_${data.roomId}`)]); + if (!isUserInRoom) { + throw new Error('[[error:no-privileges]]'); + } + users.forEach(user => { + const isSelf = parseInt(user.uid, 10) === parseInt(caller.uid, 10); + user.canKick = isOwner && !isSelf; + user.canToggleOwner = (isAdmin || isOwner) && !isSelf; + user.online = parseInt(user.uid, 10) === parseInt(caller.uid, 10) || onlineUids.includes(String(user.uid)); + }); + return { + users + }; +}; +chatsAPI.invite = async (caller, data) => { + if (!data || !data.roomId || !Array.isArray(data.uids)) { + throw new Error('[[error:invalid-data]]'); + } + const roomData = await messaging.getRoomData(data.roomId); + if (!roomData) { + throw new Error('[[error:invalid-data]]'); + } + const userCount = await messaging.getUserCountInRoom(data.roomId); + const maxUsers = meta.config.maximumUsersInChatRoom; + if (!roomData.public && maxUsers && userCount >= maxUsers) { + throw new Error('[[error:cant-add-more-users-to-chat-room]]'); + } + const uidsExist = await user.exists(data.uids); + if (!uidsExist.every(Boolean)) { + throw new Error('[[error:no-user]]'); + } + await Promise.all(data.uids.map(uid => messaging.canMessageUser(caller.uid, uid))); + await messaging.addUsersToRoom(caller.uid, data.uids, data.roomId); + delete data.uids; + return chatsAPI.users(caller, data); +}; +chatsAPI.kick = async (caller, data) => { + if (!data || !data.roomId) { + throw new Error('[[error:invalid-data]]'); + } + const uidsExist = await user.exists(data.uids); + if (!uidsExist.every(Boolean)) { + throw new Error('[[error:no-user]]'); + } + if (data.uids.length === 1 && parseInt(data.uids[0], 10) === caller.uid) { + await messaging.leaveRoom([caller.uid], data.roomId); + await socketHelpers.removeSocketsFromRoomByUids([caller.uid], data.roomId); + return []; + } + await messaging.removeUsersFromRoom(caller.uid, data.uids, data.roomId); + await socketHelpers.removeSocketsFromRoomByUids(data.uids, data.roomId); + delete data.uids; + return chatsAPI.users(caller, data); +}; +chatsAPI.toggleOwner = async (caller, { + roomId, + uid, + state +}) => { + const [isAdmin, inRoom, isRoomOwner] = await Promise.all([user.isAdministrator(caller.uid), messaging.isUserInRoom(caller.uid, roomId), messaging.isRoomOwner(caller.uid, roomId)]); + if (!isAdmin && (!inRoom || !isRoomOwner)) { + throw new Error('[[error:no-privileges]]'); + } + return await messaging.toggleOwner(uid, roomId, state); +}; +chatsAPI.listMessages = async (caller, { + uid = caller.uid, + roomId, + start = 0, + direction = null +} = {}) => { + if (!roomId) { + throw new Error('[[error:invalid-data]]'); + } + const count = 50; + let stop = start + count - 1; + if (direction === 1 || direction === -1) { + const msgCount = await db.getObjectField(`chat:room:${roomId}`, 'messageCount'); + start = msgCount - start; + if (direction === 1) { + start -= count + 1; + } + stop = start + count - 1; + start = Math.max(0, start); + if (stop <= -1) { + return { + messages: [] + }; + } + stop = Math.max(0, stop); + } + const messages = await messaging.getMessages({ + callerUid: caller.uid, + uid, + roomId, + start, + count: stop - start + 1 + }); + return { + messages + }; +}; +chatsAPI.getPinnedMessages = async (caller, { + start, + roomId +}) => { + start = parseInt(start, 10) || 0; + const isInRoom = await messaging.isUserInRoom(caller.uid, roomId); + if (!isInRoom) { + throw new Error('[[error:no-privileges]]'); + } + const messages = await messaging.getPinnedMessages(roomId, caller.uid, start, start + 49); + return { + messages + }; +}; +chatsAPI.getMessage = async (caller, { + mid, + roomId +} = {}) => { + if (!mid || !roomId) { + throw new Error('[[error:invalid-data]]'); + } + const messages = await messaging.getMessagesData([mid], caller.uid, roomId, false); + return messages.pop(); +}; +chatsAPI.getRawMessage = async (caller, { + mid, + roomId +} = {}) => { + if (!mid || !roomId) { + throw new Error('[[error:invalid-data]]'); + } + const [isAdmin, canViewMessage, inRoom] = await Promise.all([user.isAdministrator(caller.uid), messaging.canViewMessage(mid, roomId, caller.uid), messaging.isUserInRoom(caller.uid, roomId)]); + if (!isAdmin && (!inRoom || !canViewMessage)) { + throw new Error('[[error:not-allowed]]'); + } + const content = await messaging.getMessageField(mid, 'content'); + return { + content + }; +}; +chatsAPI.getIpAddress = async (caller, { + mid +}) => { + const allowed = await privileges.global.can('view:users:info', caller.uid); + if (!allowed) { + throw new Error('[[error:no-privileges]]'); + } + const ip = await messaging.getMessageField(mid, 'ip'); + return { + ip + }; +}; +chatsAPI.editMessage = async (caller, { + mid, + roomId, + message +}) => { + await messaging.canEdit(mid, caller.uid); + await messaging.editMessage(caller.uid, mid, roomId, message); +}; +chatsAPI.deleteMessage = async (caller, { + mid +}) => { + await messaging.canDelete(mid, caller.uid); + await messaging.deleteMessage(mid, caller.uid); +}; +chatsAPI.restoreMessage = async (caller, { + mid +}) => { + await messaging.canDelete(mid, caller.uid); + await messaging.restoreMessage(mid, caller.uid); +}; +chatsAPI.pinMessage = async (caller, { + roomId, + mid +}) => { + await messaging.canPin(roomId, caller.uid); + await messaging.pinMessage(mid, roomId); +}; +chatsAPI.unpinMessage = async (caller, { + roomId, + mid +}) => { + await messaging.canPin(roomId, caller.uid); + await messaging.unpinMessage(mid, roomId); +}; \ No newline at end of file diff --git a/lib/api/files.js b/lib/api/files.js new file mode 100644 index 0000000000..80441bfd33 --- /dev/null +++ b/lib/api/files.js @@ -0,0 +1,10 @@ +'use strict'; + +const fs = require('fs').promises; +const filesApi = module.exports; +filesApi.delete = async (_, { + path +}) => await fs.unlink(path); +filesApi.createFolder = async (_, { + path +}) => await fs.mkdir(path); \ No newline at end of file diff --git a/lib/api/flags.js b/lib/api/flags.js new file mode 100644 index 0000000000..bc15205cc0 --- /dev/null +++ b/lib/api/flags.js @@ -0,0 +1,103 @@ +'use strict'; + +const user = require('../user'); +const flags = require('../flags'); +const flagsApi = module.exports; +flagsApi.create = async (caller, data) => { + const required = ['type', 'id', 'reason']; + if (!required.every(prop => !!data[prop])) { + throw new Error('[[error:invalid-data]]'); + } + const { + type, + id, + reason + } = data; + await flags.validate({ + uid: caller.uid, + type: type, + id: id + }); + const flagObj = await flags.create(type, id, caller.uid, reason); + flags.notify(flagObj, caller.uid); + return flagObj; +}; +flagsApi.get = async (caller, { + flagId +}) => { + const isPrivileged = await user.isPrivileged(caller.uid); + if (!isPrivileged) { + throw new Error('[[error:no-privileges]]'); + } + return await flags.get(flagId); +}; +flagsApi.update = async (caller, data) => { + const allowed = await user.isPrivileged(caller.uid); + if (!allowed) { + throw new Error('[[error:no-privileges]]'); + } + const { + flagId + } = data; + delete data.flagId; + await flags.update(flagId, caller.uid, data); + return await flags.getHistory(flagId); +}; +flagsApi.delete = async (_, { + flagId +}) => await flags.purge([flagId]); +flagsApi.rescind = async ({ + uid +}, { + flagId +}) => { + const { + type, + targetId + } = await flags.get(flagId); + const exists = await flags.exists(type, targetId, uid); + if (!exists) { + throw new Error('[[error:no-flag]]'); + } + await flags.rescindReport(type, targetId, uid); +}; +flagsApi.appendNote = async (caller, data) => { + const allowed = await user.isPrivileged(caller.uid); + if (!allowed) { + throw new Error('[[error:no-privileges]]'); + } + if (data.datetime && data.flagId) { + try { + const note = await flags.getNote(data.flagId, data.datetime); + if (note.uid !== caller.uid) { + throw new Error('[[error:no-privileges]]'); + } + } catch (e) { + if (e.message !== '[[error:invalid-data]]') { + throw e; + } + } + } + await flags.appendNote(data.flagId, caller.uid, data.note, data.datetime); + const [notes, history] = await Promise.all([flags.getNotes(data.flagId), flags.getHistory(data.flagId)]); + return { + notes: notes, + history: history + }; +}; +flagsApi.deleteNote = async (caller, data) => { + const note = await flags.getNote(data.flagId, data.datetime); + if (note.uid !== caller.uid) { + throw new Error('[[error:no-privileges]]'); + } + await flags.deleteNote(data.flagId, data.datetime); + await flags.appendHistory(data.flagId, caller.uid, { + notes: '[[flags:note-deleted]]', + datetime: Date.now() + }); + const [notes, history] = await Promise.all([flags.getNotes(data.flagId), flags.getHistory(data.flagId)]); + return { + notes: notes, + history: history + }; +}; \ No newline at end of file diff --git a/lib/api/groups.js b/lib/api/groups.js new file mode 100644 index 0000000000..f71e00bb64 --- /dev/null +++ b/lib/api/groups.js @@ -0,0 +1,325 @@ +'use strict'; + +const validator = require('validator'); +const privileges = require('../privileges'); +const events = require('../events'); +const groups = require('../groups'); +const user = require('../user'); +const meta = require('../meta'); +const notifications = require('../notifications'); +const slugify = require('../slugify'); +const groupsAPI = module.exports; +groupsAPI.list = async (caller, data) => { + const groupsPerPage = 10; + const start = parseInt(data.after || 0, 10); + const stop = start + groupsPerPage - 1; + const groupData = await groups.getGroupsBySort(data.sort, start, stop); + return { + groups: groupData, + nextStart: stop + 1 + }; +}; +groupsAPI.create = async function (caller, data) { + if (!caller.uid) { + throw new Error('[[error:no-privileges]]'); + } else if (!data) { + throw new Error('[[error:invalid-data]]'); + } else if (typeof data.name !== 'string' || groups.isPrivilegeGroup(data.name)) { + throw new Error('[[error:invalid-group-name]]'); + } + const canCreate = await privileges.global.can('group:create', caller.uid); + if (!canCreate) { + throw new Error('[[error:no-privileges]]'); + } + data.ownerUid = caller.uid; + data.system = false; + const groupData = await groups.create(data); + logGroupEvent(caller, 'group-create', { + groupName: data.name + }); + return groupData; +}; +groupsAPI.update = async function (caller, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + const groupName = await groups.getGroupNameByGroupSlug(data.slug); + await isOwner(caller, groupName); + delete data.slug; + await groups.update(groupName, data); + return await groups.getGroupData(data.name || groupName); +}; +groupsAPI.delete = async function (caller, data) { + const groupName = await groups.getGroupNameByGroupSlug(data.slug); + await isOwner(caller, groupName); + if (groups.systemGroups.includes(groupName) || groups.ephemeralGroups.includes(groupName)) { + throw new Error('[[error:not-allowed]]'); + } + await groups.destroy(groupName); + logGroupEvent(caller, 'group-delete', { + groupName: groupName + }); +}; +groupsAPI.listMembers = async (caller, data) => { + const groupName = await groups.getGroupNameByGroupSlug(data.slug); + await canSearchMembers(caller.uid, groupName); + if (!(await privileges.global.can('search:users', caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } + const { + query + } = data; + const after = parseInt(data.after || 0, 10); + let response; + if (query && query.length) { + response = await groups.searchMembers({ + uid: caller.uid, + query, + groupName + }); + response.nextStart = null; + } else { + response = { + users: await groups.getOwnersAndMembers(groupName, caller.uid, after, after + 19), + nextStart: after + 20, + matchCount: null, + timing: null + }; + } + return response; +}; +async function canSearchMembers(uid, groupName) { + const [isHidden, isMember, hasAdminPrivilege, isGlobalMod, viewGroups] = await Promise.all([groups.isHidden(groupName), groups.isMember(uid, groupName), privileges.admin.can('admin:groups', uid), user.isGlobalModerator(uid), privileges.global.can('view:groups', uid)]); + if (!viewGroups || isHidden && !isMember && !hasAdminPrivilege && !isGlobalMod) { + throw new Error('[[error:no-privileges]]'); + } +} +groupsAPI.join = async function (caller, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + if (caller.uid <= 0 || !data.uid) { + throw new Error('[[error:invalid-uid]]'); + } + const groupName = await groups.getGroupNameByGroupSlug(data.slug); + if (!groupName) { + throw new Error('[[error:no-group]]'); + } + const isCallerAdmin = await privileges.admin.can('admin:groups', caller.uid); + if (!isCallerAdmin && (groups.systemGroups.includes(groupName) || groups.isPrivilegeGroup(groupName))) { + throw new Error('[[error:not-allowed]]'); + } + const [groupData, userExists] = await Promise.all([groups.getGroupData(groupName), user.exists(data.uid)]); + if (!userExists) { + throw new Error('[[error:invalid-uid]]'); + } + const isSelf = parseInt(caller.uid, 10) === parseInt(data.uid, 10); + if (!meta.config.allowPrivateGroups && isSelf) { + await groups.join(groupName, data.uid); + logGroupEvent(caller, 'group-join', { + groupName: groupName, + targetUid: data.uid + }); + return; + } + if (!isCallerAdmin && isSelf && groupData.private && groupData.disableJoinRequests) { + throw new Error('[[error:group-join-disabled]]'); + } + if (!groupData.private && isSelf || isCallerAdmin) { + await groups.join(groupName, data.uid); + logGroupEvent(caller, `group-${isSelf ? 'join' : 'add-member'}`, { + groupName: groupName, + targetUid: data.uid + }); + } else if (isSelf) { + await groups.requestMembership(groupName, caller.uid); + logGroupEvent(caller, 'group-request-membership', { + groupName: groupName, + targetUid: data.uid + }); + } else { + throw new Error('[[error:not-allowed]]'); + } +}; +groupsAPI.leave = async function (caller, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + if (caller.uid <= 0) { + throw new Error('[[error:invalid-uid]]'); + } + const isSelf = parseInt(caller.uid, 10) === parseInt(data.uid, 10); + const groupName = await groups.getGroupNameByGroupSlug(data.slug); + if (!groupName) { + throw new Error('[[error:no-group]]'); + } + if (typeof groupName !== 'string') { + throw new Error('[[error:invalid-group-name]]'); + } + if (groupName === 'administrators' && isSelf) { + throw new Error('[[error:cant-remove-self-as-admin]]'); + } + const [groupData, isCallerOwner, userExists, isMember] = await Promise.all([groups.getGroupData(groupName), isOwner(caller, groupName, false), user.exists(data.uid), groups.isMember(data.uid, groupName)]); + if (!isMember) { + throw new Error('[[error:group-not-member]]'); + } + if (!userExists) { + throw new Error('[[error:invalid-uid]]'); + } + if (groupData.disableLeave && isSelf) { + throw new Error('[[error:group-leave-disabled]]'); + } + if (isSelf || isCallerOwner) { + await groups.leave(groupName, data.uid); + } else { + throw new Error('[[error:no-privileges]]'); + } + const { + displayname + } = await user.getUserFields(data.uid, ['username']); + const notification = await notifications.create({ + type: 'group-leave', + bodyShort: `[[groups:membership.leave.notification-title, ${displayname}, ${groupName}]]`, + nid: `group:${validator.escape(groupName)}:uid:${data.uid}:group-leave`, + path: `/groups/${slugify(groupName)}`, + from: data.uid + }); + const uids = await groups.getOwners(groupName); + await notifications.push(notification, uids); + logGroupEvent(caller, `group-${isSelf ? 'leave' : 'kick'}`, { + groupName: groupName, + targetUid: data.uid + }); +}; +groupsAPI.grant = async (caller, data) => { + const groupName = await groups.getGroupNameByGroupSlug(data.slug); + await isOwner(caller, groupName); + await groups.ownership.grant(data.uid, groupName); + logGroupEvent(caller, 'group-owner-grant', { + groupName: groupName, + targetUid: data.uid + }); +}; +groupsAPI.rescind = async (caller, data) => { + const groupName = await groups.getGroupNameByGroupSlug(data.slug); + await isOwner(caller, groupName); + await groups.ownership.rescind(data.uid, groupName); + logGroupEvent(caller, 'group-owner-rescind', { + groupName, + targetUid: data.uid + }); +}; +groupsAPI.getPending = async (caller, { + slug +}) => { + const groupName = await groups.getGroupNameByGroupSlug(slug); + await isOwner(caller, groupName); + return await groups.getPending(groupName); +}; +groupsAPI.accept = async (caller, { + slug, + uid +}) => { + const groupName = await groups.getGroupNameByGroupSlug(slug); + await isOwner(caller, groupName); + const isPending = await groups.isPending(uid, groupName); + if (!isPending) { + throw new Error('[[error:group-user-not-pending]]'); + } + await groups.acceptMembership(groupName, uid); + logGroupEvent(caller, 'group-accept-membership', { + groupName, + targetUid: uid + }); +}; +groupsAPI.reject = async (caller, { + slug, + uid +}) => { + const groupName = await groups.getGroupNameByGroupSlug(slug); + await isOwner(caller, groupName); + const isPending = await groups.isPending(uid, groupName); + if (!isPending) { + throw new Error('[[error:group-user-not-pending]]'); + } + await groups.rejectMembership(groupName, uid); + logGroupEvent(caller, 'group-reject-membership', { + groupName, + targetUid: uid + }); +}; +groupsAPI.getInvites = async (caller, { + slug +}) => { + const groupName = await groups.getGroupNameByGroupSlug(slug); + await isOwner(caller, groupName); + return await groups.getInvites(groupName); +}; +groupsAPI.issueInvite = async (caller, { + slug, + uid +}) => { + const groupName = await groups.getGroupNameByGroupSlug(slug); + await isOwner(caller, groupName); + await groups.invite(groupName, uid); + logGroupEvent(caller, 'group-invite', { + groupName, + targetUid: uid + }); +}; +groupsAPI.acceptInvite = async (caller, { + slug, + uid +}) => { + const groupName = await groups.getGroupNameByGroupSlug(slug); + const invited = await groups.isInvited(uid, groupName); + if (caller.uid !== parseInt(uid, 10)) { + throw new Error('[[error:not-allowed]]'); + } + if (!invited) { + throw new Error('[[error:not-invited]]'); + } + await groups.acceptMembership(groupName, uid); + logGroupEvent(caller, 'group-invite-accept', { + groupName + }); +}; +groupsAPI.rejectInvite = async (caller, { + slug, + uid +}) => { + const groupName = await groups.getGroupNameByGroupSlug(slug); + const owner = await isOwner(caller, groupName, false); + const invited = await groups.isInvited(uid, groupName); + if (!owner && caller.uid !== parseInt(uid, 10)) { + throw new Error('[[error:not-allowed]]'); + } + if (!invited) { + throw new Error('[[error:not-invited]]'); + } + await groups.rejectMembership(groupName, uid); + if (!owner) { + logGroupEvent(caller, 'group-invite-reject', { + groupName + }); + } +}; +async function isOwner(caller, groupName, throwOnFalse = true) { + if (typeof groupName !== 'string') { + throw new Error('[[error:invalid-group-name]]'); + } + const [hasAdminPrivilege, isGlobalModerator, isOwner, group] = await Promise.all([privileges.admin.can('admin:groups', caller.uid), user.isGlobalModerator(caller.uid), groups.ownership.isOwner(caller.uid, groupName), groups.getGroupData(groupName)]); + const check = isOwner || hasAdminPrivilege || isGlobalModerator && !group.system; + if (!check && throwOnFalse) { + throw new Error('[[error:no-privileges]]'); + } + return check; +} +function logGroupEvent(caller, event, additional) { + events.log({ + type: event, + uid: caller.uid, + ip: caller.ip, + ...additional + }); +} \ No newline at end of file diff --git a/lib/api/helpers.js b/lib/api/helpers.js new file mode 100644 index 0000000000..b9f087067f --- /dev/null +++ b/lib/api/helpers.js @@ -0,0 +1,121 @@ +'use strict'; + +const url = require('url'); +const user = require('../user'); +const topics = require('../topics'); +const posts = require('../posts'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const socketHelpers = require('../socket.io/helpers'); +const websockets = require('../socket.io'); +const events = require('../events'); +exports.setDefaultPostData = function (reqOrSocket, data) { + data.uid = reqOrSocket.uid; + data.req = exports.buildReqObject(reqOrSocket, { + ...data + }); + data.timestamp = Date.now(); + data.fromQueue = false; +}; +exports.buildReqObject = (req, payload) => { + req = req || {}; + const headers = req.headers || req.request && req.request.headers || {}; + const session = req.session || req.request && req.request.session || {}; + const encrypted = req.connection ? !!req.connection.encrypted : false; + let { + host + } = headers; + const referer = headers.referer || ''; + if (!host) { + host = url.parse(referer).host || ''; + } + return { + uid: req.uid, + params: req.params, + method: req.method, + body: payload || req.body, + session: session, + ip: req.ip, + host: host, + protocol: encrypted ? 'https' : 'http', + secure: encrypted, + url: referer, + path: referer.slice(referer.indexOf(host) + host.length), + baseUrl: req.baseUrl, + originalUrl: req.originalUrl, + headers: headers + }; +}; +exports.doTopicAction = async function (action, event, caller, { + tids +}) { + if (!Array.isArray(tids)) { + throw new Error('[[error:invalid-tid]]'); + } + const exists = await topics.exists(tids); + if (!exists.every(Boolean)) { + throw new Error('[[error:no-topic]]'); + } + if (typeof topics.tools[action] !== 'function') { + return; + } + const uids = await user.getUidsFromSet('users:online', 0, -1); + await Promise.all(tids.map(async tid => { + const title = await topics.getTopicField(tid, 'title'); + const data = await topics.tools[action](tid, caller.uid); + const notifyUids = await privileges.categories.filterUids('topics:read', data.cid, uids); + socketHelpers.emitToUids(event, data, notifyUids); + await logTopicAction(action, caller, tid, title); + })); +}; +async function logTopicAction(action, req, tid, title) { + const actionsToLog = ['delete', 'restore', 'purge']; + if (!actionsToLog.includes(action)) { + return; + } + await events.log({ + type: `topic-${action}`, + uid: req.uid, + ip: req.ip, + tid: tid, + title: String(title) + }); +} +exports.postCommand = async function (caller, command, eventName, notification, data) { + if (!caller.uid) { + throw new Error('[[error:not-logged-in]]'); + } + if (!data || !data.pid) { + throw new Error('[[error:invalid-data]]'); + } + if (!data.room_id) { + throw new Error(`[[error:invalid-room-id, ${data.room_id}]]`); + } + const [exists, deleted] = await Promise.all([posts.exists(data.pid), posts.getPostField(data.pid, 'deleted')]); + if (!exists) { + throw new Error('[[error:invalid-pid]]'); + } + if (deleted) { + throw new Error('[[error:post-deleted]]'); + } + const filteredData = await plugins.hooks.fire(`filter:post.${command}`, { + data: data, + uid: caller.uid + }); + return await executeCommand(caller, command, eventName, notification, filteredData.data); +}; +async function executeCommand(caller, command, eventName, notification, data) { + const result = await posts[command](data.pid, caller.uid); + if (result && eventName) { + websockets.in(`uid_${caller.uid}`).emit(`posts.${command}`, result); + websockets.in(data.room_id).emit(`event:${eventName}`, result); + } + if (result && command === 'upvote') { + socketHelpers.upvote(result, notification); + } else if (result && notification) { + socketHelpers.sendNotificationToPostOwner(data.pid, caller.uid, command, notification); + } else if (result && command === 'unvote') { + socketHelpers.rescindUpvoteNotification(data.pid, caller.uid); + } + return result; +} \ No newline at end of file diff --git a/lib/api/index.js b/lib/api/index.js new file mode 100644 index 0000000000..4bcaffe719 --- /dev/null +++ b/lib/api/index.js @@ -0,0 +1,16 @@ +'use strict'; + +module.exports = { + admin: require('./admin'), + users: require('./users'), + groups: require('./groups'), + topics: require('./topics'), + tags: require('./tags'), + posts: require('./posts'), + chats: require('./chats'), + categories: require('./categories'), + search: require('./search'), + flags: require('./flags'), + files: require('./files'), + utils: require('./utils') +}; \ No newline at end of file diff --git a/lib/api/posts.js b/lib/api/posts.js new file mode 100644 index 0000000000..9246bef928 --- /dev/null +++ b/lib/api/posts.js @@ -0,0 +1,410 @@ +'use strict'; + +const validator = require('validator'); +const _ = require('lodash'); +const db = require('../database'); +const utils = require('../utils'); +const user = require('../user'); +const posts = require('../posts'); +const postsCache = require('../posts/cache'); +const topics = require('../topics'); +const groups = require('../groups'); +const plugins = require('../plugins'); +const meta = require('../meta'); +const events = require('../events'); +const privileges = require('../privileges'); +const apiHelpers = require('./helpers'); +const websockets = require('../socket.io'); +const socketHelpers = require('../socket.io/helpers'); +const postsAPI = module.exports; +postsAPI.get = async function (caller, data) { + const [userPrivileges, post, voted] = await Promise.all([privileges.posts.get([data.pid], caller.uid), posts.getPostData(data.pid), posts.hasVoted(data.pid, caller.uid)]); + const userPrivilege = userPrivileges[0]; + if (!post || !userPrivilege.read || !userPrivilege['topics:read']) { + return null; + } + Object.assign(post, voted); + post.ip = userPrivilege.isAdminOrMod ? post.ip : undefined; + const selfPost = caller.uid && caller.uid === parseInt(post.uid, 10); + if (post.deleted && !(userPrivilege.isAdminOrMod || selfPost)) { + post.content = '[[topic:post-is-deleted]]'; + } + return post; +}; +postsAPI.getIndex = async (caller, { + pid, + sort +}) => { + const tid = await posts.getPostField(pid, 'tid'); + const topicPrivileges = await privileges.topics.get(tid, caller.uid); + if (!topicPrivileges.read || !topicPrivileges['topics:read']) { + return null; + } + return await posts.getPidIndex(pid, tid, sort); +}; +postsAPI.getSummary = async (caller, { + pid +}) => { + const tid = await posts.getPostField(pid, 'tid'); + const topicPrivileges = await privileges.topics.get(tid, caller.uid); + if (!topicPrivileges.read || !topicPrivileges['topics:read']) { + return null; + } + const postsData = await posts.getPostSummaryByPids([pid], caller.uid, { + stripTags: false + }); + posts.modifyPostByPrivilege(postsData[0], topicPrivileges); + return postsData[0]; +}; +postsAPI.getRaw = async (caller, { + pid +}) => { + const userPrivileges = await privileges.posts.get([pid], caller.uid); + const userPrivilege = userPrivileges[0]; + if (!userPrivilege['topics:read']) { + return null; + } + const postData = await posts.getPostFields(pid, ['content', 'deleted']); + const selfPost = caller.uid && caller.uid === parseInt(postData.uid, 10); + if (postData.deleted && !(userPrivilege.isAdminOrMod || selfPost)) { + return null; + } + postData.pid = pid; + const result = await plugins.hooks.fire('filter:post.getRawPost', { + uid: caller.uid, + postData: postData + }); + return result.postData.content; +}; +postsAPI.edit = async function (caller, data) { + if (!data || !data.pid || meta.config.minimumPostLength !== 0 && !data.content) { + throw new Error('[[error:invalid-data]]'); + } + if (!caller.uid) { + throw new Error('[[error:not-logged-in]]'); + } + const contentLen = utils.stripHTMLTags(data.content).trim().length; + if (data.title && data.title.length < meta.config.minimumTitleLength) { + throw new Error(`[[error:title-too-short, ${meta.config.minimumTitleLength}]]`); + } else if (data.title && data.title.length > meta.config.maximumTitleLength) { + throw new Error(`[[error:title-too-long, ${meta.config.maximumTitleLength}]]`); + } else if (meta.config.minimumPostLength !== 0 && contentLen < meta.config.minimumPostLength) { + throw new Error(`[[error:content-too-short, ${meta.config.minimumPostLength}]]`); + } else if (contentLen > meta.config.maximumPostLength) { + throw new Error(`[[error:content-too-long, ${meta.config.maximumPostLength}]]`); + } else if (!(await posts.canUserPostContentWithLinks(caller.uid, data.content))) { + throw new Error(`[[error:not-enough-reputation-to-post-links, ${meta.config['min:rep:post-links']}]]`); + } + data.uid = caller.uid; + data.req = apiHelpers.buildReqObject(caller); + data.timestamp = parseInt(data.timestamp, 10) || Date.now(); + const editResult = await posts.edit(data); + if (editResult.topic.isMainPost) { + await topics.thumbs.migrate(data.uuid, editResult.topic.tid); + } + const selfPost = parseInt(caller.uid, 10) === parseInt(editResult.post.uid, 10); + if (!selfPost && editResult.post.changed) { + await events.log({ + type: `post-edit`, + uid: caller.uid, + ip: caller.ip, + pid: editResult.post.pid, + oldContent: editResult.post.oldContent, + newContent: editResult.post.newContent + }); + } + if (editResult.topic.renamed) { + await events.log({ + type: 'topic-rename', + uid: caller.uid, + ip: caller.ip, + tid: editResult.topic.tid, + oldTitle: validator.escape(String(editResult.topic.oldTitle)), + newTitle: validator.escape(String(editResult.topic.title)) + }); + } + const postObj = await posts.getPostSummaryByPids([editResult.post.pid], caller.uid, {}); + const returnData = { + ...postObj[0], + ...editResult.post + }; + returnData.topic = { + ...postObj[0].topic, + ...editResult.post.topic + }; + if (!editResult.post.deleted) { + websockets.in(`topic_${editResult.topic.tid}`).emit('event:post_edited', editResult); + return returnData; + } + const memberData = await groups.getMembersOfGroups(['administrators', 'Global Moderators', `cid:${editResult.topic.cid}:privileges:moderate`, `cid:${editResult.topic.cid}:privileges:groups:moderate`]); + const uids = _.uniq(_.flatten(memberData).concat(String(caller.uid))); + uids.forEach(uid => websockets.in(`uid_${uid}`).emit('event:post_edited', editResult)); + return returnData; +}; +postsAPI.delete = async function (caller, data) { + await deleteOrRestore(caller, data, { + command: 'delete', + event: 'event:post_deleted', + type: 'post-delete' + }); +}; +postsAPI.restore = async function (caller, data) { + await deleteOrRestore(caller, data, { + command: 'restore', + event: 'event:post_restored', + type: 'post-restore' + }); +}; +async function deleteOrRestore(caller, data, params) { + if (!data || !data.pid) { + throw new Error('[[error:invalid-data]]'); + } + const postData = await posts.tools[params.command](caller.uid, data.pid); + const results = await isMainAndLastPost(data.pid); + if (results.isMain && results.isLast) { + await deleteOrRestoreTopicOf(params.command, data.pid, caller); + } + websockets.in(`topic_${postData.tid}`).emit(params.event, postData); + await events.log({ + type: params.type, + uid: caller.uid, + pid: data.pid, + tid: postData.tid, + ip: caller.ip + }); +} +async function deleteOrRestoreTopicOf(command, pid, caller) { + const topic = await posts.getTopicFields(pid, ['tid', 'cid', 'deleted', 'scheduled']); + if (topic.scheduled) { + return; + } + await apiHelpers.doTopicAction(command, topic.deleted ? 'event:topic_restored' : 'event:topic_deleted', caller, { + tids: [topic.tid], + cid: topic.cid + }); +} +postsAPI.purge = async function (caller, data) { + if (!data || !parseInt(data.pid, 10)) { + throw new Error('[[error:invalid-data]]'); + } + const results = await isMainAndLastPost(data.pid); + if (results.isMain && !results.isLast) { + throw new Error('[[error:cant-purge-main-post]]'); + } + const isMainAndLast = results.isMain && results.isLast; + const postData = await posts.getPostFields(data.pid, ['toPid', 'tid']); + postData.pid = data.pid; + const canPurge = await privileges.posts.canPurge(data.pid, caller.uid); + if (!canPurge) { + throw new Error('[[error:no-privileges]]'); + } + postsCache.del(data.pid); + await posts.purge(data.pid, caller.uid); + websockets.in(`topic_${postData.tid}`).emit('event:post_purged', postData); + const topicData = await topics.getTopicFields(postData.tid, ['title', 'cid']); + await events.log({ + type: 'post-purge', + pid: data.pid, + uid: caller.uid, + ip: caller.ip, + tid: postData.tid, + title: String(topicData.title) + }); + if (isMainAndLast) { + await apiHelpers.doTopicAction('purge', 'event:topic_purged', caller, { + tids: [postData.tid], + cid: topicData.cid + }); + } +}; +async function isMainAndLastPost(pid) { + const [isMain, topicData] = await Promise.all([posts.isMain(pid), posts.getTopicFields(pid, ['postcount'])]); + return { + isMain: isMain, + isLast: topicData && topicData.postcount === 1 + }; +} +postsAPI.move = async function (caller, data) { + if (!caller.uid) { + throw new Error('[[error:not-logged-in]]'); + } + if (!data || !data.pid || !data.tid) { + throw new Error('[[error:invalid-data]]'); + } + const canMove = await Promise.all([privileges.topics.isAdminOrMod(data.tid, caller.uid), privileges.posts.canMove(data.pid, caller.uid)]); + if (!canMove.every(Boolean)) { + throw new Error('[[error:no-privileges]]'); + } + await topics.movePostToTopic(caller.uid, data.pid, data.tid); + const [postDeleted, topicDeleted] = await Promise.all([posts.getPostField(data.pid, 'deleted'), topics.getTopicField(data.tid, 'deleted'), await events.log({ + type: `post-move`, + uid: caller.uid, + ip: caller.ip, + pid: data.pid, + toTid: data.tid + })]); + if (!postDeleted && !topicDeleted) { + socketHelpers.sendNotificationToPostOwner(data.pid, caller.uid, 'move', 'notifications:moved-your-post'); + } +}; +postsAPI.upvote = async function (caller, data) { + return await apiHelpers.postCommand(caller, 'upvote', 'voted', 'notifications:upvoted-your-post-in', data); +}; +postsAPI.downvote = async function (caller, data) { + return await apiHelpers.postCommand(caller, 'downvote', 'voted', '', data); +}; +postsAPI.unvote = async function (caller, data) { + return await apiHelpers.postCommand(caller, 'unvote', 'voted', '', data); +}; +postsAPI.getVoters = async function (caller, data) { + if (!data || !data.pid) { + throw new Error('[[error:invalid-data]]'); + } + const { + pid + } = data; + const cid = await posts.getCidByPid(pid); + const [canSeeUpvotes, canSeeDownvotes] = await Promise.all([canSeeVotes(caller.uid, cid, 'upvoteVisibility'), canSeeVotes(caller.uid, cid, 'downvoteVisibility')]); + if (!canSeeUpvotes && !canSeeDownvotes) { + throw new Error('[[error:no-privileges]]'); + } + const repSystemDisabled = meta.config['reputation:disabled']; + const showUpvotes = canSeeUpvotes && !repSystemDisabled; + const showDownvotes = canSeeDownvotes && !meta.config['downvote:disabled'] && !repSystemDisabled; + const [upvoteUids, downvoteUids] = await Promise.all([showUpvotes ? db.getSetMembers(`pid:${data.pid}:upvote`) : [], showDownvotes ? db.getSetMembers(`pid:${data.pid}:downvote`) : []]); + const [upvoters, downvoters] = await Promise.all([user.getUsersFields(upvoteUids, ['username', 'userslug', 'picture']), user.getUsersFields(downvoteUids, ['username', 'userslug', 'picture'])]); + return { + upvoteCount: upvoters.length, + downvoteCount: downvoters.length, + showUpvotes: showUpvotes, + showDownvotes: showDownvotes, + upvoters: upvoters, + downvoters: downvoters + }; +}; +postsAPI.getUpvoters = async function (caller, data) { + if (!data.pid) { + throw new Error('[[error:invalid-data]]'); + } + const { + pid + } = data; + const cid = await posts.getCidByPid(pid); + if (!(await canSeeVotes(caller.uid, cid, 'upvoteVisibility'))) { + throw new Error('[[error:no-privileges]]'); + } + let upvotedUids = (await posts.getUpvotedUidsByPids([pid]))[0]; + const cutoff = 6; + if (!upvotedUids.length) { + return { + otherCount: 0, + usernames: [], + cutoff + }; + } + let otherCount = 0; + if (upvotedUids.length > cutoff) { + otherCount = upvotedUids.length - (cutoff - 1); + upvotedUids = upvotedUids.slice(0, cutoff - 1); + } + const usernames = await user.getUsernamesByUids(upvotedUids); + return { + otherCount, + usernames, + cutoff + }; +}; +async function canSeeVotes(uid, cids, type) { + const isArray = Array.isArray(cids); + if (!isArray) { + cids = [cids]; + } + const uniqCids = _.uniq(cids); + const [canRead, isAdmin, isMod] = await Promise.all([privileges.categories.isUserAllowedTo('topics:read', uniqCids, uid), privileges.users.isAdministrator(uid), privileges.users.isModerator(uid, cids)]); + const cidToAllowed = _.zipObject(uniqCids, canRead); + const checks = cids.map((cid, index) => isAdmin || isMod[index] || cidToAllowed[cid] && (meta.config[type] === 'all' || meta.config[type] === 'loggedin' && parseInt(uid, 10) > 0)); + return isArray ? checks : checks[0]; +} +postsAPI.bookmark = async function (caller, data) { + return await apiHelpers.postCommand(caller, 'bookmark', 'bookmarked', '', data); +}; +postsAPI.unbookmark = async function (caller, data) { + return await apiHelpers.postCommand(caller, 'unbookmark', 'bookmarked', '', data); +}; +async function diffsPrivilegeCheck(pid, uid) { + const [deleted, privilegesData] = await Promise.all([posts.getPostField(pid, 'deleted'), privileges.posts.get([pid], uid)]); + const allowed = privilegesData[0]['posts:history'] && (deleted ? privilegesData[0]['posts:view_deleted'] : true); + if (!allowed) { + throw new Error('[[error:no-privileges]]'); + } +} +postsAPI.getDiffs = async (caller, data) => { + await diffsPrivilegeCheck(data.pid, caller.uid); + const timestamps = await posts.diffs.list(data.pid); + const post = await posts.getPostFields(data.pid, ['timestamp', 'uid']); + const diffs = await posts.diffs.get(data.pid); + const uids = diffs.map(diff => diff.uid || null); + uids.push(post.uid); + let usernames = await user.getUsersFields(uids, ['username']); + usernames = usernames.map(userObj => userObj.uid ? userObj.username : null); + const cid = await posts.getCidByPid(data.pid); + const [isAdmin, isModerator] = await Promise.all([user.isAdministrator(caller.uid), privileges.users.isModerator(caller.uid, cid)]); + timestamps.push(String(post.timestamp)); + return { + timestamps: timestamps, + revisions: timestamps.map((timestamp, idx) => ({ + timestamp: timestamp, + username: usernames[idx] + })), + deletable: isAdmin || isModerator, + editable: isAdmin || isModerator || parseInt(caller.uid, 10) === parseInt(post.uid, 10) + }; +}; +postsAPI.loadDiff = async (caller, data) => { + await diffsPrivilegeCheck(data.pid, caller.uid); + return await posts.diffs.load(data.pid, data.since, caller.uid); +}; +postsAPI.restoreDiff = async (caller, data) => { + const cid = await posts.getCidByPid(data.pid); + const canEdit = await privileges.categories.can('posts:edit', cid, caller.uid); + if (!canEdit) { + throw new Error('[[error:no-privileges]]'); + } + const edit = await posts.diffs.restore(data.pid, data.since, caller.uid, apiHelpers.buildReqObject(caller)); + websockets.in(`topic_${edit.topic.tid}`).emit('event:post_edited', edit); +}; +postsAPI.deleteDiff = async (caller, { + pid, + timestamp +}) => { + const cid = await posts.getCidByPid(pid); + const [isAdmin, isModerator] = await Promise.all([privileges.users.isAdministrator(caller.uid), privileges.users.isModerator(caller.uid, cid)]); + if (!(isAdmin || isModerator)) { + throw new Error('[[error:no-privileges]]'); + } + await posts.diffs.delete(pid, timestamp, caller.uid); +}; +postsAPI.getReplies = async (caller, { + pid +}) => { + if (!utils.isNumber(pid)) { + throw new Error('[[error:invalid-data]]'); + } + const { + uid + } = caller; + const canRead = await privileges.posts.can('topics:read', pid, caller.uid); + if (!canRead) { + return null; + } + const { + topicPostSort + } = await user.getSettings(uid); + const pids = await posts.getPidsFromSet(`pid:${pid}:replies`, 0, -1, topicPostSort === 'newest_to_oldest'); + let [postData, postPrivileges] = await Promise.all([posts.getPostsByPids(pids, uid), privileges.posts.get(pids, uid)]); + postData = await topics.addPostData(postData, uid); + postData.forEach((postData, index) => posts.modifyPostByPrivilege(postData, postPrivileges[index])); + postData = postData.filter((postData, index) => postData && postPrivileges[index].read); + postData = await user.blocks.filter(uid, postData); + return postData; +}; \ No newline at end of file diff --git a/lib/api/search.js b/lib/api/search.js new file mode 100644 index 0000000000..662669f3b4 --- /dev/null +++ b/lib/api/search.js @@ -0,0 +1,170 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const user = require('../user'); +const categories = require('../categories'); +const messaging = require('../messaging'); +const privileges = require('../privileges'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const controllersHelpers = require('../controllers/helpers'); +const searchApi = module.exports; +searchApi.categories = async (caller, data) => { + let cids = []; + let matchedCids = []; + const privilege = data.privilege || 'topics:read'; + data.states = (data.states || ['watching', 'tracking', 'notwatching', 'ignoring']).map(state => categories.watchStates[state]); + data.parentCid = parseInt(data.parentCid || 0, 10); + if (data.search) { + ({ + cids, + matchedCids + } = await findMatchedCids(caller.uid, data)); + } else { + cids = await loadCids(caller.uid, data.parentCid); + } + const visibleCategories = await controllersHelpers.getVisibleCategories({ + cids, + uid: caller.uid, + states: data.states, + privilege, + showLinks: data.showLinks, + parentCid: data.parentCid + }); + if (Array.isArray(data.selectedCids)) { + data.selectedCids = data.selectedCids.map(cid => parseInt(cid, 10)); + } + let categoriesData = categories.buildForSelectCategories(visibleCategories, ['disabledClass'], data.parentCid); + categoriesData = categoriesData.slice(0, 200); + categoriesData.forEach(category => { + category.selected = data.selectedCids ? data.selectedCids.includes(category.cid) : false; + if (matchedCids.includes(category.cid)) { + category.match = true; + } + }); + const result = await plugins.hooks.fire('filter:categories.categorySearch', { + categories: categoriesData, + ...data, + uid: caller.uid + }); + return { + categories: result.categories + }; +}; +async function findMatchedCids(uid, data) { + const result = await categories.search({ + uid: uid, + query: data.search, + qs: data.query, + paginate: false + }); + let matchedCids = result.categories.map(c => c.cid); + const filterByWatchState = !Object.values(categories.watchStates).every(state => data.states.includes(state)); + if (filterByWatchState) { + const states = await categories.getWatchState(matchedCids, uid); + matchedCids = matchedCids.filter((cid, index) => data.states.includes(states[index])); + } + const rootCids = _.uniq(_.flatten(await Promise.all(matchedCids.map(categories.getParentCids)))); + const allChildCids = _.uniq(_.flatten(await Promise.all(matchedCids.map(categories.getChildrenCids)))); + return { + cids: _.uniq(rootCids.concat(allChildCids).concat(matchedCids)), + matchedCids: matchedCids + }; +} +async function loadCids(uid, parentCid) { + let resultCids = []; + async function getCidsRecursive(cids) { + const categoryData = await categories.getCategoriesFields(cids, ['subCategoriesPerPage']); + const cidToData = _.zipObject(cids, categoryData); + await Promise.all(cids.map(async cid => { + const allChildCids = await categories.getAllCidsFromSet(`cid:${cid}:children`); + if (allChildCids.length) { + const childCids = await privileges.categories.filterCids('find', allChildCids, uid); + resultCids.push(...childCids.slice(0, cidToData[cid].subCategoriesPerPage)); + await getCidsRecursive(childCids); + } + })); + } + const allRootCids = await categories.getAllCidsFromSet(`cid:${parentCid}:children`); + const rootCids = await privileges.categories.filterCids('find', allRootCids, uid); + const pageCids = rootCids.slice(0, meta.config.categoriesPerPage); + resultCids = pageCids; + await getCidsRecursive(pageCids); + return resultCids; +} +searchApi.roomUsers = async (caller, { + query, + roomId +}) => { + const [isAdmin, inRoom, isRoomOwner] = await Promise.all([user.isAdministrator(caller.uid), messaging.isUserInRoom(caller.uid, roomId), messaging.isRoomOwner(caller.uid, roomId)]); + if (!isAdmin && !inRoom) { + throw new Error('[[error:no-privileges]]'); + } + const results = await user.search({ + query, + paginate: false, + hardCap: -1, + uid: caller.uid + }); + const { + users + } = results; + const foundUids = users.map(user => user && user.uid); + const isUidInRoom = _.zipObject(foundUids, await messaging.isUsersInRoom(foundUids, roomId)); + const roomUsers = users.filter(user => isUidInRoom[user.uid]); + const isOwners = await messaging.isRoomOwner(roomUsers.map(u => u.uid), roomId); + roomUsers.forEach((user, index) => { + if (user) { + user.isOwner = isOwners[index]; + user.canKick = isRoomOwner && parseInt(user.uid, 10) !== parseInt(caller.uid, 10); + } + }); + roomUsers.sort((a, b) => { + if (a.isOwner && !b.isOwner) { + return -1; + } else if (!a.isOwner && b.isOwner) { + return 1; + } + return 0; + }); + return { + users: roomUsers + }; +}; +searchApi.roomMessages = async (caller, { + query, + roomId, + uid +}) => { + const [roomData, inRoom] = await Promise.all([messaging.getRoomData(roomId), messaging.isUserInRoom(caller.uid, roomId)]); + if (!roomData) { + throw new Error('[[error:no-room]]'); + } + if (!inRoom) { + throw new Error('[[error:no-privileges]]'); + } + const { + ids + } = await plugins.hooks.fire('filter:messaging.searchMessages', { + content: query, + roomId: [roomId], + uid: [uid], + matchWords: 'any', + ids: [] + }); + let userjoinTimestamp = 0; + if (!roomData.public) { + userjoinTimestamp = await db.sortedSetScore(`chat:room:${roomId}:uids`, caller.uid); + } + let messageData = await messaging.getMessagesData(ids, caller.uid, roomId, false); + messageData = messageData.map(msg => { + if (msg) { + msg.newSet = true; + } + return msg; + }).filter(msg => msg && !msg.deleted && msg.timestamp > userjoinTimestamp); + return { + messages: messageData + }; +}; \ No newline at end of file diff --git a/lib/api/tags.js b/lib/api/tags.js new file mode 100644 index 0000000000..0ccfacd273 --- /dev/null +++ b/lib/api/tags.js @@ -0,0 +1,10 @@ +'use strict'; + +const topics = require('../topics'); +const tagsAPI = module.exports; +tagsAPI.follow = async function (caller, data) { + await topics.followTag(data.tag, caller.uid); +}; +tagsAPI.unfollow = async function (caller, data) { + await topics.unfollowTag(data.tag, caller.uid); +}; \ No newline at end of file diff --git a/lib/api/topics.js b/lib/api/topics.js new file mode 100644 index 0000000000..776ac7e6d2 --- /dev/null +++ b/lib/api/topics.js @@ -0,0 +1,285 @@ +'use strict'; + +const validator = require('validator'); +const user = require('../user'); +const topics = require('../topics'); +const posts = require('../posts'); +const meta = require('../meta'); +const privileges = require('../privileges'); +const apiHelpers = require('./helpers'); +const { + doTopicAction +} = apiHelpers; +const websockets = require('../socket.io'); +const socketHelpers = require('../socket.io/helpers'); +const topicsAPI = module.exports; +topicsAPI._checkThumbPrivileges = async function ({ + tid, + uid +}) { + const isUUID = validator.isUUID(tid); + if (!isUUID && (isNaN(parseInt(tid, 10)) || !(await topics.exists(tid)))) { + throw new Error('[[error:no-topic]]'); + } + if (!isUUID && !(await privileges.topics.canEdit(tid, uid))) { + throw new Error('[[error:no-privileges]]'); + } +}; +topicsAPI.get = async function (caller, data) { + const [userPrivileges, topic] = await Promise.all([privileges.topics.get(data.tid, caller.uid), topics.getTopicData(data.tid)]); + if (!topic || !userPrivileges.read || !userPrivileges['topics:read'] || !privileges.topics.canViewDeletedScheduled(topic, userPrivileges)) { + return null; + } + return topic; +}; +topicsAPI.create = async function (caller, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + const payload = { + ...data + }; + payload.tags = payload.tags || []; + apiHelpers.setDefaultPostData(caller, payload); + const isScheduling = parseInt(data.timestamp, 10) > payload.timestamp; + if (isScheduling) { + if (await privileges.categories.can('topics:schedule', data.cid, caller.uid)) { + payload.timestamp = parseInt(data.timestamp, 10); + } else { + throw new Error('[[error:no-privileges]]'); + } + } + await meta.blacklist.test(caller.ip); + const shouldQueue = await posts.shouldQueue(caller.uid, payload); + if (shouldQueue) { + return await posts.addToQueue(payload); + } + const result = await topics.post(payload); + await topics.thumbs.migrate(data.uuid, result.topicData.tid); + socketHelpers.emitToUids('event:new_post', { + posts: [result.postData] + }, [caller.uid]); + socketHelpers.emitToUids('event:new_topic', result.topicData, [caller.uid]); + socketHelpers.notifyNew(caller.uid, 'newTopic', { + posts: [result.postData], + topic: result.topicData + }); + return result.topicData; +}; +topicsAPI.reply = async function (caller, data) { + if (!data || !data.tid || meta.config.minimumPostLength !== 0 && !data.content) { + throw new Error('[[error:invalid-data]]'); + } + const payload = { + ...data + }; + apiHelpers.setDefaultPostData(caller, payload); + await meta.blacklist.test(caller.ip); + const shouldQueue = await posts.shouldQueue(caller.uid, payload); + if (shouldQueue) { + return await posts.addToQueue(payload); + } + const postData = await topics.reply(payload); + const postObj = await posts.getPostSummaryByPids([postData.pid], caller.uid, {}); + const result = { + posts: [postData], + 'reputation:disabled': meta.config['reputation:disabled'] === 1, + 'downvote:disabled': meta.config['downvote:disabled'] === 1 + }; + user.updateOnlineUsers(caller.uid); + if (caller.uid) { + socketHelpers.emitToUids('event:new_post', result, [caller.uid]); + } else if (caller.uid === 0) { + websockets.in('online_guests').emit('event:new_post', result); + } + socketHelpers.notifyNew(caller.uid, 'newPost', result); + return postObj[0]; +}; +topicsAPI.delete = async function (caller, data) { + await doTopicAction('delete', 'event:topic_deleted', caller, { + tids: data.tids + }); +}; +topicsAPI.restore = async function (caller, data) { + await doTopicAction('restore', 'event:topic_restored', caller, { + tids: data.tids + }); +}; +topicsAPI.purge = async function (caller, data) { + await doTopicAction('purge', 'event:topic_purged', caller, { + tids: data.tids + }); +}; +topicsAPI.pin = async function (caller, { + tids, + expiry +}) { + await doTopicAction('pin', 'event:topic_pinned', caller, { + tids + }); + if (expiry) { + await Promise.all(tids.map(async tid => topics.tools.setPinExpiry(tid, expiry, caller.uid))); + } +}; +topicsAPI.unpin = async function (caller, data) { + await doTopicAction('unpin', 'event:topic_unpinned', caller, { + tids: data.tids + }); +}; +topicsAPI.lock = async function (caller, data) { + await doTopicAction('lock', 'event:topic_locked', caller, { + tids: data.tids + }); +}; +topicsAPI.unlock = async function (caller, data) { + await doTopicAction('unlock', 'event:topic_unlocked', caller, { + tids: data.tids + }); +}; +topicsAPI.follow = async function (caller, data) { + await topics.follow(data.tid, caller.uid); +}; +topicsAPI.ignore = async function (caller, data) { + await topics.ignore(data.tid, caller.uid); +}; +topicsAPI.unfollow = async function (caller, data) { + await topics.unfollow(data.tid, caller.uid); +}; +topicsAPI.updateTags = async (caller, { + tid, + tags +}) => { + if (!(await privileges.topics.canEdit(tid, caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } + const cid = await topics.getTopicField(tid, 'cid'); + await topics.validateTags(tags, cid, caller.uid, tid); + await topics.updateTopicTags(tid, tags); + return await topics.getTopicTagsObjects(tid); +}; +topicsAPI.addTags = async (caller, { + tid, + tags +}) => { + if (!(await privileges.topics.canEdit(tid, caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } + const cid = await topics.getTopicField(tid, 'cid'); + await topics.validateTags(tags, cid, caller.uid, tid); + tags = await topics.filterTags(tags, cid); + await topics.addTags(tags, [tid]); + return await topics.getTopicTagsObjects(tid); +}; +topicsAPI.deleteTags = async (caller, { + tid +}) => { + if (!(await privileges.topics.canEdit(tid, caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } + await topics.deleteTopicTags(tid); +}; +topicsAPI.getThumbs = async (caller, { + tid +}) => { + if (isFinite(tid)) { + const [exists, canRead] = await Promise.all([topics.exists(tid), privileges.topics.can('topics:read', tid, caller.uid)]); + if (!exists) { + throw new Error('[[error:not-found]]'); + } + if (!canRead) { + throw new Error('[[error:not-allowed]]'); + } + } + return await topics.thumbs.get(tid); +}; +topicsAPI.migrateThumbs = async (caller, { + from, + to +}) => { + await Promise.all([topicsAPI._checkThumbPrivileges({ + tid: from, + uid: caller.uid + }), topicsAPI._checkThumbPrivileges({ + tid: to, + uid: caller.uid + })]); + await topics.thumbs.migrate(from, to); +}; +topicsAPI.deleteThumb = async (caller, { + tid, + path +}) => { + await topicsAPI._checkThumbPrivileges({ + tid: tid, + uid: caller.uid + }); + await topics.thumbs.delete(tid, path); +}; +topicsAPI.reorderThumbs = async (caller, { + tid, + path, + order +}) => { + await topicsAPI._checkThumbPrivileges({ + tid: tid, + uid: caller.uid + }); + const exists = await topics.thumbs.exists(tid, path); + if (!exists) { + throw new Error('[[error:invalid-data]]'); + } + await topics.thumbs.associate({ + id: tid, + path: path, + score: order + }); +}; +topicsAPI.getEvents = async (caller, { + tid +}) => { + if (!(await privileges.topics.can('topics:read', tid, caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } + return await topics.events.get(tid, caller.uid); +}; +topicsAPI.deleteEvent = async (caller, { + tid, + eventId +}) => { + if (!(await privileges.topics.isAdminOrMod(tid, caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } + await topics.events.purge(tid, [eventId]); +}; +topicsAPI.markRead = async (caller, { + tid +}) => { + const hasMarked = await topics.markAsRead([tid], caller.uid); + const promises = [topics.markTopicNotificationsRead([tid], caller.uid)]; + if (hasMarked) { + promises.push(topics.pushUnreadCount(caller.uid)); + } + await Promise.all(promises); +}; +topicsAPI.markUnread = async (caller, { + tid +}) => { + if (!tid || caller.uid <= 0) { + throw new Error('[[error:invalid-data]]'); + } + await topics.markUnread(tid, caller.uid); + topics.pushUnreadCount(caller.uid); +}; +topicsAPI.bump = async (caller, { + tid +}) => { + if (!tid) { + throw new Error('[[error:invalid-tid]]'); + } + const isAdminOrMod = await privileges.topics.isAdminOrMod(tid, caller.uid); + if (!isAdminOrMod) { + throw new Error('[[error:no-privileges]]'); + } + await topics.markAsUnreadForAll(tid); + topics.pushUnreadCount(caller.uid); +}; \ No newline at end of file diff --git a/lib/api/users.js b/lib/api/users.js new file mode 100644 index 0000000000..24be3c2ec4 --- /dev/null +++ b/lib/api/users.js @@ -0,0 +1,717 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs').promises; +const validator = require('validator'); +const winston = require('winston'); +const db = require('../database'); +const user = require('../user'); +const groups = require('../groups'); +const meta = require('../meta'); +const messaging = require('../messaging'); +const flags = require('../flags'); +const privileges = require('../privileges'); +const notifications = require('../notifications'); +const plugins = require('../plugins'); +const events = require('../events'); +const translator = require('../translator'); +const sockets = require('../socket.io'); +const utils = require('../utils'); +const usersAPI = module.exports; +const hasAdminPrivilege = async (uid, privilege) => { + const ok = await privileges.admin.can(`admin:${privilege}`, uid); + if (!ok) { + throw new Error('[[error:no-privileges]]'); + } +}; +usersAPI.create = async function (caller, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + await hasAdminPrivilege(caller.uid, 'users'); + const uid = await user.create(data); + return await user.getUserData(uid); +}; +usersAPI.get = async (caller, { + uid +}) => { + const canView = await privileges.global.can('view:users', caller.uid); + if (!canView) { + throw new Error('[[error:no-privileges]]'); + } + const userData = await user.getUserData(uid); + return await user.hidePrivateData(userData, caller.uid); +}; +usersAPI.update = async function (caller, data) { + if (!caller.uid) { + throw new Error('[[error:invalid-uid]]'); + } + if (!data || !data.uid) { + throw new Error('[[error:invalid-data]]'); + } + const oldUserData = await user.getUserFields(data.uid, ['email', 'username']); + if (!oldUserData || !oldUserData.username) { + throw new Error('[[error:invalid-data]]'); + } + const [isAdminOrGlobalMod, canEdit] = await Promise.all([user.isAdminOrGlobalMod(caller.uid), privileges.users.canEdit(caller.uid, data.uid)]); + if (data.hasOwnProperty('email') || data.hasOwnProperty('username')) { + await isPrivilegedOrSelfAndPasswordMatch(caller, data); + } + if (!canEdit) { + throw new Error('[[error:no-privileges]]'); + } + if (!isAdminOrGlobalMod && meta.config['username:disableEdit']) { + data.username = oldUserData.username; + } + if (!isAdminOrGlobalMod && meta.config['email:disableEdit']) { + data.email = oldUserData.email; + } + await user.updateProfile(caller.uid, data); + const userData = await user.getUserData(data.uid); + if (userData.username !== oldUserData.username) { + await events.log({ + type: 'username-change', + uid: caller.uid, + targetUid: data.uid, + ip: caller.ip, + oldUsername: oldUserData.username, + newUsername: userData.username + }); + } + return userData; +}; +usersAPI.delete = async function (caller, { + uid, + password +}) { + await processDeletion({ + uid: uid, + method: 'delete', + password, + caller + }); +}; +usersAPI.deleteContent = async function (caller, { + uid, + password +}) { + await processDeletion({ + uid, + method: 'deleteContent', + password, + caller + }); +}; +usersAPI.deleteAccount = async function (caller, { + uid, + password +}) { + await processDeletion({ + uid, + method: 'deleteAccount', + password, + caller + }); +}; +usersAPI.deleteMany = async function (caller, data) { + await hasAdminPrivilege(caller.uid, 'users'); + if (await canDeleteUids(data.uids)) { + await Promise.all(data.uids.map(uid => processDeletion({ + uid, + method: 'delete', + caller + }))); + } +}; +usersAPI.updateSettings = async function (caller, data) { + if (!caller.uid || !data || !data.settings) { + throw new Error('[[error:invalid-data]]'); + } + const canEdit = await privileges.users.canEdit(caller.uid, data.uid); + if (!canEdit) { + throw new Error('[[error:no-privileges]]'); + } + let defaults = await user.getSettings(0); + defaults = { + postsPerPage: defaults.postsPerPage, + topicsPerPage: defaults.topicsPerPage, + userLang: defaults.userLang, + acpLang: defaults.acpLang + }; + const current = await db.getObject(`user:${data.uid}:settings`); + const payload = { + ...defaults, + ...current, + ...data.settings + }; + delete payload.uid; + return await user.saveSettings(data.uid, payload); +}; +usersAPI.getStatus = async (caller, { + uid +}) => { + const status = await db.getObjectField(`user:${uid}`, 'status'); + return { + status + }; +}; +usersAPI.getPrivateRoomId = async (caller, { + uid +} = {}) => { + if (!uid) { + throw new Error('[[error:invalid-data]]'); + } + let roomId = await messaging.hasPrivateChat(caller.uid, uid); + roomId = parseInt(roomId, 10); + return { + roomId: roomId > 0 ? roomId : null + }; +}; +usersAPI.changePassword = async function (caller, data) { + await user.changePassword(caller.uid, Object.assign(data, { + ip: caller.ip + })); + await events.log({ + type: 'password-change', + uid: caller.uid, + targetUid: data.uid, + ip: caller.ip + }); +}; +usersAPI.follow = async function (caller, data) { + await user.follow(caller.uid, data.uid); + plugins.hooks.fire('action:user.follow', { + fromUid: caller.uid, + toUid: data.uid + }); + const userData = await user.getUserFields(caller.uid, ['username', 'userslug']); + const { + displayname + } = userData; + const notifObj = await notifications.create({ + type: 'follow', + bodyShort: `[[notifications:user-started-following-you, ${displayname}]]`, + nid: `follow:${data.uid}:uid:${caller.uid}`, + from: caller.uid, + path: `/uid/${data.uid}/followers`, + mergeId: 'notifications:user-started-following-you' + }); + if (!notifObj) { + return; + } + notifObj.user = userData; + await notifications.push(notifObj, [data.uid]); +}; +usersAPI.unfollow = async function (caller, data) { + await user.unfollow(caller.uid, data.uid); + plugins.hooks.fire('action:user.unfollow', { + fromUid: caller.uid, + toUid: data.uid + }); +}; +usersAPI.ban = async function (caller, data) { + if (!(await privileges.users.hasBanPrivilege(caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } else if (await user.isAdministrator(data.uid)) { + throw new Error('[[error:cant-ban-other-admins]]'); + } + const banData = await user.bans.ban(data.uid, data.until, data.reason); + await db.setObjectField(`uid:${data.uid}:ban:${banData.timestamp}`, 'fromUid', caller.uid); + if (!data.reason) { + data.reason = await translator.translate('[[user:info.banned-no-reason]]'); + } + sockets.in(`uid_${data.uid}`).emit('event:banned', { + until: data.until, + reason: validator.escape(String(data.reason || '')) + }); + await flags.resolveFlag('user', data.uid, caller.uid); + await flags.resolveUserPostFlags(data.uid, caller.uid); + await events.log({ + type: 'user-ban', + uid: caller.uid, + targetUid: data.uid, + ip: caller.ip, + reason: data.reason || undefined + }); + plugins.hooks.fire('action:user.banned', { + callerUid: caller.uid, + ip: caller.ip, + uid: data.uid, + until: data.until > 0 ? data.until : undefined, + reason: data.reason || undefined + }); + const canLoginIfBanned = await user.bans.canLoginIfBanned(data.uid); + if (!canLoginIfBanned) { + await user.auth.revokeAllSessions(data.uid); + } +}; +usersAPI.unban = async function (caller, data) { + if (!(await privileges.users.hasBanPrivilege(caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } + const unbanData = await user.bans.unban(data.uid, data.reason); + await db.setObjectField(`uid:${data.uid}:unban:${unbanData.timestamp}`, 'fromUid', caller.uid); + sockets.in(`uid_${data.uid}`).emit('event:unbanned'); + await events.log({ + type: 'user-unban', + uid: caller.uid, + targetUid: data.uid, + ip: caller.ip + }); + plugins.hooks.fire('action:user.unbanned', { + callerUid: caller.uid, + ip: caller.ip, + uid: data.uid + }); +}; +usersAPI.mute = async function (caller, data) { + if (!(await privileges.users.hasMutePrivilege(caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } else if (await user.isAdministrator(data.uid)) { + throw new Error('[[error:cant-mute-other-admins]]'); + } + const reason = data.reason || '[[user:info.muted-no-reason]]'; + await db.setObject(`user:${data.uid}`, { + mutedUntil: data.until, + mutedReason: reason + }); + const now = Date.now(); + const muteKey = `uid:${data.uid}:mute:${now}`; + const muteData = { + type: 'mute', + fromUid: caller.uid, + uid: data.uid, + timestamp: now, + expire: data.until + }; + if (data.reason) { + muteData.reason = reason; + } + await db.sortedSetAdd(`uid:${data.uid}:mutes:timestamp`, now, muteKey); + await db.setObject(muteKey, muteData); + await events.log({ + type: 'user-mute', + uid: caller.uid, + targetUid: data.uid, + ip: caller.ip, + reason: data.reason || undefined + }); + plugins.hooks.fire('action:user.muted', { + callerUid: caller.uid, + ip: caller.ip, + uid: data.uid, + until: data.until > 0 ? data.until : undefined, + reason: data.reason || undefined + }); +}; +usersAPI.unmute = async function (caller, data) { + if (!(await privileges.users.hasMutePrivilege(caller.uid))) { + throw new Error('[[error:no-privileges]]'); + } + await db.deleteObjectFields(`user:${data.uid}`, ['mutedUntil', 'mutedReason']); + const now = Date.now(); + const unmuteKey = `uid:${data.uid}:unmute:${now}`; + const unmuteData = { + type: 'unmute', + fromUid: caller.uid, + uid: data.uid, + timestamp: now + }; + if (data.reason) { + unmuteData.reason = data.reason; + } + await db.sortedSetAdd(`uid:${data.uid}:unmutes:timestamp`, now, unmuteKey); + await db.setObject(unmuteKey, unmuteData); + await events.log({ + type: 'user-unmute', + uid: caller.uid, + targetUid: data.uid, + ip: caller.ip + }); + plugins.hooks.fire('action:user.unmuted', { + callerUid: caller.uid, + ip: caller.ip, + uid: data.uid + }); +}; +usersAPI.generateToken = async (caller, { + uid, + description +}) => { + const api = require('.'); + await hasAdminPrivilege(caller.uid, 'settings'); + if (parseInt(uid, 10) !== parseInt(caller.uid, 10)) { + throw new Error('[[error:invalid-uid]]'); + } + const tokenObj = await api.utils.tokens.generate({ + uid, + description + }); + return tokenObj.token; +}; +usersAPI.deleteToken = async (caller, { + uid, + token +}) => { + const api = require('.'); + await hasAdminPrivilege(caller.uid, 'settings'); + if (parseInt(uid, 10) !== parseInt(caller.uid, 10)) { + throw new Error('[[error:invalid-uid]]'); + } + await api.utils.tokens.delete(token); + return true; +}; +usersAPI.revokeSession = async (caller, { + uid, + uuid +}) => { + if (parseInt(uid, 10) !== caller.uid && !(await user.isAdminOrGlobalMod(caller.uid))) { + throw new Error('[[error:invalid-uid]]'); + } + const sids = await db.getSortedSetRange(`uid:${uid}:sessions`, 0, -1); + let _id; + for (const sid of sids) { + const sessionObj = await db.sessionStoreGet(sid); + if (sessionObj && sessionObj.meta && sessionObj.meta.uuid === uuid) { + _id = sid; + break; + } + } + if (!_id) { + throw new Error('[[error:no-session-found]]'); + } + await user.auth.revokeSession(_id, uid); +}; +usersAPI.invite = async (caller, { + emails, + groupsToJoin, + uid +}) => { + if (!emails || !Array.isArray(groupsToJoin)) { + throw new Error('[[error:invalid-data]]'); + } + if (parseInt(caller.uid, 10) !== parseInt(uid, 10)) { + throw new Error('[[error:no-privileges]]'); + } + const canInvite = await privileges.users.hasInvitePrivilege(caller.uid); + if (!canInvite) { + throw new Error('[[error:no-privileges]]'); + } + const { + registrationType + } = meta.config; + const isAdmin = await user.isAdministrator(caller.uid); + if (registrationType === 'admin-invite-only' && !isAdmin) { + throw new Error('[[error:no-privileges]]'); + } + const inviteGroups = (await groups.getUserInviteGroups(caller.uid)).map(group => group.name); + const cannotInvite = groupsToJoin.some(group => !inviteGroups.includes(group)); + if (groupsToJoin.length > 0 && cannotInvite) { + throw new Error('[[error:no-privileges]]'); + } + const max = meta.config.maximumInvites; + const emailsArr = emails.split(',').map(email => email.trim()).filter(Boolean); + for (const email of emailsArr) { + let invites = 0; + if (max) { + invites = await user.getInvitesNumber(caller.uid); + } + if (!isAdmin && max && invites >= max) { + throw new Error(`[[error:invite-maximum-met, ${invites}, ${max}]]`); + } + await user.sendInvitationEmail(caller.uid, email, groupsToJoin); + } +}; +usersAPI.getInviteGroups = async (caller, { + uid +}) => { + if (parseInt(uid, 10) !== parseInt(caller.uid, 10)) { + throw new Error('[[error:no-privileges]]'); + } + const userInviteGroups = await groups.getUserInviteGroups(uid); + return userInviteGroups.map(group => group.displayName); +}; +usersAPI.addEmail = async (caller, { + email, + skipConfirmation, + uid +}) => { + const isSelf = parseInt(caller.uid, 10) === parseInt(uid, 10); + const canEdit = await privileges.users.canEdit(caller.uid, uid); + if (skipConfirmation && canEdit && !isSelf) { + if (!email.length) { + await user.email.remove(uid); + } else { + if (!(await user.email.available(email))) { + throw new Error('[[error:email-taken]]'); + } + await user.setUserField(uid, 'email', email); + await user.email.confirmByUid(uid, caller.uid); + } + } else { + await usersAPI.update(caller, { + uid, + email + }); + } + return await db.getSortedSetRangeByScore('email:uid', 0, 500, uid, uid); +}; +usersAPI.listEmails = async (caller, { + uid +}) => { + const [isPrivileged, { + showemail + }] = await Promise.all([user.isPrivileged(caller.uid), user.getSettings(uid)]); + const isSelf = caller.uid === parseInt(uid, 10); + if (isSelf || isPrivileged || showemail) { + return await db.getSortedSetRangeByScore('email:uid', 0, 500, uid, uid); + } + return null; +}; +usersAPI.getEmail = async (caller, { + uid, + email +}) => { + const [isPrivileged, { + showemail + }, exists] = await Promise.all([user.isPrivileged(caller.uid), user.getSettings(uid), db.isSortedSetMember('email:uid', email.toLowerCase())]); + const isSelf = caller.uid === parseInt(uid, 10); + return exists && (isSelf || isPrivileged || showemail); +}; +usersAPI.confirmEmail = async (caller, { + uid, + email, + sessionId +}) => { + const [pending, current, canManage] = await Promise.all([user.email.isValidationPending(uid, email), user.getUserField(uid, 'email'), privileges.admin.can('admin:users', caller.uid)]); + if (!canManage) { + throw new Error('[[error:no-privileges]]'); + } + if (pending) { + const code = await db.get(`confirm:byUid:${uid}`); + await user.email.confirmByCode(code, sessionId); + return true; + } else if (current && current === email) { + await user.email.confirmByUid(uid, caller.uid); + return true; + } + return false; +}; +async function isPrivilegedOrSelfAndPasswordMatch(caller, data) { + const { + uid + } = caller; + const isSelf = parseInt(uid, 10) === parseInt(data.uid, 10); + const canEdit = await privileges.users.canEdit(uid, data.uid); + if (!canEdit) { + throw new Error('[[error:no-privileges]]'); + } + const [hasPassword, passwordMatch] = await Promise.all([user.hasPassword(data.uid), data.password ? user.isPasswordCorrect(data.uid, data.password, caller.ip) : false]); + if (isSelf && hasPassword && !passwordMatch) { + throw new Error('[[error:invalid-password]]'); + } +} +async function processDeletion({ + uid, + method, + password, + caller +}) { + const isTargetAdmin = await user.isAdministrator(uid); + const isSelf = parseInt(uid, 10) === parseInt(caller.uid, 10); + const hasAdminPrivilege = await privileges.admin.can('admin:users', caller.uid); + if (isSelf && meta.config.allowAccountDelete !== 1) { + throw new Error('[[error:account-deletion-disabled]]'); + } else if (!isSelf && !hasAdminPrivilege) { + throw new Error('[[error:no-privileges]]'); + } else if (isTargetAdmin) { + throw new Error('[[error:cant-delete-admin]'); + } + if (!hasAdminPrivilege && ['delete', 'deleteContent'].includes(method)) { + throw new Error('[[error:no-privileges]]'); + } + const hasPassword = await user.hasPassword(uid); + if (isSelf && hasPassword) { + const ok = await user.isPasswordCorrect(uid, password, caller.ip); + if (!ok) { + throw new Error('[[error:invalid-password]]'); + } + } + await flags.resolveFlag('user', uid, caller.uid); + let userData; + if (method === 'deleteAccount') { + userData = await user[method](uid); + } else { + userData = await user[method](caller.uid, uid); + } + userData = userData || {}; + sockets.server.sockets.emit('event:user_status_change', { + uid: caller.uid, + status: 'offline' + }); + plugins.hooks.fire('action:user.delete', { + callerUid: caller.uid, + uid: uid, + ip: caller.ip, + user: userData + }); + await events.log({ + type: `user-${method}`, + uid: caller.uid, + targetUid: uid, + ip: caller.ip, + username: userData.username, + email: userData.email + }); +} +async function canDeleteUids(uids) { + if (!Array.isArray(uids)) { + throw new Error('[[error:invalid-data]]'); + } + const isMembers = await groups.isMembers(uids, 'administrators'); + if (isMembers.includes(true)) { + throw new Error('[[error:cant-delete-other-admins]]'); + } + return true; +} +usersAPI.search = async function (caller, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + const [allowed, isPrivileged] = await Promise.all([privileges.global.can('search:users', caller.uid), user.isPrivileged(caller.uid)]); + let filters = data.filters || []; + filters = Array.isArray(filters) ? filters : [filters]; + if (!allowed || (data.searchBy === 'ip' || data.searchBy === 'email' || filters.includes('banned') || filters.includes('flagged')) && !isPrivileged) { + throw new Error('[[error:no-privileges]]'); + } + return await user.search({ + uid: caller.uid, + query: data.query, + searchBy: data.searchBy || 'username', + page: data.page || 1, + sortBy: data.sortBy || 'lastonline', + filters: filters + }); +}; +usersAPI.changePicture = async (caller, data) => { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + const { + type, + url + } = data; + let picture = ''; + await user.checkMinReputation(caller.uid, data.uid, 'min:rep:profile-picture'); + const canEdit = await privileges.users.canEdit(caller.uid, data.uid); + if (!canEdit) { + throw new Error('[[error:no-privileges]]'); + } + if (type === 'default') { + picture = ''; + } else if (type === 'uploaded') { + picture = await user.getUserField(data.uid, 'uploadedpicture'); + } else if (type === 'external' && url) { + picture = validator.escape(url); + } else { + const returnData = await plugins.hooks.fire('filter:user.getPicture', { + uid: caller.uid, + type: type, + picture: undefined + }); + picture = returnData && returnData.picture; + } + const validBackgrounds = await user.getIconBackgrounds(); + if (!validBackgrounds.includes(data.bgColor)) { + data.bgColor = validBackgrounds[0]; + } + await user.updateProfile(caller.uid, { + uid: data.uid, + picture: picture, + 'icon:bgColor': data.bgColor + }, ['picture', 'icon:bgColor']); +}; +const exportMetadata = new Map([['posts', ['csv', 'text/csv']], ['uploads', ['zip', 'application/zip']], ['profile', ['json', 'application/json']]]); +const prepareExport = async ({ + uid, + type +}) => { + const [extension] = exportMetadata.get(type); + const filename = `${uid}_${type}.${extension}`; + try { + const stat = await fs.stat(path.join(__dirname, '../../build/export', filename)); + return stat; + } catch (e) { + return false; + } +}; +usersAPI.checkExportByType = async (caller, { + uid, + type +}) => await prepareExport({ + uid, + type +}); +usersAPI.getExportByType = async (caller, { + uid, + type +}) => { + const [extension, mime] = exportMetadata.get(type); + const filename = `${uid}_${type}.${extension}`; + const exists = await prepareExport({ + uid, + type + }); + if (exists) { + return { + filename, + mime + }; + } + return false; +}; +usersAPI.generateExport = async (caller, { + uid, + type +}) => { + const validTypes = ['profile', 'posts', 'uploads']; + if (!validTypes.includes(type)) { + throw new Error('[[error:invalid-data]]'); + } + if (!utils.isNumber(uid) || !(parseInt(uid, 10) > 0)) { + throw new Error('[[error:invalid-uid]]'); + } + const count = await db.incrObjectField('locks', `export:${uid}${type}`); + if (count > 1) { + throw new Error('[[error:already-exporting]]'); + } + const child = require('child_process').fork(`./src/user/jobs/export-${type}.js`, [], { + env: process.env + }); + child.send({ + uid + }); + child.on('error', async err => { + winston.error(err.stack); + await db.deleteObjectField('locks', `export:${uid}${type}`); + }); + child.on('exit', async () => { + await db.deleteObjectField('locks', `export:${uid}${type}`); + const { + displayname + } = await user.getUserFields(uid, ['username']); + const n = await notifications.create({ + bodyShort: `[[notifications:${type}-exported, ${displayname}]]`, + path: `/api/v3/users/${uid}/exports/${type}`, + nid: `${type}:export:${uid}`, + from: uid + }); + await notifications.push(n, [caller.uid]); + await events.log({ + type: `export:${type}`, + uid: caller.uid, + targetUid: uid, + ip: caller.ip + }); + }); +}; \ No newline at end of file diff --git a/lib/api/utils.js b/lib/api/utils.js new file mode 100644 index 0000000000..b447767d26 --- /dev/null +++ b/lib/api/utils.js @@ -0,0 +1,96 @@ +'use strict'; + +const db = require('../database'); +const user = require('../user'); +const srcUtils = require('../utils'); +const utils = module.exports; +utils.tokens = {}; +utils.tokens.list = async (start = 0, stop = -1) => { + const tokens = await db.getSortedSetRange(`tokens:createtime`, start, stop); + return await utils.tokens.get(tokens); +}; +utils.tokens.count = async () => await db.sortedSetCard('tokens:createtime'); +utils.tokens.get = async tokens => { + if (!tokens) { + throw new Error('[[error:invalid-data]]'); + } + let singular = false; + if (!Array.isArray(tokens)) { + tokens = [tokens]; + singular = true; + } + let [tokenObjs, lastSeen] = await Promise.all([db.getObjects(tokens.map(t => `token:${t}`)), utils.tokens.getLastSeen(tokens)]); + tokenObjs = tokenObjs.map((tokenObj, idx) => { + if (!tokenObj) { + return null; + } + tokenObj.token = tokens[idx]; + tokenObj.lastSeen = lastSeen[idx]; + tokenObj.lastSeenISO = lastSeen[idx] ? new Date(lastSeen[idx]).toISOString() : null; + tokenObj.timestampISO = new Date(parseInt(tokenObj.timestamp, 10)).toISOString(); + return tokenObj; + }); + return singular ? tokenObjs[0] : tokenObjs; +}; +utils.tokens.generate = async ({ + uid, + description +}) => { + if (parseInt(uid, 10) !== 0) { + const uidExists = await user.exists(uid); + if (!uidExists) { + throw new Error('[[error:no-user]]'); + } + } + const token = srcUtils.generateUUID(); + const timestamp = Date.now(); + return utils.tokens.add({ + token, + uid, + description, + timestamp + }); +}; +utils.tokens.add = async ({ + token, + uid, + description = '', + timestamp = Date.now() +}) => { + if (!token || uid === undefined) { + throw new Error('[[error:invalid-data]]'); + } + await Promise.all([db.setObject(`token:${token}`, { + uid, + description, + timestamp + }), db.sortedSetAdd(`tokens:createtime`, timestamp, token), db.sortedSetAdd(`tokens:uid`, uid, token)]); + return token; +}; +utils.tokens.update = async (token, { + uid, + description +}) => { + await Promise.all([db.setObject(`token:${token}`, { + uid, + description + }), db.sortedSetAdd(`tokens:uid`, uid, token)]); + return await utils.tokens.get(token); +}; +utils.tokens.roll = async token => { + const [createTime, uid, lastSeen] = await db.sortedSetsScore([`tokens:createtime`, `tokens:uid`, `tokens:lastSeen`], token); + const newToken = srcUtils.generateUUID(); + const updates = [db.rename(`token:${token}`, `token:${newToken}`), db.sortedSetsRemove([`tokens:createtime`, `tokens:uid`, `tokens:lastSeen`], token), db.sortedSetAdd(`tokens:createtime`, createTime, newToken), db.sortedSetAdd(`tokens:uid`, uid, newToken)]; + if (lastSeen) { + updates.push(db.sortedSetAdd(`tokens:lastSeen`, lastSeen, newToken)); + } + await Promise.all(updates); + return newToken; +}; +utils.tokens.delete = async token => { + await Promise.all([db.delete(`token:${token}`), db.sortedSetsRemove([`tokens:createtime`, `tokens:uid`, `tokens:lastSeen`], token)]); +}; +utils.tokens.log = async token => { + await db.sortedSetAdd('tokens:lastSeen', Date.now(), token); +}; +utils.tokens.getLastSeen = async tokens => await db.sortedSetScores('tokens:lastSeen', tokens); \ No newline at end of file diff --git a/lib/batch.js b/lib/batch.js new file mode 100644 index 0000000000..69ff462201 --- /dev/null +++ b/lib/batch.js @@ -0,0 +1,74 @@ +'use strict'; + +const util = require('util'); +const db = require('./database'); +const utils = require('./utils'); +const DEFAULT_BATCH_SIZE = 100; +const sleep = util.promisify(setTimeout); +exports.processSortedSet = async function (setKey, process, options) { + options = options || {}; + if (typeof process !== 'function') { + throw new Error('[[error:process-not-a-function]]'); + } + if (options.progress) { + options.progress.total = await db.sortedSetCard(setKey); + } + options.batch = options.batch || DEFAULT_BATCH_SIZE; + options.reverse = options.reverse || false; + if (db.processSortedSet && typeof options.doneIf !== 'function' && !utils.isNumber(options.alwaysStartAt)) { + return await db.processSortedSet(setKey, process, options); + } + options.doneIf = typeof options.doneIf === 'function' ? options.doneIf : function () {}; + let start = 0; + let stop = options.batch - 1; + if (process && process.constructor && process.constructor.name !== 'AsyncFunction') { + process = util.promisify(process); + } + const method = options.reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'; + const isByScore = options.min && options.min !== '-inf' || options.max && options.max !== '+inf'; + const byScore = isByScore ? 'ByScore' : ''; + const withScores = options.withScores ? 'WithScores' : ''; + let iteration = 1; + const getFn = db[`${method}${byScore}${withScores}`]; + while (true) { + const ids = await getFn(setKey, start, isByScore ? stop - start + 1 : stop, options.reverse ? options.max : options.min, options.reverse ? options.min : options.max); + if (!ids.length || options.doneIf(start, stop, ids)) { + return; + } + if (iteration > 1 && options.interval) { + await sleep(options.interval); + } + await process(ids); + iteration += 1; + start += utils.isNumber(options.alwaysStartAt) ? options.alwaysStartAt : options.batch; + stop = start + options.batch - 1; + } +}; +exports.processArray = async function (array, process, options) { + options = options || {}; + if (!Array.isArray(array) || !array.length) { + return; + } + if (typeof process !== 'function') { + throw new Error('[[error:process-not-a-function]]'); + } + const batch = options.batch || DEFAULT_BATCH_SIZE; + let start = 0; + if (process && process.constructor && process.constructor.name !== 'AsyncFunction') { + process = util.promisify(process); + } + let iteration = 1; + while (true) { + const currentBatch = array.slice(start, start + batch); + if (!currentBatch.length) { + return; + } + if (iteration > 1 && options.interval) { + await sleep(options.interval); + } + await process(currentBatch); + start += batch; + iteration += 1; + } +}; +require('./promisify')(exports); \ No newline at end of file diff --git a/lib/cache.js b/lib/cache.js new file mode 100644 index 0000000000..6d9b8d3d9f --- /dev/null +++ b/lib/cache.js @@ -0,0 +1,8 @@ +'use strict'; + +const cacheCreate = require('./cache/lru'); +module.exports = cacheCreate({ + name: 'local', + max: 40000, + ttl: 0 +}); \ No newline at end of file diff --git a/lib/cache/lru.js b/lib/cache/lru.js new file mode 100644 index 0000000000..c2550d55fb --- /dev/null +++ b/lib/cache/lru.js @@ -0,0 +1,114 @@ +'use strict'; + +module.exports = function (opts) { + const { + LRUCache + } = require('lru-cache'); + const pubsub = require('../pubsub'); + const winston = require('winston'); + const chalk = require('chalk'); + if (opts.hasOwnProperty('length') && !opts.hasOwnProperty('maxSize')) { + winston.warn(`[cache/init(${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} ${chalk.yellow('length')} was passed in without a corresponding ${chalk.yellow('maxSize')}. Both are now required as of lru-cache@7.0.0.`); + delete opts.length; + } + const deprecations = new Map([['stale', 'allowStale'], ['maxAge', 'ttl'], ['length', 'sizeCalculation']]); + deprecations.forEach((newProp, oldProp) => { + if (opts.hasOwnProperty(oldProp) && !opts.hasOwnProperty(newProp)) { + winston.warn(`[cache/init(${opts.name})] ${chalk.white.bgRed.bold('DEPRECATION')} The option ${chalk.yellow(oldProp)} has been deprecated as of lru-cache@7.0.0. Please change this to ${chalk.yellow(newProp)} instead.`); + opts[newProp] = opts[oldProp]; + delete opts[oldProp]; + } + }); + const lruCache = new LRUCache(opts); + const cache = {}; + cache.name = opts.name; + cache.hits = 0; + cache.misses = 0; + cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true; + const cacheSet = lruCache.set; + const propertyMap = new Map([['length', 'calculatedSize'], ['calculatedSize', 'calculatedSize'], ['max', 'max'], ['maxSize', 'maxSize'], ['itemCount', 'size'], ['size', 'size'], ['ttl', 'ttl']]); + propertyMap.forEach((lruProp, cacheProp) => { + Object.defineProperty(cache, cacheProp, { + get: function () { + return lruCache[lruProp]; + }, + configurable: true, + enumerable: true + }); + }); + cache.set = function (key, value, ttl) { + if (!cache.enabled) { + return; + } + const opts = {}; + if (ttl) { + opts.ttl = ttl; + } + cacheSet.apply(lruCache, [key, value, opts]); + }; + cache.get = function (key) { + if (!cache.enabled) { + return undefined; + } + const data = lruCache.get(key); + if (data === undefined) { + cache.misses += 1; + } else { + cache.hits += 1; + } + return data; + }; + cache.del = function (keys) { + if (!Array.isArray(keys)) { + keys = [keys]; + } + pubsub.publish(`${cache.name}:lruCache:del`, keys); + keys.forEach(key => lruCache.delete(key)); + }; + cache.delete = cache.del; + cache.reset = function () { + pubsub.publish(`${cache.name}:lruCache:reset`); + localReset(); + }; + cache.clear = cache.reset; + function localReset() { + lruCache.clear(); + cache.hits = 0; + cache.misses = 0; + } + pubsub.on(`${cache.name}:lruCache:reset`, () => { + localReset(); + }); + pubsub.on(`${cache.name}:lruCache:del`, keys => { + if (Array.isArray(keys)) { + keys.forEach(key => lruCache.delete(key)); + } + }); + cache.getUnCachedKeys = function (keys, cachedData) { + if (!cache.enabled) { + return keys; + } + let data; + let isCached; + const unCachedKeys = keys.filter(key => { + data = cache.get(key); + isCached = data !== undefined; + if (isCached) { + cachedData[key] = data; + } + return !isCached; + }); + const hits = keys.length - unCachedKeys.length; + const misses = keys.length - hits; + cache.hits += hits; + cache.misses += misses; + return unCachedKeys; + }; + cache.dump = function () { + return lruCache.dump(); + }; + cache.peek = function (key) { + return lruCache.peek(key); + }; + return cache; +}; \ No newline at end of file diff --git a/lib/cache/ttl.js b/lib/cache/ttl.js new file mode 100644 index 0000000000..b9da5d239e --- /dev/null +++ b/lib/cache/ttl.js @@ -0,0 +1,106 @@ +'use strict'; + +module.exports = function (opts) { + const TTLCache = require('@isaacs/ttlcache'); + const pubsub = require('../pubsub'); + const ttlCache = new TTLCache(opts); + const cache = {}; + cache.name = opts.name; + cache.hits = 0; + cache.misses = 0; + cache.enabled = opts.hasOwnProperty('enabled') ? opts.enabled : true; + const cacheSet = ttlCache.set; + const propertyMap = new Map([['max', 'max'], ['itemCount', 'size'], ['size', 'size'], ['ttl', 'ttl']]); + propertyMap.forEach((ttlProp, cacheProp) => { + Object.defineProperty(cache, cacheProp, { + get: function () { + return ttlCache[ttlProp]; + }, + configurable: true, + enumerable: true + }); + }); + cache.has = key => { + if (!cache.enabled) { + return false; + } + return ttlCache.has(key); + }; + cache.set = function (key, value, ttl) { + if (!cache.enabled) { + return; + } + const opts = {}; + if (ttl) { + opts.ttl = ttl; + } + cacheSet.apply(ttlCache, [key, value, opts]); + }; + cache.get = function (key) { + if (!cache.enabled) { + return undefined; + } + const data = ttlCache.get(key); + if (data === undefined) { + cache.misses += 1; + } else { + cache.hits += 1; + } + return data; + }; + cache.del = function (keys) { + if (!Array.isArray(keys)) { + keys = [keys]; + } + pubsub.publish(`${cache.name}:ttlCache:del`, keys); + keys.forEach(key => ttlCache.delete(key)); + }; + cache.delete = cache.del; + cache.reset = function () { + pubsub.publish(`${cache.name}:ttlCache:reset`); + localReset(); + }; + cache.clear = cache.reset; + function localReset() { + ttlCache.clear(); + cache.hits = 0; + cache.misses = 0; + } + pubsub.on(`${cache.name}:ttlCache:reset`, () => { + localReset(); + }); + pubsub.on(`${cache.name}:ttlCache:del`, keys => { + if (Array.isArray(keys)) { + keys.forEach(key => ttlCache.delete(key)); + } + }); + cache.getUnCachedKeys = function (keys, cachedData) { + if (!cache.enabled) { + return keys; + } + let data; + let isCached; + const unCachedKeys = keys.filter(key => { + data = cache.get(key); + isCached = data !== undefined; + if (isCached) { + cachedData[key] = data; + } + return !isCached; + }); + const hits = keys.length - unCachedKeys.length; + const misses = keys.length - hits; + cache.hits += hits; + cache.misses += misses; + return unCachedKeys; + }; + cache.dump = function () { + return Array.from(ttlCache.entries()); + }; + cache.peek = function (key) { + return ttlCache.get(key, { + updateAgeOnGet: false + }); + }; + return cache; +}; \ No newline at end of file diff --git a/lib/cacheCreate.js b/lib/cacheCreate.js new file mode 100644 index 0000000000..5f1c96f14e --- /dev/null +++ b/lib/cacheCreate.js @@ -0,0 +1,3 @@ +'use strict'; + +module.exports = require('./cache/lru'); \ No newline at end of file diff --git a/lib/categories/activeusers.js b/lib/categories/activeusers.js new file mode 100644 index 0000000000..154e596f48 --- /dev/null +++ b/lib/categories/activeusers.js @@ -0,0 +1,15 @@ +'use strict'; + +const _ = require('lodash'); +const posts = require('../posts'); +const db = require('../database'); +module.exports = function (Categories) { + Categories.getActiveUsers = async function (cids) { + if (!Array.isArray(cids)) { + cids = [cids]; + } + const pids = await db.getSortedSetRevRange(cids.map(cid => `cid:${cid}:pids`), 0, 24); + const postData = await posts.getPostsFields(pids, ['uid']); + return _.uniq(postData.map(post => post.uid).filter(uid => uid)); + }; +}; \ No newline at end of file diff --git a/lib/categories/create.js b/lib/categories/create.js new file mode 100644 index 0000000000..7314463e46 --- /dev/null +++ b/lib/categories/create.js @@ -0,0 +1,188 @@ +'use strict'; + +const async = require('async'); +const _ = require('lodash'); +const db = require('../database'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const utils = require('../utils'); +const slugify = require('../slugify'); +const cache = require('../cache'); +module.exports = function (Categories) { + Categories.create = async function (data) { + const parentCid = data.parentCid ? data.parentCid : 0; + const [cid, firstChild] = await Promise.all([db.incrObjectField('global', 'nextCid'), db.getSortedSetRangeWithScores(`cid:${parentCid}:children`, 0, 0)]); + data.name = String(data.name || `Category ${cid}`); + const slug = `${cid}/${slugify(data.name)}`; + const smallestOrder = firstChild.length ? firstChild[0].score - 1 : 1; + const order = data.order || smallestOrder; + const colours = Categories.assignColours(); + let category = { + cid: cid, + name: data.name, + description: data.description ? data.description : '', + descriptionParsed: data.descriptionParsed ? data.descriptionParsed : '', + icon: data.icon ? data.icon : '', + bgColor: data.bgColor || colours[0], + color: data.color || colours[1], + slug: slug, + parentCid: parentCid, + topic_count: 0, + post_count: 0, + disabled: data.disabled ? 1 : 0, + order: order, + link: data.link || '', + numRecentReplies: 1, + class: data.class ? data.class : 'col-md-3 col-6', + imageClass: 'cover', + isSection: 0, + subCategoriesPerPage: 10 + }; + if (data.backgroundImage) { + category.backgroundImage = data.backgroundImage; + } + const defaultPrivileges = ['groups:find', 'groups:read', 'groups:topics:read', 'groups:topics:create', 'groups:topics:reply', 'groups:topics:tag', 'groups:posts:edit', 'groups:posts:history', 'groups:posts:delete', 'groups:posts:upvote', 'groups:posts:downvote', 'groups:topics:delete']; + const modPrivileges = defaultPrivileges.concat(['groups:topics:schedule', 'groups:posts:view_deleted', 'groups:purge']); + const guestPrivileges = ['groups:find', 'groups:read', 'groups:topics:read']; + const result = await plugins.hooks.fire('filter:category.create', { + category: category, + data: data, + defaultPrivileges: defaultPrivileges, + modPrivileges: modPrivileges, + guestPrivileges: guestPrivileges + }); + category = result.category; + await db.setObject(`category:${category.cid}`, category); + if (!category.descriptionParsed) { + await Categories.parseDescription(category.cid, category.description); + } + await db.sortedSetAddBulk([['categories:cid', category.order, category.cid], [`cid:${parentCid}:children`, category.order, category.cid], ['categories:name', 0, `${data.name.slice(0, 200).toLowerCase()}:${category.cid}`]]); + await privileges.categories.give(result.defaultPrivileges, category.cid, 'registered-users'); + await privileges.categories.give(result.modPrivileges, category.cid, ['administrators', 'Global Moderators']); + await privileges.categories.give(result.guestPrivileges, category.cid, ['guests', 'spiders']); + cache.del('categories:cid'); + await clearParentCategoryCache(parentCid); + if (data.cloneFromCid && parseInt(data.cloneFromCid, 10)) { + category = await Categories.copySettingsFrom(data.cloneFromCid, category.cid, !data.parentCid); + } + if (data.cloneChildren) { + await duplicateCategoriesChildren(category.cid, data.cloneFromCid, data.uid); + } + plugins.hooks.fire('action:category.create', { + category: category + }); + return category; + }; + async function clearParentCategoryCache(parentCid) { + while (parseInt(parentCid, 10) >= 0) { + cache.del([`cid:${parentCid}:children`, `cid:${parentCid}:children:all`]); + if (parseInt(parentCid, 10) === 0) { + return; + } + parentCid = await Categories.getCategoryField(parentCid, 'parentCid'); + } + } + async function duplicateCategoriesChildren(parentCid, cid, uid) { + let children = await Categories.getChildren([cid], uid); + if (!children.length) { + return; + } + children = children[0]; + children.forEach(child => { + child.parentCid = parentCid; + child.cloneFromCid = child.cid; + child.cloneChildren = true; + child.name = utils.decodeHTMLEntities(child.name); + child.description = utils.decodeHTMLEntities(child.description); + child.uid = uid; + }); + await async.each(children, Categories.create); + } + Categories.assignColours = function () { + const backgrounds = ['#AB4642', '#DC9656', '#F7CA88', '#A1B56C', '#86C1B9', '#7CAFC2', '#BA8BAF', '#A16946']; + const text = ['#ffffff', '#ffffff', '#333333', '#ffffff', '#333333', '#ffffff', '#ffffff', '#ffffff']; + const index = Math.floor(Math.random() * backgrounds.length); + return [backgrounds[index], text[index]]; + }; + Categories.copySettingsFrom = async function (fromCid, toCid, copyParent) { + const [source, destination] = await Promise.all([db.getObject(`category:${fromCid}`), db.getObject(`category:${toCid}`)]); + if (!source) { + throw new Error('[[error:invalid-cid]]'); + } + const oldParent = parseInt(destination.parentCid, 10) || 0; + const newParent = parseInt(source.parentCid, 10) || 0; + if (copyParent && newParent !== parseInt(toCid, 10)) { + await db.sortedSetRemove(`cid:${oldParent}:children`, toCid); + await db.sortedSetAdd(`cid:${newParent}:children`, source.order, toCid); + cache.del([`cid:${oldParent}:children`, `cid:${oldParent}:children:all`, `cid:${newParent}:children`, `cid:${newParent}:children:all`]); + } + destination.description = source.description; + destination.descriptionParsed = source.descriptionParsed; + destination.icon = source.icon; + destination.bgColor = source.bgColor; + destination.color = source.color; + destination.link = source.link; + destination.numRecentReplies = source.numRecentReplies; + destination.class = source.class; + destination.image = source.image; + destination.imageClass = source.imageClass; + destination.minTags = source.minTags; + destination.maxTags = source.maxTags; + if (copyParent) { + destination.parentCid = source.parentCid || 0; + } + await plugins.hooks.fire('filter:categories.copySettingsFrom', { + source: source, + destination: destination, + copyParent: copyParent + }); + await db.setObject(`category:${toCid}`, destination); + await copyTagWhitelist(fromCid, toCid); + await Categories.copyPrivilegesFrom(fromCid, toCid); + return destination; + }; + async function copyTagWhitelist(fromCid, toCid) { + const data = await db.getSortedSetRangeWithScores(`cid:${fromCid}:tag:whitelist`, 0, -1); + await db.delete(`cid:${toCid}:tag:whitelist`); + await db.sortedSetAdd(`cid:${toCid}:tag:whitelist`, data.map(item => item.score), data.map(item => item.value)); + cache.del(`cid:${toCid}:tag:whitelist`); + } + Categories.copyPrivilegesFrom = async function (fromCid, toCid, group, filter) { + group = group || ''; + let privsToCopy = privileges.categories.getPrivilegesByFilter(filter); + if (group) { + privsToCopy = privsToCopy.map(priv => `groups:${priv}`); + } else { + privsToCopy = privsToCopy.concat(privsToCopy.map(priv => `groups:${priv}`)); + } + const data = await plugins.hooks.fire('filter:categories.copyPrivilegesFrom', { + privileges: privsToCopy, + fromCid: fromCid, + toCid: toCid, + group: group + }); + if (group) { + await copyPrivilegesByGroup(data.privileges, data.fromCid, data.toCid, group); + } else { + await copyPrivileges(data.privileges, data.fromCid, data.toCid); + } + }; + async function copyPrivileges(privileges, fromCid, toCid) { + const toGroups = privileges.map(privilege => `group:cid:${toCid}:privileges:${privilege}:members`); + const fromGroups = privileges.map(privilege => `group:cid:${fromCid}:privileges:${privilege}:members`); + const currentMembers = await db.getSortedSetsMembers(toGroups.concat(fromGroups)); + const copyGroups = _.uniq(_.flatten(currentMembers)); + await async.each(copyGroups, async group => { + await copyPrivilegesByGroup(privileges, fromCid, toCid, group); + }); + } + async function copyPrivilegesByGroup(privilegeList, fromCid, toCid, group) { + const fromGroups = privilegeList.map(privilege => `group:cid:${fromCid}:privileges:${privilege}:members`); + const toGroups = privilegeList.map(privilege => `group:cid:${toCid}:privileges:${privilege}:members`); + const [fromChecks, toChecks] = await Promise.all([db.isMemberOfSortedSets(fromGroups, group), db.isMemberOfSortedSets(toGroups, group)]); + const givePrivs = privilegeList.filter((priv, index) => fromChecks[index] && !toChecks[index]); + const rescindPrivs = privilegeList.filter((priv, index) => !fromChecks[index] && toChecks[index]); + await privileges.categories.give(givePrivs, toCid, group); + await privileges.categories.rescind(rescindPrivs, toCid, group); + } +}; \ No newline at end of file diff --git a/lib/categories/data.js b/lib/categories/data.js new file mode 100644 index 0000000000..ca59ece503 --- /dev/null +++ b/lib/categories/data.js @@ -0,0 +1,84 @@ +'use strict'; + +const validator = require('validator'); +const db = require('../database'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const intFields = ['cid', 'parentCid', 'disabled', 'isSection', 'order', 'topic_count', 'post_count', 'numRecentReplies', 'minTags', 'maxTags', 'postQueue', 'subCategoriesPerPage']; +module.exports = function (Categories) { + Categories.getCategoriesFields = async function (cids, fields) { + if (!Array.isArray(cids) || !cids.length) { + return []; + } + const keys = cids.map(cid => `category:${cid}`); + const categories = await db.getObjects(keys, fields); + const result = await plugins.hooks.fire('filter:category.getFields', { + cids: cids, + categories: categories, + fields: fields, + keys: keys + }); + result.categories.forEach(category => modifyCategory(category, fields)); + return result.categories; + }; + Categories.getCategoryData = async function (cid) { + const categories = await Categories.getCategoriesFields([cid], []); + return categories && categories.length ? categories[0] : null; + }; + Categories.getCategoriesData = async function (cids) { + return await Categories.getCategoriesFields(cids, []); + }; + Categories.getCategoryField = async function (cid, field) { + const category = await Categories.getCategoryFields(cid, [field]); + return category ? category[field] : null; + }; + Categories.getCategoryFields = async function (cid, fields) { + const categories = await Categories.getCategoriesFields([cid], fields); + return categories ? categories[0] : null; + }; + Categories.getAllCategoryFields = async function (fields) { + const cids = await Categories.getAllCidsFromSet('categories:cid'); + return await Categories.getCategoriesFields(cids, fields); + }; + Categories.setCategoryField = async function (cid, field, value) { + await db.setObjectField(`category:${cid}`, field, value); + }; + Categories.incrementCategoryFieldBy = async function (cid, field, value) { + await db.incrObjectFieldBy(`category:${cid}`, field, value); + }; +}; +function defaultIntField(category, fields, fieldName, defaultField) { + if (!fields.length || fields.includes(fieldName)) { + const useDefault = !category.hasOwnProperty(fieldName) || category[fieldName] === null || category[fieldName] === '' || !utils.isNumber(category[fieldName]); + category[fieldName] = useDefault ? meta.config[defaultField] : category[fieldName]; + } +} +function modifyCategory(category, fields) { + if (!category) { + return; + } + defaultIntField(category, fields, 'minTags', 'minimumTagsPerTopic'); + defaultIntField(category, fields, 'maxTags', 'maximumTagsPerTopic'); + defaultIntField(category, fields, 'postQueue', 'postQueue'); + db.parseIntFields(category, intFields, fields); + const escapeFields = ['name', 'color', 'bgColor', 'backgroundImage', 'imageClass', 'class', 'link']; + escapeFields.forEach(field => { + if (category.hasOwnProperty(field)) { + category[field] = validator.escape(String(category[field] || '')); + } + }); + if (category.hasOwnProperty('icon')) { + category.icon = category.icon || 'hidden'; + } + if (category.hasOwnProperty('post_count')) { + category.totalPostCount = category.post_count; + } + if (category.hasOwnProperty('topic_count')) { + category.totalTopicCount = category.topic_count; + } + if (category.description) { + category.description = validator.escape(String(category.description)); + category.descriptionParsed = category.descriptionParsed || category.description; + } +} \ No newline at end of file diff --git a/lib/categories/delete.js b/lib/categories/delete.js new file mode 100644 index 0000000000..9351b3c11a --- /dev/null +++ b/lib/categories/delete.js @@ -0,0 +1,59 @@ +'use strict'; + +const async = require('async'); +const db = require('../database'); +const batch = require('../batch'); +const plugins = require('../plugins'); +const topics = require('../topics'); +const groups = require('../groups'); +const privileges = require('../privileges'); +const cache = require('../cache'); +module.exports = function (Categories) { + Categories.purge = async function (cid, uid) { + await batch.processSortedSet(`cid:${cid}:tids`, async tids => { + await async.eachLimit(tids, 10, async tid => { + await topics.purgePostsAndTopic(tid, uid); + }); + }, { + alwaysStartAt: 0 + }); + const pinnedTids = await db.getSortedSetRevRange(`cid:${cid}:tids:pinned`, 0, -1); + await async.eachLimit(pinnedTids, 10, async tid => { + await topics.purgePostsAndTopic(tid, uid); + }); + const categoryData = await Categories.getCategoryData(cid); + await purgeCategory(cid, categoryData); + plugins.hooks.fire('action:category.delete', { + cid: cid, + uid: uid, + category: categoryData + }); + }; + async function purgeCategory(cid, categoryData) { + const bulkRemove = [['categories:cid', cid]]; + if (categoryData && categoryData.name) { + bulkRemove.push(['categories:name', `${categoryData.name.slice(0, 200).toLowerCase()}:${cid}`]); + } + await db.sortedSetRemoveBulk(bulkRemove); + await removeFromParent(cid); + await deleteTags(cid); + await db.deleteAll([`cid:${cid}:tids`, `cid:${cid}:tids:pinned`, `cid:${cid}:tids:posts`, `cid:${cid}:tids:votes`, `cid:${cid}:tids:views`, `cid:${cid}:tids:lastposttime`, `cid:${cid}:recent_tids`, `cid:${cid}:pids`, `cid:${cid}:read_by_uid`, `cid:${cid}:uid:watch:state`, `cid:${cid}:children`, `cid:${cid}:tag:whitelist`, `category:${cid}`]); + const privilegeList = await privileges.categories.getPrivilegeList(); + await groups.destroy(privilegeList.map(privilege => `cid:${cid}:privileges:${privilege}`)); + } + async function removeFromParent(cid) { + const [parentCid, children] = await Promise.all([Categories.getCategoryField(cid, 'parentCid'), db.getSortedSetRange(`cid:${cid}:children`, 0, -1)]); + const bulkAdd = []; + const childrenKeys = children.map(cid => { + bulkAdd.push(['cid:0:children', cid, cid]); + return `category:${cid}`; + }); + await Promise.all([db.sortedSetRemove(`cid:${parentCid}:children`, cid), db.setObjectField(childrenKeys, 'parentCid', 0), db.sortedSetAddBulk(bulkAdd)]); + cache.del(['categories:cid', 'cid:0:children', `cid:${parentCid}:children`, `cid:${parentCid}:children:all`, `cid:${cid}:children`, `cid:${cid}:children:all`, `cid:${cid}:tag:whitelist`]); + } + async function deleteTags(cid) { + const tags = await db.getSortedSetMembers(`cid:${cid}:tags`); + await db.deleteAll(tags.map(tag => `cid:${cid}:tag:${tag}:topics`)); + await db.delete(`cid:${cid}:tags`); + } +}; \ No newline at end of file diff --git a/lib/categories/index.js b/lib/categories/index.js new file mode 100644 index 0000000000..ea14605a80 --- /dev/null +++ b/lib/categories/index.js @@ -0,0 +1,348 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const user = require('../user'); +const topics = require('../topics'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const cache = require('../cache'); +const meta = require('../meta'); +const Categories = module.exports; +require('./data')(Categories); +require('./create')(Categories); +require('./delete')(Categories); +require('./topics')(Categories); +require('./unread')(Categories); +require('./activeusers')(Categories); +require('./recentreplies')(Categories); +require('./update')(Categories); +require('./watch')(Categories); +require('./search')(Categories); +Categories.exists = async function (cids) { + return await db.exists(Array.isArray(cids) ? cids.map(cid => `category:${cid}`) : `category:${cids}`); +}; +Categories.getCategoryById = async function (data) { + const categories = await Categories.getCategories([data.cid]); + if (!categories[0]) { + return null; + } + const category = categories[0]; + data.category = category; + const promises = [Categories.getCategoryTopics(data), Categories.getTopicCount(data), Categories.getWatchState([data.cid], data.uid), getChildrenTree(category, data.uid)]; + if (category.parentCid) { + promises.push(Categories.getCategoryData(category.parentCid)); + } + const [topics, topicCount, watchState,, parent] = await Promise.all(promises); + category.topics = topics.topics; + category.nextStart = topics.nextStart; + category.topic_count = topicCount; + category.isWatched = watchState[0] === Categories.watchStates.watching; + category.isTracked = watchState[0] === Categories.watchStates.tracking; + category.isNotWatched = watchState[0] === Categories.watchStates.notwatching; + category.isIgnored = watchState[0] === Categories.watchStates.ignoring; + category.parent = parent; + calculateTopicPostCount(category); + const result = await plugins.hooks.fire('filter:category.get', { + category: category, + ...data + }); + return { + ...result.category + }; +}; +Categories.getAllCidsFromSet = async function (key) { + let cids = cache.get(key); + if (cids) { + return cids.slice(); + } + cids = await db.getSortedSetRange(key, 0, -1); + cids = cids.map(cid => parseInt(cid, 10)); + cache.set(key, cids); + return cids.slice(); +}; +Categories.getAllCategories = async function () { + const cids = await Categories.getAllCidsFromSet('categories:cid'); + return await Categories.getCategories(cids); +}; +Categories.getCidsByPrivilege = async function (set, uid, privilege) { + const cids = await Categories.getAllCidsFromSet(set); + return await privileges.categories.filterCids(privilege, cids, uid); +}; +Categories.getCategoriesByPrivilege = async function (set, uid, privilege) { + const cids = await Categories.getCidsByPrivilege(set, uid, privilege); + return await Categories.getCategories(cids); +}; +Categories.getModerators = async function (cid) { + const uids = await Categories.getModeratorUids([cid]); + return await user.getUsersFields(uids[0], ['uid', 'username', 'userslug', 'picture']); +}; +Categories.getModeratorUids = async function (cids) { + return await privileges.categories.getUidsWithPrivilege(cids, 'moderate'); +}; +Categories.getCategories = async function (cids) { + if (!Array.isArray(cids)) { + throw new Error('[[error:invalid-cid]]'); + } + if (!cids.length) { + return []; + } + const [categories, tagWhitelist] = await Promise.all([Categories.getCategoriesData(cids), Categories.getTagWhitelist(cids)]); + categories.forEach((category, i) => { + if (category) { + category.tagWhitelist = tagWhitelist[i]; + } + }); + return categories; +}; +Categories.setUnread = async function (tree, cids, uid) { + if (uid <= 0) { + return; + } + const { + unreadCids + } = await topics.getUnreadData({ + uid: uid, + cid: cids + }); + if (!unreadCids.length) { + return; + } + function setCategoryUnread(category) { + if (category) { + category.unread = false; + if (unreadCids.includes(category.cid)) { + category.unread = category.topic_count > 0 && true; + } else if (category.children.length) { + category.children.forEach(setCategoryUnread); + category.unread = category.children.some(c => c && c.unread); + } + category['unread-class'] = category.unread ? 'unread' : ''; + } + } + tree.forEach(setCategoryUnread); +}; +Categories.getTagWhitelist = async function (cids) { + const cachedData = {}; + const nonCachedCids = cids.filter(cid => { + const data = cache.get(`cid:${cid}:tag:whitelist`); + const isInCache = data !== undefined; + if (isInCache) { + cachedData[cid] = data; + } + return !isInCache; + }); + if (!nonCachedCids.length) { + return cids.map(cid => cachedData[cid]); + } + const keys = nonCachedCids.map(cid => `cid:${cid}:tag:whitelist`); + const data = await db.getSortedSetsMembers(keys); + nonCachedCids.forEach((cid, index) => { + cachedData[cid] = data[index]; + cache.set(`cid:${cid}:tag:whitelist`, data[index]); + }); + return cids.map(cid => cachedData[cid]); +}; +Categories.filterTagWhitelist = function (tagWhitelist, isAdminOrMod) { + const systemTags = (meta.config.systemTags || '').split(','); + if (!isAdminOrMod && systemTags.length) { + return tagWhitelist.filter(tag => !systemTags.includes(tag)); + } + return tagWhitelist; +}; +function calculateTopicPostCount(category) { + if (!category) { + return; + } + let postCount = category.post_count; + let topicCount = category.topic_count; + if (Array.isArray(category.children)) { + category.children.forEach(child => { + calculateTopicPostCount(child); + postCount += parseInt(child.totalPostCount, 10) || 0; + topicCount += parseInt(child.totalTopicCount, 10) || 0; + }); + } + category.totalPostCount = postCount; + category.totalTopicCount = topicCount; +} +Categories.calculateTopicPostCount = calculateTopicPostCount; +Categories.getParents = async function (cids) { + const categoriesData = await Categories.getCategoriesFields(cids, ['parentCid']); + const parentCids = categoriesData.filter(c => c && c.parentCid).map(c => c.parentCid); + if (!parentCids.length) { + return cids.map(() => null); + } + const parentData = await Categories.getCategoriesData(parentCids); + const cidToParent = _.zipObject(parentCids, parentData); + return categoriesData.map(category => cidToParent[category.parentCid]); +}; +Categories.getChildren = async function (cids, uid) { + const categoryData = await Categories.getCategoriesFields(cids, ['parentCid']); + const categories = categoryData.map((category, index) => ({ + cid: cids[index], + parentCid: category.parentCid + })); + await Promise.all(categories.map(c => getChildrenTree(c, uid))); + return categories.map(c => c && c.children); +}; +async function getChildrenTree(category, uid) { + let childrenCids = await Categories.getChildrenCids(category.cid); + childrenCids = await privileges.categories.filterCids('find', childrenCids, uid); + childrenCids = childrenCids.filter(cid => parseInt(category.cid, 10) !== parseInt(cid, 10)); + if (!childrenCids.length) { + category.children = []; + return; + } + let childrenData = await Categories.getCategoriesData(childrenCids); + childrenData = childrenData.filter(Boolean); + childrenCids = childrenData.map(child => child.cid); + Categories.getTree([category].concat(childrenData), category.parentCid); +} +Categories.getChildrenTree = getChildrenTree; +Categories.getParentCids = async function (currentCid) { + let cid = currentCid; + const parents = []; + while (parseInt(cid, 10)) { + cid = await Categories.getCategoryField(cid, 'parentCid'); + if (cid) { + parents.unshift(cid); + } + } + return parents; +}; +Categories.getChildrenCids = async function (rootCid) { + let allCids = []; + async function recursive(keys) { + let childrenCids = await db.getSortedSetRange(keys, 0, -1); + childrenCids = childrenCids.filter(cid => !allCids.includes(parseInt(cid, 10))); + if (!childrenCids.length) { + return; + } + keys = childrenCids.map(cid => `cid:${cid}:children`); + childrenCids.forEach(cid => allCids.push(parseInt(cid, 10))); + await recursive(keys); + } + const key = `cid:${rootCid}:children`; + const cacheKey = `${key}:all`; + const childrenCids = cache.get(cacheKey); + if (childrenCids) { + return childrenCids.slice(); + } + await recursive(key); + allCids = _.uniq(allCids); + cache.set(cacheKey, allCids); + return allCids.slice(); +}; +Categories.flattenCategories = function (allCategories, categoryData) { + categoryData.forEach(category => { + if (category) { + allCategories.push(category); + if (Array.isArray(category.children) && category.children.length) { + Categories.flattenCategories(allCategories, category.children); + } + } + }); +}; +Categories.getTree = function (categories, parentCid) { + parentCid = parentCid || 0; + const cids = categories.map(category => category && category.cid); + const cidToCategory = {}; + const parents = {}; + cids.forEach((cid, index) => { + if (cid) { + categories[index].children = undefined; + cidToCategory[cid] = categories[index]; + parents[cid] = { + ...categories[index] + }; + } + }); + const tree = []; + categories.forEach(category => { + if (category) { + category.children = category.children || []; + if (!category.cid) { + return; + } + if (!category.hasOwnProperty('parentCid') || category.parentCid === null) { + category.parentCid = 0; + } + if (category.parentCid === parentCid) { + tree.push(category); + category.parent = parents[parentCid]; + } else { + const parent = cidToCategory[category.parentCid]; + if (parent && parent.cid !== category.cid) { + category.parent = parents[category.parentCid]; + parent.children = parent.children || []; + parent.children.push(category); + } + } + } + }); + function sortTree(tree) { + tree.sort((a, b) => { + if (a.order !== b.order) { + return a.order - b.order; + } + return a.cid - b.cid; + }); + tree.forEach(category => { + if (category && Array.isArray(category.children)) { + sortTree(category.children); + } + }); + } + sortTree(tree); + categories.forEach(c => calculateTopicPostCount(c)); + return tree; +}; +Categories.buildForSelect = async function (uid, privilege, fields) { + const cids = await Categories.getCidsByPrivilege('categories:cid', uid, privilege); + return await getSelectData(cids, fields); +}; +Categories.buildForSelectAll = async function (fields) { + const cids = await Categories.getAllCidsFromSet('categories:cid'); + return await getSelectData(cids, fields); +}; +async function getSelectData(cids, fields) { + const categoryData = await Categories.getCategoriesData(cids); + const tree = Categories.getTree(categoryData); + return Categories.buildForSelectCategories(tree, fields); +} +Categories.buildForSelectCategories = function (categories, fields, parentCid) { + function recursive({ + ...category + }, categoriesData, level, depth) { + const bullet = level ? '• ' : ''; + category.value = category.cid; + category.level = level; + category.text = level + bullet + category.name; + category.depth = depth; + categoriesData.push(category); + if (Array.isArray(category.children)) { + category.children.forEach(child => recursive(child, categoriesData, `    ${level}`, depth + 1)); + } + } + parentCid = parentCid || 0; + const categoriesData = []; + const rootCategories = categories.filter(category => category && category.parentCid === parentCid); + rootCategories.sort((a, b) => { + if (a.order !== b.order) { + return a.order - b.order; + } + return a.cid - b.cid; + }); + rootCategories.forEach(category => recursive(category, categoriesData, '', 0)); + const pickFields = ['cid', 'name', 'level', 'icon', 'parentCid', 'color', 'bgColor', 'backgroundImage', 'imageClass']; + fields = fields || []; + if (fields.includes('text') && fields.includes('value')) { + return categoriesData.map(category => _.pick(category, fields)); + } + if (fields.length) { + pickFields.push(...fields); + } + return categoriesData.map(category => _.pick(category, pickFields)); +}; +require('../promisify')(Categories); \ No newline at end of file diff --git a/lib/categories/recentreplies.js b/lib/categories/recentreplies.js new file mode 100644 index 0000000000..95f0958bb7 --- /dev/null +++ b/lib/categories/recentreplies.js @@ -0,0 +1,162 @@ +'use strict'; + +const winston = require('winston'); +const _ = require('lodash'); +const db = require('../database'); +const posts = require('../posts'); +const topics = require('../topics'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const batch = require('../batch'); +module.exports = function (Categories) { + Categories.getRecentReplies = async function (cid, uid, start, stop) { + if (stop === undefined && start > 0) { + winston.warn('[Categories.getRecentReplies] 3 params deprecated please use Categories.getRecentReplies(cid, uid, start, stop)'); + stop = start - 1; + start = 0; + } + let pids = await db.getSortedSetRevRange(`cid:${cid}:pids`, start, stop); + pids = await privileges.posts.filter('topics:read', pids, uid); + return await posts.getPostSummaryByPids(pids, uid, { + stripTags: true + }); + }; + Categories.updateRecentTid = async function (cid, tid) { + const [count, numRecentReplies] = await Promise.all([db.sortedSetCard(`cid:${cid}:recent_tids`), db.getObjectField(`category:${cid}`, 'numRecentReplies')]); + if (count >= numRecentReplies) { + const data = await db.getSortedSetRangeWithScores(`cid:${cid}:recent_tids`, 0, count - numRecentReplies); + const shouldRemove = !(data.length === 1 && count === 1 && data[0].value === String(tid)); + if (data.length && shouldRemove) { + await db.sortedSetsRemoveRangeByScore([`cid:${cid}:recent_tids`], '-inf', data[data.length - 1].score); + } + } + if (numRecentReplies > 0) { + await db.sortedSetAdd(`cid:${cid}:recent_tids`, Date.now(), tid); + } + await plugins.hooks.fire('action:categories.updateRecentTid', { + cid: cid, + tid: tid + }); + }; + Categories.updateRecentTidForCid = async function (cid) { + let postData; + let topicData; + let index = 0; + do { + const pids = await db.getSortedSetRevRange(`cid:${cid}:pids`, index, index); + if (!pids.length) { + return; + } + postData = await posts.getPostFields(pids[0], ['tid', 'deleted']); + if (postData && postData.tid && !postData.deleted) { + topicData = await topics.getTopicData(postData.tid); + } + index += 1; + } while (!topicData || topicData.deleted || topicData.scheduled); + if (postData && postData.tid) { + await Categories.updateRecentTid(cid, postData.tid); + } + }; + Categories.getRecentTopicReplies = async function (categoryData, uid, query) { + if (!Array.isArray(categoryData) || !categoryData.length) { + return; + } + const categoriesToLoad = categoryData.filter(c => c && c.numRecentReplies && parseInt(c.numRecentReplies, 10) > 0); + let keys = []; + if (plugins.hooks.hasListeners('filter:categories.getRecentTopicReplies')) { + const result = await plugins.hooks.fire('filter:categories.getRecentTopicReplies', { + categories: categoriesToLoad, + uid: uid, + query: query, + keys: [] + }); + keys = result.keys; + } else { + keys = categoriesToLoad.map(c => `cid:${c.cid}:recent_tids`); + } + const results = await db.getSortedSetsMembers(keys); + let tids = _.uniq(_.flatten(results).filter(Boolean)); + tids = await privileges.topics.filterTids('topics:read', tids, uid); + const topics = await getTopics(tids, uid); + assignTopicsToCategories(categoryData, topics); + bubbleUpChildrenPosts(categoryData); + }; + async function getTopics(tids, uid) { + const topicData = await topics.getTopicsFields(tids, ['tid', 'mainPid', 'slug', 'title', 'teaserPid', 'cid', 'postcount']); + topicData.forEach(topic => { + if (topic) { + topic.teaserPid = topic.teaserPid || topic.mainPid; + } + }); + const cids = _.uniq(topicData.map(t => t && t.cid).filter(cid => parseInt(cid, 10))); + const getToRoot = async () => await Promise.all(cids.map(Categories.getParentCids)); + const [toRoot, teasers] = await Promise.all([getToRoot(), topics.getTeasers(topicData, uid)]); + const cidToRoot = _.zipObject(cids, toRoot); + teasers.forEach((teaser, index) => { + if (teaser) { + teaser.cid = topicData[index].cid; + teaser.parentCids = cidToRoot[teaser.cid]; + teaser.tid = topicData[index].tid; + teaser.uid = topicData[index].uid; + teaser.topic = { + tid: topicData[index].tid, + slug: topicData[index].slug, + title: topicData[index].title + }; + } + }); + return teasers.filter(Boolean); + } + function assignTopicsToCategories(categories, topics) { + categories.forEach(category => { + if (category) { + category.posts = topics.filter(t => t.cid && (t.cid === category.cid || t.parentCids.includes(category.cid))).sort((a, b) => b.timestamp - a.timestamp).slice(0, parseInt(category.numRecentReplies, 10)); + } + }); + topics.forEach(t => { + t.parentCids = undefined; + }); + } + function bubbleUpChildrenPosts(categoryData) { + categoryData.forEach(category => { + if (category) { + if (category.posts.length) { + return; + } + const posts = []; + getPostsRecursive(category, posts); + posts.sort((a, b) => b.timestamp - a.timestamp); + if (posts.length) { + category.posts = [posts[0]]; + } + } + }); + } + function getPostsRecursive(category, posts) { + if (Array.isArray(category.posts)) { + category.posts.forEach(p => posts.push(p)); + } + category.children.forEach(child => getPostsRecursive(child, posts)); + } + Categories.moveRecentReplies = async function (tid, oldCid, cid) { + const [pids, topicDeleted] = await Promise.all([topics.getPids(tid), topics.getTopicField(tid, 'deleted')]); + await batch.processArray(pids, async pids => { + const postData = await posts.getPostsFields(pids, ['pid', 'deleted', 'uid', 'timestamp', 'upvotes', 'downvotes']); + const bulkRemove = []; + const bulkAdd = []; + postData.forEach(post => { + bulkRemove.push([`cid:${oldCid}:uid:${post.uid}:pids`, post.pid]); + bulkRemove.push([`cid:${oldCid}:uid:${post.uid}:pids:votes`, post.pid]); + bulkAdd.push([`cid:${cid}:uid:${post.uid}:pids`, post.timestamp, post.pid]); + if (post.votes > 0 || post.votes < 0) { + bulkAdd.push([`cid:${cid}:uid:${post.uid}:pids:votes`, post.votes, post.pid]); + } + }); + const postsToReAdd = postData.filter(p => !p.deleted && !topicDeleted); + const timestamps = postsToReAdd.map(p => p && p.timestamp); + await Promise.all([db.sortedSetRemove(`cid:${oldCid}:pids`, pids), db.sortedSetAdd(`cid:${cid}:pids`, timestamps, postsToReAdd.map(p => p.pid)), db.sortedSetRemoveBulk(bulkRemove), db.sortedSetAddBulk(bulkAdd)]); + }, { + batch: 500 + }); + }; +}; \ No newline at end of file diff --git a/lib/categories/search.js b/lib/categories/search.js new file mode 100644 index 0000000000..d903318100 --- /dev/null +++ b/lib/categories/search.js @@ -0,0 +1,69 @@ +'use strict'; + +const _ = require('lodash'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const db = require('../database'); +module.exports = function (Categories) { + Categories.search = async function (data) { + const query = data.query || ''; + const page = data.page || 1; + const uid = data.uid || 0; + const paginate = data.hasOwnProperty('paginate') ? data.paginate : true; + const startTime = process.hrtime(); + let cids = await findCids(query, data.hardCap); + const result = await plugins.hooks.fire('filter:categories.search', { + data: data, + cids: cids, + uid: uid + }); + cids = await privileges.categories.filterCids('find', result.cids, uid); + const searchResult = { + matchCount: cids.length + }; + if (paginate) { + const resultsPerPage = data.resultsPerPage || 50; + const start = Math.max(0, page - 1) * resultsPerPage; + const stop = start + resultsPerPage; + searchResult.pageCount = Math.ceil(cids.length / resultsPerPage); + cids = cids.slice(start, stop); + } + const childrenCids = await getChildrenCids(cids, uid); + const uniqCids = _.uniq(cids.concat(childrenCids)); + const categoryData = await Categories.getCategories(uniqCids); + Categories.getTree(categoryData, 0); + await Categories.getRecentTopicReplies(categoryData, uid, data.qs); + categoryData.forEach(category => { + if (category && Array.isArray(category.children)) { + category.children = category.children.slice(0, category.subCategoriesPerPage); + category.children.forEach(child => { + child.children = undefined; + }); + } + }); + categoryData.sort((c1, c2) => { + if (c1.parentCid !== c2.parentCid) { + return c1.parentCid - c2.parentCid; + } + return c1.order - c2.order; + }); + searchResult.timing = (process.elapsedTimeSince(startTime) / 1000).toFixed(2); + searchResult.categories = categoryData.filter(c => cids.includes(c.cid)); + return searchResult; + }; + async function findCids(query, hardCap) { + if (!query || String(query).length < 2) { + return []; + } + const data = await db.getSortedSetScan({ + key: 'categories:name', + match: `*${String(query).toLowerCase()}*`, + limit: hardCap || 500 + }); + return data.map(data => parseInt(data.split(':').pop(), 10)); + } + async function getChildrenCids(cids, uid) { + const childrenCids = await Promise.all(cids.map(cid => Categories.getChildrenCids(cid))); + return await privileges.categories.filterCids('find', _.flatten(childrenCids), uid); + } +}; \ No newline at end of file diff --git a/lib/categories/topics.js b/lib/categories/topics.js new file mode 100644 index 0000000000..293dc27f19 --- /dev/null +++ b/lib/categories/topics.js @@ -0,0 +1,228 @@ +'use strict'; + +const db = require('../database'); +const topics = require('../topics'); +const plugins = require('../plugins'); +const meta = require('../meta'); +const privileges = require('../privileges'); +const user = require('../user'); +const notifications = require('../notifications'); +const translator = require('../translator'); +const batch = require('../batch'); +module.exports = function (Categories) { + Categories.getCategoryTopics = async function (data) { + let results = await plugins.hooks.fire('filter:category.topics.prepare', data); + const tids = await Categories.getTopicIds(results); + let topicsData = await topics.getTopicsByTids(tids, data.uid); + topicsData = await user.blocks.filter(data.uid, topicsData); + if (!topicsData.length) { + return { + topics: [], + uid: data.uid + }; + } + topics.calculateTopicIndices(topicsData, data.start); + results = await plugins.hooks.fire('filter:category.topics.get', { + cid: data.cid, + topics: topicsData, + uid: data.uid + }); + return { + topics: results.topics, + nextStart: data.stop + 1 + }; + }; + Categories.getTopicIds = async function (data) { + const [pinnedTids, set] = await Promise.all([Categories.getPinnedTids({ + ...data, + start: 0, + stop: -1 + }), Categories.buildTopicsSortedSet(data)]); + const totalPinnedCount = pinnedTids.length; + const pinnedTidsOnPage = pinnedTids.slice(data.start, data.stop !== -1 ? data.stop + 1 : undefined); + const pinnedCountOnPage = pinnedTidsOnPage.length; + const topicsPerPage = data.stop - data.start + 1; + const normalTidsToGet = Math.max(0, topicsPerPage - pinnedCountOnPage); + if (!normalTidsToGet && data.stop !== -1) { + return pinnedTidsOnPage; + } + if (plugins.hooks.hasListeners('filter:categories.getTopicIds')) { + const result = await plugins.hooks.fire('filter:categories.getTopicIds', { + tids: [], + data: data, + pinnedTids: pinnedTidsOnPage, + allPinnedTids: pinnedTids, + totalPinnedCount: totalPinnedCount, + normalTidsToGet: normalTidsToGet + }); + return result && result.tids; + } + let { + start + } = data; + if (start > 0 && totalPinnedCount) { + start -= totalPinnedCount - pinnedCountOnPage; + } + const stop = data.stop === -1 ? data.stop : start + normalTidsToGet - 1; + let normalTids; + if (Array.isArray(set)) { + const weights = set.map((s, index) => index ? 0 : 1); + normalTids = await db.getSortedSetRevIntersect({ + sets: set, + start: start, + stop: stop, + weights: weights + }); + } else { + normalTids = await db.getSortedSetRevRange(set, start, stop); + } + normalTids = normalTids.filter(tid => !pinnedTids.includes(tid)); + return pinnedTidsOnPage.concat(normalTids); + }; + Categories.getTopicCount = async function (data) { + if (plugins.hooks.hasListeners('filter:categories.getTopicCount')) { + const result = await plugins.hooks.fire('filter:categories.getTopicCount', { + topicCount: data.category.topic_count, + data: data + }); + return result && result.topicCount; + } + const set = await Categories.buildTopicsSortedSet(data); + if (Array.isArray(set)) { + return await db.sortedSetIntersectCard(set); + } else if (data.targetUid && set) { + return await db.sortedSetCard(set); + } + return data.category.topic_count; + }; + Categories.buildTopicsSortedSet = async function (data) { + const { + cid + } = data; + const sort = data.sort || data.settings && data.settings.categoryTopicSort || meta.config.categoryTopicSort || 'recently_replied'; + const sortToSet = { + recently_replied: `cid:${cid}:tids`, + recently_created: `cid:${cid}:tids:create`, + most_posts: `cid:${cid}:tids:posts`, + most_votes: `cid:${cid}:tids:votes`, + most_views: `cid:${cid}:tids:views` + }; + let set = sortToSet.hasOwnProperty(sort) ? sortToSet[sort] : `cid:${cid}:tids`; + if (data.tag) { + if (Array.isArray(data.tag)) { + set = [set].concat(data.tag.map(tag => `tag:${tag}:topics`)); + } else { + set = [set, `tag:${data.tag}:topics`]; + } + } + if (data.targetUid) { + set = (Array.isArray(set) ? set : [set]).concat([`cid:${cid}:uid:${data.targetUid}:tids`]); + } + const result = await plugins.hooks.fire('filter:categories.buildTopicsSortedSet', { + set: set, + data: data + }); + return result && result.set; + }; + Categories.getSortedSetRangeDirection = async function (sort) { + console.warn('[deprecated] Will be removed in 4.x'); + sort = sort || 'recently_replied'; + const direction = ['newest_to_oldest', 'most_posts', 'most_votes', 'most_views'].includes(sort) ? 'highest-to-lowest' : 'lowest-to-highest'; + const result = await plugins.hooks.fire('filter:categories.getSortedSetRangeDirection', { + sort: sort, + direction: direction + }); + return result && result.direction; + }; + Categories.getAllTopicIds = async function (cid, start, stop) { + return await db.getSortedSetRange([`cid:${cid}:tids:pinned`, `cid:${cid}:tids`], start, stop); + }; + Categories.getPinnedTids = async function (data) { + if (plugins.hooks.hasListeners('filter:categories.getPinnedTids')) { + const result = await plugins.hooks.fire('filter:categories.getPinnedTids', { + pinnedTids: [], + data: data + }); + return result && result.pinnedTids; + } + const [allPinnedTids, canSchedule] = await Promise.all([db.getSortedSetRevRange(`cid:${data.cid}:tids:pinned`, data.start, data.stop), privileges.categories.can('topics:schedule', data.cid, data.uid)]); + const pinnedTids = canSchedule ? allPinnedTids : await filterScheduledTids(allPinnedTids); + return await topics.tools.checkPinExpiry(pinnedTids); + }; + Categories.modifyTopicsByPrivilege = function (topics, privileges) { + if (!Array.isArray(topics) || !topics.length || privileges.view_deleted) { + return; + } + topics.forEach(topic => { + if (!topic.scheduled && topic.deleted && !topic.isOwner) { + topic.title = '[[topic:topic-is-deleted]]'; + if (topic.hasOwnProperty('titleRaw')) { + topic.titleRaw = '[[topic:topic-is-deleted]]'; + } + topic.slug = topic.tid; + topic.teaser = null; + topic.noAnchor = true; + topic.unread = false; + topic.tags = []; + } + }); + }; + Categories.onNewPostMade = async function (cid, pinned, postData) { + if (!cid || !postData) { + return; + } + const promises = [db.sortedSetAdd(`cid:${cid}:pids`, postData.timestamp, postData.pid), db.incrObjectField(`category:${cid}`, 'post_count')]; + if (!pinned) { + promises.push(db.sortedSetIncrBy(`cid:${cid}:tids:posts`, 1, postData.tid)); + } + await Promise.all(promises); + await Categories.updateRecentTidForCid(cid); + }; + Categories.onTopicsMoved = async cids => { + await Promise.all(cids.map(async cid => { + await Promise.all([Categories.setCategoryField(cid, 'topic_count', await db.sortedSetCard(`cid:${cid}:tids:lastposttime`)), Categories.setCategoryField(cid, 'post_count', await db.sortedSetCard(`cid:${cid}:pids`)), Categories.updateRecentTidForCid(cid)]); + })); + }; + async function filterScheduledTids(tids) { + const scores = await db.sortedSetScores('topics:scheduled', tids); + const now = Date.now(); + return tids.filter((tid, index) => tid && (!scores[index] || scores[index] <= now)); + } + Categories.notifyCategoryFollowers = async (postData, exceptUid) => { + const { + cid + } = postData.topic; + const followers = []; + await batch.processSortedSet(`cid:${cid}:uid:watch:state`, async uids => { + followers.push(...(await privileges.categories.filterUids('topics:read', cid, uids))); + }, { + batch: 500, + min: Categories.watchStates.watching, + max: Categories.watchStates.watching + }); + const index = followers.indexOf(String(exceptUid)); + if (index !== -1) { + followers.splice(index, 1); + } + if (!followers.length) { + return; + } + const { + displayname + } = postData.user; + const categoryName = await Categories.getCategoryField(cid, 'name'); + const notifBase = 'notifications:user-posted-topic-in-category'; + const bodyShort = translator.compile(notifBase, displayname, categoryName); + const notification = await notifications.create({ + type: 'new-topic-in-category', + nid: `new_topic:tid:${postData.topic.tid}:uid:${exceptUid}`, + bodyShort: bodyShort, + bodyLong: postData.content, + pid: postData.pid, + path: `/post/${postData.pid}`, + tid: postData.topic.tid, + from: exceptUid + }); + notifications.push(notification, followers); + }; +}; \ No newline at end of file diff --git a/lib/categories/unread.js b/lib/categories/unread.js new file mode 100644 index 0000000000..4f4fcbc16e --- /dev/null +++ b/lib/categories/unread.js @@ -0,0 +1,37 @@ +'use strict'; + +const db = require('../database'); +module.exports = function (Categories) { + Categories.markAsRead = async function (cids, uid) { + console.warn('[deprecated] Categories.markAsRead deprecated'); + if (!Array.isArray(cids) || !cids.length || parseInt(uid, 10) <= 0) { + return; + } + let keys = cids.map(cid => `cid:${cid}:read_by_uid`); + const hasRead = await db.isMemberOfSets(keys, uid); + keys = keys.filter((key, index) => !hasRead[index]); + await db.setsAdd(keys, uid); + }; + Categories.markAsUnreadForAll = async function (cid) { + console.warn('[deprecated] Categories.markAsUnreadForAll deprecated'); + if (!parseInt(cid, 10)) { + return; + } + await db.delete(`cid:${cid}:read_by_uid`); + }; + Categories.hasReadCategories = async function (cids, uid) { + console.warn('[deprecated] Categories.hasReadCategories deprecated, see Categories.setUnread'); + if (parseInt(uid, 10) <= 0) { + return cids.map(() => false); + } + const sets = cids.map(cid => `cid:${cid}:read_by_uid`); + return await db.isMemberOfSets(sets, uid); + }; + Categories.hasReadCategory = async function (cid, uid) { + console.warn('[deprecated] Categories.hasReadCategory deprecated, see Categories.setUnread'); + if (parseInt(uid, 10) <= 0) { + return false; + } + return await db.isSetMember(`cid:${cid}:read_by_uid`, uid); + }; +}; \ No newline at end of file diff --git a/lib/categories/update.js b/lib/categories/update.js new file mode 100644 index 0000000000..26385c8a07 --- /dev/null +++ b/lib/categories/update.js @@ -0,0 +1,111 @@ +'use strict'; + +const db = require('../database'); +const meta = require('../meta'); +const utils = require('../utils'); +const slugify = require('../slugify'); +const translator = require('../translator'); +const plugins = require('../plugins'); +const cache = require('../cache'); +module.exports = function (Categories) { + Categories.update = async function (modified) { + const cids = Object.keys(modified); + await Promise.all(cids.map(cid => updateCategory(cid, modified[cid]))); + return cids; + }; + async function updateCategory(cid, modifiedFields) { + const exists = await Categories.exists(cid); + if (!exists) { + return; + } + if (modifiedFields.hasOwnProperty('name')) { + const translated = await translator.translate(modifiedFields.name); + modifiedFields.slug = `${cid}/${slugify(translated)}`; + } + const result = await plugins.hooks.fire('filter:category.update', { + cid: cid, + category: modifiedFields + }); + const { + category + } = result; + const fields = Object.keys(category); + const parentCidIndex = fields.indexOf('parentCid'); + if (parentCidIndex !== -1 && fields.length > 1) { + fields.splice(0, 0, fields.splice(parentCidIndex, 1)[0]); + } + for (const key of fields) { + await updateCategoryField(cid, key, category[key]); + } + plugins.hooks.fire('action:category.update', { + cid: cid, + modified: category + }); + } + async function updateCategoryField(cid, key, value) { + if (key === 'parentCid') { + return await updateParent(cid, value); + } else if (key === 'tagWhitelist') { + return await updateTagWhitelist(cid, value); + } else if (key === 'name') { + return await updateName(cid, value); + } else if (key === 'order') { + return await updateOrder(cid, value); + } + await db.setObjectField(`category:${cid}`, key, value); + if (key === 'description') { + await Categories.parseDescription(cid, value); + } + } + async function updateParent(cid, newParent) { + newParent = parseInt(newParent, 10) || 0; + if (parseInt(cid, 10) === newParent) { + throw new Error('[[error:cant-set-self-as-parent]]'); + } + const childrenCids = await Categories.getChildrenCids(cid); + if (childrenCids.includes(newParent)) { + throw new Error('[[error:cant-set-child-as-parent]]'); + } + const categoryData = await Categories.getCategoryFields(cid, ['parentCid', 'order']); + const oldParent = categoryData.parentCid; + if (oldParent === newParent) { + return; + } + await Promise.all([db.sortedSetRemove(`cid:${oldParent}:children`, cid), db.sortedSetAdd(`cid:${newParent}:children`, categoryData.order, cid), db.setObjectField(`category:${cid}`, 'parentCid', newParent)]); + cache.del([`cid:${oldParent}:children`, `cid:${newParent}:children`, `cid:${oldParent}:children:all`, `cid:${newParent}:children:all`]); + } + async function updateTagWhitelist(cid, tags) { + tags = tags.split(',').map(tag => utils.cleanUpTag(tag, meta.config.maximumTagLength)).filter(Boolean); + await db.delete(`cid:${cid}:tag:whitelist`); + const scores = tags.map((tag, index) => index); + await db.sortedSetAdd(`cid:${cid}:tag:whitelist`, scores, tags); + cache.del(`cid:${cid}:tag:whitelist`); + } + async function updateOrder(cid, order) { + const parentCid = await Categories.getCategoryField(cid, 'parentCid'); + await db.sortedSetsAdd('categories:cid', order, cid); + const childrenCids = await db.getSortedSetRange(`cid:${parentCid}:children`, 0, -1); + const currentIndex = childrenCids.indexOf(String(cid)); + if (currentIndex === -1) { + throw new Error('[[error:no-category]]'); + } + if (childrenCids.length > 1) { + childrenCids.splice(Math.max(0, order - 1), 0, childrenCids.splice(currentIndex, 1)[0]); + } + await db.sortedSetAdd(`cid:${parentCid}:children`, childrenCids.map((cid, index) => index + 1), childrenCids); + await db.setObjectBulk(childrenCids.map((cid, index) => [`category:${cid}`, { + order: index + 1 + }])); + cache.del(['categories:cid', `cid:${parentCid}:children`, `cid:${parentCid}:children:all`]); + } + Categories.parseDescription = async function (cid, description) { + const parsedDescription = await plugins.hooks.fire('filter:parse.raw', description); + await Categories.setCategoryField(cid, 'descriptionParsed', parsedDescription); + }; + async function updateName(cid, newName) { + const oldName = await Categories.getCategoryField(cid, 'name'); + await db.sortedSetRemove('categories:name', `${oldName.slice(0, 200).toLowerCase()}:${cid}`); + await db.sortedSetAdd('categories:name', 0, `${newName.slice(0, 200).toLowerCase()}:${cid}`); + await db.setObjectField(`category:${cid}`, 'name', newName); + } +}; \ No newline at end of file diff --git a/lib/categories/watch.js b/lib/categories/watch.js new file mode 100644 index 0000000000..c7d551087f --- /dev/null +++ b/lib/categories/watch.js @@ -0,0 +1,43 @@ +'use strict'; + +const db = require('../database'); +const user = require('../user'); +module.exports = function (Categories) { + Categories.watchStates = { + ignoring: 1, + notwatching: 2, + tracking: 3, + watching: 4 + }; + Categories.isIgnored = async function (cids, uid) { + if (!(parseInt(uid, 10) > 0)) { + return cids.map(() => false); + } + const states = await Categories.getWatchState(cids, uid); + return states.map(state => state === Categories.watchStates.ignoring); + }; + Categories.getWatchState = async function (cids, uid) { + if (!(parseInt(uid, 10) > 0)) { + return cids.map(() => Categories.watchStates.notwatching); + } + if (!Array.isArray(cids) || !cids.length) { + return []; + } + const keys = cids.map(cid => `cid:${cid}:uid:watch:state`); + const [userSettings, states] = await Promise.all([user.getSettings(uid), db.sortedSetsScore(keys, uid)]); + return states.map(state => state || Categories.watchStates[userSettings.categoryWatchState]); + }; + Categories.getIgnorers = async function (cid, start, stop) { + const count = stop === -1 ? -1 : stop - start + 1; + return await db.getSortedSetRevRangeByScore(`cid:${cid}:uid:watch:state`, start, count, Categories.watchStates.ignoring, Categories.watchStates.ignoring); + }; + Categories.filterIgnoringUids = async function (cid, uids) { + const states = await Categories.getUidsWatchStates(cid, uids); + const readingUids = uids.filter((uid, index) => uid && states[index] !== Categories.watchStates.ignoring); + return readingUids; + }; + Categories.getUidsWatchStates = async function (cid, uids) { + const [userSettings, states] = await Promise.all([user.getMultipleUserSettings(uids), db.sortedSetScores(`cid:${cid}:uid:watch:state`, uids)]); + return states.map((state, index) => state || Categories.watchStates[userSettings[index].categoryWatchState]); + }; +}; \ No newline at end of file diff --git a/lib/cli/colors.js b/lib/cli/colors.js new file mode 100644 index 0000000000..4e72381102 --- /dev/null +++ b/lib/cli/colors.js @@ -0,0 +1,115 @@ +'use strict'; + +const { + Command +} = require('commander'); +const chalk = require('chalk'); +const colors = [{ + command: 'yellow', + option: 'cyan', + arg: 'magenta' +}, { + command: 'green', + option: 'blue', + arg: 'red' +}, { + command: 'yellow', + option: 'cyan', + arg: 'magenta' +}, { + command: 'green', + option: 'blue', + arg: 'red' +}]; +function humanReadableArgName(arg) { + const nameOutput = arg.name() + (arg.variadic === true ? '...' : ''); + return arg.required ? `<${nameOutput}>` : `[${nameOutput}]`; +} +function getControlCharacterSpaces(term) { + const matches = term.match(/.\[\d+m/g); + return matches ? matches.length * 5 : 0; +} +Command.prototype.depth = function () { + if (this._depth === undefined) { + let depth = 0; + let { + parent + } = this; + while (parent) { + depth += 1; + parent = parent.parent; + } + this._depth = depth; + } + return this._depth; +}; +module.exports = { + commandUsage(cmd) { + const depth = cmd.depth(); + let cmdName = cmd._name; + if (cmd._aliases[0]) { + cmdName = `${cmdName}|${cmd._aliases[0]}`; + } + let parentCmdNames = ''; + let parentCmd = cmd.parent; + let parentDepth = depth - 1; + while (parentCmd) { + parentCmdNames = `${chalk[colors[parentDepth].command](parentCmd.name())} ${parentCmdNames}`; + parentCmd = parentCmd.parent; + parentDepth -= 1; + } + const args = cmd._args.map(arg => chalk[colors[depth].arg](humanReadableArgName(arg))); + const cmdUsage = [].concat(cmd.options.length || cmd._hasHelpOption ? chalk[colors[depth].option]('[options]') : [], cmd.commands.length ? chalk[colors[depth + 1].command]('[command]') : [], cmd._args.length ? args : []).join(' '); + return `${parentCmdNames}${chalk[colors[depth].command](cmdName)} ${cmdUsage}`; + }, + subcommandTerm(cmd) { + const depth = cmd.depth(); + const args = cmd._args.map(arg => humanReadableArgName(arg)).join(' '); + return chalk[colors[depth].command](cmd._name + (cmd._aliases[0] ? `|${cmd._aliases[0]}` : '')) + chalk[colors[depth].option](cmd.options.length ? ' [options]' : '') + chalk[colors[depth].arg](args ? ` ${args}` : ''); + }, + longestOptionTermLength(cmd, helper) { + return helper.visibleOptions(cmd).reduce((max, option) => Math.max(max, helper.optionTerm(option).length - getControlCharacterSpaces(helper.optionTerm(option))), 0); + }, + longestSubcommandTermLength(cmd, helper) { + return helper.visibleCommands(cmd).reduce((max, command) => Math.max(max, helper.subcommandTerm(command).length - getControlCharacterSpaces(helper.subcommandTerm(command))), 0); + }, + longestArgumentTermLength(cmd, helper) { + return helper.visibleArguments(cmd).reduce((max, argument) => Math.max(max, helper.argumentTerm(argument).length - getControlCharacterSpaces(helper.argumentTerm(argument))), 0); + }, + formatHelp(cmd, helper) { + const depth = cmd.depth(); + const termWidth = helper.padWidth(cmd, helper); + const helpWidth = helper.helpWidth || 80; + const itemIndentWidth = 2; + const itemSeparatorWidth = 2; + function formatItem(term, description) { + const padding = ' '.repeat(termWidth + itemSeparatorWidth - (term.length - getControlCharacterSpaces(term))); + if (description) { + const fullText = `${term}${padding}${description}`; + return helper.wrap(fullText, helpWidth - itemIndentWidth, termWidth + itemSeparatorWidth); + } + return term; + } + function formatList(textArray) { + return textArray.join('\n').replace(/^/gm, ' '.repeat(itemIndentWidth)); + } + let output = [`Usage: ${helper.commandUsage(cmd)}`, '']; + const commandDescription = helper.commandDescription(cmd); + if (commandDescription.length > 0) { + output = output.concat([commandDescription, '']); + } + const argumentList = helper.visibleArguments(cmd).map(argument => formatItem(chalk[colors[depth].arg](argument.term), argument.description)); + if (argumentList.length > 0) { + output = output.concat(['Arguments:', formatList(argumentList), '']); + } + const optionList = helper.visibleOptions(cmd).map(option => formatItem(chalk[colors[depth].option](helper.optionTerm(option)), helper.optionDescription(option))); + if (optionList.length > 0) { + output = output.concat(['Options:', formatList(optionList), '']); + } + const commandList = helper.visibleCommands(cmd).map(cmd => formatItem(helper.subcommandTerm(cmd), helper.subcommandDescription(cmd))); + if (commandList.length > 0) { + output = output.concat(['Commands:', formatList(commandList), '']); + } + return output.join('\n'); + } +}; \ No newline at end of file diff --git a/lib/cli/index.js b/lib/cli/index.js new file mode 100644 index 0000000000..a1cd192350 --- /dev/null +++ b/lib/cli/index.js @@ -0,0 +1,215 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +require('../../require-main'); +const packageInstall = require('./package-install'); +const { + paths +} = require('../constants'); +try { + fs.accessSync(paths.currentPackage, fs.constants.R_OK); + try { + fs.accessSync(paths.nodeModules, fs.constants.R_OK); + } catch (e) { + if (e.code === 'ENOENT') { + packageInstall.installAll(); + } else { + throw e; + } + } + fs.accessSync(path.join(paths.nodeModules, 'semver/package.json'), fs.constants.R_OK); + const semver = require('semver'); + const defaultPackage = require('../../install/package.json'); + const checkVersion = function (packageName) { + const { + version + } = JSON.parse(fs.readFileSync(path.join(paths.nodeModules, packageName, 'package.json'), 'utf8')); + if (!semver.satisfies(version, defaultPackage.dependencies[packageName])) { + const e = new TypeError(`Incorrect dependency version: ${packageName}`); + e.code = 'DEP_WRONG_VERSION'; + throw e; + } + }; + checkVersion('nconf'); + checkVersion('async'); + checkVersion('commander'); + checkVersion('chalk'); + checkVersion('lodash'); + checkVersion('lru-cache'); +} catch (e) { + if (['ENOENT', 'DEP_WRONG_VERSION', 'MODULE_NOT_FOUND'].includes(e.code)) { + console.warn('Dependencies outdated or not yet installed.'); + console.log('Installing them now...\n'); + packageInstall.updatePackageFile(); + packageInstall.preserveExtraneousPlugins(); + packageInstall.installAll(); + const packages = ['nconf', 'async', 'commander', 'chalk', 'lodash', 'lru-cache']; + packages.forEach(packageName => { + const resolvedModule = require.resolve(packageName); + if (require.cache[resolvedModule]) { + delete require.cache[resolvedModule]; + } + }); + const chalk = require('chalk'); + console.log(`${chalk.green('OK')}\n`); + } else { + throw e; + } +} +const chalk = require('chalk'); +const nconf = require('nconf'); +const { + program +} = require('commander'); +const yargs = require('yargs'); +const pkg = require('../../install/package.json'); +const file = require('../file'); +const prestart = require('../prestart'); +program.configureHelp(require('./colors')); +program.name('./nodebb').description('Welcome to NodeBB').version(pkg.version).option('--json-logging', 'Output to logs in JSON format', false).option('--log-level ', 'Default logging level to use', 'info').option('--config ', 'Specify a config file', 'config.json').option('-d, --dev', 'Development mode, including verbose logging', false).option('-l, --log', 'Log subprocess output to console', false); +const opts = yargs(process.argv.slice(2)).help(false).exitProcess(false); +nconf.argv(opts).env({ + separator: '__' +}); +process.env.NODE_ENV = process.env.NODE_ENV || 'production'; +global.env = process.env.NODE_ENV || 'production'; +prestart.setupWinston(); +const configFile = path.resolve(paths.baseDir, nconf.get('config') || 'config.json'); +const configExists = file.existsSync(configFile) || nconf.get('url') && nconf.get('secret') && nconf.get('database'); +prestart.loadConfig(configFile); +prestart.versionCheck(); +if (!configExists && process.argv[2] !== 'setup') { + require('./setup').webInstall(); +} +if (configExists) { + process.env.CONFIG = configFile; +} +program.command('start').description('Start the NodeBB server').action(() => { + require('./running').start(program.opts()); +}); +program.command('slog', null, { + noHelp: true +}).description('Start the NodeBB server and view the live output log').action(() => { + require('./running').start({ + ...program.opts(), + log: true + }); +}); +program.command('dev', null, { + noHelp: true +}).description('Start NodeBB in verbose development mode').action(() => { + process.env.NODE_ENV = 'development'; + global.env = 'development'; + require('./running').start({ + ...program.opts(), + dev: true + }); +}); +program.command('stop').description('Stop the NodeBB server').action(() => { + require('./running').stop(program.opts()); +}); +program.command('restart').description('Restart the NodeBB server').action(() => { + require('./running').restart(program.opts()); +}); +program.command('status').description('Check the running status of the NodeBB server').action(() => { + require('./running').status(program.opts()); +}); +program.command('log').description('Open the output log (useful for debugging)').action(() => { + require('./running').log(program.opts()); +}); +program.command('setup [config]').description('Run the NodeBB setup script, or setup with an initial config').option('--skip-build', 'Run setup without building assets').action(initConfig => { + if (initConfig) { + try { + initConfig = JSON.parse(initConfig); + } catch (e) { + console.warn(chalk.red('Invalid JSON passed as initial config value.')); + console.log('If you meant to pass in an initial config value, please try again.\n'); + throw e; + } + } + require('./setup').setup(initConfig); +}); +program.command('install [plugin]').description('Launch the NodeBB web installer for configuration setup or install a plugin').option('-f, --force', 'Force plugin installation even if it may be incompatible with currently installed NodeBB version').action((plugin, options) => { + if (plugin) { + require('./manage').install(plugin, options); + } else { + require('./setup').webInstall(); + } +}); +program.command('build [targets...]').description(`Compile static assets ${chalk.red('(JS, CSS, templates, languages)')}`).option('-s, --series', 'Run builds in series without extra processes').option('-w, --webpack', 'Bundle assets with webpack', true).action((targets, options) => { + if (program.opts().dev) { + process.env.NODE_ENV = 'development'; + global.env = 'development'; + } + require('./manage').build(targets.length ? targets : true, options); +}).on('--help', () => { + require('../meta/aliases').buildTargets(); +}); +program.command('activate [plugin]').description('Activate a plugin for the next startup of NodeBB (nodebb-plugin- prefix is optional)').action(plugin => { + require('./manage').activate(plugin); +}); +program.command('plugins').action(() => { + require('./manage').listPlugins(); +}).description('List all installed plugins'); +program.command('events [count]').description('Outputs the most recent administrative events recorded by NodeBB').action(count => { + require('./manage').listEvents(count); +}); +program.command('info').description('Outputs various system info').action(() => { + require('./manage').info(); +}); +program.command('maintenance ').description('Toggle maintenance mode true/false').action(toggle => { + require('./manage').maintenance(toggle); +}); +const resetCommand = program.command('reset'); +resetCommand.description('Reset plugins, themes, settings, etc').option('-t, --theme [theme]', 'Reset to [theme] or to the default theme').option('-p, --plugin [plugin]', 'Disable [plugin] or all plugins').option('-w, --widgets', 'Disable all widgets').option('-s, --settings', 'Reset settings to their default values').option('-a, --all', 'All of the above').action(options => { + const valid = ['theme', 'plugin', 'widgets', 'settings', 'all'].some(x => options[x]); + if (!valid) { + console.warn(`\n${chalk.red('No valid options passed in, so nothing was reset.')}`); + resetCommand.help(); + } + require('./reset').reset(options, err => { + if (err) { + return process.exit(1); + } + process.exit(0); + }); +}); +program.addCommand(require('./user')()); +program.command('upgrade [scripts...]').description('Run NodeBB upgrade scripts and ensure packages are up-to-date, or run a particular upgrade script').option('-m, --package', 'Update package.json from defaults', false).option('-i, --install', 'Bringing base dependencies up to date', false).option('-p, --plugins', 'Check installed plugins for updates', false).option('-s, --schema', 'Update NodeBB data store schema', false).option('-b, --build', 'Rebuild assets', false).on('--help', () => { + console.log(`\n${['When running particular upgrade scripts, options are ignored.', 'By default all options are enabled. Passing any options disables that default.', '\nExamples:', ` Only package and dependency updates: ${chalk.yellow('./nodebb upgrade -mi')}`, ` Only database update: ${chalk.yellow('./nodebb upgrade -s')}`].join('\n')}`); +}).action((scripts, options) => { + if (program.opts().dev) { + process.env.NODE_ENV = 'development'; + global.env = 'development'; + } + require('./upgrade').upgrade(scripts.length ? scripts : true, options); +}); +program.command('upgrade-plugins', null, { + noHelp: true +}).alias('upgradePlugins').description('Upgrade plugins').action(() => { + require('./upgrade-plugins').upgradePlugins(err => { + if (err) { + throw err; + } + console.log(chalk.green('OK')); + process.exit(); + }); +}); +program.command('help [command]').description('Display help for [command]').action(name => { + if (!name) { + return program.help(); + } + const command = program.commands.find(command => command._name === name); + if (command) { + command.help(); + } else { + console.log(`error: unknown command '${command}'.`); + program.help(); + } +}); +if (process.argv.length === 2) { + program.help(); +} +program.executables = false; +program.parse(); \ No newline at end of file diff --git a/lib/cli/manage.js b/lib/cli/manage.js new file mode 100644 index 0000000000..f1cc6fab1a --- /dev/null +++ b/lib/cli/manage.js @@ -0,0 +1,195 @@ +'use strict'; + +const winston = require('winston'); +const childProcess = require('child_process'); +const CliGraph = require('cli-graph'); +const chalk = require('chalk'); +const nconf = require('nconf'); +const build = require('../meta/build'); +const db = require('../database'); +const plugins = require('../plugins'); +const events = require('../events'); +const analytics = require('../analytics'); +const reset = require('./reset'); +const { + pluginNamePattern, + themeNamePattern, + paths +} = require('../constants'); +async function install(plugin, options) { + if (!options) { + options = {}; + } + try { + await db.init(); + if (!pluginNamePattern.test(plugin)) { + plugin = `nodebb-plugin-${plugin}`; + } + plugin = await plugins.autocomplete(plugin); + const isInstalled = await plugins.isInstalled(plugin); + if (isInstalled) { + throw new Error('plugin already installed'); + } + const nbbVersion = require(paths.currentPackage).version; + const suggested = await plugins.suggest(plugin, nbbVersion); + if (!suggested.version) { + if (!options.force) { + throw new Error(suggested.message); + } + winston.warn(`${suggested.message} Proceeding with installation anyway due to force option being provided`); + suggested.version = 'latest'; + } + winston.info('Installing Plugin `%s@%s`', plugin, suggested.version); + await plugins.toggleInstall(plugin, suggested.version); + process.exit(0); + } catch (err) { + winston.error(`An error occurred during plugin installation\n${err.stack}`); + process.exit(1); + } +} +async function activate(plugin) { + if (themeNamePattern.test(plugin)) { + await reset.reset({ + theme: plugin + }); + process.exit(); + } + try { + await db.init(); + if (!pluginNamePattern.test(plugin)) { + plugin = `nodebb-plugin-${plugin}`; + } + plugin = await plugins.autocomplete(plugin); + const isInstalled = await plugins.isInstalled(plugin); + if (!isInstalled) { + throw new Error('plugin not installed'); + } + const isActive = await plugins.isActive(plugin); + if (isActive) { + winston.info('Plugin `%s` already active', plugin); + process.exit(0); + } + if (nconf.get('plugins:active')) { + winston.error('Cannot activate plugins while plugin state configuration is set, please change your active configuration (config.json, environmental variables or terminal arguments) instead'); + process.exit(1); + } + const numPlugins = await db.sortedSetCard('plugins:active'); + winston.info('Activating plugin `%s`', plugin); + await db.sortedSetAdd('plugins:active', numPlugins, plugin); + await events.log({ + type: 'plugin-activate', + text: plugin + }); + process.exit(0); + } catch (err) { + winston.error(`An error occurred during plugin activation\n${err.stack}`); + process.exit(1); + } +} +async function listPlugins() { + await db.init(); + const installed = await plugins.showInstalled(); + const installedList = installed.map(plugin => plugin.name); + const active = await plugins.getActive(); + const combined = installed.concat(active.reduce((memo, cur) => { + if (!installedList.includes(cur)) { + memo.push({ + id: cur, + active: true, + installed: false + }); + } + return memo; + }, [])); + combined.sort((a, b) => a.id > b.id ? 1 : -1); + process.stdout.write('Active plugins:\n'); + combined.forEach(plugin => { + process.stdout.write(`\t* ${plugin.id}${plugin.version ? `@${plugin.version}` : ''} (`); + process.stdout.write(plugin.installed ? chalk.green('installed') : chalk.red('not installed')); + process.stdout.write(', '); + process.stdout.write(plugin.active ? chalk.green('enabled') : chalk.yellow('disabled')); + process.stdout.write(')\n'); + }); + process.exit(); +} +async function listEvents(count = 10) { + await db.init(); + const eventData = await events.getEvents({ + filter: '', + start: 0, + stop: count - 1 + }); + console.log(chalk.bold(`\nDisplaying last ${count} administrative events...`)); + eventData.forEach(event => { + console.log(` * ${chalk.green(String(event.timestampISO))} ${chalk.yellow(String(event.type))}${event.text ? ` ${event.text}` : ''} (uid: ${event.uid ? event.uid : 0})`); + }); + process.exit(); +} +async function info() { + console.log(''); + const { + version + } = require('../../package.json'); + console.log(` version: ${version}`); + console.log(` Node ver: ${process.version}`); + const hash = childProcess.execSync('git rev-parse HEAD'); + console.log(` git hash: ${hash}`); + console.log(` database: ${nconf.get('database')}`); + await db.init(); + const info = await db.info(db.client); + switch (nconf.get('database')) { + case 'redis': + console.log(` version: ${info.redis_version}`); + console.log(` disk sync: ${info.rdb_last_bgsave_status}`); + break; + case 'mongo': + console.log(` version: ${info.version}`); + console.log(` engine: ${info.storageEngine}`); + break; + case 'postgres': + console.log(` version: ${info.version}`); + console.log(` uptime: ${info.uptime}`); + break; + } + const analyticsData = await analytics.getHourlyStatsForSet('analytics:pageviews', Date.now(), 24); + const graph = new CliGraph({ + height: 12, + width: 25, + center: { + x: 0, + y: 11 + } + }); + const min = Math.min(...analyticsData); + const max = Math.max(...analyticsData); + analyticsData.forEach((point, idx) => { + graph.addPoint(idx + 1, Math.round(point / max * 10)); + }); + console.log(''); + console.log(graph.toString()); + console.log(`Pageviews, last 24h (min: ${min} max: ${max})`); + process.exit(); +} +async function maintenance(toggle) { + const turnOnMaintenance = toggle === 'true'; + await db.init(); + await db.setObjectField('config', 'maintenanceMode', turnOnMaintenance ? 1 : 0); + console.log(`Maintenance mode turned ${turnOnMaintenance ? 'on' : 'off'}`); + process.exit(); +} +async function buildWrapper(targets, options) { + try { + await build.build(targets, options); + process.exit(0); + } catch (err) { + winston.error(err.stack); + process.exit(1); + } +} +exports.build = buildWrapper; +exports.install = install; +exports.activate = activate; +exports.listPlugins = listPlugins; +exports.listEvents = listEvents; +exports.info = info; +exports.maintenance = maintenance; \ No newline at end of file diff --git a/lib/cli/package-install.js b/lib/cli/package-install.js new file mode 100644 index 0000000000..a7efb40f3b --- /dev/null +++ b/lib/cli/package-install.js @@ -0,0 +1,147 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const cproc = require('child_process'); +const { + paths, + pluginNamePattern +} = require('../constants'); +const pkgInstall = module.exports; +function sortDependencies(dependencies) { + return Object.entries(dependencies).sort((a, b) => a < b ? -1 : 1).reduce((memo, pkg) => { + memo[pkg[0]] = pkg[1]; + return memo; + }, {}); +} +pkgInstall.updatePackageFile = () => { + let oldPackageContents; + try { + oldPackageContents = JSON.parse(fs.readFileSync(paths.currentPackage, 'utf8')); + } catch (e) { + if (e.code !== 'ENOENT') { + throw e; + } else { + fs.copyFileSync(paths.installPackage, paths.currentPackage); + return; + } + } + const _ = require('lodash'); + const defaultPackageContents = JSON.parse(fs.readFileSync(paths.installPackage, 'utf8')); + let dependencies = {}; + Object.entries(oldPackageContents.dependencies || {}).forEach(([dep, version]) => { + if (pluginNamePattern.test(dep)) { + dependencies[dep] = version; + } + }); + const { + devDependencies + } = defaultPackageContents; + dependencies = sortDependencies({ + ...dependencies, + ...defaultPackageContents.dependencies + }); + const packageContents = { + ..._.merge(oldPackageContents, defaultPackageContents), + dependencies, + devDependencies + }; + fs.writeFileSync(paths.currentPackage, JSON.stringify(packageContents, null, 4)); +}; +pkgInstall.supportedPackageManager = ['npm', 'cnpm', 'pnpm', 'yarn']; +pkgInstall.getPackageManager = () => { + try { + const packageContents = require(paths.currentPackage); + const pmRegex = new RegExp(`^(?${pkgInstall.supportedPackageManager.join('|')})@?[\\d\\w\\.\\-]*$`); + const packageManager = packageContents.packageManager ? packageContents.packageManager.match(pmRegex) : false; + if (packageManager) { + return packageManager.groups.packageManager; + } + fs.accessSync(path.join(paths.nodeModules, 'nconf/package.json'), fs.constants.R_OK); + const nconf = require('nconf'); + if (!Object.keys(nconf.stores).length) { + const configFile = path.resolve(__dirname, '../../', nconf.any(['config', 'CONFIG']) || 'config.json'); + nconf.env().file({ + file: configFile + }); + } + if (nconf.get('package_manager') && !pkgInstall.supportedPackageManager.includes(nconf.get('package_manager'))) { + nconf.clear('package_manager'); + } + if (!nconf.get('package_manager')) { + nconf.set('package_manager', getPackageManagerByLockfile()); + } + return nconf.get('package_manager') || 'npm'; + } catch (e) { + return getPackageManagerByLockfile() || 'npm'; + } +}; +function getPackageManagerByLockfile() { + for (const [packageManager, lockfile] of Object.entries({ + npm: 'package-lock.json', + yarn: 'yarn.lock', + pnpm: 'pnpm-lock.yaml' + })) { + try { + fs.accessSync(path.resolve(__dirname, `../../${lockfile}`), fs.constants.R_OK); + return packageManager; + } catch (e) {} + } +} +pkgInstall.installAll = () => { + const prod = process.env.NODE_ENV !== 'development'; + let command = 'npm install'; + const supportedPackageManagerList = exports.supportedPackageManager; + const packageManager = pkgInstall.getPackageManager(); + if (supportedPackageManagerList.indexOf(packageManager) >= 0) { + switch (packageManager) { + case 'yarn': + command = `yarn${prod ? ' --production' : ''}`; + break; + case 'pnpm': + command = 'pnpm install'; + break; + case 'cnpm': + command = `cnpm install ${prod ? ' --production' : ''}`; + break; + default: + command += prod ? ' --omit=dev' : ''; + break; + } + } + try { + cproc.execSync(command, { + cwd: path.join(__dirname, '../../'), + stdio: [0, 1, 2] + }); + } catch (e) { + console.log('Error installing dependencies!'); + console.log(`message: ${e.message}`); + console.log(`stdout: ${e.stdout}`); + console.log(`stderr: ${e.stderr}`); + throw e; + } +}; +pkgInstall.preserveExtraneousPlugins = () => { + try { + fs.accessSync(paths.nodeModules, fs.constants.R_OK); + } catch (e) { + return; + } + const packages = fs.readdirSync(paths.nodeModules).filter(pkgName => pluginNamePattern.test(pkgName)); + const packageContents = JSON.parse(fs.readFileSync(paths.currentPackage, 'utf8')); + const extraneous = packages.filter(pkgName => { + const extraneous = !packageContents.dependencies.hasOwnProperty(pkgName); + const isLink = fs.lstatSync(path.join(paths.nodeModules, pkgName)).isSymbolicLink(); + return extraneous && !isLink; + }).reduce((map, pkgName) => { + const pkgConfig = JSON.parse(fs.readFileSync(path.join(paths.nodeModules, pkgName, 'package.json'), 'utf8')); + map[pkgName] = pkgConfig.version; + return map; + }, {}); + packageContents.dependencies = sortDependencies({ + ...packageContents.dependencies, + ...extraneous + }); + fs.writeFileSync(paths.currentPackage, JSON.stringify(packageContents, null, 4)); +}; \ No newline at end of file diff --git a/lib/cli/reset.js b/lib/cli/reset.js new file mode 100644 index 0000000000..fe4ae84f67 --- /dev/null +++ b/lib/cli/reset.js @@ -0,0 +1,131 @@ +'use strict'; + +const path = require('path'); +const winston = require('winston'); +const fs = require('fs'); +const chalk = require('chalk'); +const nconf = require('nconf'); +const db = require('../database'); +const events = require('../events'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const widgets = require('../widgets'); +const privileges = require('../privileges'); +const { + paths, + pluginNamePattern, + themeNamePattern +} = require('../constants'); +exports.reset = async function (options) { + const map = { + theme: async function () { + let themeId = options.theme; + if (themeId === true) { + await resetThemes(); + } else { + if (!themeNamePattern.test(themeId)) { + themeId = `nodebb-theme-${themeId}`; + } + themeId = await plugins.autocomplete(themeId); + await resetTheme(themeId); + } + }, + plugin: async function () { + let pluginId = options.plugin; + if (pluginId === true) { + await resetPlugins(); + } else { + if (!pluginNamePattern.test(pluginId)) { + pluginId = `nodebb-plugin-${pluginId}`; + } + pluginId = await plugins.autocomplete(pluginId); + await resetPlugin(pluginId); + } + }, + widgets: resetWidgets, + settings: resetSettings, + all: async function () { + await resetWidgets(); + await resetThemes(); + await resetPlugin(); + await resetSettings(); + } + }; + const tasks = Object.keys(map).filter(x => options[x]).map(x => map[x]); + if (!tasks.length) { + console.log([chalk.yellow('No arguments passed in, so nothing was reset.\n'), `Use ./nodebb reset ${chalk.red('{-t|-p|-w|-s|-a}')}`, ' -t\tthemes', ' -p\tplugins', ' -w\twidgets', ' -s\tsettings', ' -a\tall of the above', '', 'Plugin and theme reset flags (-p & -t) can take a single argument', ' e.g. ./nodebb reset -p nodebb-plugin-mentions, ./nodebb reset -t nodebb-theme-harmony', ' Prefix is optional, e.g. ./nodebb reset -p markdown, ./nodebb reset -t harmony'].join('\n')); + process.exit(0); + } + try { + await db.init(); + for (const task of tasks) { + await task(); + } + winston.info('[reset] Reset complete. Please run `./nodebb build` to rebuild assets.'); + process.exit(0); + } catch (err) { + winston.error(`[reset] Errors were encountered during reset -- ${err.message}`); + process.exit(1); + } +}; +async function resetSettings() { + await privileges.global.give(['groups:local:login'], 'registered-users'); + winston.info('[reset] registered-users given login privilege'); + winston.info('[reset] Settings reset to default'); +} +async function resetTheme(themeId) { + try { + await fs.promises.access(path.join(paths.nodeModules, themeId, 'package.json')); + } catch (err) { + winston.warn('[reset] Theme `%s` is not installed on this forum', themeId); + throw new Error('theme-not-found'); + } + await resetThemeTo(themeId); +} +async function resetThemes() { + await resetThemeTo('nodebb-theme-harmony'); +} +async function resetThemeTo(themeId) { + await meta.themes.set({ + type: 'local', + id: themeId + }); + await meta.configs.set('bootswatchSkin', ''); + winston.info(`[reset] Theme reset to ${themeId} and default skin`); +} +async function resetPlugin(pluginId) { + try { + if (nconf.get('plugins:active')) { + winston.error('Cannot reset plugins while plugin state is set in the configuration (config.json, environmental variables or terminal arguments), please modify the configuration instead'); + process.exit(1); + } + const isActive = await db.isSortedSetMember('plugins:active', pluginId); + if (isActive) { + await db.sortedSetRemove('plugins:active', pluginId); + await events.log({ + type: 'plugin-deactivate', + text: pluginId + }); + winston.info('[reset] Plugin `%s` disabled', pluginId); + } else { + winston.warn('[reset] Plugin `%s` was not active on this forum', pluginId); + winston.info('[reset] No action taken.'); + } + } catch (err) { + winston.error(`[reset] Could not disable plugin: ${pluginId} encountered error %s\n${err.stack}`); + throw err; + } +} +async function resetPlugins() { + if (nconf.get('plugins:active')) { + winston.error('Cannot reset plugins while plugin state is set in the configuration (config.json, environmental variables or terminal arguments), please modify the configuration instead'); + process.exit(1); + } + await db.delete('plugins:active'); + winston.info('[reset] All Plugins De-activated'); +} +async function resetWidgets() { + await plugins.reload(); + await widgets.reset(); + winston.info('[reset] All Widgets moved to Draft Zone'); +} \ No newline at end of file diff --git a/lib/cli/running.js b/lib/cli/running.js new file mode 100644 index 0000000000..8f89987cc3 --- /dev/null +++ b/lib/cli/running.js @@ -0,0 +1,97 @@ +'use strict'; + +const fs = require('fs'); +const childProcess = require('child_process'); +const chalk = require('chalk'); +const fork = require('../meta/debugFork'); +const { + paths +} = require('../constants'); +const cwd = paths.baseDir; +function getRunningPid(callback) { + fs.readFile(paths.pidfile, { + encoding: 'utf-8' + }, (err, pid) => { + if (err) { + return callback(err); + } + pid = parseInt(pid, 10); + try { + process.kill(pid, 0); + callback(null, pid); + } catch (e) { + callback(e); + } + }); +} +function start(options) { + if (options.dev) { + process.env.NODE_ENV = 'development'; + fork(paths.loader, ['--no-daemon', '--no-silent'], { + env: process.env, + stdio: 'inherit', + cwd + }); + return; + } + if (options.log) { + console.log(`\n${[chalk.bold('Starting NodeBB with logging output'), chalk.red('Hit ') + chalk.bold('Ctrl-C ') + chalk.red('to exit'), 'The NodeBB process will continue to run in the background', `Use "${chalk.yellow('./nodebb stop')}" to stop the NodeBB server`].join('\n')}`); + } else if (!options.silent) { + console.log(`\n${[chalk.bold('Starting NodeBB'), ` "${chalk.yellow('./nodebb stop')}" to stop the NodeBB server`, ` "${chalk.yellow('./nodebb log')}" to view server output`, ` "${chalk.yellow('./nodebb help')}" for more commands\n`].join('\n')}`); + } + const child = fork(paths.loader, process.argv.slice(3), { + env: process.env, + cwd + }); + if (options.log) { + childProcess.spawn('tail', ['-F', './logs/output.log'], { + stdio: 'inherit', + cwd + }); + } + return child; +} +function stop() { + getRunningPid((err, pid) => { + if (!err) { + process.kill(pid, 'SIGTERM'); + console.log('Stopping NodeBB. Goodbye!'); + } else { + console.log('NodeBB is already stopped.'); + } + }); +} +function restart(options) { + getRunningPid((err, pid) => { + if (!err) { + console.log(chalk.bold('\nRestarting NodeBB')); + process.kill(pid, 'SIGTERM'); + options.silent = true; + start(options); + } else { + console.warn('NodeBB could not be restarted, as a running instance could not be found.'); + } + }); +} +function status() { + getRunningPid((err, pid) => { + if (!err) { + console.log(`\n${[chalk.bold('NodeBB Running ') + chalk.cyan(`(pid ${pid.toString()})`), `\t"${chalk.yellow('./nodebb stop')}" to stop the NodeBB server`, `\t"${chalk.yellow('./nodebb log')}" to view server output`, `\t"${chalk.yellow('./nodebb restart')}" to restart NodeBB\n`].join('\n')}`); + } else { + console.log(chalk.bold('\nNodeBB is not running')); + console.log(`\t"${chalk.yellow('./nodebb start')}" to launch the NodeBB server\n`); + } + }); +} +function log() { + console.log(`${chalk.red('\nHit ') + chalk.bold('Ctrl-C ') + chalk.red('to exit\n')}\n`); + childProcess.spawn('tail', ['-F', './logs/output.log'], { + stdio: 'inherit', + cwd + }); +} +exports.start = start; +exports.stop = stop; +exports.restart = restart; +exports.status = status; +exports.log = log; \ No newline at end of file diff --git a/lib/cli/setup.js b/lib/cli/setup.js new file mode 100644 index 0000000000..0f6c017a92 --- /dev/null +++ b/lib/cli/setup.js @@ -0,0 +1,53 @@ +'use strict'; + +const winston = require('winston'); +const path = require('path'); +const nconf = require('nconf'); +const { + install +} = require('../../install/web'); +async function setup(initConfig) { + const { + paths + } = require('../constants'); + const install = require('../install'); + const build = require('../meta/build'); + const prestart = require('../prestart'); + const pkg = require('../../package.json'); + winston.info('NodeBB Setup Triggered via Command Line'); + console.log(`\nWelcome to NodeBB v${pkg.version}!`); + console.log('\nThis looks like a new installation, so you\'ll have to answer a few questions about your environment before we can proceed.'); + console.log('Press enter to accept the default setting (shown in brackets).'); + install.values = initConfig; + let configFile = paths.config; + const config = nconf.any(['config', 'CONFIG']); + if (config) { + nconf.set('config', config); + configFile = path.resolve(paths.baseDir, config); + } + const data = await install.setup(); + prestart.loadConfig(configFile); + if (!nconf.get('skip-build')) { + await build.buildAll(); + } + let separator = ' '; + if (process.stdout.columns > 10) { + for (let x = 0, cols = process.stdout.columns - 10; x < cols; x += 1) { + separator += '='; + } + } + console.log(`\n${separator}\n`); + if (data.hasOwnProperty('password')) { + console.log('An administrative user was automatically created for you:'); + console.log(` Username: ${data.username}`); + console.log(` Password: ${data.password}`); + console.log(''); + } + console.log('NodeBB Setup Completed. Run "./nodebb start" to manually start your NodeBB server.'); + if (process.send) { + process.send(data); + } + process.exit(); +} +exports.setup = setup; +exports.webInstall = install; \ No newline at end of file diff --git a/lib/cli/upgrade-plugins.js b/lib/cli/upgrade-plugins.js new file mode 100644 index 0000000000..d850cbd33c --- /dev/null +++ b/lib/cli/upgrade-plugins.js @@ -0,0 +1,138 @@ +'use strict'; + +const prompt = require('prompt'); +const cproc = require('child_process'); +const semver = require('semver'); +const fs = require('fs'); +const path = require('path'); +const chalk = require('chalk'); +const { + paths, + pluginNamePattern +} = require('../constants'); +const pkgInstall = require('./package-install'); +const packageManager = pkgInstall.getPackageManager(); +let packageManagerExecutable = packageManager; +const packageManagerInstallArgs = packageManager === 'yarn' ? ['add'] : ['install', '--save']; +if (process.platform === 'win32') { + packageManagerExecutable += '.cmd'; +} +async function getModuleVersions(modules) { + const versionHash = {}; + const batch = require('../batch'); + await batch.processArray(modules, async moduleNames => { + await Promise.all(moduleNames.map(async module => { + let pkg = await fs.promises.readFile(path.join(paths.nodeModules, module, 'package.json'), { + encoding: 'utf-8' + }); + pkg = JSON.parse(pkg); + versionHash[module] = pkg.version; + })); + }, { + batch: 50 + }); + return versionHash; +} +async function getInstalledPlugins() { + let [deps, bundled] = await Promise.all([fs.promises.readFile(paths.currentPackage, { + encoding: 'utf-8' + }), fs.promises.readFile(paths.installPackage, { + encoding: 'utf-8' + })]); + deps = Object.keys(JSON.parse(deps).dependencies).filter(pkgName => pluginNamePattern.test(pkgName)); + bundled = Object.keys(JSON.parse(bundled).dependencies).filter(pkgName => pluginNamePattern.test(pkgName)); + const checklist = deps.filter(pkgName => { + if (bundled.includes(pkgName)) { + return false; + } + try { + fs.accessSync(path.join(paths.nodeModules, pkgName, '.git')); + return false; + } catch (e) { + return true; + } + }); + return await getModuleVersions(checklist); +} +async function getCurrentVersion() { + let pkg = await fs.promises.readFile(paths.installPackage, { + encoding: 'utf-8' + }); + pkg = JSON.parse(pkg); + return pkg.version; +} +async function getSuggestedModules(nbbVersion, toCheck) { + const request = require('../request'); + let { + response, + body + } = await request.get(`https://packages.nodebb.org/api/v1/suggest?version=${nbbVersion}&package[]=${toCheck.join('&package[]=')}`); + if (!response.ok) { + throw new Error(`Unable to get suggested module for NodeBB(${nbbVersion}) ${toCheck.join(',')}`); + } + if (!Array.isArray(body) && toCheck.length === 1) { + body = [body]; + } + return body; +} +async function checkPlugins() { + process.stdout.write('Checking installed plugins and themes for updates... '); + const [plugins, nbbVersion] = await Promise.all([getInstalledPlugins(), getCurrentVersion()]); + const toCheck = Object.keys(plugins); + if (!toCheck.length) { + process.stdout.write(chalk.green(' OK')); + return []; + } + const suggestedModules = await getSuggestedModules(nbbVersion, toCheck); + process.stdout.write(chalk.green(' OK')); + let current; + let suggested; + const upgradable = suggestedModules.map(suggestObj => { + current = plugins[suggestObj.package]; + suggested = suggestObj.version; + if (suggestObj.code === 'match-found' && semver.valid(current) && semver.valid(suggested) && semver.gt(suggested, current)) { + return { + name: suggestObj.package, + current: current, + suggested: suggested + }; + } + return null; + }).filter(Boolean); + return upgradable; +} +async function upgradePlugins() { + try { + const found = await checkPlugins(); + if (found && found.length) { + process.stdout.write(`\n\nA total of ${chalk.bold(String(found.length))} package(s) can be upgraded:\n\n`); + found.forEach(suggestObj => { + process.stdout.write(`${chalk.yellow(' * ') + suggestObj.name} (${chalk.yellow(suggestObj.current)} -> ${chalk.green(suggestObj.suggested)})\n`); + }); + } else { + console.log(chalk.green('\nAll packages up-to-date!')); + return; + } + prompt.message = ''; + prompt.delimiter = ''; + prompt.start(); + const result = await prompt.get({ + name: 'upgrade', + description: '\nProceed with upgrade (y|n)?', + type: 'string' + }); + if (['y', 'Y', 'yes', 'YES'].includes(result.upgrade)) { + console.log('\nUpgrading packages...'); + const args = packageManagerInstallArgs.concat(found.map(suggestObj => `${suggestObj.name}@${suggestObj.suggested}`)); + cproc.execFileSync(packageManagerExecutable, args, { + stdio: 'ignore' + }); + } else { + console.log(`${chalk.yellow('Package upgrades skipped')}. Check for upgrades at any time by running "${chalk.green('./nodebb upgrade -p')}".`); + } + } catch (err) { + console.log(`${chalk.yellow('Warning')}: An unexpected error occured when attempting to verify plugin upgradability`); + throw err; + } +} +exports.upgradePlugins = upgradePlugins; \ No newline at end of file diff --git a/lib/cli/upgrade.js b/lib/cli/upgrade.js new file mode 100644 index 0000000000..549be5bcd3 --- /dev/null +++ b/lib/cli/upgrade.js @@ -0,0 +1,94 @@ +'use strict'; + +const nconf = require('nconf'); +const chalk = require('chalk'); +const packageInstall = require('./package-install'); +const { + upgradePlugins +} = require('./upgrade-plugins'); +const steps = { + package: { + message: 'Updating package.json file with defaults...', + handler: function () { + packageInstall.updatePackageFile(); + packageInstall.preserveExtraneousPlugins(); + process.stdout.write(chalk.green(' OK\n')); + } + }, + install: { + message: 'Bringing base dependencies up to date...', + handler: function () { + process.stdout.write(chalk.green(' started\n')); + packageInstall.installAll(); + } + }, + plugins: { + message: 'Checking installed plugins for updates...', + handler: async function () { + await require('../database').init(); + await upgradePlugins(); + } + }, + schema: { + message: 'Updating NodeBB data store schema...', + handler: async function () { + await require('../database').init(); + await require('../meta').configs.init(); + await require('../upgrade').run(); + } + }, + build: { + message: 'Rebuilding assets...', + handler: async function () { + await require('../meta/build').buildAll(); + } + } +}; +async function runSteps(tasks) { + try { + for (let i = 0; i < tasks.length; i++) { + const step = steps[tasks[i]]; + if (step && step.message && step.handler) { + process.stdout.write(`\n${chalk.bold(`${i + 1}. `)}${chalk.yellow(step.message)}`); + await step.handler(); + } + } + const message = 'NodeBB Upgrade Complete!'; + const { + columns + } = process.stdout; + const spaces = columns ? new Array(Math.floor(columns / 2) - message.length / 2 + 1).join(' ') : ' '; + console.log(`\n\n${spaces}${chalk.green.bold(message)}\n`); + process.exit(); + } catch (err) { + console.error(`Error occurred during upgrade: ${err.stack}`); + throw err; + } +} +async function runUpgrade(upgrades, options) { + const winston = require('winston'); + const path = require('path'); + winston.configure({ + transports: [new winston.transports.Console({ + handleExceptions: true + }), new winston.transports.File({ + filename: path.join(__dirname, '../../', nconf.get('logFile') || 'logs/output.log') + })] + }); + console.log(chalk.cyan('\nUpdating NodeBB...')); + options = options || {}; + nconf.set('mongo:options:socketTimeoutMS', 0); + if (upgrades === true) { + let tasks = Object.keys(steps); + if (options.package || options.install || options.plugins || options.schema || options.build) { + tasks = tasks.filter(key => options[key]); + } + await runSteps(tasks); + return; + } + await require('../database').init(); + await require('../meta').configs.init(); + await require('../upgrade').runParticular(upgrades); + process.exit(0); +} +exports.upgrade = runUpgrade; \ No newline at end of file diff --git a/lib/cli/user.js b/lib/cli/user.js new file mode 100644 index 0000000000..fe6901e28c --- /dev/null +++ b/lib/cli/user.js @@ -0,0 +1,239 @@ +'use strict'; + +const { + Command, + Option +} = require('commander'); +module.exports = () => { + const userCmd = new Command('user').description('Manage users').arguments('[command]'); + userCmd.configureHelp(require('./colors')); + const userCommands = UserCommands(); + userCmd.command('info').description('Display user info by uid/username/userslug.').option('-i, --uid ', 'Retrieve user by uid').option('-u, --username ', 'Retrieve user by username').option('-s, --userslug ', 'Retrieve user by userslug').action((...args) => execute(userCommands.info, args)); + userCmd.command('create').description('Create a new user.').arguments('').option('-p, --password ', 'Set a new password. (Auto-generates if omitted)').option('-e, --email ', 'Associate with an email.').action((...args) => execute(userCommands.create, args)); + userCmd.command('reset').description('Reset a user\'s password or send a password reset email.').arguments('').option('-p, --password ', 'Set a new password. (Auto-generates if passed empty)', false).option('-s, --send-reset-email', 'Send a password reset email.', false).action((...args) => execute(userCommands.reset, args)); + userCmd.command('delete').description('Delete user(s) and/or their content').arguments('').addOption(new Option('-t, --type [operation]', 'Delete user content ([purge]), leave content ([account]), or delete content only ([content])').choices(['purge', 'account', 'content']).default('purge')).action((...args) => execute(userCommands.deleteUser, args)); + const make = userCmd.command('make').description('Make user(s) admin, global mod, moderator or a regular user.').arguments('[command]'); + make.command('admin').description('Make user(s) an admin').arguments('').action((...args) => execute(userCommands.makeAdmin, args)); + make.command('global-mod').description('Make user(s) a global moderator').arguments('').action((...args) => execute(userCommands.makeGlobalMod, args)); + make.command('mod').description('Make uid(s) of user(s) moderator of given category IDs (cids)').arguments('').requiredOption('-c, --cid ', 'ID(s) of categories to make the user a moderator of').action((...args) => execute(userCommands.makeMod, args)); + make.command('regular').description('Make user(s) a non-privileged user').arguments('').action((...args) => execute(userCommands.makeRegular, args)); + return userCmd; +}; +let db; +let user; +let groups; +let privileges; +let privHelpers; +let utils; +let winston; +async function init() { + db = require('../database'); + await db.init(); + await db.initSessionStore(); + user = require('../user'); + groups = require('../groups'); + privileges = require('../privileges'); + privHelpers = require('../privileges/helpers'); + utils = require('../utils'); + winston = require('winston'); +} +async function execute(cmd, args) { + await init(); + try { + await cmd(...args); + } catch (err) { + const userError = err.name === 'UserError'; + winston.error(`[userCmd/${cmd.name}] ${userError ? `${err.message}` : 'Command failed.'}`, userError ? '' : err); + process.exit(1); + } + process.exit(); +} +function UserCmdHelpers() { + async function getAdminUidOrFail() { + const adminUid = await user.getFirstAdminUid(); + if (!adminUid) { + const err = new Error('An admin account does not exists to execute the operation.'); + err.name = 'UserError'; + throw err; + } + return adminUid; + } + async function setupApp() { + const nconf = require('nconf'); + const Benchpress = require('benchpressjs'); + const meta = require('../meta'); + await meta.configs.init(); + const webserver = require('../webserver'); + const viewsDir = nconf.get('views_dir'); + webserver.app.engine('tpl', (filepath, data, next) => { + filepath = filepath.replace(/\.tpl$/, '.js'); + Benchpress.__express(filepath, data, next); + }); + webserver.app.set('view engine', 'tpl'); + webserver.app.set('views', viewsDir); + const emailer = require('../emailer'); + emailer.registerApp(webserver.app); + } + const argParsers = { + intParse: (value, varName) => { + const parsedValue = parseInt(value, 10); + if (isNaN(parsedValue)) { + const err = new Error(`"${varName}" expected to be a number.`); + err.name = 'UserError'; + throw err; + } + return parsedValue; + }, + intArrayParse: (values, varName) => values.map(value => argParsers.intParse(value, varName)) + }; + return { + argParsers, + getAdminUidOrFail, + setupApp + }; +} +function UserCommands() { + const { + argParsers, + getAdminUidOrFail, + setupApp + } = UserCmdHelpers(); + async function info({ + uid, + username, + userslug + }) { + if (!uid && !username && !userslug) { + return winston.error('[userCmd/info] At least one option has to be passed (--uid, --username or --userslug).'); + } + if (uid) { + uid = argParsers.intParse(uid, 'uid'); + } else if (username) { + uid = await user.getUidByUsername(username); + } else { + uid = await user.getUidByUserslug(userslug); + } + const userData = await user.getUserData(uid); + winston.info('[userCmd/info] User info retrieved:'); + console.log(userData); + } + async function create(username, { + password, + email + }) { + let pwGenerated = false; + if (password === undefined) { + password = utils.generateUUID().slice(0, 8); + pwGenerated = true; + } + const userExists = await user.getUidByUsername(username); + if (userExists) { + return winston.error(`[userCmd/create] A user with username '${username}' already exists`); + } + const uid = await user.create({ + username, + password, + email + }); + winston.info(`[userCmd/create] User '${username}'${password ? '' : ' without a password'} has been created with uid: ${uid}.\ +${pwGenerated ? ` Generated password: ${password}` : ''}`); + } + async function reset(uid, { + password, + sendResetEmail + }) { + uid = argParsers.intParse(uid, 'uid'); + if (password === false && sendResetEmail === false) { + return winston.error('[userCmd/reset] At least one option has to be passed (--password or --send-reset-email).'); + } + const userExists = await user.exists(uid); + if (!userExists) { + return winston.error(`[userCmd/reset] A user with given uid does not exists.`); + } + let pwGenerated = false; + if (password === '') { + password = utils.generateUUID().slice(0, 8); + pwGenerated = true; + } + const adminUid = await getAdminUidOrFail(); + if (password) { + await user.setUserField(uid, 'password', ''); + await user.changePassword(adminUid, { + newPassword: password, + uid + }); + winston.info(`[userCmd/reset] ${password ? 'User password changed.' : ''}${pwGenerated ? ` Generated password: ${password}` : ''}`); + } + if (sendResetEmail) { + const userEmail = await user.getUserField(uid, 'email'); + if (!userEmail) { + return winston.error('User doesn\'t have an email address to send reset email.'); + } + await setupApp(); + await user.reset.send(userEmail); + winston.info('[userCmd/reset] Password reset email has been sent.'); + } + } + async function deleteUser(uids, { + type + }) { + uids = argParsers.intArrayParse(uids, 'uids'); + const userExists = await user.exists(uids); + if (!userExists || userExists.some(r => r === false)) { + return winston.error(`[userCmd/reset] A user with given uid does not exists.`); + } + await db.initSessionStore(); + const adminUid = await getAdminUidOrFail(); + switch (type) { + case 'purge': + await Promise.all(uids.map(uid => user.delete(adminUid, uid))); + winston.info(`[userCmd/delete] User(s) with their content has been deleted.`); + break; + case 'account': + await Promise.all(uids.map(uid => user.deleteAccount(uid))); + winston.info(`[userCmd/delete] User(s) has been deleted, their content left intact.`); + break; + case 'content': + await Promise.all(uids.map(uid => user.deleteContent(adminUid, uid))); + winston.info(`[userCmd/delete] User(s)' content has been deleted.`); + break; + } + } + async function makeAdmin(uids) { + uids = argParsers.intArrayParse(uids, 'uids'); + await Promise.all(uids.map(uid => groups.join('administrators', uid))); + winston.info('[userCmd/make/admin] User(s) added as administrators.'); + } + async function makeGlobalMod(uids) { + uids = argParsers.intArrayParse(uids, 'uids'); + await Promise.all(uids.map(uid => groups.join('Global Moderators', uid))); + winston.info('[userCmd/make/globalMod] User(s) added as global moderators.'); + } + async function makeMod(uids, { + cid: cids + }) { + uids = argParsers.intArrayParse(uids, 'uids'); + cids = argParsers.intArrayParse(cids, 'cids'); + const categoryPrivList = await privileges.categories.getPrivilegeList(); + await privHelpers.giveOrRescind(groups.join, categoryPrivList, cids, uids); + winston.info('[userCmd/make/mod] User(s) added as moderators to given categories.'); + } + async function makeRegular(uids) { + uids = argParsers.intArrayParse(uids, 'uids'); + await Promise.all(uids.map(uid => groups.leave(['administrators', 'Global Moderators'], uid))); + const categoryPrivList = await privileges.categories.getPrivilegeList(); + const cids = await db.getSortedSetRevRange('categories:cid', 0, -1); + await privHelpers.giveOrRescind(groups.leave, categoryPrivList, cids, uids); + winston.info('[userCmd/make/regular] User(s) made regular/non-privileged.'); + } + return { + info, + create, + reset, + deleteUser, + makeAdmin, + makeGlobalMod, + makeMod, + makeRegular + }; +} \ No newline at end of file diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000000..7e3c7bc55e --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,23 @@ +'use strict'; + +const path = require('path'); +const baseDir = path.join(__dirname, '../'); +const loader = path.join(baseDir, 'loader.js'); +const app = path.join(baseDir, 'app.js'); +const pidfile = path.join(baseDir, 'pidfile'); +const config = path.join(baseDir, 'config.json'); +const currentPackage = path.join(baseDir, 'package.json'); +const installPackage = path.join(baseDir, 'install/package.json'); +const nodeModules = path.join(baseDir, 'node_modules'); +exports.paths = { + baseDir, + loader, + app, + pidfile, + config, + currentPackage, + installPackage, + nodeModules +}; +exports.pluginNamePattern = /^(@[\w-]+\/)?nodebb-(theme|plugin|widget|rewards)-[\w-]+$/; +exports.themeNamePattern = /^(@[\w-]+\/)?nodebb-theme-[\w-]+$/; \ No newline at end of file diff --git a/lib/controllers/404.js b/lib/controllers/404.js new file mode 100644 index 0000000000..1f6082a5da --- /dev/null +++ b/lib/controllers/404.js @@ -0,0 +1,53 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const validator = require('validator'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const middleware = require('../middleware'); +const helpers = require('../middleware/helpers'); +exports.handle404 = helpers.try(async (req, res) => { + const relativePath = nconf.get('relative_path'); + const isClientScript = new RegExp(`^${relativePath}\\/assets\\/src\\/.+\\.js(\\?v=\\w+)?$`); + if (plugins.hooks.hasListeners('action:meta.override404')) { + return plugins.hooks.fire('action:meta.override404', { + req: req, + res: res, + error: {} + }); + } + if (isClientScript.test(req.url)) { + res.type('text/javascript').status(404).send('Not Found'); + } else if (!res.locals.isAPI && (req.path.startsWith(`${relativePath}/assets/uploads`) || req.get('accept') && !req.get('accept').includes('text/html') || req.path === '/favicon.ico')) { + meta.errors.log404(req.path || ''); + res.sendStatus(404); + } else if (req.accepts('html')) { + if (process.env.NODE_ENV === 'development') { + winston.warn(`Route requested but not found: ${req.url}`); + } + meta.errors.log404(req.path.replace(/^\/api/, '') || ''); + await exports.send404(req, res); + } else { + res.status(404).type('txt').send('Not found'); + } +}); +exports.send404 = helpers.try(async (req, res) => { + res.status(404); + const path = String(req.path || ''); + if (res.locals.isAPI) { + return res.json({ + path: validator.escape(path.replace(/^\/api/, '')), + title: '[[global:404.title]]', + bodyClass: helpers.buildBodyClass(req, res) + }); + } + const icons = ['fa-hippo', 'fa-cat', 'fa-otter', 'fa-dog', 'fa-cow', 'fa-fish', 'fa-dragon', 'fa-horse', 'fa-dove']; + await middleware.buildHeaderAsync(req, res); + res.render('404', { + path: validator.escape(path), + title: '[[global:404.title]]', + bodyClass: helpers.buildBodyClass(req, res), + icon: icons[Math.floor(Math.random() * icons.length)] + }); +}); \ No newline at end of file diff --git a/lib/controllers/accounts.js b/lib/controllers/accounts.js new file mode 100644 index 0000000000..4d7148f624 --- /dev/null +++ b/lib/controllers/accounts.js @@ -0,0 +1,20 @@ +'use strict'; + +const accountsController = { + profile: require('./accounts/profile'), + edit: require('./accounts/edit'), + info: require('./accounts/info'), + categories: require('./accounts/categories'), + tags: require('./accounts/tags'), + settings: require('./accounts/settings'), + groups: require('./accounts/groups'), + follow: require('./accounts/follow'), + posts: require('./accounts/posts'), + notifications: require('./accounts/notifications'), + chats: require('./accounts/chats'), + sessions: require('./accounts/sessions'), + blocks: require('./accounts/blocks'), + uploads: require('./accounts/uploads'), + consent: require('./accounts/consent') +}; +module.exports = accountsController; \ No newline at end of file diff --git a/lib/controllers/accounts/blocks.js b/lib/controllers/accounts/blocks.js new file mode 100644 index 0000000000..79d88b84b6 --- /dev/null +++ b/lib/controllers/accounts/blocks.js @@ -0,0 +1,39 @@ +'use strict'; + +const helpers = require('../helpers'); +const pagination = require('../../pagination'); +const user = require('../../user'); +const plugins = require('../../plugins'); +const blocksController = module.exports; +blocksController.getBlocks = async function (req, res) { + const page = parseInt(req.query.page, 10) || 1; + const resultsPerPage = 50; + const start = Math.max(0, page - 1) * resultsPerPage; + const stop = start + resultsPerPage - 1; + const payload = res.locals.userData; + const { + uid, + username, + userslug, + blocksCount + } = payload; + const uids = await user.blocks.list(uid); + const data = await plugins.hooks.fire('filter:user.getBlocks', { + uids: uids, + uid: uid, + start: start, + stop: stop + }); + data.uids = data.uids.slice(start, stop + 1); + payload.users = await user.getUsers(data.uids, req.uid); + payload.title = `[[pages:account/blocks, ${username}]]`; + const pageCount = Math.ceil(blocksCount / resultsPerPage); + payload.pagination = pagination.create(page, pageCount); + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[user:blocks]]' + }]); + res.render('account/blocks', payload); +}; \ No newline at end of file diff --git a/lib/controllers/accounts/categories.js b/lib/controllers/accounts/categories.js new file mode 100644 index 0000000000..20c75ca6a3 --- /dev/null +++ b/lib/controllers/accounts/categories.js @@ -0,0 +1,39 @@ +'use strict'; + +const user = require('../../user'); +const categories = require('../../categories'); +const helpers = require('../helpers'); +const pagination = require('../../pagination'); +const meta = require('../../meta'); +const categoriesController = module.exports; +categoriesController.get = async function (req, res) { + const payload = res.locals.userData; + const { + username, + userslug + } = payload; + const [states, allCategoriesData] = await Promise.all([user.getCategoryWatchState(res.locals.uid), categories.buildForSelect(res.locals.uid, 'find', ['descriptionParsed', 'depth', 'slug'])]); + const pageCount = Math.max(1, Math.ceil(allCategoriesData.length / meta.config.categoriesPerPage)); + const page = Math.min(parseInt(req.query.page, 10) || 1, pageCount); + const start = Math.max(0, (page - 1) * meta.config.categoriesPerPage); + const stop = start + meta.config.categoriesPerPage - 1; + const categoriesData = allCategoriesData.slice(start, stop + 1); + categoriesData.forEach(category => { + if (category) { + category.isWatched = states[category.cid] === categories.watchStates.watching; + category.isTracked = states[category.cid] === categories.watchStates.tracking; + category.isNotWatched = states[category.cid] === categories.watchStates.notwatching; + category.isIgnored = states[category.cid] === categories.watchStates.ignoring; + } + }); + payload.categories = categoriesData; + payload.title = `[[pages:account/watched-categories, ${username}]]`; + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[pages:categories]]' + }]); + payload.pagination = pagination.create(page, pageCount, req.query); + res.render('account/categories', payload); +}; \ No newline at end of file diff --git a/lib/controllers/accounts/chats.js b/lib/controllers/accounts/chats.js new file mode 100644 index 0000000000..c193e10e1f --- /dev/null +++ b/lib/controllers/accounts/chats.js @@ -0,0 +1,93 @@ +'use strict'; + +const db = require('../../database'); +const messaging = require('../../messaging'); +const meta = require('../../meta'); +const user = require('../../user'); +const privileges = require('../../privileges'); +const helpers = require('../helpers'); +const chatsController = module.exports; +chatsController.get = async function (req, res, next) { + if (meta.config.disableChat) { + return next(); + } + const uid = await user.getUidByUserslug(req.params.userslug); + if (!uid) { + return next(); + } + const canChat = await privileges.global.can(['chat', 'chat:privileged'], req.uid); + if (!canChat.includes(true)) { + return helpers.notAllowed(req, res); + } + const payload = { + title: '[[pages:chats]]', + uid: uid, + userslug: req.params.userslug + }; + const isSwitch = res.locals.isAPI && parseInt(req.query.switch, 10) === 1; + if (!isSwitch) { + const [recentChats, publicRooms, privateRoomCount] = await Promise.all([messaging.getRecentChats(req.uid, uid, 0, 29), messaging.getPublicRooms(req.uid, uid), db.sortedSetCard(`uid:${uid}:chat:rooms`)]); + if (!recentChats) { + return next(); + } + payload.rooms = recentChats.rooms; + payload.nextStart = recentChats.nextStart; + payload.publicRooms = publicRooms; + payload.privateRoomCount = privateRoomCount; + } + if (!req.params.roomid) { + return res.render('chats', payload); + } + const { + index + } = req.params; + let start = 0; + payload.scrollToIndex = null; + if (index) { + const msgCount = await db.getObjectField(`chat:room:${req.params.roomid}`, 'messageCount'); + start = Math.max(0, parseInt(msgCount, 10) - index - 49); + payload.scrollToIndex = Math.min(msgCount, Math.max(0, parseInt(index, 10) || 1)); + } + const room = await messaging.loadRoom(req.uid, { + uid: uid, + roomId: req.params.roomid, + start: start + }); + if (!room) { + return next(); + } + room.title = room.roomName || room.usernames || '[[pages:chats]]'; + room.bodyClasses = ['chat-loaded']; + room.canViewInfo = await privileges.global.can('view:users:info', uid); + res.render('chats', { + ...payload, + ...room + }); +}; +chatsController.redirectToChat = async function (req, res, next) { + if (!req.loggedIn) { + return next(); + } + const userslug = await user.getUserField(req.uid, 'userslug'); + if (!userslug) { + return next(); + } + const roomid = parseInt(req.params.roomid, 10); + const index = parseInt(req.params.index, 10); + helpers.redirect(res, `/user/${userslug}/chats${roomid ? `/${roomid}` : ''}${index ? `/${index}` : ''}`); +}; +chatsController.redirectToMessage = async function (req, res, next) { + const mid = parseInt(req.params.mid, 10); + if (!mid) { + return next(); + } + const [userslug, roomId] = await Promise.all([user.getUserField(req.uid, 'userslug'), messaging.getMessageField(mid, 'roomId')]); + if (!userslug || !roomId) { + return next(); + } + const index = await db.sortedSetRank(`chat:room:${roomId}:mids`, mid); + if (!(parseInt(index, 10) >= 0)) { + return next(); + } + helpers.redirect(res, `/user/${userslug}/chats/${roomId}${index ? `/${index + 1}` : ''}`, true); +}; \ No newline at end of file diff --git a/lib/controllers/accounts/consent.js b/lib/controllers/accounts/consent.js new file mode 100644 index 0000000000..a3f33ba7c7 --- /dev/null +++ b/lib/controllers/accounts/consent.js @@ -0,0 +1,30 @@ +'use strict'; + +const db = require('../../database'); +const meta = require('../../meta'); +const helpers = require('../helpers'); +const consentController = module.exports; +consentController.get = async function (req, res, next) { + if (!meta.config.gdpr_enabled) { + return next(); + } + const payload = res.locals.userData; + const { + username, + userslug + } = payload; + const consented = await db.getObjectField(`user:${res.locals.uid}`, 'gdpr_consent'); + payload.gdpr_consent = parseInt(consented, 10) === 1; + payload.digest = { + frequency: meta.config.dailyDigestFreq || 'off', + enabled: meta.config.dailyDigestFreq !== 'off' + }; + payload.title = '[[user:consent.title]]'; + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[user:consent.title]]' + }]); + res.render('account/consent', payload); +}; \ No newline at end of file diff --git a/lib/controllers/accounts/edit.js b/lib/controllers/accounts/edit.js new file mode 100644 index 0000000000..52b0377a1e --- /dev/null +++ b/lib/controllers/accounts/edit.js @@ -0,0 +1,144 @@ +'use strict'; + +const user = require('../../user'); +const meta = require('../../meta'); +const helpers = require('../helpers'); +const groups = require('../../groups'); +const privileges = require('../../privileges'); +const plugins = require('../../plugins'); +const file = require('../../file'); +const editController = module.exports; +editController.get = async function (req, res, next) { + const { + userData + } = res.locals; + if (!userData) { + return next(); + } + const { + username, + userslug, + isSelf, + reputation, + groups: _groups, + groupTitleArray, + allowMultipleBadges + } = userData; + const [canUseSignature, canManageUsers] = await Promise.all([privileges.global.can('signature', req.uid), privileges.admin.can('admin:users', req.uid)]); + userData.maximumSignatureLength = meta.config.maximumSignatureLength; + userData.maximumAboutMeLength = meta.config.maximumAboutMeLength; + userData.maximumProfileImageSize = meta.config.maximumProfileImageSize; + userData.allowMultipleBadges = meta.config.allowMultipleBadges === 1; + userData.allowAccountDelete = meta.config.allowAccountDelete === 1; + userData.allowWebsite = !isSelf || !!meta.config['reputation:disabled'] || reputation >= meta.config['min:rep:website']; + userData.allowAboutMe = !isSelf || !!meta.config['reputation:disabled'] || reputation >= meta.config['min:rep:aboutme']; + userData.allowSignature = canUseSignature && (!isSelf || !!meta.config['reputation:disabled'] || reputation >= meta.config['min:rep:signature']); + userData.profileImageDimension = meta.config.profileImageDimension; + userData.defaultAvatar = user.getDefaultAvatar(); + userData.groups = _groups.filter(g => g && g.userTitleEnabled && !groups.isPrivilegeGroup(g.name) && g.name !== 'registered-users'); + if (req.uid === res.locals.uid || canManageUsers) { + const { + associations + } = await plugins.hooks.fire('filter:auth.list', { + uid: res.locals.uid, + associations: [] + }); + userData.sso = associations; + } + if (!allowMultipleBadges) { + userData.groupTitle = groupTitleArray[0]; + } + userData.groups.sort((a, b) => { + const i1 = groupTitleArray.indexOf(a.name); + const i2 = groupTitleArray.indexOf(b.name); + if (i1 === -1) { + return 1; + } else if (i2 === -1) { + return -1; + } + return i1 - i2; + }); + userData.groups.forEach(group => { + group.userTitle = group.userTitle || group.displayName; + group.selected = groupTitleArray.includes(group.name); + }); + userData.groupSelectSize = Math.min(10, Math.max(5, userData.groups.length + 1)); + userData.title = `[[pages:account/edit, ${username}]]`; + userData.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[user:edit]]' + }]); + userData.editButtons = []; + res.render('account/edit', userData); +}; +editController.password = async function (req, res, next) { + await renderRoute('password', req, res, next); +}; +editController.username = async function (req, res, next) { + await renderRoute('username', req, res, next); +}; +editController.email = async function (req, res, next) { + const targetUid = await user.getUidByUserslug(req.params.userslug); + if (!targetUid || req.uid !== parseInt(targetUid, 10)) { + return next(); + } + req.session.returnTo = `/uid/${targetUid}`; + req.session.registration = req.session.registration || {}; + req.session.registration.updateEmail = true; + req.session.registration.uid = targetUid; + helpers.redirect(res, '/register/complete'); +}; +async function renderRoute(name, req, res) { + const { + userData + } = res.locals; + const [isAdmin, { + username, + userslug + }, hasPassword] = await Promise.all([privileges.admin.can('admin:users', req.uid), user.getUserFields(res.locals.uid, ['username', 'userslug']), user.hasPassword(res.locals.uid)]); + if (meta.config[`${name}:disableEdit`] && !isAdmin) { + return helpers.notAllowed(req, res); + } + userData.hasPassword = hasPassword; + if (name === 'password') { + userData.minimumPasswordLength = meta.config.minimumPasswordLength; + userData.minimumPasswordStrength = meta.config.minimumPasswordStrength; + } + userData.title = `[[pages:account/edit/${name}, ${username}]]`; + userData.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[user:edit]]', + url: `/user/${userslug}/edit` + }, { + text: `[[user:${name}]]` + }]); + res.render(`account/edit/${name}`, userData); +} +editController.uploadPicture = async function (req, res, next) { + const userPhoto = req.files.files[0]; + try { + const updateUid = await user.getUidByUserslug(req.params.userslug); + const isAllowed = await privileges.users.canEdit(req.uid, updateUid); + if (!isAllowed) { + return helpers.notAllowed(req, res); + } + await user.checkMinReputation(req.uid, updateUid, 'min:rep:profile-picture'); + const image = await user.uploadCroppedPictureFile({ + callerUid: req.uid, + uid: updateUid, + file: userPhoto + }); + res.json([{ + name: userPhoto.name, + url: image.url + }]); + } catch (err) { + next(err); + } finally { + await file.delete(userPhoto.path); + } +}; \ No newline at end of file diff --git a/lib/controllers/accounts/follow.js b/lib/controllers/accounts/follow.js new file mode 100644 index 0000000000..4ece948036 --- /dev/null +++ b/lib/controllers/accounts/follow.js @@ -0,0 +1,43 @@ +'use strict'; + +const user = require('../../user'); +const helpers = require('../helpers'); +const pagination = require('../../pagination'); +const followController = module.exports; +followController.getFollowing = async function (req, res, next) { + await getFollow('account/following', 'following', req, res, next); +}; +followController.getFollowers = async function (req, res, next) { + await getFollow('account/followers', 'followers', req, res, next); +}; +async function getFollow(tpl, name, req, res, next) { + const { + userData: payload + } = res.locals; + if (!payload) { + return next(); + } + const { + username, + userslug, + followerCount, + followingCount + } = payload; + const page = parseInt(req.query.page, 10) || 1; + const resultsPerPage = 50; + const start = Math.max(0, page - 1) * resultsPerPage; + const stop = start + resultsPerPage - 1; + payload.title = `[[pages:${tpl}, ${username}]]`; + const method = name === 'following' ? 'getFollowing' : 'getFollowers'; + payload.users = await user[method](res.locals.uid, start, stop); + const count = name === 'following' ? followingCount : followerCount; + const pageCount = Math.ceil(count / resultsPerPage); + payload.pagination = pagination.create(page, pageCount); + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: `[[user:${name}]]` + }]); + res.render(tpl, payload); +} \ No newline at end of file diff --git a/lib/controllers/accounts/groups.js b/lib/controllers/accounts/groups.js new file mode 100644 index 0000000000..01bf2d541b --- /dev/null +++ b/lib/controllers/accounts/groups.js @@ -0,0 +1,29 @@ +'use strict'; + +const user = require('../../user'); +const groups = require('../../groups'); +const helpers = require('../helpers'); +const groupsController = module.exports; +groupsController.get = async function (req, res) { + const { + username, + userslug + } = await user.getUserFields(res.locals.uid, ['username', 'userslug']); + const payload = res.locals.userData; + let groupsData = await groups.getUserGroups([res.locals.uid]); + groupsData = groupsData[0]; + const groupNames = groupsData.filter(Boolean).map(group => group.name); + const members = await groups.getMemberUsers(groupNames, 0, 3); + groupsData.forEach((group, index) => { + group.members = members[index]; + }); + payload.groups = groupsData; + payload.title = `[[pages:account/groups, ${username}]]`; + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[global:header.groups]]' + }]); + res.render('account/groups', payload); +}; \ No newline at end of file diff --git a/lib/controllers/accounts/helpers.js b/lib/controllers/accounts/helpers.js new file mode 100644 index 0000000000..aad98f8b2c --- /dev/null +++ b/lib/controllers/accounts/helpers.js @@ -0,0 +1,270 @@ +'use strict'; + +const validator = require('validator'); +const nconf = require('nconf'); +const db = require('../../database'); +const user = require('../../user'); +const groups = require('../../groups'); +const plugins = require('../../plugins'); +const meta = require('../../meta'); +const utils = require('../../utils'); +const privileges = require('../../privileges'); +const translator = require('../../translator'); +const messaging = require('../../messaging'); +const categories = require('../../categories'); +const relative_path = nconf.get('relative_path'); +const helpers = module.exports; +helpers.getUserDataByUserSlug = async function (userslug, callerUID, query = {}) { + const uid = await user.getUidByUserslug(userslug); + if (!uid) { + return null; + } + const results = await getAllData(uid, callerUID); + if (!results.userData) { + throw new Error('[[error:invalid-uid]]'); + } + await parseAboutMe(results.userData); + let { + userData + } = results; + const { + userSettings, + isAdmin, + isGlobalModerator, + isModerator, + canViewInfo + } = results; + const isSelf = parseInt(callerUID, 10) === parseInt(userData.uid, 10); + if (meta.config['reputation:disabled']) { + delete userData.reputation; + } + userData.age = Math.max(0, userData.birthday ? Math.floor((new Date().getTime() - new Date(userData.birthday).getTime()) / 31536000000) : 0); + userData = await user.hidePrivateData(userData, callerUID); + userData.emailHidden = !userSettings.showemail; + userData.emailClass = userSettings.showemail ? 'hide' : ''; + if (!userData['email:confirmed']) { + userData.email = ''; + } + if (isAdmin || isSelf || canViewInfo && !results.isTargetAdmin) { + userData.ips = results.ips; + } + if (!isAdmin && !isGlobalModerator && !isModerator) { + userData.moderationNote = undefined; + } + userData.isBlocked = results.isBlocked; + userData.yourid = callerUID; + userData.theirid = userData.uid; + userData.isTargetAdmin = results.isTargetAdmin; + userData.isAdmin = isAdmin; + userData.isGlobalModerator = isGlobalModerator; + userData.isModerator = isModerator; + userData.isAdminOrGlobalModerator = isAdmin || isGlobalModerator; + userData.isAdminOrGlobalModeratorOrModerator = isAdmin || isGlobalModerator || isModerator; + userData.isSelfOrAdminOrGlobalModerator = isSelf || isAdmin || isGlobalModerator; + userData.canEdit = results.canEdit; + userData.canBan = results.canBanUser; + userData.canMute = results.canMuteUser; + userData.canFlag = (await privileges.users.canFlag(callerUID, userData.uid)).flag; + userData.canChangePassword = isAdmin || isSelf && !meta.config['password:disableEdit']; + userData.isSelf = isSelf; + userData.isFollowing = results.isFollowing; + userData.canChat = results.canChat; + userData.hasPrivateChat = results.hasPrivateChat; + userData.iconBackgrounds = results.iconBackgrounds; + userData.showHidden = results.canEdit; + userData.allowProfilePicture = !userData.isSelf || !!meta.config['reputation:disabled'] || userData.reputation >= meta.config['min:rep:profile-picture']; + userData.allowCoverPicture = !userData.isSelf || !!meta.config['reputation:disabled'] || userData.reputation >= meta.config['min:rep:cover-picture']; + userData.allowProfileImageUploads = meta.config.allowProfileImageUploads; + userData.allowedProfileImageExtensions = user.getAllowedProfileImageExtensions().map(ext => `.${ext}`).join(', '); + userData.groups = Array.isArray(results.groups) && results.groups.length ? results.groups[0] : []; + userData.selectedGroup = userData.groups.filter(group => group && userData.groupTitleArray.includes(group.name)).sort((a, b) => userData.groupTitleArray.indexOf(a.name) - userData.groupTitleArray.indexOf(b.name)); + userData.disableSignatures = meta.config.disableSignatures === 1; + userData['reputation:disabled'] = meta.config['reputation:disabled'] === 1; + userData['downvote:disabled'] = meta.config['downvote:disabled'] === 1; + userData['email:confirmed'] = !!userData['email:confirmed']; + userData.profile_links = filterLinks(results.profile_menu.links, { + self: isSelf, + other: !isSelf, + moderator: isModerator, + globalMod: isGlobalModerator, + admin: isAdmin, + canViewInfo: canViewInfo + }); + userData.banned = Boolean(userData.banned); + userData.muted = parseInt(userData.mutedUntil, 10) > Date.now(); + userData.website = escape(userData.website); + userData.websiteLink = !userData.website.startsWith('http') ? `http://${userData.website}` : userData.website; + userData.websiteName = userData.website.replace(validator.escape('http://'), '').replace(validator.escape('https://'), ''); + userData.fullname = escape(userData.fullname); + userData.location = escape(userData.location); + userData.signature = escape(userData.signature); + userData.birthday = validator.escape(String(userData.birthday || '')); + userData.moderationNote = validator.escape(String(userData.moderationNote || '')); + if (userData['cover:url']) { + userData['cover:url'] = userData['cover:url'].startsWith('http') ? userData['cover:url'] : nconf.get('relative_path') + userData['cover:url']; + } else { + userData['cover:url'] = require('../../coverPhoto').getDefaultProfileCover(userData.uid); + } + userData['cover:position'] = validator.escape(String(userData['cover:position'] || '50% 50%')); + userData['username:disableEdit'] = !userData.isAdmin && meta.config['username:disableEdit']; + userData['email:disableEdit'] = !userData.isAdmin && meta.config['email:disableEdit']; + await getCounts(userData, callerUID); + const hookData = await plugins.hooks.fire('filter:helpers.getUserDataByUserSlug', { + userData: userData, + callerUID: callerUID, + query: query + }); + return hookData.userData; +}; +function escape(value) { + return translator.escape(validator.escape(String(value || ''))); +} +async function getAllData(uid, callerUID) { + const [[isTargetAdmin, isCallerAdmin], isGlobalModerator] = await Promise.all([user.isAdministrator([uid, callerUID]), user.isGlobalModerator(callerUID)]); + return await utils.promiseParallel({ + userData: user.getUserData(uid), + isTargetAdmin: isTargetAdmin, + userSettings: user.getSettings(uid), + isAdmin: isCallerAdmin, + isGlobalModerator: isGlobalModerator, + isModerator: user.isModeratorOfAnyCategory(callerUID), + isFollowing: user.isFollowing(callerUID, uid), + ips: user.getIPs(uid, 4), + profile_menu: getProfileMenu(uid, callerUID), + groups: groups.getUserGroups([uid]), + canEdit: privileges.users.canEdit(callerUID, uid), + canBanUser: privileges.users.canBanUser(callerUID, uid), + canMuteUser: privileges.users.canMuteUser(callerUID, uid), + isBlocked: user.blocks.is(uid, callerUID), + canViewInfo: privileges.global.can('view:users:info', callerUID), + canChat: canChat(callerUID, uid), + hasPrivateChat: messaging.hasPrivateChat(callerUID, uid), + iconBackgrounds: user.getIconBackgrounds() + }); +} +async function canChat(callerUID, uid) { + try { + await messaging.canMessageUser(callerUID, uid); + } catch (err) { + if (err.message.startsWith('[[error:')) { + return false; + } + throw err; + } + return true; +} +async function getCounts(userData, callerUID) { + const { + uid + } = userData; + const cids = await categories.getCidsByPrivilege('categories:cid', callerUID, 'topics:read'); + const promises = { + posts: db.sortedSetsCardSum(cids.map(c => `cid:${c}:uid:${uid}:pids`)), + best: db.sortedSetsCardSum(cids.map(c => `cid:${c}:uid:${uid}:pids:votes`), 1, '+inf'), + controversial: db.sortedSetsCardSum(cids.map(c => `cid:${c}:uid:${uid}:pids:votes`), '-inf', -1), + topics: db.sortedSetsCardSum(cids.map(c => `cid:${c}:uid:${uid}:tids`)) + }; + if (userData.isAdmin || userData.isSelf) { + promises.ignored = db.sortedSetCard(`uid:${uid}:ignored_tids`); + promises.watched = db.sortedSetCard(`uid:${uid}:followed_tids`); + promises.upvoted = db.sortedSetCard(`uid:${uid}:upvote`); + promises.downvoted = db.sortedSetCard(`uid:${uid}:downvote`); + promises.bookmarks = db.sortedSetCard(`uid:${uid}:bookmarks`); + promises.uploaded = db.sortedSetCard(`uid:${uid}:uploads`); + promises.categoriesWatched = user.getWatchedCategories(uid); + promises.tagsWatched = db.sortedSetCard(`uid:${uid}:followed_tags`); + promises.blocks = user.getUserField(userData.uid, 'blocksCount'); + } + const counts = await utils.promiseParallel(promises); + counts.categoriesWatched = counts.categoriesWatched && counts.categoriesWatched.length; + counts.groups = userData.groups.length; + counts.following = userData.followingCount; + counts.followers = userData.followerCount; + userData.blocksCount = counts.blocks || 0; + userData.counts = counts; +} +async function getProfileMenu(uid, callerUID) { + const links = [{ + id: 'info', + route: 'info', + name: '[[user:account-info]]', + icon: 'fa-info', + visibility: { + self: false, + other: false, + moderator: false, + globalMod: false, + admin: true, + canViewInfo: true + } + }, { + id: 'sessions', + route: 'sessions', + name: '[[pages:account/sessions]]', + icon: 'fa-group', + visibility: { + self: true, + other: false, + moderator: false, + globalMod: false, + admin: false, + canViewInfo: false + } + }]; + if (meta.config.gdpr_enabled) { + links.push({ + id: 'consent', + route: 'consent', + name: '[[user:consent.title]]', + icon: 'fa-thumbs-o-up', + visibility: { + self: true, + other: false, + moderator: false, + globalMod: false, + admin: false, + canViewInfo: false + } + }); + } + const data = await plugins.hooks.fire('filter:user.profileMenu', { + uid: uid, + callerUID: callerUID, + links: links + }); + const userslug = await user.getUserField(uid, 'userslug'); + data.links.forEach(link => { + if (!link.hasOwnProperty('url')) { + link.url = `${relative_path}/user/${userslug}/${link.route}`; + } + }); + return data; +} +async function parseAboutMe(userData) { + if (!userData.aboutme) { + userData.aboutme = ''; + userData.aboutmeParsed = ''; + return; + } + userData.aboutme = validator.escape(String(userData.aboutme || '')); + const parsed = await plugins.hooks.fire('filter:parse.aboutme', userData.aboutme); + userData.aboutme = translator.escape(userData.aboutme); + userData.aboutmeParsed = translator.escape(parsed); +} +function filterLinks(links, states) { + return links.filter((link, index) => { + link.visibility = { + self: true, + other: true, + moderator: true, + globalMod: true, + admin: true, + canViewInfo: true, + ...link.visibility + }; + const permit = Object.keys(states).some(state => states[state] && link.visibility[state]); + links[index].public = permit; + return permit; + }); +} +require('../../promisify')(helpers); \ No newline at end of file diff --git a/lib/controllers/accounts/info.js b/lib/controllers/accounts/info.js new file mode 100644 index 0000000000..48a9c8d3c9 --- /dev/null +++ b/lib/controllers/accounts/info.js @@ -0,0 +1,53 @@ +'use strict'; + +const db = require('../../database'); +const user = require('../../user'); +const helpers = require('../helpers'); +const pagination = require('../../pagination'); +const infoController = module.exports; +infoController.get = async function (req, res) { + const page = Math.max(1, req.query.page || 1); + const itemsPerPage = 10; + const start = (page - 1) * itemsPerPage; + const stop = start + itemsPerPage - 1; + const payload = res.locals.userData; + const { + username, + userslug + } = payload; + const [isPrivileged, history, sessions, usernames, emails] = await Promise.all([user.isPrivileged(req.uid), user.getModerationHistory(res.locals.uid), user.auth.getSessions(res.locals.uid, req.sessionID), user.getHistory(`user:${res.locals.uid}:usernames`), user.getHistory(`user:${res.locals.uid}:emails`)]); + const notes = await getNotes({ + uid: res.locals.uid, + isPrivileged + }, start, stop); + payload.history = history; + payload.sessions = sessions; + payload.usernames = usernames; + payload.emails = emails; + if (isPrivileged) { + payload.moderationNotes = notes.notes; + const pageCount = Math.ceil(notes.count / itemsPerPage); + payload.pagination = pagination.create(page, pageCount, req.query); + } + payload.title = '[[pages:account/info]]'; + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[user:account-info]]' + }]); + res.render('account/info', payload); +}; +async function getNotes({ + uid, + isPrivileged +}, start, stop) { + if (!isPrivileged) { + return; + } + const [notes, count] = await Promise.all([user.getModerationNotes(uid, start, stop), db.sortedSetCard(`uid:${uid}:moderation:notes`)]); + return { + notes: notes, + count: count + }; +} \ No newline at end of file diff --git a/lib/controllers/accounts/notifications.js b/lib/controllers/accounts/notifications.js new file mode 100644 index 0000000000..d59d8e418b --- /dev/null +++ b/lib/controllers/accounts/notifications.js @@ -0,0 +1,97 @@ +'use strict'; + +const user = require('../../user'); +const helpers = require('../helpers'); +const plugins = require('../../plugins'); +const pagination = require('../../pagination'); +const notificationsController = module.exports; +notificationsController.get = async function (req, res, next) { + const regularFilters = [{ + name: '[[notifications:all]]', + filter: '' + }, { + name: '[[global:topics]]', + filter: 'new-topic' + }, { + name: '[[notifications:replies]]', + filter: 'new-reply' + }, { + name: '[[notifications:tags]]', + filter: 'new-topic-with-tag' + }, { + name: '[[notifications:categories]]', + filter: 'new-topic-in-category' + }, { + name: '[[notifications:chat]]', + filter: 'new-chat' + }, { + name: '[[notifications:group-chat]]', + filter: 'new-group-chat' + }, { + name: '[[notifications:public-chat]]', + filter: 'new-public-chat' + }, { + name: '[[notifications:follows]]', + filter: 'follow' + }, { + name: '[[notifications:upvote]]', + filter: 'upvote' + }, { + name: '[[notifications:awards]]', + filter: 'new-reward' + }]; + const moderatorFilters = [{ + name: '[[notifications:new-flags]]', + filter: 'new-post-flag' + }, { + name: '[[notifications:my-flags]]', + filter: 'my-flags' + }, { + name: '[[notifications:bans]]', + filter: 'ban' + }]; + const filter = req.query.filter || ''; + const page = Math.max(1, req.query.page || 1); + const itemsPerPage = 20; + const start = (page - 1) * itemsPerPage; + const stop = start + itemsPerPage - 1; + const [filters, isPrivileged] = await Promise.all([plugins.hooks.fire('filter:notifications.addFilters', { + regularFilters: regularFilters, + moderatorFilters: moderatorFilters, + uid: req.uid + }), user.isPrivileged(req.uid)]); + let allFilters = filters.regularFilters; + if (isPrivileged) { + allFilters = allFilters.concat([{ + separator: true + }]).concat(filters.moderatorFilters); + } + allFilters.forEach(filterData => { + filterData.selected = filterData.filter === filter; + }); + const selectedFilter = allFilters.find(filterData => filterData.selected); + if (!selectedFilter) { + return next(); + } + const data = await user.notifications.getAllWithCounts(req.uid, selectedFilter.filter); + let notifications = await user.notifications.getNotifications(data.nids, req.uid); + allFilters.forEach(filterData => { + if (filterData && filterData.filter) { + filterData.count = data.counts[filterData.filter] || 0; + } + }); + const pageCount = Math.max(1, Math.ceil(notifications.length / itemsPerPage)); + notifications = notifications.slice(start, stop + 1); + res.render('notifications', { + notifications: notifications, + pagination: pagination.create(page, pageCount, req.query), + filters: allFilters, + regularFilters: regularFilters, + moderatorFilters: moderatorFilters, + selectedFilter: selectedFilter, + title: '[[pages:notifications]]', + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[pages:notifications]]' + }]) + }); +}; \ No newline at end of file diff --git a/lib/controllers/accounts/posts.js b/lib/controllers/accounts/posts.js new file mode 100644 index 0000000000..4b6b1f6e03 --- /dev/null +++ b/lib/controllers/accounts/posts.js @@ -0,0 +1,271 @@ +'use strict'; + +const db = require('../../database'); +const user = require('../../user'); +const posts = require('../../posts'); +const topics = require('../../topics'); +const categories = require('../../categories'); +const privileges = require('../../privileges'); +const pagination = require('../../pagination'); +const helpers = require('../helpers'); +const plugins = require('../../plugins'); +const utils = require('../../utils'); +const postsController = module.exports; +const templateToData = { + 'account/bookmarks': { + type: 'posts', + noItemsFoundKey: '[[topic:bookmarks.has-no-bookmarks]]', + crumb: '[[user:bookmarks]]', + getSets: function (callerUid, userData) { + return `uid:${userData.uid}:bookmarks`; + } + }, + 'account/posts': { + type: 'posts', + noItemsFoundKey: '[[user:has-no-posts]]', + crumb: '[[global:posts]]', + getSets: async function (callerUid, userData) { + const cids = await categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read'); + return cids.map(c => `cid:${c}:uid:${userData.uid}:pids`); + } + }, + 'account/upvoted': { + type: 'posts', + noItemsFoundKey: '[[user:has-no-upvoted-posts]]', + crumb: '[[global:upvoted]]', + getSets: function (callerUid, userData) { + return `uid:${userData.uid}:upvote`; + } + }, + 'account/downvoted': { + type: 'posts', + noItemsFoundKey: '[[user:has-no-downvoted-posts]]', + crumb: '[[global:downvoted]]', + getSets: function (callerUid, userData) { + return `uid:${userData.uid}:downvote`; + } + }, + 'account/best': { + type: 'posts', + noItemsFoundKey: '[[user:has-no-best-posts]]', + crumb: '[[global:best]]', + getSets: async function (callerUid, userData) { + const cids = await categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read'); + return cids.map(c => `cid:${c}:uid:${userData.uid}:pids:votes`); + }, + getTopics: async (sets, req, start, stop) => { + let pids = await db.getSortedSetRevRangeByScore(sets, start, stop - start + 1, '+inf', 1); + pids = await privileges.posts.filter('topics:read', pids, req.uid); + const postObjs = await posts.getPostSummaryByPids(pids, req.uid, { + stripTags: false + }); + return { + posts: postObjs, + nextStart: stop + 1 + }; + }, + getItemCount: async sets => { + const counts = await Promise.all(sets.map(set => db.sortedSetCount(set, 1, '+inf'))); + return counts.reduce((acc, val) => acc + val, 0); + } + }, + 'account/controversial': { + type: 'posts', + noItemsFoundKey: '[[user:has-no-controversial-posts]]', + crumb: '[[global:controversial]]', + getSets: async function (callerUid, userData) { + const cids = await categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read'); + return cids.map(c => `cid:${c}:uid:${userData.uid}:pids:votes`); + }, + getTopics: async (sets, req, start, stop) => { + let pids = await db.getSortedSetRangeByScore(sets, start, stop - start + 1, '-inf', -1); + pids = await privileges.posts.filter('topics:read', pids, req.uid); + const postObjs = await posts.getPostSummaryByPids(pids, req.uid, { + stripTags: false + }); + return { + posts: postObjs, + nextStart: stop + 1 + }; + }, + getItemCount: async sets => { + const counts = await Promise.all(sets.map(set => db.sortedSetCount(set, '-inf', -1))); + return counts.reduce((acc, val) => acc + val, 0); + } + }, + 'account/watched': { + type: 'topics', + noItemsFoundKey: '[[user:has-no-watched-topics]]', + crumb: '[[user:watched]]', + getSets: function (callerUid, userData) { + return `uid:${userData.uid}:followed_tids`; + }, + getTopics: async function (set, req, start, stop) { + const { + sort + } = req.query; + const map = { + votes: 'topics:votes', + posts: 'topics:posts', + views: 'topics:views', + lastpost: 'topics:recent', + firstpost: 'topics:tid' + }; + if (!sort || !map[sort]) { + return await topics.getTopicsFromSet(set, req.uid, start, stop); + } + const sortSet = map[sort]; + let tids = await db.getSortedSetRevRange(set, 0, -1); + const scores = await db.sortedSetScores(sortSet, tids); + tids = tids.map((tid, i) => ({ + tid: tid, + score: scores[i] + })).sort((a, b) => b.score - a.score).slice(start, stop + 1).map(t => t.tid); + const topicsData = await topics.getTopics(tids, req.uid); + topics.calculateTopicIndices(topicsData, start); + return { + topics: topicsData, + nextStart: stop + 1 + }; + } + }, + 'account/ignored': { + type: 'topics', + noItemsFoundKey: '[[user:has-no-ignored-topics]]', + crumb: '[[user:ignored]]', + getSets: function (callerUid, userData) { + return `uid:${userData.uid}:ignored_tids`; + } + }, + 'account/topics': { + type: 'topics', + noItemsFoundKey: '[[user:has-no-topics]]', + crumb: '[[global:topics]]', + getSets: async function (callerUid, userData) { + const cids = await categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read'); + return cids.map(c => `cid:${c}:uid:${userData.uid}:tids`); + } + } +}; +postsController.getBookmarks = async function (req, res, next) { + await getPostsFromUserSet('account/bookmarks', req, res, next); +}; +postsController.getPosts = async function (req, res, next) { + await getPostsFromUserSet('account/posts', req, res, next); +}; +postsController.getUpVotedPosts = async function (req, res, next) { + await getPostsFromUserSet('account/upvoted', req, res, next); +}; +postsController.getDownVotedPosts = async function (req, res, next) { + await getPostsFromUserSet('account/downvoted', req, res, next); +}; +postsController.getBestPosts = async function (req, res, next) { + await getPostsFromUserSet('account/best', req, res, next); +}; +postsController.getControversialPosts = async function (req, res, next) { + await getPostsFromUserSet('account/controversial', req, res, next); +}; +postsController.getWatchedTopics = async function (req, res, next) { + await getPostsFromUserSet('account/watched', req, res, next); +}; +postsController.getIgnoredTopics = async function (req, res, next) { + await getPostsFromUserSet('account/ignored', req, res, next); +}; +postsController.getTopics = async function (req, res, next) { + await getPostsFromUserSet('account/topics', req, res, next); +}; +async function getPostsFromUserSet(template, req, res) { + const data = templateToData[template]; + const page = Math.max(1, parseInt(req.query.page, 10) || 1); + const payload = res.locals.userData; + const { + username, + userslug + } = payload; + const settings = await user.getSettings(req.uid); + const itemsPerPage = data.type === 'topics' ? settings.topicsPerPage : settings.postsPerPage; + const start = (page - 1) * itemsPerPage; + const stop = start + itemsPerPage - 1; + const sets = await data.getSets(req.uid, { + uid: res.locals.uid, + username, + userslug + }); + let result; + if (plugins.hooks.hasListeners('filter:account.getPostsFromUserSet')) { + result = await plugins.hooks.fire('filter:account.getPostsFromUserSet', { + req: req, + template: template, + userData: { + uid: res.locals.uid, + username, + userslug + }, + settings: settings, + data: data, + start: start, + stop: stop, + itemCount: 0, + itemData: [] + }); + } else { + result = await utils.promiseParallel({ + itemCount: getItemCount(sets, data, settings), + itemData: getItemData(sets, data, req, start, stop) + }); + } + const { + itemCount, + itemData + } = result; + payload[data.type] = itemData[data.type]; + payload.nextStart = itemData.nextStart; + const pageCount = Math.ceil(itemCount / itemsPerPage); + payload.pagination = pagination.create(page, pageCount, req.query); + payload.noItemsFoundKey = data.noItemsFoundKey; + payload.title = `[[pages:${template}, ${username}]]`; + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: data.crumb + }]); + payload.showSort = template === 'account/watched'; + const baseUrl = req.baseUrl + req.path.replace(/^\/api/, ''); + payload.sortOptions = [{ + url: `${baseUrl}?sort=votes`, + name: '[[global:votes]]' + }, { + url: `${baseUrl}?sort=posts`, + name: '[[global:posts]]' + }, { + url: `${baseUrl}?sort=views`, + name: '[[global:views]]' + }, { + url: `${baseUrl}?sort=lastpost`, + name: '[[global:lastpost]]' + }, { + url: `${baseUrl}?sort=firstpost`, + name: '[[global:firstpost]]' + }]; + payload.sortOptions.forEach(option => { + option.selected = option.url.includes(`sort=${req.query.sort}`); + }); + res.render(template, payload); +} +async function getItemData(sets, data, req, start, stop) { + if (data.getTopics) { + return await data.getTopics(sets, req, start, stop); + } + const method = data.type === 'topics' ? topics.getTopicsFromSet : posts.getPostSummariesFromSet; + return await method(sets, req.uid, start, stop); +} +async function getItemCount(sets, data, settings) { + if (!settings.usePagination) { + return 0; + } + if (data.getItemCount) { + return await data.getItemCount(sets); + } + return await db.sortedSetsCardSum(sets); +} \ No newline at end of file diff --git a/lib/controllers/accounts/profile.js b/lib/controllers/accounts/profile.js new file mode 100644 index 0000000000..02a83e9f2a --- /dev/null +++ b/lib/controllers/accounts/profile.js @@ -0,0 +1,113 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../../database'); +const user = require('../../user'); +const posts = require('../../posts'); +const categories = require('../../categories'); +const plugins = require('../../plugins'); +const privileges = require('../../privileges'); +const helpers = require('../helpers'); +const utils = require('../../utils'); +const profileController = module.exports; +profileController.get = async function (req, res, next) { + const { + userData + } = res.locals; + if (!userData) { + return next(); + } + await incrementProfileViews(req, userData); + const [latestPosts, bestPosts] = await Promise.all([getLatestPosts(req.uid, userData), getBestPosts(req.uid, userData), posts.parseSignature(userData, req.uid)]); + userData.posts = latestPosts; + userData.latestPosts = latestPosts; + userData.bestPosts = bestPosts; + userData.breadcrumbs = helpers.buildBreadcrumbs([{ + text: userData.username + }]); + userData.title = userData.username; + userData.emailChanged = req.session.emailChanged; + delete req.session.emailChanged; + if (!userData.profileviews) { + userData.profileviews = 1; + } + addMetaTags(res, userData); + res.render('account/profile', userData); +}; +async function incrementProfileViews(req, userData) { + if (req.uid >= 1) { + req.session.uids_viewed = req.session.uids_viewed || {}; + if (req.uid !== userData.uid && (!req.session.uids_viewed[userData.uid] || req.session.uids_viewed[userData.uid] < Date.now() - 3600000)) { + await user.incrementUserFieldBy(userData.uid, 'profileviews', 1); + req.session.uids_viewed[userData.uid] = Date.now(); + } + } +} +async function getLatestPosts(callerUid, userData) { + return await getPosts(callerUid, userData, 'pids'); +} +async function getBestPosts(callerUid, userData) { + return await getPosts(callerUid, userData, 'pids:votes'); +} +async function getPosts(callerUid, userData, setSuffix) { + const cids = await categories.getCidsByPrivilege('categories:cid', callerUid, 'topics:read'); + const keys = cids.map(c => `cid:${c}:uid:${userData.uid}:${setSuffix}`); + let hasMorePosts = true; + let start = 0; + const count = 10; + const postData = []; + const [isAdmin, isModOfCids, canSchedule] = await Promise.all([user.isAdministrator(callerUid), user.isModerator(callerUid, cids), privileges.categories.isUserAllowedTo('topics:schedule', cids, callerUid)]); + const isModOfCid = _.zipObject(cids, isModOfCids); + const cidToCanSchedule = _.zipObject(cids, canSchedule); + do { + let pids = await db.getSortedSetRevRange(keys, start, start + count - 1); + if (!pids.length || pids.length < count) { + hasMorePosts = false; + } + if (pids.length) { + ({ + pids + } = await plugins.hooks.fire('filter:account.profile.getPids', { + uid: callerUid, + userData, + setSuffix, + pids + })); + const p = await posts.getPostSummaryByPids(pids, callerUid, { + stripTags: false + }); + postData.push(...p.filter(p => p && p.topic && (isAdmin || isModOfCid[p.topic.cid] || p.topic.scheduled && cidToCanSchedule[p.topic.cid] || !p.deleted && !p.topic.deleted))); + } + start += count; + } while (postData.length < count && hasMorePosts); + return postData.slice(0, count); +} +function addMetaTags(res, userData) { + const plainAboutMe = userData.aboutme ? utils.stripHTMLTags(utils.decodeHTMLEntities(userData.aboutme)) : ''; + res.locals.metaTags = [{ + name: 'title', + content: userData.fullname || userData.username, + noEscape: true + }, { + name: 'description', + content: plainAboutMe + }, { + property: 'og:title', + content: userData.fullname || userData.username, + noEscape: true + }, { + property: 'og:description', + content: plainAboutMe + }]; + if (userData.picture) { + res.locals.metaTags.push({ + property: 'og:image', + content: userData.picture, + noEscape: true + }, { + property: 'og:image:url', + content: userData.picture, + noEscape: true + }); + } +} \ No newline at end of file diff --git a/lib/controllers/accounts/sessions.js b/lib/controllers/accounts/sessions.js new file mode 100644 index 0000000000..6a211830f6 --- /dev/null +++ b/lib/controllers/accounts/sessions.js @@ -0,0 +1,21 @@ +'use strict'; + +const user = require('../../user'); +const helpers = require('../helpers'); +const sessionController = module.exports; +sessionController.get = async function (req, res) { + const payload = res.locals.userData; + const { + username, + userslug + } = payload; + payload.sessions = await user.auth.getSessions(res.locals.uid, req.sessionID); + payload.title = '[[pages:account/sessions]]'; + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[pages:account/sessions]]' + }]); + res.render('account/sessions', payload); +}; \ No newline at end of file diff --git a/lib/controllers/accounts/settings.js b/lib/controllers/accounts/settings.js new file mode 100644 index 0000000000..ce0055affb --- /dev/null +++ b/lib/controllers/accounts/settings.js @@ -0,0 +1,220 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const _ = require('lodash'); +const jwt = require('jsonwebtoken'); +const util = require('util'); +const user = require('../../user'); +const languages = require('../../languages'); +const meta = require('../../meta'); +const plugins = require('../../plugins'); +const notifications = require('../../notifications'); +const db = require('../../database'); +const helpers = require('../helpers'); +const slugify = require('../../slugify'); +const settingsController = module.exports; +settingsController.get = async function (req, res, next) { + const { + userData + } = res.locals; + if (!userData) { + return next(); + } + const [settings, languagesData] = await Promise.all([user.getSettings(userData.uid), languages.list()]); + userData.settings = settings; + userData.languages = languagesData; + if (userData.isAdmin && userData.isSelf) { + userData.acpLanguages = _.cloneDeep(languagesData); + } + const data = await plugins.hooks.fire('filter:user.customSettings', { + settings: settings, + customSettings: [], + uid: req.uid + }); + const [notificationSettings, routes, bsSkinOptions] = await Promise.all([getNotificationSettings(userData), getHomePageRoutes(userData), getSkinOptions(userData)]); + userData.customSettings = data.customSettings; + userData.homePageRoutes = routes; + userData.bootswatchSkinOptions = bsSkinOptions; + userData.notificationSettings = notificationSettings; + userData.disableEmailSubscriptions = meta.config.disableEmailSubscriptions; + userData.dailyDigestFreqOptions = [{ + value: 'off', + name: '[[user:digest-off]]', + selected: userData.settings.dailyDigestFreq === 'off' + }, { + value: 'day', + name: '[[user:digest-daily]]', + selected: userData.settings.dailyDigestFreq === 'day' + }, { + value: 'week', + name: '[[user:digest-weekly]]', + selected: userData.settings.dailyDigestFreq === 'week' + }, { + value: 'biweek', + name: '[[user:digest-biweekly]]', + selected: userData.settings.dailyDigestFreq === 'biweek' + }, { + value: 'month', + name: '[[user:digest-monthly]]', + selected: userData.settings.dailyDigestFreq === 'month' + }]; + userData.languages.forEach(language => { + language.selected = language.code === userData.settings.userLang; + }); + if (userData.isAdmin && userData.isSelf) { + userData.acpLanguages.forEach(language => { + language.selected = language.code === userData.settings.acpLang; + }); + } + const notifFreqOptions = ['all', 'first', 'everyTen', 'threshold', 'logarithmic', 'disabled']; + userData.upvoteNotifFreq = notifFreqOptions.map(name => ({ + name: name, + selected: name === userData.settings.upvoteNotifFreq + })); + userData.categoryWatchState = { + [userData.settings.categoryWatchState]: true + }; + userData.disableCustomUserSkins = meta.config.disableCustomUserSkins || 0; + userData.allowUserHomePage = meta.config.allowUserHomePage === 1 ? 1 : 0; + userData.hideFullname = meta.config.hideFullname || 0; + userData.hideEmail = meta.config.hideEmail || 0; + userData.inTopicSearchAvailable = plugins.hooks.hasListeners('filter:topic.search'); + userData.maxTopicsPerPage = meta.config.maxTopicsPerPage; + userData.maxPostsPerPage = meta.config.maxPostsPerPage; + userData.title = '[[pages:account/settings]]'; + userData.breadcrumbs = helpers.buildBreadcrumbs([{ + text: userData.username, + url: `/user/${userData.userslug}` + }, { + text: '[[user:settings]]' + }]); + res.render('account/settings', userData); +}; +const unsubscribable = ['digest', 'notification']; +const jwtVerifyAsync = util.promisify((token, callback) => { + jwt.verify(token, nconf.get('secret'), (err, payload) => callback(err, payload)); +}); +const doUnsubscribe = async payload => { + if (payload.template === 'digest') { + await Promise.all([user.setSetting(payload.uid, 'dailyDigestFreq', 'off'), user.updateDigestSetting(payload.uid, 'off')]); + } else if (payload.template === 'notification') { + const current = await db.getObjectField(`user:${payload.uid}:settings`, `notificationType_${payload.type}`); + await user.setSetting(payload.uid, `notificationType_${payload.type}`, current === 'notificationemail' ? 'notification' : 'none'); + } + return true; +}; +settingsController.unsubscribe = async (req, res) => { + try { + const payload = await jwtVerifyAsync(req.params.token); + if (!payload || !unsubscribable.includes(payload.template)) { + return; + } + await doUnsubscribe(payload); + res.render('unsubscribe', { + payload + }); + } catch (err) { + res.render('unsubscribe', { + error: err.message + }); + } +}; +settingsController.unsubscribePost = async function (req, res) { + let payload; + try { + payload = await jwtVerifyAsync(req.params.token); + if (!payload || !unsubscribable.includes(payload.template)) { + return res.sendStatus(404); + } + } catch (err) { + return res.sendStatus(403); + } + try { + await doUnsubscribe(payload); + res.sendStatus(200); + } catch (err) { + winston.error(`[settings/unsubscribe] One-click unsubscribe failed with error: ${err.message}`); + res.sendStatus(500); + } +}; +async function getNotificationSettings(userData) { + const privilegedTypes = []; + const privileges = await user.getPrivileges(userData.uid); + if (privileges.isAdmin) { + privilegedTypes.push('notificationType_new-register'); + } + if (privileges.isAdmin || privileges.isGlobalMod || privileges.isModeratorOfAnyCategory) { + privilegedTypes.push('notificationType_post-queue', 'notificationType_new-post-flag'); + } + if (privileges.isAdmin || privileges.isGlobalMod) { + privilegedTypes.push('notificationType_new-user-flag'); + } + const results = await plugins.hooks.fire('filter:user.notificationTypes', { + types: notifications.baseTypes.slice(), + privilegedTypes: privilegedTypes + }); + function modifyType(type) { + const setting = userData.settings[type]; + return { + name: type, + label: `[[notifications:${type.replace(/_/g, '-')}]]`, + none: setting === 'none', + notification: setting === 'notification', + email: setting === 'email', + notificationemail: setting === 'notificationemail' + }; + } + if (meta.config.disableChat) { + results.types = results.types.filter(type => type !== 'notificationType_new-chat'); + } + return results.types.map(modifyType).concat(results.privilegedTypes.map(modifyType)); +} +async function getHomePageRoutes(userData) { + let routes = await helpers.getHomePageRoutes(userData.uid); + let customIdx; + let hasSelected = false; + routes = routes.map((route, idx) => { + if (route.route === userData.settings.homePageRoute) { + route.selected = true; + hasSelected = true; + } else { + route.selected = false; + } + if (route.route === 'custom') { + customIdx = idx; + } + return route; + }); + if (!hasSelected && customIdx && userData.settings.homePageRoute !== 'none') { + routes[customIdx].selected = true; + } + return routes; +} +async function getSkinOptions(userData) { + const defaultSkin = _.capitalize(meta.config.bootswatchSkin) || '[[user:no-skin]]'; + const bootswatchSkinOptions = [{ + name: '[[user:no-skin]]', + value: 'noskin' + }, { + name: `[[user:default, ${defaultSkin}]]`, + value: '' + }]; + const customSkins = await meta.settings.get('custom-skins'); + if (customSkins && Array.isArray(customSkins['custom-skin-list'])) { + customSkins['custom-skin-list'].forEach(customSkin => { + bootswatchSkinOptions.push({ + name: customSkin['custom-skin-name'], + value: slugify(customSkin['custom-skin-name']) + }); + }); + } + bootswatchSkinOptions.push(...meta.css.supportedSkins.map(skin => ({ + name: _.capitalize(skin), + value: skin + }))); + bootswatchSkinOptions.forEach(skin => { + skin.selected = skin.value === userData.settings.bootswatchSkin; + }); + return bootswatchSkinOptions; +} \ No newline at end of file diff --git a/lib/controllers/accounts/tags.js b/lib/controllers/accounts/tags.js new file mode 100644 index 0000000000..b5103a08d9 --- /dev/null +++ b/lib/controllers/accounts/tags.js @@ -0,0 +1,25 @@ +'use strict'; + +const db = require('../../database'); +const helpers = require('../helpers'); +const tagsController = module.exports; +tagsController.get = async function (req, res) { + if (req.uid !== res.locals.uid) { + return helpers.notAllowed(req, res); + } + const payload = res.locals.userData; + const { + username, + userslug + } = payload; + const tagData = await db.getSortedSetRange(`uid:${res.locals.uid}:followed_tags`, 0, -1); + payload.tags = tagData; + payload.title = `[[pages:account/watched-tags, ${username}]]`; + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[pages:tags]]' + }]); + res.render('account/tags', payload); +}; \ No newline at end of file diff --git a/lib/controllers/accounts/uploads.js b/lib/controllers/accounts/uploads.js new file mode 100644 index 0000000000..6265a60c49 --- /dev/null +++ b/lib/controllers/accounts/uploads.js @@ -0,0 +1,36 @@ +'use strict'; + +const path = require('path'); +const nconf = require('nconf'); +const db = require('../../database'); +const helpers = require('../helpers'); +const meta = require('../../meta'); +const pagination = require('../../pagination'); +const uploadsController = module.exports; +uploadsController.get = async function (req, res) { + const payload = res.locals.userData; + const { + username, + userslug + } = payload; + const page = Math.max(1, parseInt(req.query.page, 10) || 1); + const itemsPerPage = 25; + const start = (page - 1) * itemsPerPage; + const stop = start + itemsPerPage - 1; + const [itemCount, uploadNames] = await Promise.all([db.sortedSetCard(`uid:${res.locals.uid}:uploads`), db.getSortedSetRevRange(`uid:${res.locals.uid}:uploads`, start, stop)]); + payload.uploads = uploadNames.map(uploadName => ({ + name: uploadName, + url: path.resolve(nconf.get('upload_url'), uploadName) + })); + const pageCount = Math.ceil(itemCount / itemsPerPage); + payload.pagination = pagination.create(page, pageCount, req.query); + payload.privateUploads = meta.config.privateUploads === 1; + payload.title = `[[pages:account/uploads, ${username}]]`; + payload.breadcrumbs = helpers.buildBreadcrumbs([{ + text: username, + url: `/user/${userslug}` + }, { + text: '[[global:uploads]]' + }]); + res.render('account/uploads', payload); +}; \ No newline at end of file diff --git a/lib/controllers/admin.js b/lib/controllers/admin.js new file mode 100644 index 0000000000..7e50c0fdcd --- /dev/null +++ b/lib/controllers/admin.js @@ -0,0 +1,64 @@ +'use strict'; + +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const helpers = require('./helpers'); +const apiController = require('./api'); +const adminController = { + dashboard: require('./admin/dashboard'), + categories: require('./admin/categories'), + privileges: require('./admin/privileges'), + adminsMods: require('./admin/admins-mods'), + tags: require('./admin/tags'), + groups: require('./admin/groups'), + digest: require('./admin/digest'), + appearance: require('./admin/appearance'), + extend: { + widgets: require('./admin/widgets'), + rewards: require('./admin/rewards') + }, + events: require('./admin/events'), + hooks: require('./admin/hooks'), + logs: require('./admin/logs'), + errors: require('./admin/errors'), + database: require('./admin/database'), + cache: require('./admin/cache'), + plugins: require('./admin/plugins'), + settings: require('./admin/settings'), + logger: require('./admin/logger'), + themes: require('./admin/themes'), + users: require('./admin/users'), + uploads: require('./admin/uploads'), + info: require('./admin/info') +}; +adminController.routeIndex = async (req, res) => { + const privilegeSet = await privileges.admin.get(req.uid); + if (privilegeSet.superadmin || privilegeSet['admin:dashboard']) { + return adminController.dashboard.get(req, res); + } else if (privilegeSet['admin:categories']) { + return helpers.redirect(res, 'admin/manage/categories'); + } else if (privilegeSet['admin:privileges']) { + return helpers.redirect(res, 'admin/manage/privileges'); + } else if (privilegeSet['admin:users']) { + return helpers.redirect(res, 'admin/manage/users'); + } else if (privilegeSet['admin:groups']) { + return helpers.redirect(res, 'admin/manage/groups'); + } else if (privilegeSet['admin:admins-mods']) { + return helpers.redirect(res, 'admin/manage/admins-mods'); + } else if (privilegeSet['admin:tags']) { + return helpers.redirect(res, 'admin/manage/tags'); + } else if (privilegeSet['admin:settings']) { + return helpers.redirect(res, 'admin/settings/general'); + } + return helpers.notAllowed(req, res); +}; +adminController.loadConfig = async function (req) { + const config = await apiController.loadConfig(req); + await plugins.hooks.fire('filter:config.get.admin', config); + return config; +}; +adminController.getConfig = async (req, res) => { + const config = await adminController.loadConfig(req); + res.json(config); +}; +module.exports = adminController; \ No newline at end of file diff --git a/lib/controllers/admin/admins-mods.js b/lib/controllers/admin/admins-mods.js new file mode 100644 index 0000000000..6d6d3cdbce --- /dev/null +++ b/lib/controllers/admin/admins-mods.js @@ -0,0 +1,46 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../../database'); +const groups = require('../../groups'); +const categories = require('../../categories'); +const user = require('../../user'); +const meta = require('../../meta'); +const pagination = require('../../pagination'); +const categoriesController = require('./categories'); +const AdminsMods = module.exports; +AdminsMods.get = async function (req, res) { + const rootCid = parseInt(req.query.cid, 10) || 0; + const cidsCount = await db.sortedSetCard(`cid:${rootCid}:children`); + const pageCount = Math.max(1, Math.ceil(cidsCount / meta.config.categoriesPerPage)); + const page = Math.min(parseInt(req.query.page, 10) || 1, pageCount); + const start = Math.max(0, (page - 1) * meta.config.categoriesPerPage); + const stop = start + meta.config.categoriesPerPage - 1; + const cids = await db.getSortedSetRange(`cid:${rootCid}:children`, start, stop); + const selectedCategory = rootCid ? await categories.getCategoryData(rootCid) : null; + const pageCategories = await categories.getCategoriesData(cids); + const [admins, globalMods, moderators, crumbs] = await Promise.all([groups.get('administrators', { + uid: req.uid + }), groups.get('Global Moderators', { + uid: req.uid + }), getModeratorsOfCategories(pageCategories), categoriesController.buildBreadCrumbs(selectedCategory, '/admin/manage/admins-mods')]); + res.render('admin/manage/admins-mods', { + admins: admins, + globalMods: globalMods, + categoryMods: moderators, + selectedCategory: selectedCategory, + pagination: pagination.create(page, pageCount, req.query), + breadcrumbs: crumbs + }); +}; +async function getModeratorsOfCategories(categoryData) { + const [moderatorUids, childrenCounts] = await Promise.all([categories.getModeratorUids(categoryData.map(c => c.cid)), db.sortedSetsCard(categoryData.map(c => `cid:${c.cid}:children`))]); + const uids = _.uniq(_.flatten(moderatorUids)); + const moderatorData = await user.getUsersFields(uids, ['uid', 'username', 'userslug', 'picture']); + const moderatorMap = _.zipObject(uids, moderatorData); + categoryData.forEach((c, index) => { + c.moderators = moderatorUids[index].map(uid => moderatorMap[uid]); + c.subCategoryCount = childrenCounts[index]; + }); + return categoryData; +} \ No newline at end of file diff --git a/lib/controllers/admin/appearance.js b/lib/controllers/admin/appearance.js new file mode 100644 index 0000000000..72cab999a2 --- /dev/null +++ b/lib/controllers/admin/appearance.js @@ -0,0 +1,7 @@ +'use strict'; + +const appearanceController = module.exports; +appearanceController.get = function (req, res) { + const term = req.params.term ? req.params.term : 'themes'; + res.render(`admin/appearance/${term}`, {}); +}; \ No newline at end of file diff --git a/lib/controllers/admin/cache.js b/lib/controllers/admin/cache.js new file mode 100644 index 0000000000..c6f64b3808 --- /dev/null +++ b/lib/controllers/admin/cache.js @@ -0,0 +1,65 @@ +'use strict'; + +const cacheController = module.exports; +const utils = require('../../utils'); +const plugins = require('../../plugins'); +cacheController.get = async function (req, res) { + const postCache = require('../../posts/cache').getOrCreate(); + const groupCache = require('../../groups').cache; + const { + objectCache + } = require('../../database'); + const localCache = require('../../cache'); + const uptimeInSeconds = process.uptime(); + function getInfo(cache) { + return { + length: cache.length, + max: cache.max, + maxSize: cache.maxSize, + itemCount: cache.itemCount, + percentFull: cache.name === 'post' ? (cache.length / cache.maxSize * 100).toFixed(2) : (cache.itemCount / cache.max * 100).toFixed(2), + hits: utils.addCommas(String(cache.hits)), + hitsPerSecond: (cache.hits / uptimeInSeconds).toFixed(2), + misses: utils.addCommas(String(cache.misses)), + hitRatio: (cache.hits / (cache.hits + cache.misses) || 0).toFixed(4), + enabled: cache.enabled, + ttl: cache.ttl + }; + } + let caches = { + post: postCache, + group: groupCache, + local: localCache + }; + if (objectCache) { + caches.object = objectCache; + } + caches = await plugins.hooks.fire('filter:admin.cache.get', caches); + for (const [key, value] of Object.entries(caches)) { + caches[key] = getInfo(value); + } + res.render('admin/advanced/cache', { + caches + }); +}; +cacheController.dump = async function (req, res, next) { + let caches = { + post: require('../../posts/cache').getOrCreate(), + object: require('../../database').objectCache, + group: require('../../groups').cache, + local: require('../../cache') + }; + caches = await plugins.hooks.fire('filter:admin.cache.get', caches); + if (!caches[req.query.name]) { + return next(); + } + const data = JSON.stringify(caches[req.query.name].dump(), null, 4); + res.setHeader('Content-disposition', `attachment; filename= ${req.query.name}-cache.json`); + res.setHeader('Content-type', 'application/json'); + res.write(data, err => { + if (err) { + return next(err); + } + res.end(); + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/categories.js b/lib/controllers/admin/categories.js new file mode 100644 index 0000000000..b12bba3e1a --- /dev/null +++ b/lib/controllers/admin/categories.js @@ -0,0 +1,117 @@ +'use strict'; + +const _ = require('lodash'); +const nconf = require('nconf'); +const categories = require('../../categories'); +const analytics = require('../../analytics'); +const plugins = require('../../plugins'); +const translator = require('../../translator'); +const meta = require('../../meta'); +const helpers = require('../helpers'); +const pagination = require('../../pagination'); +const categoriesController = module.exports; +categoriesController.get = async function (req, res, next) { + const [categoryData, parent, selectedData] = await Promise.all([categories.getCategories([req.params.category_id]), categories.getParents([req.params.category_id]), helpers.getSelectedCategory(req.params.category_id)]); + const category = categoryData[0]; + if (!category) { + return next(); + } + category.parent = parent[0]; + const data = await plugins.hooks.fire('filter:admin.category.get', { + req: req, + res: res, + category: category, + customClasses: [] + }); + data.category.name = translator.escape(String(data.category.name)); + data.category.description = translator.escape(String(data.category.description)); + res.render('admin/manage/category', { + category: data.category, + selectedCategory: selectedData.selectedCategory, + customClasses: data.customClasses, + postQueueEnabled: !!meta.config.postQueue + }); +}; +categoriesController.getAll = async function (req, res) { + const rootCid = parseInt(req.query.cid, 10) || 0; + async function getRootAndChildren() { + const rootChildren = await categories.getAllCidsFromSet(`cid:${rootCid}:children`); + const childCids = _.flatten(await Promise.all(rootChildren.map(cid => categories.getChildrenCids(cid)))); + return [rootCid].concat(rootChildren.concat(childCids)); + } + const cids = await (rootCid ? getRootAndChildren() : categories.getAllCidsFromSet('categories:cid')); + let rootParent = 0; + if (rootCid) { + rootParent = (await categories.getCategoryField(rootCid, 'parentCid')) || 0; + } + const fields = ['cid', 'name', 'icon', 'parentCid', 'disabled', 'link', 'order', 'color', 'bgColor', 'backgroundImage', 'imageClass', 'subCategoriesPerPage', 'description']; + const categoriesData = await categories.getCategoriesFields(cids, fields); + const result = await plugins.hooks.fire('filter:admin.categories.get', { + categories: categoriesData, + fields: fields + }); + let tree = categories.getTree(result.categories, rootParent); + const cidsCount = rootCid && tree[0] ? tree[0].children.length : tree.length; + const pageCount = Math.max(1, Math.ceil(cidsCount / meta.config.categoriesPerPage)); + const page = Math.min(parseInt(req.query.page, 10) || 1, pageCount); + const start = Math.max(0, (page - 1) * meta.config.categoriesPerPage); + const stop = start + meta.config.categoriesPerPage; + function trim(c) { + if (c.children) { + c.subCategoriesLeft = Math.max(0, c.children.length - c.subCategoriesPerPage); + c.hasMoreSubCategories = c.children.length > c.subCategoriesPerPage; + c.showMorePage = Math.ceil(c.subCategoriesPerPage / meta.config.categoriesPerPage); + c.children = c.children.slice(0, c.subCategoriesPerPage); + c.children.forEach(c => trim(c)); + } + } + if (rootCid && tree[0] && Array.isArray(tree[0].children)) { + tree[0].children = tree[0].children.slice(start, stop); + tree[0].children.forEach(trim); + } else { + tree = tree.slice(start, stop); + tree.forEach(trim); + } + let selectedCategory; + if (rootCid) { + selectedCategory = await categories.getCategoryData(rootCid); + } + const crumbs = await buildBreadcrumbs(selectedCategory, '/admin/manage/categories'); + res.render('admin/manage/categories', { + categoriesTree: tree, + selectedCategory: selectedCategory, + breadcrumbs: crumbs, + pagination: pagination.create(page, pageCount, req.query), + categoriesPerPage: meta.config.categoriesPerPage, + selectCategoryLabel: '[[admin/manage/categories:jump-to]]' + }); +}; +async function buildBreadcrumbs(categoryData, url) { + if (!categoryData) { + return; + } + const breadcrumbs = [{ + text: categoryData.name, + url: `${nconf.get('relative_path')}${url}?cid=${categoryData.cid}`, + cid: categoryData.cid + }]; + const allCrumbs = await helpers.buildCategoryBreadcrumbs(categoryData.parentCid); + const crumbs = allCrumbs.filter(c => c.cid); + crumbs.forEach(c => { + c.url = `${url}?cid=${c.cid}`; + }); + crumbs.unshift({ + text: '[[admin/manage/categories:top-level]]', + url: url + }); + return crumbs.concat(breadcrumbs); +} +categoriesController.buildBreadCrumbs = buildBreadcrumbs; +categoriesController.getAnalytics = async function (req, res) { + const [name, analyticsData, selectedData] = await Promise.all([categories.getCategoryField(req.params.category_id, 'name'), analytics.getCategoryAnalytics(req.params.category_id), helpers.getSelectedCategory(req.params.category_id)]); + res.render('admin/manage/category-analytics', { + name: name, + analytics: analyticsData, + selectedCategory: selectedData.selectedCategory + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/dashboard.js b/lib/controllers/admin/dashboard.js new file mode 100644 index 0000000000..f9442fd29e --- /dev/null +++ b/lib/controllers/admin/dashboard.js @@ -0,0 +1,334 @@ +'use strict'; + +const nconf = require('nconf'); +const semver = require('semver'); +const winston = require('winston'); +const _ = require('lodash'); +const validator = require('validator'); +const versions = require('../../admin/versions'); +const db = require('../../database'); +const meta = require('../../meta'); +const analytics = require('../../analytics'); +const plugins = require('../../plugins'); +const user = require('../../user'); +const topics = require('../../topics'); +const utils = require('../../utils'); +const emailer = require('../../emailer'); +const dashboardController = module.exports; +dashboardController.get = async function (req, res) { + const [stats, notices, latestVersion, lastrestart, isAdmin, popularSearches] = await Promise.all([getStats(), getNotices(), getLatestVersion(), getLastRestart(), user.isAdministrator(req.uid), getPopularSearches()]); + const version = nconf.get('version'); + res.render('admin/dashboard', { + version: version, + lookupFailed: latestVersion === null, + latestVersion: latestVersion, + upgradeAvailable: latestVersion && semver.gt(latestVersion, version), + currentPrerelease: versions.isPrerelease.test(version), + notices: notices, + stats: stats, + canRestart: !!process.send, + lastrestart: lastrestart, + showSystemControls: isAdmin, + popularSearches: popularSearches + }); +}; +async function getNotices() { + const notices = [{ + done: !meta.reloadRequired, + doneText: '[[admin/dashboard:restart-not-required]]', + notDoneText: '[[admin/dashboard:restart-required]]' + }, { + done: plugins.hooks.hasListeners('filter:search.query'), + doneText: '[[admin/dashboard:search-plugin-installed]]', + notDoneText: '[[admin/dashboard:search-plugin-not-installed]]', + tooltip: '[[admin/dashboard:search-plugin-tooltip]]', + link: '/admin/extend/plugins' + }]; + if (emailer.fallbackNotFound) { + notices.push({ + done: false, + notDoneText: '[[admin/dashboard:fallback-emailer-not-found]]' + }); + } + if (global.env !== 'production') { + notices.push({ + done: false, + notDoneText: '[[admin/dashboard:running-in-development]]' + }); + } + return await plugins.hooks.fire('filter:admin.notices', notices); +} +async function getLatestVersion() { + try { + return await versions.getLatestVersion(); + } catch (err) { + winston.error(`[acp] Failed to fetch latest version\n${err.stack}`); + } + return null; +} +dashboardController.getAnalytics = async (req, res, next) => { + const validUnits = ['days', 'hours']; + const validSets = ['uniquevisitors', 'pageviews', 'pageviews:registered', 'pageviews:bot', 'pageviews:guest']; + const until = req.query.until ? new Date(parseInt(req.query.until, 10)) : Date.now(); + const count = req.query.count || (req.query.units === 'hours' ? 24 : 30); + if (isNaN(until) || !validUnits.includes(req.query.units)) { + return next(new Error('[[error:invalid-data]]')); + } + let sets; + if (req.query.sets) { + sets = Array.isArray(req.query.sets) ? req.query.sets : [req.query.sets]; + sets = sets.filter(set => validSets.includes(set)); + } else { + sets = validSets; + } + const method = req.query.units === 'days' ? analytics.getDailyStatsForSet : analytics.getHourlyStatsForSet; + let payload = await Promise.all(sets.map(set => method(`analytics:${set}`, until, count))); + payload = _.zipObject(sets, payload); + res.json({ + query: { + set: req.query.set, + units: req.query.units, + until: until, + count: count + }, + result: payload + }); +}; +async function getStats() { + const cache = require('../../cache'); + const cachedStats = cache.get('admin:stats'); + if (cachedStats !== undefined) { + return cachedStats; + } + let results = await Promise.all([getStatsFromAnalytics('uniquevisitors', 'uniqueIPCount'), getStatsFromAnalytics('logins', 'loginCount'), getStatsForSet('users:joindate', 'userCount'), getStatsForSet('posts:pid', 'postCount'), getStatsForSet('topics:tid', 'topicCount')]); + results[0].name = '[[admin/dashboard:unique-visitors]]'; + results[1].name = '[[admin/dashboard:logins]]'; + results[1].href = `${nconf.get('relative_path')}/admin/dashboard/logins`; + results[2].name = '[[admin/dashboard:new-users]]'; + results[2].href = `${nconf.get('relative_path')}/admin/dashboard/users`; + results[3].name = '[[admin/dashboard:posts]]'; + results[4].name = '[[admin/dashboard:topics]]'; + results[4].href = `${nconf.get('relative_path')}/admin/dashboard/topics`; + ({ + results + } = await plugins.hooks.fire('filter:admin.getStats', { + results, + helpers: { + getStatsForSet, + getStatsFromAnalytics + } + })); + cache.set('admin:stats', results, 600000); + return results; +} +async function getStatsForSet(set, field) { + const terms = { + day: 86400000, + week: 604800000, + month: 2592000000 + }; + const now = Date.now(); + const results = await utils.promiseParallel({ + yesterday: db.sortedSetCount(set, now - terms.day * 2, '+inf'), + today: db.sortedSetCount(set, now - terms.day, '+inf'), + lastweek: db.sortedSetCount(set, now - terms.week * 2, '+inf'), + thisweek: db.sortedSetCount(set, now - terms.week, '+inf'), + lastmonth: db.sortedSetCount(set, now - terms.month * 2, '+inf'), + thismonth: db.sortedSetCount(set, now - terms.month, '+inf'), + alltime: getGlobalField(field) + }); + return calculateDeltas(results); +} +async function getStatsFromAnalytics(set, field) { + const today = new Date(); + today.setHours(0, 0, 0, 0); + const data = await analytics.getDailyStatsForSet(`analytics:${set}`, today, 60); + const sum = arr => arr.reduce((memo, cur) => memo + cur, 0); + const results = { + yesterday: sum(data.slice(-2)), + today: data.slice(-1)[0], + lastweek: sum(data.slice(-14)), + thisweek: sum(data.slice(-7)), + lastmonth: sum(data.slice(0)), + thismonth: sum(data.slice(-30)), + alltime: await getGlobalField(field) + }; + return calculateDeltas(results); +} +function calculateDeltas(results) { + function textClass(num) { + if (num > 0) { + return 'text-success'; + } else if (num < 0) { + return 'text-danger'; + } + return 'text-warning'; + } + function increasePercent(last, now) { + const percent = last ? (now - last) / last * 100 : 0; + return percent.toFixed(1); + } + results.yesterday -= results.today; + results.dayIncrease = increasePercent(results.yesterday, results.today); + results.dayTextClass = textClass(results.dayIncrease); + results.lastweek -= results.thisweek; + results.weekIncrease = increasePercent(results.lastweek, results.thisweek); + results.weekTextClass = textClass(results.weekIncrease); + results.lastmonth -= results.thismonth; + results.monthIncrease = increasePercent(results.lastmonth, results.thismonth); + results.monthTextClass = textClass(results.monthIncrease); + return results; +} +async function getGlobalField(field) { + const count = await db.getObjectField('global', field); + return parseInt(count, 10) || 0; +} +async function getLastRestart() { + const lastrestart = await db.getObject('lastrestart'); + if (!lastrestart) { + return null; + } + const userData = await user.getUserData(lastrestart.uid); + lastrestart.user = userData; + lastrestart.timestampISO = utils.toISOString(lastrestart.timestamp); + return lastrestart; +} +async function getPopularSearches() { + const searches = await db.getSortedSetRevRangeWithScores('searches:all', 0, 9); + return searches.map(s => ({ + value: validator.escape(String(s.value)), + score: s.score + })); +} +dashboardController.getLogins = async (req, res) => { + let stats = await getStats(); + stats = stats.filter(stat => stat.name === '[[admin/dashboard:logins]]').map(({ + ...stat + }) => { + delete stat.href; + return stat; + }); + const summary = { + day: stats[0].today, + week: stats[0].thisweek, + month: stats[0].thismonth + }; + const start = Date.now() - 1000 * 60 * 60 * 24 * meta.config.loginDays; + const uids = await db.getSortedSetRangeByScore('users:online', 0, 500, start, Date.now()); + const usersData = await user.getUsersData(uids); + let sessions = await Promise.all(uids.map(async uid => { + const sessions = await user.auth.getSessions(uid); + sessions.forEach(session => { + session.user = usersData[uids.indexOf(uid)]; + }); + return sessions; + })); + sessions = _.flatten(sessions).sort((a, b) => b.datetime - a.datetime); + res.render('admin/dashboard/logins', { + set: 'logins', + query: req.query, + stats, + summary, + sessions, + loginDays: meta.config.loginDays + }); +}; +dashboardController.getUsers = async (req, res) => { + let stats = await getStats(); + stats = stats.filter(stat => stat.name === '[[admin/dashboard:new-users]]').map(({ + ...stat + }) => { + delete stat.href; + return stat; + }); + const summary = { + day: stats[0].today, + week: stats[0].thisweek, + month: stats[0].thismonth + }; + const end = parseInt(req.query.until, 10) || Date.now(); + const start = end - 1000 * 60 * 60 * (req.query.units === 'days' ? 24 : 1) * (req.query.count || (req.query.units === 'days' ? 30 : 24)); + const uids = await db.getSortedSetRangeByScore('users:joindate', 0, 500, start, end); + const users = await user.getUsersData(uids); + res.render('admin/dashboard/users', { + set: 'registrations', + query: req.query, + stats, + summary, + users + }); +}; +dashboardController.getTopics = async (req, res) => { + let stats = await getStats(); + stats = stats.filter(stat => stat.name === '[[admin/dashboard:topics]]').map(({ + ...stat + }) => { + delete stat.href; + return stat; + }); + const summary = { + day: stats[0].today, + week: stats[0].thisweek, + month: stats[0].thismonth + }; + const end = parseInt(req.query.until, 10) || Date.now(); + const start = end - 1000 * 60 * 60 * (req.query.units === 'days' ? 24 : 1) * (req.query.count || (req.query.units === 'days' ? 30 : 24)); + const tids = await db.getSortedSetRangeByScore('topics:tid', 0, 500, start, end); + const topicData = await topics.getTopicsByTids(tids); + res.render('admin/dashboard/topics', { + set: 'topics', + query: req.query, + stats, + summary, + topics: topicData + }); +}; +dashboardController.getSearches = async (req, res) => { + let start = 0; + let end = 0; + if (req.query.start) { + start = new Date(req.query.start); + start.setHours(24, 0, 0, 0); + end = new Date(); + end.setHours(24, 0, 0, 0); + } + if (req.query.end) { + end = new Date(req.query.end); + end.setHours(24, 0, 0, 0); + } + let searches; + if (start && end && start <= end) { + const daysArr = [start]; + const nextDay = new Date(start.getTime()); + while (nextDay < end) { + nextDay.setDate(nextDay.getDate() + 1); + nextDay.setHours(0, 0, 0, 0); + daysArr.push(new Date(nextDay.getTime())); + } + const daysData = await Promise.all(daysArr.map(async d => db.getSortedSetRevRangeWithScores(`searches:${d.getTime()}`, 0, -1))); + const map = {}; + daysData.forEach(d => { + d.forEach(search => { + if (!map[search.value]) { + map[search.value] = search.score; + } else { + map[search.value] += search.score; + } + }); + }); + searches = Object.keys(map).map(key => ({ + value: key, + score: map[key] + })).sort((a, b) => b.score - a.score); + } else { + searches = await db.getSortedSetRevRangeWithScores('searches:all', 0, 99); + } + res.render('admin/dashboard/searches', { + searches: searches.map(s => ({ + value: validator.escape(String(s.value)), + score: s.score + })), + startDate: req.query.start ? validator.escape(String(req.query.start)) : null, + endDate: req.query.end ? validator.escape(String(req.query.end)) : null + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/database.js b/lib/controllers/admin/database.js new file mode 100644 index 0000000000..ee353d2d4f --- /dev/null +++ b/lib/controllers/admin/database.js @@ -0,0 +1,20 @@ +'use strict'; + +const nconf = require('nconf'); +const databaseController = module.exports; +databaseController.get = async function (req, res) { + const results = {}; + if (nconf.get('redis')) { + const rdb = require('../../database/redis'); + results.redis = await rdb.info(rdb.client); + } + if (nconf.get('mongo')) { + const mdb = require('../../database/mongo'); + results.mongo = await mdb.info(mdb.client); + } + if (nconf.get('postgres')) { + const pdb = require('../../database/postgres'); + results.postgres = await pdb.info(pdb.pool); + } + res.render('admin/advanced/database', results); +}; \ No newline at end of file diff --git a/lib/controllers/admin/digest.js b/lib/controllers/admin/digest.js new file mode 100644 index 0000000000..f1b4971fef --- /dev/null +++ b/lib/controllers/admin/digest.js @@ -0,0 +1,20 @@ +'use strict'; + +const meta = require('../../meta'); +const digest = require('../../user/digest'); +const pagination = require('../../pagination'); +const digestController = module.exports; +digestController.get = async function (req, res) { + const page = parseInt(req.query.page, 10) || 1; + const resultsPerPage = 50; + const start = Math.max(0, page - 1) * resultsPerPage; + const stop = start + resultsPerPage - 1; + const delivery = await digest.getDeliveryTimes(start, stop); + const pageCount = Math.ceil(delivery.count / resultsPerPage); + res.render('admin/manage/digest', { + title: '[[admin/menu:manage/digest]]', + delivery: delivery.users, + default: meta.config.dailyDigestFreq, + pagination: pagination.create(page, pageCount) + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/errors.js b/lib/controllers/admin/errors.js new file mode 100644 index 0000000000..cee1282557 --- /dev/null +++ b/lib/controllers/admin/errors.js @@ -0,0 +1,23 @@ +'use strict'; + +const json2csvAsync = require('json2csv').parseAsync; +const meta = require('../../meta'); +const analytics = require('../../analytics'); +const utils = require('../../utils'); +const errorsController = module.exports; +errorsController.get = async function (req, res) { + const data = await utils.promiseParallel({ + 'not-found': meta.errors.get(true), + analytics: analytics.getErrorAnalytics() + }); + res.render('admin/advanced/errors', data); +}; +errorsController.export = async function (req, res) { + const data = await meta.errors.get(false); + const fields = data.length ? Object.keys(data[0]) : []; + const opts = { + fields + }; + const csv = await json2csvAsync(data, opts); + res.set('Content-Type', 'text/csv').set('Content-Disposition', 'attachment; filename="404.csv"').send(csv); +}; \ No newline at end of file diff --git a/lib/controllers/admin/events.js b/lib/controllers/admin/events.js new file mode 100644 index 0000000000..d21ff1c9d4 --- /dev/null +++ b/lib/controllers/admin/events.js @@ -0,0 +1,51 @@ +'use strict'; + +const db = require('../../database'); +const events = require('../../events'); +const pagination = require('../../pagination'); +const user = require('../../user'); +const groups = require('../../groups'); +const eventsController = module.exports; +eventsController.get = async function (req, res) { + const page = parseInt(req.query.page, 10) || 1; + const itemsPerPage = parseInt(req.query.perPage, 10) || 20; + const start = (page - 1) * itemsPerPage; + const stop = start + itemsPerPage - 1; + let uids; + if (req.query.username) { + uids = [await user.getUidByUsername(req.query.username)]; + } else if (req.query.group) { + uids = await groups.getMembers(req.query.group, 0, -1); + } + let from = req.query.start ? new Date(req.query.start) || undefined : undefined; + let to = req.query.end ? new Date(req.query.end) || undefined : new Date(); + from = from && from.setUTCHours(0, 0, 0, 0); + to = to && to.setUTCHours(23, 59, 59, 999); + const currentFilter = req.query.type || ''; + const [eventCount, eventData, counts] = await Promise.all([events.getEventCount({ + filter: currentFilter, + uids, + from: from || '-inf', + to + }), events.getEvents({ + filter: currentFilter, + uids, + start, + stop, + from: from || '-inf', + to + }), db.sortedSetsCard([''].concat(events.types).map(type => `events:time${type ? `:${type}` : ''}`))]); + const types = [''].concat(events.types).map((type, index) => ({ + value: type, + name: type || 'all', + selected: type === currentFilter, + count: counts[index] + })); + const pageCount = Math.max(1, Math.ceil(eventCount / itemsPerPage)); + res.render('admin/advanced/events', { + events: eventData, + pagination: pagination.create(page, pageCount, req.query), + types: types, + query: req.query + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/groups.js b/lib/controllers/admin/groups.js new file mode 100644 index 0000000000..37bba9fcac --- /dev/null +++ b/lib/controllers/admin/groups.js @@ -0,0 +1,84 @@ +'use strict'; + +const nconf = require('nconf'); +const validator = require('validator'); +const db = require('../../database'); +const user = require('../../user'); +const groups = require('../../groups'); +const meta = require('../../meta'); +const pagination = require('../../pagination'); +const events = require('../../events'); +const slugify = require('../../slugify'); +const groupsController = module.exports; +groupsController.list = async function (req, res) { + const page = parseInt(req.query.page, 10) || 1; + const groupsPerPage = 20; + let groupNames = await getGroupNames(); + const pageCount = Math.ceil(groupNames.length / groupsPerPage); + const start = (page - 1) * groupsPerPage; + const stop = start + groupsPerPage - 1; + groupNames = groupNames.slice(start, stop + 1); + const groupData = await groups.getGroupsData(groupNames); + res.render('admin/manage/groups', { + groups: groupData, + pagination: pagination.create(page, pageCount), + yourid: req.uid + }); +}; +groupsController.get = async function (req, res, next) { + const slug = slugify(req.params.name); + const groupName = await groups.getGroupNameByGroupSlug(slug); + if (!groupName) { + return next(); + } + const [groupNames, group] = await Promise.all([getGroupNames(), groups.get(groupName, { + uid: req.uid, + truncateUserList: true, + userListCount: 20 + })]); + if (!group || groupName === groups.BANNED_USERS) { + return next(); + } + const groupNameData = groupNames.map(name => ({ + encodedName: encodeURIComponent(name), + displayName: validator.escape(String(name)), + selected: name === groupName + })); + res.render('admin/manage/group', { + group: group, + groupNames: groupNameData, + allowPrivateGroups: meta.config.allowPrivateGroups, + maximumGroupNameLength: meta.config.maximumGroupNameLength, + maximumGroupTitleLength: meta.config.maximumGroupTitleLength + }); +}; +async function getGroupNames() { + const groupNames = Object.values(await db.getObject('groupslug:groupname')); + return groupNames.filter(name => name !== 'registered-users' && name !== 'verified-users' && name !== 'unverified-users' && name !== groups.BANNED_USERS).sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase())); +} +groupsController.getCSV = async function (req, res) { + const { + referer + } = req.headers; + if (!referer || !referer.replace(nconf.get('url'), '').startsWith('/admin/manage/groups')) { + return res.status(403).send('[[error:invalid-origin]]'); + } + await events.log({ + type: 'getGroupCSV', + uid: req.uid, + ip: req.ip, + group: req.params.groupname + }); + const groupName = req.params.groupname; + const members = (await groups.getMembersOfGroups([groupName]))[0]; + const fields = ['email', 'username', 'uid']; + const userData = await user.getUsersFields(members, fields); + let csvContent = `${fields.join(',')}\n`; + csvContent += userData.reduce((memo, user) => { + memo += `${user.email},${user.username},${user.uid}\n`; + return memo; + }, ''); + res.attachment(`${validator.escape(groupName)}_members.csv`); + res.setHeader('Content-Type', 'text/csv'); + res.end(csvContent); +}; \ No newline at end of file diff --git a/lib/controllers/admin/hooks.js b/lib/controllers/admin/hooks.js new file mode 100644 index 0000000000..24ea612aba --- /dev/null +++ b/lib/controllers/admin/hooks.js @@ -0,0 +1,29 @@ +'use strict'; + +const validator = require('validator'); +const plugins = require('../../plugins'); +const hooksController = module.exports; +hooksController.get = function (req, res) { + const hooks = []; + Object.keys(plugins.loadedHooks).forEach((key, hookIndex) => { + const current = { + hookName: key, + methods: [], + index: `hook-${hookIndex}`, + count: plugins.loadedHooks[key].length + }; + plugins.loadedHooks[key].forEach((hookData, methodIndex) => { + current.methods.push({ + id: hookData.id, + priority: hookData.priority, + method: hookData.method ? validator.escape(hookData.method.toString()) : 'No plugin function!', + index: `hook-${hookIndex}-code-${methodIndex}` + }); + }); + hooks.push(current); + }); + hooks.sort((a, b) => b.count - a.count); + res.render('admin/advanced/hooks', { + hooks: hooks + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/info.js b/lib/controllers/admin/info.js new file mode 100644 index 0000000000..a661808c9a --- /dev/null +++ b/lib/controllers/admin/info.js @@ -0,0 +1,134 @@ +'use strict'; + +const os = require('os'); +const winston = require('winston'); +const nconf = require('nconf'); +const { + exec +} = require('child_process'); +const pubsub = require('../../pubsub'); +const rooms = require('../../socket.io/admin/rooms'); +const infoController = module.exports; +let info = {}; +let previousUsage = process.cpuUsage(); +let usageStartDate = Date.now(); +infoController.get = function (req, res) { + info = {}; + pubsub.publish('sync:node:info:start'); + const timeoutMS = 1000; + setTimeout(() => { + const data = []; + Object.keys(info).forEach(key => data.push(info[key])); + data.sort((a, b) => { + if (a.id < b.id) { + return -1; + } + if (a.id > b.id) { + return 1; + } + return 0; + }); + let port = nconf.get('port'); + if (!Array.isArray(port) && !isNaN(parseInt(port, 10))) { + port = [port]; + } + res.render('admin/development/info', { + info: data, + infoJSON: JSON.stringify(data, null, 4), + host: os.hostname(), + port: port, + nodeCount: data.length, + timeout: timeoutMS, + ip: req.ip + }); + }, timeoutMS); +}; +pubsub.on('sync:node:info:start', async () => { + try { + const data = await getNodeInfo(); + data.id = `${os.hostname()}:${nconf.get('port')}`; + pubsub.publish('sync:node:info:end', { + data: data, + id: data.id + }); + } catch (err) { + winston.error(err.stack); + } +}); +pubsub.on('sync:node:info:end', data => { + info[data.id] = data.data; +}); +async function getNodeInfo() { + const data = { + process: { + port: nconf.get('port'), + pid: process.pid, + title: process.title, + version: process.version, + memoryUsage: process.memoryUsage(), + uptime: process.uptime(), + cpuUsage: getCpuUsage() + }, + os: { + hostname: os.hostname(), + type: os.type(), + platform: os.platform(), + arch: os.arch(), + release: os.release(), + load: os.loadavg().map(load => load.toFixed(2)).join(', '), + freemem: os.freemem(), + totalmem: os.totalmem() + }, + nodebb: { + isCluster: nconf.get('isCluster'), + isPrimary: nconf.get('isPrimary'), + runJobs: nconf.get('runJobs'), + jobsDisabled: nconf.get('jobsDisabled') + } + }; + data.process.memoryUsage.humanReadable = (data.process.memoryUsage.rss / (1024 * 1024 * 1024)).toFixed(3); + data.process.uptimeHumanReadable = humanReadableUptime(data.process.uptime); + data.os.freemem = (data.os.freemem / (1024 * 1024 * 1024)).toFixed(2); + data.os.totalmem = (data.os.totalmem / (1024 * 1024 * 1024)).toFixed(2); + data.os.usedmem = (data.os.totalmem - data.os.freemem).toFixed(2); + const [stats, gitInfo] = await Promise.all([rooms.getLocalStats(), getGitInfo()]); + data.git = gitInfo; + data.stats = stats; + return data; +} +function getCpuUsage() { + const newUsage = process.cpuUsage(); + const diff = newUsage.user + newUsage.system - (previousUsage.user + previousUsage.system); + const now = Date.now(); + const result = diff / ((now - usageStartDate) * 1000) * 100; + previousUsage = newUsage; + usageStartDate = now; + return result.toFixed(2); +} +function humanReadableUptime(seconds) { + if (seconds < 60) { + return `${Math.floor(seconds)}s`; + } else if (seconds < 3600) { + return `${Math.floor(seconds / 60)}m`; + } else if (seconds < 3600 * 24) { + return `${Math.floor(seconds / (60 * 60))}h`; + } + return `${Math.floor(seconds / (60 * 60 * 24))}d`; +} +async function getGitInfo() { + function get(cmd, callback) { + exec(cmd, (err, stdout) => { + if (err) { + winston.error(err.stack); + } + callback(null, stdout ? stdout.replace(/\n$/, '') : 'no-git-info'); + }); + } + const getAsync = require('util').promisify(get); + const [hash, branch] = await Promise.all([getAsync('git rev-parse HEAD'), getAsync('git rev-parse --abbrev-ref HEAD')]); + return { + hash: hash, + hashShort: hash.slice(0, 6), + branch: branch + }; +} \ No newline at end of file diff --git a/lib/controllers/admin/logger.js b/lib/controllers/admin/logger.js new file mode 100644 index 0000000000..912ad46614 --- /dev/null +++ b/lib/controllers/admin/logger.js @@ -0,0 +1,6 @@ +'use strict'; + +const loggerController = module.exports; +loggerController.get = function (req, res) { + res.render('admin/development/logger', {}); +}; \ No newline at end of file diff --git a/lib/controllers/admin/logs.js b/lib/controllers/admin/logs.js new file mode 100644 index 0000000000..32b401373a --- /dev/null +++ b/lib/controllers/admin/logs.js @@ -0,0 +1,17 @@ +'use strict'; + +const validator = require('validator'); +const winston = require('winston'); +const meta = require('../../meta'); +const logsController = module.exports; +logsController.get = async function (req, res) { + let logs = ''; + try { + logs = await meta.logs.get(); + } catch (err) { + winston.error(err.stack); + } + res.render('admin/advanced/logs', { + data: validator.escape(logs) + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/plugins.js b/lib/controllers/admin/plugins.js new file mode 100644 index 0000000000..06c2e7a709 --- /dev/null +++ b/lib/controllers/admin/plugins.js @@ -0,0 +1,54 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const plugins = require('../../plugins'); +const meta = require('../../meta'); +const pluginsController = module.exports; +pluginsController.get = async function (req, res) { + const [compatible, all, trending] = await Promise.all([getCompatiblePlugins(), getAllPlugins(), plugins.listTrending()]); + const compatiblePkgNames = compatible.map(pkgData => pkgData.name); + const installedPlugins = compatible.filter(plugin => plugin && (plugin.installed || nconf.get('plugins:active') && plugin.active)); + const activePlugins = all.filter(plugin => plugin && (plugin.installed || nconf.get('plugins:active')) && plugin.active); + const trendingScores = trending.reduce((memo, cur) => { + memo[cur.label] = cur.value; + return memo; + }, {}); + const trendingPlugins = all.filter(plugin => plugin && Object.keys(trendingScores).includes(plugin.id)).sort((a, b) => trendingScores[b.id] - trendingScores[a.id]).map(plugin => { + plugin.downloads = trendingScores[plugin.id]; + return plugin; + }); + res.render('admin/extend/plugins', { + installed: installedPlugins, + installedCount: installedPlugins.length, + activeCount: activePlugins.length, + inactiveCount: Math.max(0, installedPlugins.length - activePlugins.length), + canChangeState: !nconf.get('plugins:active'), + upgradeCount: compatible.reduce((count, current) => { + if (current.installed && current.outdated) { + count += 1; + } + return count; + }, 0), + download: compatible.filter(plugin => !plugin.installed), + incompatible: all.filter(plugin => !compatiblePkgNames.includes(plugin.name)), + trending: trendingPlugins, + submitPluginUsage: meta.config.submitPluginUsage, + version: nconf.get('version') + }); +}; +async function getCompatiblePlugins() { + return await getPlugins(true); +} +async function getAllPlugins() { + return await getPlugins(false); +} +async function getPlugins(matching) { + try { + const pluginsData = await plugins.list(matching); + return pluginsData || []; + } catch (err) { + winston.error(err.stack); + return []; + } +} \ No newline at end of file diff --git a/lib/controllers/admin/privileges.js b/lib/controllers/admin/privileges.js new file mode 100644 index 0000000000..ed98693328 --- /dev/null +++ b/lib/controllers/admin/privileges.js @@ -0,0 +1,45 @@ +'use strict'; + +const categories = require('../../categories'); +const privileges = require('../../privileges'); +const privilegesController = module.exports; +privilegesController.get = async function (req, res) { + const cid = req.params.cid ? parseInt(req.params.cid, 10) || 0 : 0; + const isAdminPriv = req.params.cid === 'admin'; + let privilegesData; + if (cid > 0) { + privilegesData = await privileges.categories.list(cid); + } else if (cid === 0) { + privilegesData = await (isAdminPriv ? privileges.admin.list(req.uid) : privileges.global.list()); + } + const categoriesData = [{ + cid: 0, + name: '[[admin/manage/privileges:global]]', + icon: 'fa-list' + }, { + cid: 'admin', + name: '[[admin/manage/privileges:admin]]', + icon: 'fa-lock' + }]; + let selectedCategory; + categoriesData.forEach(category => { + if (category) { + category.selected = category.cid === (!isAdminPriv ? cid : 'admin'); + if (category.selected) { + selectedCategory = category; + } + } + }); + if (!selectedCategory) { + selectedCategory = await categories.getCategoryFields(cid, ['cid', 'name', 'icon', 'bgColor', 'color']); + } + const group = req.query.group ? req.query.group : ''; + res.render('admin/manage/privileges', { + privileges: privilegesData, + categories: categoriesData, + selectedCategory, + cid, + group, + isAdminPriv + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/rewards.js b/lib/controllers/admin/rewards.js new file mode 100644 index 0000000000..5d65be6a3e --- /dev/null +++ b/lib/controllers/admin/rewards.js @@ -0,0 +1,8 @@ +'use strict'; + +const admin = require('../../rewards/admin'); +const rewardsController = module.exports; +rewardsController.get = async function (req, res) { + const data = await admin.get(); + res.render('admin/extend/rewards', data); +}; \ No newline at end of file diff --git a/lib/controllers/admin/settings.js b/lib/controllers/admin/settings.js new file mode 100644 index 0000000000..60903f07d0 --- /dev/null +++ b/lib/controllers/admin/settings.js @@ -0,0 +1,105 @@ +'use strict'; + +const validator = require('validator'); +const meta = require('../../meta'); +const emailer = require('../../emailer'); +const notifications = require('../../notifications'); +const groups = require('../../groups'); +const languages = require('../../languages'); +const navigationAdmin = require('../../navigation/admin'); +const social = require('../../social'); +const api = require('../../api'); +const pagination = require('../../pagination'); +const helpers = require('../helpers'); +const translator = require('../../translator'); +const settingsController = module.exports; +settingsController.get = async function (req, res) { + const term = req.params.term || 'general'; + const payload = { + title: `[[admin/menu:settings/${term}]]` + }; + if (term === 'general') { + payload.routes = await helpers.getHomePageRoutes(req.uid); + payload.postSharing = await social.getPostSharing(); + const languageData = await languages.list(); + languageData.forEach(language => { + language.selected = language.code === meta.config.defaultLang; + }); + payload.languages = languageData; + payload.autoDetectLang = meta.config.autoDetectLang; + } + res.render(`admin/settings/${term}`, payload); +}; +settingsController.email = async (req, res) => { + const emails = await emailer.getTemplates(meta.config); + res.render('admin/settings/email', { + title: '[[admin/menu:settings/email]]', + emails: emails, + sendable: emails.filter(e => !e.path.includes('_plaintext') && !e.path.includes('partials')).map(tpl => tpl.path), + services: emailer.listServices() + }); +}; +settingsController.user = async (req, res) => { + const [notificationTypes, groupData] = await Promise.all([notifications.getAllNotificationTypes(), groups.getNonPrivilegeGroups('groups:createtime', 0, -1)]); + const notificationSettings = notificationTypes.map(type => ({ + name: type, + label: `[[notifications:${type.replace(/_/g, '-')}]]` + })); + res.render('admin/settings/user', { + title: '[[admin/menu:settings/user]]', + notificationSettings: notificationSettings, + groupsExemptFromNewUserRestrictions: groupData + }); +}; +settingsController.post = async (req, res) => { + const groupData = await groups.getNonPrivilegeGroups('groups:createtime', 0, -1); + res.render('admin/settings/post', { + title: '[[admin/menu:settings/post]]', + groupsExemptFromPostQueue: groupData + }); +}; +settingsController.advanced = async (req, res) => { + const groupData = await groups.getNonPrivilegeGroups('groups:createtime', 0, -1); + res.render('admin/settings/advanced', { + title: '[[admin/menu:settings/advanced]]', + groupsExemptFromMaintenanceMode: groupData + }); +}; +settingsController.navigation = async function (req, res) { + const [admin, allGroups] = await Promise.all([navigationAdmin.getAdmin(), groups.getNonPrivilegeGroups('groups:createtime', 0, -1)]); + allGroups.sort((a, b) => b.system - a.system); + admin.groups = allGroups.map(group => ({ + name: group.name, + displayName: group.displayName + })); + admin.enabled.forEach((enabled, index) => { + enabled.index = index; + enabled.selected = index === 0; + enabled.title = translator.escape(enabled.title); + enabled.text = translator.escape(enabled.text); + enabled.dropdownContent = translator.escape(validator.escape(String(enabled.dropdownContent || ''))); + enabled.groups = admin.groups.map(group => ({ + displayName: group.displayName, + selected: enabled.groups.includes(group.name) + })); + }); + admin.available.forEach(available => { + available.groups = admin.groups; + }); + admin.navigation = admin.enabled.slice(); + admin.title = '[[admin/menu:settings/navigation]]'; + res.render('admin/settings/navigation', admin); +}; +settingsController.api = async (req, res) => { + const page = parseInt(req.query.page, 10) || 1; + const resultsPerPage = 50; + const start = Math.max(0, page - 1) * resultsPerPage; + const stop = start + resultsPerPage - 1; + const [tokens, count] = await Promise.all([api.utils.tokens.list(start, stop), api.utils.tokens.count()]); + const pageCount = Math.ceil(count / resultsPerPage); + res.render('admin/settings/api', { + title: '[[admin/menu:settings/api]]', + tokens, + pagination: pagination.create(page, pageCount, req.query) + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/tags.js b/lib/controllers/admin/tags.js new file mode 100644 index 0000000000..0ca22a7e46 --- /dev/null +++ b/lib/controllers/admin/tags.js @@ -0,0 +1,10 @@ +'use strict'; + +const topics = require('../../topics'); +const tagsController = module.exports; +tagsController.get = async function (req, res) { + const tags = await topics.getTags(0, 199); + res.render('admin/manage/tags', { + tags: tags + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/themes.js b/lib/controllers/admin/themes.js new file mode 100644 index 0000000000..95afaf2c9c --- /dev/null +++ b/lib/controllers/admin/themes.js @@ -0,0 +1,27 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const file = require('../../file'); +const { + paths +} = require('../../constants'); +const themesController = module.exports; +const defaultScreenshotPath = path.join(__dirname, '../../../public/images/themes/default.png'); +themesController.get = async function (req, res, next) { + const themeDir = path.join(paths.nodeModules, req.params.theme); + const themeConfigPath = path.join(themeDir, 'theme.json'); + let themeConfig; + try { + themeConfig = await fs.promises.readFile(themeConfigPath, 'utf8'); + themeConfig = JSON.parse(themeConfig); + } catch (err) { + if (err.code === 'ENOENT') { + return next(Error('invalid-data')); + } + return next(err); + } + const screenshotPath = themeConfig.screenshot ? path.join(themeDir, themeConfig.screenshot) : defaultScreenshotPath; + const exists = await file.exists(screenshotPath); + res.sendFile(exists ? screenshotPath : defaultScreenshotPath); +}; \ No newline at end of file diff --git a/lib/controllers/admin/uploads.js b/lib/controllers/admin/uploads.js new file mode 100644 index 0000000000..23d8fb8bba --- /dev/null +++ b/lib/controllers/admin/uploads.js @@ -0,0 +1,248 @@ +'use strict'; + +const path = require('path'); +const nconf = require('nconf'); +const fs = require('fs'); +const meta = require('../../meta'); +const posts = require('../../posts'); +const file = require('../../file'); +const image = require('../../image'); +const plugins = require('../../plugins'); +const pagination = require('../../pagination'); +const allowedImageTypes = ['image/png', 'image/jpeg', 'image/pjpeg', 'image/jpg', 'image/gif', 'image/svg+xml']; +const uploadsController = module.exports; +uploadsController.get = async function (req, res, next) { + const currentFolder = path.join(nconf.get('upload_path'), req.query.dir || ''); + if (!currentFolder.startsWith(nconf.get('upload_path'))) { + return next(new Error('[[error:invalid-path]]')); + } + const itemsPerPage = 20; + const page = parseInt(req.query.page, 10) || 1; + try { + let files = await fs.promises.readdir(currentFolder); + files = files.filter(filename => filename !== '.gitignore'); + const itemCount = files.length; + const start = Math.max(0, (page - 1) * itemsPerPage); + const stop = start + itemsPerPage; + files = files.slice(start, stop); + files = await filesToData(currentFolder, files); + files.sort((a, b) => { + if (a.isDirectory && !b.isDirectory) { + return -1; + } else if (!a.isDirectory && b.isDirectory) { + return 1; + } else if (!a.isDirectory && !b.isDirectory) { + return a.mtime < b.mtime ? -1 : 1; + } + return 0; + }); + if (['files', '/files', '/files/'].includes(req.query.dir)) { + const usage = await posts.uploads.getUsage(files); + files.forEach((file, idx) => { + file.inPids = usage[idx].map(pid => parseInt(pid, 10)); + }); + } + res.render('admin/manage/uploads', { + currentFolder: currentFolder.replace(nconf.get('upload_path'), ''), + showPids: files.length && files[0].hasOwnProperty('inPids'), + files: files, + breadcrumbs: buildBreadcrumbs(currentFolder), + pagination: pagination.create(page, Math.ceil(itemCount / itemsPerPage), req.query) + }); + } catch (err) { + next(err); + } +}; +function buildBreadcrumbs(currentFolder) { + const crumbs = []; + const parts = currentFolder.replace(nconf.get('upload_path'), '').split(path.sep); + let currentPath = ''; + parts.forEach((part, i) => { + const dir = path.join(currentPath, part); + const crumb = { + text: part || 'Uploads' + }; + if (i < parts.length - 1) { + crumb.url = part ? `${nconf.get('relative_path')}/admin/manage/uploads?dir=${dir}` : `${nconf.get('relative_path')}/admin/manage/uploads`; + } + crumbs.push(crumb); + currentPath = dir; + }); + return crumbs; +} +async function filesToData(currentDir, files) { + return await Promise.all(files.map(file => getFileData(currentDir, file))); +} +async function getFileData(currentDir, file) { + const pathToFile = path.join(currentDir, file); + const stat = await fs.promises.stat(pathToFile); + let filesInDir = []; + if (stat.isDirectory()) { + filesInDir = await fs.promises.readdir(pathToFile); + } + const url = `${nconf.get('upload_url') + currentDir.replace(nconf.get('upload_path'), '')}/${file}`; + return { + name: file, + path: pathToFile.replace(path.join(nconf.get('upload_path'), '/'), ''), + url: url, + fileCount: Math.max(0, filesInDir.length - 1), + size: stat.size, + sizeHumanReadable: `${(stat.size / 1024).toFixed(1)}KiB`, + isDirectory: stat.isDirectory(), + isFile: stat.isFile(), + mtime: stat.mtimeMs + }; +} +uploadsController.uploadCategoryPicture = async function (req, res, next) { + const uploadedFile = req.files.files[0]; + let params = null; + try { + params = JSON.parse(req.body.params); + } catch (e) { + file.delete(uploadedFile.path); + return next(new Error('[[error:invalid-json]]')); + } + await validateUpload(uploadedFile, allowedImageTypes); + const filename = `category-${params.cid}${path.extname(uploadedFile.name)}`; + await uploadImage(filename, 'category', uploadedFile, req, res, next); +}; +uploadsController.uploadFavicon = async function (req, res, next) { + const uploadedFile = req.files.files[0]; + const allowedTypes = ['image/x-icon', 'image/vnd.microsoft.icon']; + await validateUpload(uploadedFile, allowedTypes); + try { + const imageObj = await file.saveFileToLocal('favicon.ico', 'system', uploadedFile.path); + res.json([{ + name: uploadedFile.name, + url: imageObj.url + }]); + } catch (err) { + next(err); + } finally { + file.delete(uploadedFile.path); + } +}; +uploadsController.uploadTouchIcon = async function (req, res, next) { + const uploadedFile = req.files.files[0]; + const allowedTypes = ['image/png']; + const sizes = [36, 48, 72, 96, 144, 192, 512]; + await validateUpload(uploadedFile, allowedTypes); + try { + const imageObj = await file.saveFileToLocal('touchicon-orig.png', 'system', uploadedFile.path); + for (const size of sizes) { + await image.resizeImage({ + path: uploadedFile.path, + target: path.join(nconf.get('upload_path'), 'system', `touchicon-${size}.png`), + width: size, + height: size + }); + } + res.json([{ + name: uploadedFile.name, + url: imageObj.url + }]); + } catch (err) { + next(err); + } finally { + file.delete(uploadedFile.path); + } +}; +uploadsController.uploadMaskableIcon = async function (req, res, next) { + const uploadedFile = req.files.files[0]; + const allowedTypes = ['image/png']; + await validateUpload(uploadedFile, allowedTypes); + try { + const imageObj = await file.saveFileToLocal('maskableicon-orig.png', 'system', uploadedFile.path); + res.json([{ + name: uploadedFile.name, + url: imageObj.url + }]); + } catch (err) { + next(err); + } finally { + file.delete(uploadedFile.path); + } +}; +uploadsController.uploadLogo = async function (req, res, next) { + await upload('site-logo', req, res, next); +}; +uploadsController.uploadFile = async function (req, res, next) { + const uploadedFile = req.files.files[0]; + let params; + try { + params = JSON.parse(req.body.params); + } catch (e) { + file.delete(uploadedFile.path); + return next(new Error('[[error:invalid-json]]')); + } + try { + const data = await file.saveFileToLocal(uploadedFile.name, params.folder, uploadedFile.path); + res.json([{ + url: data.url + }]); + } catch (err) { + next(err); + } finally { + file.delete(uploadedFile.path); + } +}; +uploadsController.uploadDefaultAvatar = async function (req, res, next) { + await upload('avatar-default', req, res, next); +}; +uploadsController.uploadOgImage = async function (req, res, next) { + await upload('og:image', req, res, next); +}; +async function upload(name, req, res, next) { + const uploadedFile = req.files.files[0]; + await validateUpload(uploadedFile, allowedImageTypes); + const filename = name + path.extname(uploadedFile.name); + await uploadImage(filename, 'system', uploadedFile, req, res, next); +} +async function validateUpload(uploadedFile, allowedTypes) { + if (!allowedTypes.includes(uploadedFile.type)) { + file.delete(uploadedFile.path); + throw new Error(`[[error:invalid-image-type, ${allowedTypes.join(', ')}]]`); + } +} +async function uploadImage(filename, folder, uploadedFile, req, res, next) { + let imageData; + try { + if (plugins.hooks.hasListeners('filter:uploadImage')) { + imageData = await plugins.hooks.fire('filter:uploadImage', { + image: uploadedFile, + uid: req.uid, + folder: folder + }); + } else { + imageData = await file.saveFileToLocal(filename, folder, uploadedFile.path); + } + if (path.basename(filename, path.extname(filename)) === 'site-logo' && folder === 'system') { + const uploadPath = path.join(nconf.get('upload_path'), folder, 'site-logo-x50.png'); + await image.resizeImage({ + path: uploadedFile.path, + target: uploadPath, + height: 50 + }); + await meta.configs.set('brand:emailLogo', path.join(nconf.get('upload_url'), 'system/site-logo-x50.png')); + const size = await image.size(uploadedFile.path); + await meta.configs.setMultiple({ + 'brand:logo:width': size.width, + 'brand:logo:height': size.height + }); + } else if (path.basename(filename, path.extname(filename)) === 'og:image' && folder === 'system') { + const size = await image.size(uploadedFile.path); + await meta.configs.setMultiple({ + 'og:image:width': size.width, + 'og:image:height': size.height + }); + } + res.json([{ + name: uploadedFile.name, + url: imageData.url.startsWith('http') ? imageData.url : nconf.get('relative_path') + imageData.url + }]); + } catch (err) { + next(err); + } finally { + file.delete(uploadedFile.path); + } +} \ No newline at end of file diff --git a/lib/controllers/admin/users.js b/lib/controllers/admin/users.js new file mode 100644 index 0000000000..1bc208ae15 --- /dev/null +++ b/lib/controllers/admin/users.js @@ -0,0 +1,260 @@ +'use strict'; + +const validator = require('validator'); +const user = require('../../user'); +const meta = require('../../meta'); +const db = require('../../database'); +const pagination = require('../../pagination'); +const events = require('../../events'); +const plugins = require('../../plugins'); +const privileges = require('../../privileges'); +const utils = require('../../utils'); +const usersController = module.exports; +const userFields = ['uid', 'username', 'userslug', 'email', 'postcount', 'joindate', 'banned', 'reputation', 'picture', 'flags', 'lastonline', 'email:confirmed']; +usersController.index = async function (req, res) { + if (req.query.query) { + await usersController.search(req, res); + } else { + await getUsers(req, res); + } +}; +async function getUsers(req, res) { + const sortDirection = req.query.sortDirection || 'desc'; + const reverse = sortDirection === 'desc'; + const page = parseInt(req.query.page, 10) || 1; + let resultsPerPage = parseInt(req.query.resultsPerPage, 10) || 50; + if (![50, 100, 250, 500].includes(resultsPerPage)) { + resultsPerPage = 50; + } + let sortBy = validator.escape(req.query.sortBy || ''); + const filterBy = Array.isArray(req.query.filters || []) ? req.query.filters || [] : [req.query.filters]; + const start = Math.max(0, page - 1) * resultsPerPage; + const stop = start + resultsPerPage - 1; + function buildSet() { + const sortToSet = { + postcount: 'users:postcount', + reputation: 'users:reputation', + joindate: 'users:joindate', + lastonline: 'users:online', + flags: 'users:flags' + }; + const set = []; + if (sortBy) { + set.push(sortToSet[sortBy]); + } + if (filterBy.includes('unverified')) { + set.push('group:unverified-users:members'); + } + if (filterBy.includes('verified')) { + set.push('group:verified-users:members'); + } + if (filterBy.includes('banned')) { + set.push('users:banned'); + } + if (!set.length) { + set.push('users:online'); + sortBy = 'lastonline'; + } + return set.length > 1 ? set : set[0]; + } + async function getCount(set) { + if (Array.isArray(set)) { + return await db.sortedSetIntersectCard(set); + } + return await db.sortedSetCard(set); + } + async function getUids(set) { + let uids = []; + if (Array.isArray(set)) { + const weights = set.map((s, index) => index ? 0 : 1); + uids = await db[reverse ? 'getSortedSetRevIntersect' : 'getSortedSetIntersect']({ + sets: set, + start: start, + stop: stop, + weights: weights + }); + } else { + uids = await db[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](set, start, stop); + } + return uids; + } + const set = buildSet(); + const uids = await getUids(set); + const [count, users] = await Promise.all([getCount(set), loadUserInfo(req.uid, uids)]); + await render(req, res, { + users: users.filter(user => user && parseInt(user.uid, 10)), + page: page, + pageCount: Math.max(1, Math.ceil(count / resultsPerPage)), + resultsPerPage: resultsPerPage, + reverse: reverse, + sortBy: sortBy + }); +} +usersController.search = async function (req, res) { + const sortDirection = req.query.sortDirection || 'desc'; + const reverse = sortDirection === 'desc'; + const page = parseInt(req.query.page, 10) || 1; + let resultsPerPage = parseInt(req.query.resultsPerPage, 10) || 50; + if (![50, 100, 250, 500].includes(resultsPerPage)) { + resultsPerPage = 50; + } + const searchData = await user.search({ + uid: req.uid, + query: req.query.query, + searchBy: req.query.searchBy, + sortBy: req.query.sortBy, + sortDirection: sortDirection, + filters: req.query.filters, + page: page, + resultsPerPage: resultsPerPage, + findUids: async function (query, searchBy, hardCap) { + if (!query || query.length < 2) { + return []; + } + query = String(query).toLowerCase(); + if (!query.endsWith('*')) { + query += '*'; + } + const data = await db.getSortedSetScan({ + key: `${searchBy}:sorted`, + match: query, + limit: hardCap || resultsPerPage * 10 + }); + return data.map(data => data.split(':').pop()); + } + }); + const uids = searchData.users.map(user => user && user.uid); + searchData.users = await loadUserInfo(req.uid, uids); + if (req.query.searchBy === 'ip') { + searchData.users.forEach(user => { + user.ip = user.ips.find(ip => ip.includes(String(req.query.query))); + }); + } + searchData.query = validator.escape(String(req.query.query || '')); + searchData.page = page; + searchData.resultsPerPage = resultsPerPage; + searchData.sortBy = req.query.sortBy; + searchData.reverse = reverse; + await render(req, res, searchData); +}; +async function loadUserInfo(callerUid, uids) { + async function getIPs() { + return await Promise.all(uids.map(uid => db.getSortedSetRevRange(`uid:${uid}:ip`, 0, 4))); + } + async function getConfirmObjs() { + const keys = uids.map(uid => `confirm:byUid:${uid}`); + const codes = await db.mget(keys); + const confirmObjs = await db.getObjects(codes.map(code => `confirm:${code}`)); + return uids.map((uid, index) => confirmObjs[index]); + } + const [isAdmin, userData, lastonline, confirmObjs, ips] = await Promise.all([user.isAdministrator(uids), user.getUsersWithFields(uids, userFields, callerUid), db.sortedSetScores('users:online', uids), getConfirmObjs(), getIPs()]); + userData.forEach((user, index) => { + if (user) { + user.administrator = isAdmin[index]; + user.flags = userData[index].flags || 0; + const timestamp = lastonline[index] || user.joindate; + user.lastonline = timestamp; + user.lastonlineISO = utils.toISOString(timestamp); + user.ips = ips[index]; + user.ip = ips[index] && ips[index][0] ? ips[index][0] : null; + user.emailToConfirm = user.email; + if (confirmObjs[index] && confirmObjs[index].email) { + const confirmObj = confirmObjs[index]; + user['email:expired'] = !confirmObj.expires || Date.now() >= confirmObj.expires; + user['email:pending'] = confirmObj.expires && Date.now() < confirmObj.expires; + user.emailToConfirm = confirmObj.email; + } + } + }); + return userData; +} +usersController.registrationQueue = async function (req, res) { + const page = parseInt(req.query.page, 10) || 1; + const itemsPerPage = 20; + const start = (page - 1) * 20; + const stop = start + itemsPerPage - 1; + const data = await utils.promiseParallel({ + registrationQueueCount: db.sortedSetCard('registration:queue'), + users: user.getRegistrationQueue(start, stop), + customHeaders: plugins.hooks.fire('filter:admin.registrationQueue.customHeaders', { + headers: [] + }), + invites: getInvites() + }); + const pageCount = Math.max(1, Math.ceil(data.registrationQueueCount / itemsPerPage)); + data.pagination = pagination.create(page, pageCount); + data.customHeaders = data.customHeaders.headers; + data.title = '[[pages:registration-queue]]'; + res.render('admin/manage/registration', data); +}; +async function getInvites() { + const invitations = await user.getAllInvites(); + const uids = invitations.map(invite => invite.uid); + let usernames = await user.getUsersFields(uids, ['username']); + usernames = usernames.map(user => user.username); + invitations.forEach((invites, index) => { + invites.username = usernames[index]; + }); + async function getUsernamesByEmails(emails) { + const uids = await db.sortedSetScores('email:uid', emails.map(email => String(email).toLowerCase())); + const usernames = await user.getUsersFields(uids, ['username']); + return usernames.map(user => user.username); + } + usernames = await Promise.all(invitations.map(invites => getUsernamesByEmails(invites.invitations))); + invitations.forEach((invites, index) => { + invites.invitations = invites.invitations.map((email, i) => ({ + email: email, + username: usernames[index][i] === '[[global:guest]]' ? '' : usernames[index][i] + })); + }); + return invitations; +} +async function render(req, res, data) { + data.pagination = pagination.create(data.page, data.pageCount, req.query); + const { + registrationType + } = meta.config; + data.inviteOnly = registrationType === 'invite-only' || registrationType === 'admin-invite-only'; + data.adminInviteOnly = registrationType === 'admin-invite-only'; + data[`sort_${data.sortBy}`] = true; + if (req.query.searchBy) { + data[`searchBy_${validator.escape(String(req.query.searchBy))}`] = true; + } + const filterBy = Array.isArray(req.query.filters || []) ? req.query.filters || [] : [req.query.filters]; + filterBy.forEach(filter => { + data[`filterBy_${validator.escape(String(filter))}`] = true; + }); + data.userCount = parseInt(await db.getObjectField('global', 'userCount'), 10); + if (data.adminInviteOnly) { + data.showInviteButton = await privileges.users.isAdministrator(req.uid); + } else { + data.showInviteButton = await privileges.users.hasInvitePrivilege(req.uid); + } + res.render('admin/manage/users', data); +} +usersController.getCSV = async function (req, res, next) { + await events.log({ + type: 'getUsersCSV', + uid: req.uid, + ip: req.ip + }); + const path = require('path'); + const { + baseDir + } = require('../../constants').paths; + res.sendFile('users.csv', { + root: path.join(baseDir, 'build/export'), + headers: { + 'Content-Type': 'text/csv', + 'Content-Disposition': 'attachment; filename=users.csv' + } + }, err => { + if (err) { + if (err.code === 'ENOENT') { + res.locals.isAPI = false; + return next(); + } + return next(err); + } + }); +}; \ No newline at end of file diff --git a/lib/controllers/admin/widgets.js b/lib/controllers/admin/widgets.js new file mode 100644 index 0000000000..f176e960a2 --- /dev/null +++ b/lib/controllers/admin/widgets.js @@ -0,0 +1,8 @@ +'use strict'; + +const widgetsController = module.exports; +const admin = require('../../widgets/admin'); +widgetsController.get = async function (req, res) { + const data = await admin.get(); + res.render('admin/extend/widgets', data); +}; \ No newline at end of file diff --git a/lib/controllers/api.js b/lib/controllers/api.js new file mode 100644 index 0000000000..ab4927bb18 --- /dev/null +++ b/lib/controllers/api.js @@ -0,0 +1,137 @@ +'use strict'; + +const validator = require('validator'); +const nconf = require('nconf'); +const meta = require('../meta'); +const user = require('../user'); +const categories = require('../categories'); +const plugins = require('../plugins'); +const translator = require('../translator'); +const languages = require('../languages'); +const { + generateToken +} = require('../middleware/csrf'); +const utils = require('../utils'); +const apiController = module.exports; +const relative_path = nconf.get('relative_path'); +const upload_url = nconf.get('upload_url'); +const asset_base_url = nconf.get('asset_base_url'); +const socketioTransports = nconf.get('socket.io:transports') || ['polling', 'websocket']; +const socketioOrigins = nconf.get('socket.io:origins'); +const websocketAddress = nconf.get('socket.io:address') || ''; +const fontawesome_pro = nconf.get('fontawesome:pro') || false; +const fontawesome_styles = utils.getFontawesomeStyles(); +const fontawesome_version = utils.getFontawesomeVersion(); +apiController.loadConfig = async function (req) { + const config = { + relative_path, + upload_url, + asset_base_url, + assetBaseUrl: asset_base_url, + siteTitle: validator.escape(String(meta.config.title || meta.config.browserTitle || 'NodeBB')), + browserTitle: validator.escape(String(meta.config.browserTitle || meta.config.title || 'NodeBB')), + titleLayout: (meta.config.titleLayout || '{pageTitle} | {browserTitle}').replace(/{/g, '{').replace(/}/g, '}'), + showSiteTitle: meta.config.showSiteTitle === 1, + maintenanceMode: meta.config.maintenanceMode === 1, + postQueue: meta.config.postQueue, + minimumTitleLength: meta.config.minimumTitleLength, + maximumTitleLength: meta.config.maximumTitleLength, + minimumPostLength: meta.config.minimumPostLength, + maximumPostLength: meta.config.maximumPostLength, + minimumTagsPerTopic: meta.config.minimumTagsPerTopic || 0, + maximumTagsPerTopic: meta.config.maximumTagsPerTopic || 5, + minimumTagLength: meta.config.minimumTagLength || 3, + maximumTagLength: meta.config.maximumTagLength || 15, + undoTimeout: meta.config.undoTimeout || 0, + useOutgoingLinksPage: meta.config.useOutgoingLinksPage === 1, + outgoingLinksWhitelist: meta.config.useOutgoingLinksPage === 1 ? meta.config['outgoingLinks:whitelist'] : undefined, + allowGuestHandles: meta.config.allowGuestHandles === 1, + allowTopicsThumbnail: meta.config.allowTopicsThumbnail === 1, + usePagination: meta.config.usePagination === 1, + disableChat: meta.config.disableChat === 1, + disableChatMessageEditing: meta.config.disableChatMessageEditing === 1, + maximumChatMessageLength: meta.config.maximumChatMessageLength || 1000, + socketioTransports, + socketioOrigins, + websocketAddress, + maxReconnectionAttempts: meta.config.maxReconnectionAttempts, + reconnectionDelay: meta.config.reconnectionDelay, + topicsPerPage: meta.config.topicsPerPage || 20, + postsPerPage: meta.config.postsPerPage || 20, + maximumFileSize: meta.config.maximumFileSize, + 'theme:id': meta.config['theme:id'], + 'theme:src': meta.config['theme:src'], + defaultLang: meta.config.defaultLang || 'en-GB', + userLang: req.query.lang ? validator.escape(String(req.query.lang)) : meta.config.defaultLang || 'en-GB', + loggedIn: !!req.user, + uid: req.uid, + 'cache-buster': meta.config['cache-buster'] || '', + topicPostSort: meta.config.topicPostSort || 'oldest_to_newest', + categoryTopicSort: meta.config.categoryTopicSort || 'recently_replied', + csrf_token: req.uid >= 0 ? generateToken(req) : false, + searchEnabled: plugins.hooks.hasListeners('filter:search.query'), + searchDefaultInQuick: meta.config.searchDefaultInQuick || 'titles', + bootswatchSkin: meta.config.bootswatchSkin || '', + 'composer:showHelpTab': meta.config['composer:showHelpTab'] === 1, + enablePostHistory: meta.config.enablePostHistory === 1, + timeagoCutoff: meta.config.timeagoCutoff !== '' ? Math.max(0, parseInt(meta.config.timeagoCutoff, 10)) : meta.config.timeagoCutoff, + timeagoCodes: languages.timeagoCodes, + cookies: { + enabled: meta.config.cookieConsentEnabled === 1, + message: translator.escape(validator.escape(meta.config.cookieConsentMessage || '[[global:cookies.message]]')).replace(/\\/g, '\\\\'), + dismiss: translator.escape(validator.escape(meta.config.cookieConsentDismiss || '[[global:cookies.accept]]')).replace(/\\/g, '\\\\'), + link: translator.escape(validator.escape(meta.config.cookieConsentLink || '[[global:cookies.learn-more]]')).replace(/\\/g, '\\\\'), + link_url: translator.escape(validator.escape(meta.config.cookieConsentLinkUrl || 'https://www.cookiesandyou.com')).replace(/\\/g, '\\\\') + }, + thumbs: { + size: meta.config.topicThumbSize + }, + emailPrompt: meta.config.emailPrompt, + useragent: { + isSafari: req.useragent.isSafari + }, + fontawesome: { + pro: fontawesome_pro, + styles: fontawesome_styles, + version: fontawesome_version + } + }; + let settings = config; + let isAdminOrGlobalMod; + if (req.loggedIn) { + [settings, isAdminOrGlobalMod] = await Promise.all([user.getSettings(req.uid), user.isAdminOrGlobalMod(req.uid)]); + } + const oldSkins = ['default']; + settings.bootswatchSkin = oldSkins.includes(settings.bootswatchSkin) ? '' : settings.bootswatchSkin; + config.usePagination = settings.usePagination; + config.topicsPerPage = settings.topicsPerPage; + config.postsPerPage = settings.postsPerPage; + config.userLang = validator.escape(String((req.query.lang ? req.query.lang : null) || settings.userLang || config.defaultLang)); + config.acpLang = validator.escape(String((req.query.lang ? req.query.lang : null) || settings.acpLang)); + config.openOutgoingLinksInNewTab = settings.openOutgoingLinksInNewTab; + config.topicPostSort = settings.topicPostSort || config.topicPostSort; + config.categoryTopicSort = settings.categoryTopicSort || config.categoryTopicSort; + config.topicSearchEnabled = settings.topicSearchEnabled || false; + config.disableCustomUserSkins = meta.config.disableCustomUserSkins === 1; + config.defaultBootswatchSkin = config.bootswatchSkin; + if (!config.disableCustomUserSkins && settings.bootswatchSkin) { + if (settings.bootswatchSkin === 'noskin') { + config.bootswatchSkin = ''; + } else if (settings.bootswatchSkin !== '' && (await meta.css.isSkinValid(settings.bootswatchSkin))) { + config.bootswatchSkin = settings.bootswatchSkin; + } + } + config.disableChatMessageEditing = isAdminOrGlobalMod ? false : config.disableChatMessageEditing; + return await plugins.hooks.fire('filter:config.get', config); +}; +apiController.getConfig = async function (req, res) { + const config = await apiController.loadConfig(req); + res.json(config); +}; +apiController.getModerators = async function (req, res) { + const moderators = await categories.getModerators(req.params.cid); + res.json({ + moderators: moderators + }); +}; +require('../promisify')(apiController, ['getConfig', 'getObject', 'getModerators']); \ No newline at end of file diff --git a/lib/controllers/authentication.js b/lib/controllers/authentication.js new file mode 100644 index 0000000000..723f3f7729 --- /dev/null +++ b/lib/controllers/authentication.js @@ -0,0 +1,459 @@ +'use strict'; + +const winston = require('winston'); +const passport = require('passport'); +const nconf = require('nconf'); +const validator = require('validator'); +const _ = require('lodash'); +const util = require('util'); +const db = require('../database'); +const meta = require('../meta'); +const analytics = require('../analytics'); +const user = require('../user'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const slugify = require('../slugify'); +const helpers = require('./helpers'); +const privileges = require('../privileges'); +const sockets = require('../socket.io'); +const authenticationController = module.exports; +async function registerAndLoginUser(req, res, userData) { + if (!userData.hasOwnProperty('email')) { + userData.updateEmail = true; + } + const data = await user.interstitials.get(req, userData); + const deferRegistration = data.interstitials.length; + if (deferRegistration) { + userData.register = true; + req.session.registration = userData; + if (req.body.noscript === 'true') { + res.redirect(`${nconf.get('relative_path')}/register/complete`); + return; + } + res.json({ + next: `${nconf.get('relative_path')}/register/complete` + }); + return; + } + const queue = await user.shouldQueueUser(req.ip); + const result = await plugins.hooks.fire('filter:register.shouldQueue', { + req: req, + res: res, + userData: userData, + queue: queue + }); + if (result.queue) { + return await addToApprovalQueue(req, userData); + } + const uid = await user.create(userData); + if (res.locals.processLogin) { + await authenticationController.doLogin(req, uid); + } + if (userData.token) { + await Promise.all([user.confirmIfInviteEmailIsUsed(userData.token, userData.email, uid), user.joinGroupsFromInvitation(uid, userData.token)]); + } + await user.deleteInvitationKey(userData.email, userData.token); + const next = req.session.returnTo || `${nconf.get('relative_path')}/`; + const complete = await plugins.hooks.fire('filter:register.complete', { + uid: uid, + next: next + }); + req.session.returnTo = complete.next; + return complete; +} +authenticationController.register = async function (req, res) { + const registrationType = meta.config.registrationType || 'normal'; + if (registrationType === 'disabled') { + return res.sendStatus(403); + } + const userData = req.body; + try { + if (userData.token || registrationType === 'invite-only' || registrationType === 'admin-invite-only') { + await user.verifyInvitation(userData); + } + if (!userData.username || userData.username.length < meta.config.minimumUsernameLength || slugify(userData.username).length < meta.config.minimumUsernameLength) { + throw new Error('[[error:username-too-short]]'); + } + if (userData.username.length > meta.config.maximumUsernameLength) { + throw new Error('[[error:username-too-long]]'); + } + if (userData.password !== userData['password-confirm']) { + throw new Error('[[user:change-password-error-match]]'); + } + if (userData.password.length > 512) { + throw new Error('[[error:password-too-long]]'); + } + user.isPasswordValid(userData.password); + await plugins.hooks.fire('filter:password.check', { + password: userData.password, + uid: 0, + userData: userData + }); + res.locals.processLogin = true; + await plugins.hooks.fire('filter:register.check', { + req: req, + res: res, + userData: userData + }); + const data = await registerAndLoginUser(req, res, userData); + if (data) { + if (data.uid && req.body.userLang) { + await user.setSetting(data.uid, 'userLang', req.body.userLang); + } + res.json(data); + } + } catch (err) { + helpers.noScriptErrors(req, res, err.message, 400); + } +}; +async function addToApprovalQueue(req, userData) { + userData.ip = req.ip; + await user.addToApprovalQueue(userData); + let message = '[[register:registration-added-to-queue]]'; + if (meta.config.showAverageApprovalTime) { + const average_time = await db.getObjectField('registration:queue:approval:times', 'average'); + if (average_time > 0) { + message += ` [[register:registration-queue-average-time, ${Math.floor(average_time / 60)}, ${Math.floor(average_time % 60)}]]`; + } + } + if (meta.config.autoApproveTime > 0) { + message += ` [[register:registration-queue-auto-approve-time, ${meta.config.autoApproveTime}]]`; + } + return { + message: message + }; +} +authenticationController.registerComplete = async function (req, res) { + try { + const data = await user.interstitials.get(req, req.session.registration); + const callbacks = data.interstitials.reduce((memo, cur) => { + if (cur.hasOwnProperty('callback') && typeof cur.callback === 'function') { + req.body.files = req.files; + if (cur.callback.constructor && cur.callback.constructor.name === 'AsyncFunction' || cur.callback.length === 2) { + memo.push(cur.callback); + } else { + memo.push(util.promisify(cur.callback)); + } + } + return memo; + }, []); + const done = function (data) { + delete req.session.registration; + const relative_path = nconf.get('relative_path'); + if (data && data.message) { + return res.redirect(`${relative_path}/?register=${encodeURIComponent(data.message)}`); + } + if (req.session.returnTo) { + res.redirect(relative_path + req.session.returnTo.replace(new RegExp(`^${relative_path}`), '')); + } else { + res.redirect(`${relative_path}/`); + } + }; + const results = await Promise.allSettled(callbacks.map(async cb => { + await cb(req.session.registration, req.body); + })); + const errors = results.map(result => result.status === 'rejected' && result.reason && result.reason.message).filter(Boolean); + if (errors.length) { + req.flash('errors', errors); + return req.session.save(() => { + res.redirect(`${nconf.get('relative_path')}/register/complete`); + }); + } + if (req.session.registration.register === true) { + res.locals.processLogin = true; + req.body.noscript = 'true'; + const data = await registerAndLoginUser(req, res, req.session.registration); + if (!data) { + return winston.warn('[register] Interstitial callbacks processed with no errors, but one or more interstitials remain. This is likely an issue with one of the interstitials not properly handling a null case or invalid value.'); + } + done(data); + } else { + const payload = req.session.registration; + const { + uid + } = payload; + delete payload.uid; + delete payload.returnTo; + Object.keys(payload).forEach(prop => { + if (typeof payload[prop] === 'boolean') { + payload[prop] = payload[prop] ? 1 : 0; + } + }); + await user.setUserFields(uid, payload); + done(); + } + } catch (err) { + delete req.session.registration; + res.redirect(`${nconf.get('relative_path')}/?register=${encodeURIComponent(err.message)}`); + } +}; +authenticationController.registerAbort = async (req, res) => { + if (req.uid && req.session.registration) { + delete req.session.registration.updateEmail; + const { + interstitials + } = await user.interstitials.get(req, req.session.registration); + if (!interstitials.length) { + delete req.session.registration; + return res.redirect(nconf.get('relative_path') + (req.session.returnTo || '/')); + } + } + req.session.destroy(() => { + res.clearCookie(nconf.get('sessionKey'), meta.configs.cookie.get()); + res.redirect(`${nconf.get('relative_path')}/`); + }); +}; +authenticationController.login = async (req, res, next) => { + let { + strategy + } = await plugins.hooks.fire('filter:login.override', { + req, + strategy: 'local' + }); + if (!passport._strategy(strategy)) { + winston.error(`[auth/override] Requested login strategy "${strategy}" not found, reverting back to local login strategy.`); + strategy = 'local'; + } + if (plugins.hooks.hasListeners('action:auth.overrideLogin')) { + return continueLogin(strategy, req, res, next); + } + const loginWith = meta.config.allowLoginWith || 'username-email'; + req.body.username = String(req.body.username).trim(); + const errorHandler = res.locals.noScriptErrors || helpers.noScriptErrors; + try { + await plugins.hooks.fire('filter:login.check', { + req: req, + res: res, + userData: req.body + }); + } catch (err) { + return errorHandler(req, res, err.message, 403); + } + try { + const isEmailLogin = loginWith.includes('email') && req.body.username && utils.isEmailValid(req.body.username); + const isUsernameLogin = loginWith.includes('username') && !validator.isEmail(req.body.username); + if (isEmailLogin) { + const username = await user.getUsernameByEmail(req.body.username); + if (username !== '[[global:guest]]') { + req.body.username = username; + } + } + if (isEmailLogin || isUsernameLogin) { + continueLogin(strategy, req, res, next); + } else { + errorHandler(req, res, `[[error:wrong-login-type-${loginWith}]]`, 400); + } + } catch (err) { + return errorHandler(req, res, err.message, 500); + } +}; +function continueLogin(strategy, req, res, next) { + passport.authenticate(strategy, async (err, userData, info) => { + if (err) { + plugins.hooks.fire('action:login.continue', { + req, + strategy, + userData, + error: err + }); + return helpers.noScriptErrors(req, res, err.data || err.message, 403); + } + if (!userData) { + if (info instanceof Error) { + info = info.message; + } else if (typeof info === 'object') { + info = '[[error:invalid-username-or-password]]'; + } + plugins.hooks.fire('action:login.continue', { + req, + strategy, + userData, + error: new Error(info) + }); + return helpers.noScriptErrors(req, res, info, 403); + } + if (req.body.remember === 'on') { + const duration = meta.getSessionTTLSeconds() * 1000; + req.session.cookie.maxAge = duration; + req.session.cookie.expires = new Date(Date.now() + duration); + } else { + const duration = meta.config.sessionDuration * 1000; + req.session.cookie.maxAge = duration || false; + req.session.cookie.expires = duration ? new Date(Date.now() + duration) : false; + } + plugins.hooks.fire('action:login.continue', { + req, + strategy, + userData, + error: null + }); + if (userData.passwordExpiry && userData.passwordExpiry < Date.now()) { + winston.verbose(`[auth] Triggering password reset for uid ${userData.uid} due to password policy`); + req.session.passwordExpired = true; + const code = await user.reset.generate(userData.uid); + (res.locals.redirectAfterLogin || redirectAfterLogin)(req, res, `${nconf.get('relative_path')}/reset/${code}`); + } else { + delete req.query.lang; + await authenticationController.doLogin(req, userData.uid); + let destination; + if (req.session.returnTo) { + destination = req.session.returnTo.startsWith('http') ? req.session.returnTo : nconf.get('relative_path') + req.session.returnTo; + delete req.session.returnTo; + } else { + destination = `${nconf.get('relative_path')}/`; + } + (res.locals.redirectAfterLogin || redirectAfterLogin)(req, res, destination); + } + })(req, res, next); +} +function redirectAfterLogin(req, res, destination) { + if (req.body.noscript === 'true') { + res.redirect(`${destination}?loggedin`); + } else { + res.status(200).send({ + next: destination + }); + } +} +authenticationController.doLogin = async function (req, uid) { + if (!uid) { + return; + } + const loginAsync = util.promisify(req.login).bind(req); + await loginAsync({ + uid: uid + }, { + keepSessionInfo: req.res.locals.reroll !== false + }); + await authenticationController.onSuccessfulLogin(req, uid); +}; +authenticationController.onSuccessfulLogin = async function (req, uid, trackSession = true) { + if (req.loggedIn && !req.session.forceLogin) { + return true; + } + try { + const uuid = utils.generateUUID(); + req.uid = uid; + req.loggedIn = true; + await meta.blacklist.test(req.ip); + await user.logIP(uid, req.ip); + await user.bans.unbanIfExpired([uid]); + await user.reset.cleanByUid(uid); + req.session.meta = {}; + delete req.session.forceLogin; + req.session.meta.ip = req.ip; + req.session.meta = _.extend(req.session.meta, { + uuid: uuid, + datetime: Date.now(), + platform: req.useragent.platform, + browser: req.useragent.browser, + version: req.useragent.version + }); + await Promise.all([new Promise(resolve => { + req.session.save(resolve); + }), trackSession ? user.auth.addSession(uid, req.sessionID) : undefined, user.updateLastOnlineTime(uid), user.onUserOnline(uid, Date.now()), analytics.increment('logins'), db.incrObjectFieldBy('global', 'loginCount', 1)]); + sockets.in(`sess_${req.sessionID}`).emit('checkSession', uid); + plugins.hooks.fire('action:user.loggedIn', { + uid: uid, + req: req + }); + } catch (err) { + req.session.destroy(); + throw err; + } +}; +const destroyAsync = util.promisify((req, callback) => req.session.destroy(callback)); +const logoutAsync = util.promisify((req, callback) => req.logout(callback)); +authenticationController.localLogin = async function (req, username, password, next) { + if (!username) { + return next(new Error('[[error:invalid-username]]')); + } + if (!password || !utils.isPasswordValid(password)) { + return next(new Error('[[error:invalid-password]]')); + } + if (password.length > 512) { + return next(new Error('[[error:password-too-long]]')); + } + const userslug = slugify(username); + const uid = await user.getUidByUserslug(userslug); + try { + const [userData, isAdminOrGlobalMod, canLoginIfBanned] = await Promise.all([user.getUserFields(uid, ['uid', 'passwordExpiry']), user.isAdminOrGlobalMod(uid), user.bans.canLoginIfBanned(uid)]); + userData.isAdminOrGlobalMod = isAdminOrGlobalMod; + if (!canLoginIfBanned) { + return next(await getBanError(uid)); + } + const hasLoginPrivilege = await privileges.global.can('local:login', uid); + if (parseInt(uid, 10) && !hasLoginPrivilege) { + return next(new Error('[[error:local-login-disabled]]')); + } + try { + const passwordMatch = await user.isPasswordCorrect(uid, password, req.ip); + if (!passwordMatch) { + return next(new Error('[[error:invalid-login-credentials]]')); + } + } catch (e) { + if (req.loggedIn) { + await logoutAsync(req); + await destroyAsync(req); + } + throw e; + } + next(null, userData, '[[success:authentication-successful]]'); + } catch (err) { + next(err); + } +}; +authenticationController.logout = async function (req, res, next) { + if (!req.loggedIn || !req.sessionID) { + res.clearCookie(nconf.get('sessionKey'), meta.configs.cookie.get()); + return res.status(200).send('not-logged-in'); + } + const { + uid + } = req; + const { + sessionID + } = req; + try { + await user.auth.revokeSession(sessionID, uid); + await logoutAsync(req); + await destroyAsync(req); + res.clearCookie(nconf.get('sessionKey'), meta.configs.cookie.get()); + await user.setUserField(uid, 'lastonline', Date.now() - meta.config.onlineCutoff * 60000); + await db.sortedSetAdd('users:online', Date.now() - meta.config.onlineCutoff * 60000, uid); + await plugins.hooks.fire('static:user.loggedOut', { + req: req, + res: res, + uid: uid, + sessionID: sessionID + }); + sockets.in(`sess_${sessionID}`).emit('checkSession', 0); + const payload = { + next: `${nconf.get('relative_path')}/` + }; + plugins.hooks.fire('filter:user.logout', payload); + if (req.body.noscript === 'true') { + return res.redirect(payload.next); + } + res.status(200).send(payload); + } catch (err) { + next(err); + } +}; +async function getBanError(uid) { + try { + const banInfo = await user.getLatestBanInfo(uid); + if (!banInfo.reason) { + banInfo.reason = '[[user:info.banned-no-reason]]'; + } + const err = new Error(banInfo.reason); + err.data = banInfo; + return err; + } catch (err) { + if (err.message === 'no-ban-info') { + return new Error('[[error:user-banned]]'); + } + throw err; + } +} +require('../promisify')(authenticationController, ['register', 'registerComplete', 'registerAbort', 'login', 'localLogin', 'logout']); \ No newline at end of file diff --git a/lib/controllers/categories.js b/lib/controllers/categories.js new file mode 100644 index 0000000000..21b9477968 --- /dev/null +++ b/lib/controllers/categories.js @@ -0,0 +1,54 @@ +'use strict'; + +const nconf = require('nconf'); +const _ = require('lodash'); +const categories = require('../categories'); +const meta = require('../meta'); +const pagination = require('../pagination'); +const helpers = require('./helpers'); +const privileges = require('../privileges'); +const categoriesController = module.exports; +categoriesController.list = async function (req, res) { + res.locals.metaTags = [{ + name: 'title', + content: String(meta.config.title || 'NodeBB') + }, { + property: 'og:type', + content: 'website' + }]; + const allRootCids = await categories.getAllCidsFromSet('cid:0:children'); + const rootCids = await privileges.categories.filterCids('find', allRootCids, req.uid); + const pageCount = Math.max(1, Math.ceil(rootCids.length / meta.config.categoriesPerPage)); + const page = Math.min(parseInt(req.query.page, 10) || 1, pageCount); + const start = Math.max(0, (page - 1) * meta.config.categoriesPerPage); + const stop = start + meta.config.categoriesPerPage - 1; + const pageCids = rootCids.slice(start, stop + 1); + const allChildCids = _.flatten(await Promise.all(pageCids.map(categories.getChildrenCids))); + const childCids = await privileges.categories.filterCids('find', allChildCids, req.uid); + const categoryData = await categories.getCategories(pageCids.concat(childCids)); + const tree = categories.getTree(categoryData, 0); + await Promise.all([categories.getRecentTopicReplies(categoryData, req.uid, req.query), categories.setUnread(tree, pageCids.concat(childCids), req.uid)]); + const data = { + title: meta.config.homePageTitle || '[[pages:home]]', + selectCategoryLabel: '[[pages:categories]]', + categories: tree, + pagination: pagination.create(page, pageCount, req.query) + }; + data.categories.forEach(category => { + if (category) { + helpers.trimChildren(category); + helpers.setCategoryTeaser(category); + } + }); + if (req.originalUrl.startsWith(`${nconf.get('relative_path')}/api/categories`) || req.originalUrl.startsWith(`${nconf.get('relative_path')}/categories`)) { + data.title = '[[pages:categories]]'; + data.breadcrumbs = helpers.buildBreadcrumbs([{ + text: data.title + }]); + res.locals.metaTags.push({ + property: 'og:title', + content: '[[pages:categories]]' + }); + } + res.render('categories', data); +}; \ No newline at end of file diff --git a/lib/controllers/category.js b/lib/controllers/category.js new file mode 100644 index 0000000000..97e4ff1ebe --- /dev/null +++ b/lib/controllers/category.js @@ -0,0 +1,175 @@ +'use strict'; + +const nconf = require('nconf'); +const validator = require('validator'); +const qs = require('querystring'); +const db = require('../database'); +const privileges = require('../privileges'); +const user = require('../user'); +const categories = require('../categories'); +const meta = require('../meta'); +const pagination = require('../pagination'); +const helpers = require('./helpers'); +const utils = require('../utils'); +const translator = require('../translator'); +const analytics = require('../analytics'); +const categoryController = module.exports; +const url = nconf.get('url'); +const relative_path = nconf.get('relative_path'); +const validSorts = ['recently_replied', 'recently_created', 'most_posts', 'most_votes', 'most_views']; +categoryController.get = async function (req, res, next) { + const cid = req.params.category_id; + let currentPage = parseInt(req.query.page, 10) || 1; + let topicIndex = utils.isNumber(req.params.topic_index) ? parseInt(req.params.topic_index, 10) - 1 : 0; + if (req.params.topic_index && !utils.isNumber(req.params.topic_index) || !utils.isNumber(cid)) { + return next(); + } + const [categoryFields, userPrivileges, tagData, userSettings, rssToken] = await Promise.all([categories.getCategoryFields(cid, ['slug', 'disabled', 'link']), privileges.categories.get(cid, req.uid), helpers.getSelectedTag(req.query.tag), user.getSettings(req.uid), user.auth.getFeedToken(req.uid)]); + if (!categoryFields.slug || categoryFields && categoryFields.disabled || userSettings.usePagination && currentPage < 1) { + return next(); + } + if (topicIndex < 0) { + return helpers.redirect(res, `/category/${categoryFields.slug}?${qs.stringify(req.query)}`); + } + if (!userPrivileges.read) { + return helpers.notAllowed(req, res); + } + if (!res.locals.isAPI && !req.params.slug && categoryFields.slug && categoryFields.slug !== `${cid}/`) { + return helpers.redirect(res, `/category/${categoryFields.slug}?${qs.stringify(req.query)}`, true); + } + if (categoryFields.link) { + await db.incrObjectField(`category:${cid}`, 'timesClicked'); + return helpers.redirect(res, validator.unescape(categoryFields.link)); + } + if (!userSettings.usePagination) { + topicIndex = Math.max(0, topicIndex - (Math.ceil(userSettings.topicsPerPage / 2) - 1)); + } else if (!req.query.page) { + const index = Math.max(parseInt(topicIndex || 0, 10), 0); + currentPage = Math.ceil((index + 1) / userSettings.topicsPerPage); + topicIndex = 0; + } + const targetUid = await user.getUidByUserslug(req.query.author); + const start = (currentPage - 1) * userSettings.topicsPerPage + topicIndex; + const stop = start + userSettings.topicsPerPage - 1; + const sort = validSorts.includes(req.query.sort) ? req.query.sort : userSettings.categoryTopicSort; + const categoryData = await categories.getCategoryById({ + uid: req.uid, + cid: cid, + start: start, + stop: stop, + sort: sort, + settings: userSettings, + query: req.query, + tag: req.query.tag, + targetUid: targetUid + }); + if (!categoryData) { + return next(); + } + if (topicIndex > Math.max(categoryData.topic_count - 1, 0)) { + return helpers.redirect(res, `/category/${categoryData.slug}/${categoryData.topic_count}?${qs.stringify(req.query)}`); + } + const pageCount = Math.max(1, Math.ceil(categoryData.topic_count / userSettings.topicsPerPage)); + if (userSettings.usePagination && currentPage > pageCount) { + return next(); + } + categories.modifyTopicsByPrivilege(categoryData.topics, userPrivileges); + categoryData.tagWhitelist = categories.filterTagWhitelist(categoryData.tagWhitelist, userPrivileges.isAdminOrMod); + const allCategories = []; + categories.flattenCategories(allCategories, categoryData.children); + await Promise.all([buildBreadcrumbs(req, categoryData), categories.setUnread([categoryData], allCategories.map(c => c.cid).concat(cid), req.uid)]); + if (categoryData.children.length) { + await categories.getRecentTopicReplies(allCategories, req.uid, req.query); + categoryData.subCategoriesLeft = Math.max(0, categoryData.children.length - categoryData.subCategoriesPerPage); + categoryData.hasMoreSubCategories = categoryData.children.length > categoryData.subCategoriesPerPage; + categoryData.nextSubCategoryStart = categoryData.subCategoriesPerPage; + categoryData.children = categoryData.children.slice(0, categoryData.subCategoriesPerPage); + categoryData.children.forEach(child => { + if (child) { + helpers.trimChildren(child); + helpers.setCategoryTeaser(child); + } + }); + } + categoryData.title = translator.escape(categoryData.name); + categoryData.selectCategoryLabel = '[[category:subcategories]]'; + categoryData.description = translator.escape(categoryData.description); + categoryData.privileges = userPrivileges; + categoryData.showSelect = userPrivileges.editable; + categoryData.showTopicTools = userPrivileges.editable; + categoryData.topicIndex = topicIndex; + categoryData.selectedTag = tagData.selectedTag; + categoryData.selectedTags = tagData.selectedTags; + categoryData.sortOptionLabel = `[[topic:${validator.escape(String(sort)).replace(/_/g, '-')}]]`; + if (!meta.config['feeds:disableRSS']) { + categoryData.rssFeedUrl = `${url}/category/${categoryData.cid}.rss`; + if (req.loggedIn) { + categoryData.rssFeedUrl += `?uid=${req.uid}&token=${rssToken}`; + } + } + addTags(categoryData, res, currentPage); + categoryData['feeds:disableRSS'] = meta.config['feeds:disableRSS'] || 0; + categoryData['reputation:disabled'] = meta.config['reputation:disabled']; + categoryData.pagination = pagination.create(currentPage, pageCount, req.query); + categoryData.pagination.rel.forEach(rel => { + rel.href = `${url}/category/${categoryData.slug}${rel.href}`; + res.locals.linkTags.push(rel); + }); + analytics.increment([`pageviews:byCid:${categoryData.cid}`]); + res.render('category', categoryData); +}; +async function buildBreadcrumbs(req, categoryData) { + const breadcrumbs = [{ + text: categoryData.name, + url: `${url}/category/${categoryData.slug}`, + cid: categoryData.cid + }]; + const crumbs = await helpers.buildCategoryBreadcrumbs(categoryData.parentCid); + if (req.originalUrl.startsWith(`${relative_path}/api/category`) || req.originalUrl.startsWith(`${relative_path}/category`)) { + categoryData.breadcrumbs = crumbs.concat(breadcrumbs); + } +} +function addTags(categoryData, res, currentPage) { + res.locals.metaTags = [{ + name: 'title', + content: categoryData.name, + noEscape: true + }, { + property: 'og:title', + content: categoryData.name, + noEscape: true + }, { + name: 'description', + content: categoryData.description, + noEscape: true + }, { + property: 'og:type', + content: 'website' + }]; + if (categoryData.backgroundImage) { + if (!categoryData.backgroundImage.startsWith('http')) { + categoryData.backgroundImage = url + categoryData.backgroundImage; + } + res.locals.metaTags.push({ + property: 'og:image', + content: categoryData.backgroundImage, + noEscape: true + }); + } + const page = currentPage > 1 ? `?page=${currentPage}` : ''; + res.locals.linkTags = [{ + rel: 'up', + href: url + }, { + rel: 'canonical', + href: `${url}/category/${categoryData.slug}${page}`, + noEscape: true + }]; + if (!categoryData['feeds:disableRSS']) { + res.locals.linkTags.push({ + rel: 'alternate', + type: 'application/rss+xml', + href: categoryData.rssFeedUrl + }); + } +} \ No newline at end of file diff --git a/lib/controllers/composer.js b/lib/controllers/composer.js new file mode 100644 index 0000000000..c2eb403a7b --- /dev/null +++ b/lib/controllers/composer.js @@ -0,0 +1,91 @@ +'use strict'; + +const nconf = require('nconf'); +const user = require('../user'); +const plugins = require('../plugins'); +const topics = require('../topics'); +const posts = require('../posts'); +const helpers = require('./helpers'); +exports.get = async function (req, res, callback) { + res.locals.metaTags = { + ...res.locals.metaTags, + name: 'robots', + content: 'noindex' + }; + const data = await plugins.hooks.fire('filter:composer.build', { + req: req, + res: res, + next: callback, + templateData: {} + }); + if (res.headersSent) { + return; + } + if (!data || !data.templateData) { + return callback(new Error('[[error:invalid-data]]')); + } + if (data.templateData.disabled) { + res.render('', { + title: '[[modules:composer.compose]]' + }); + } else { + data.templateData.title = '[[modules:composer.compose]]'; + res.render('compose', data.templateData); + } +}; +exports.post = async function (req, res) { + const { + body + } = req; + const data = { + uid: req.uid, + req: req, + timestamp: Date.now(), + content: body.content, + handle: body.handle, + fromQueue: false + }; + req.body.noscript = 'true'; + if (!data.content) { + return helpers.noScriptErrors(req, res, '[[error:invalid-data]]', 400); + } + async function queueOrPost(postFn, data) { + const shouldQueue = await posts.shouldQueue(req.uid, data); + if (shouldQueue) { + delete data.req; + return await posts.addToQueue(data); + } + return await postFn(data); + } + try { + let result; + if (body.tid) { + data.tid = body.tid; + result = await queueOrPost(topics.reply, data); + } else if (body.cid) { + data.cid = body.cid; + data.title = body.title; + data.tags = []; + data.thumb = ''; + result = await queueOrPost(topics.post, data); + } else { + throw new Error('[[error:invalid-data]]'); + } + if (!result) { + throw new Error('[[error:invalid-data]]'); + } + if (result.queued) { + return res.redirect(`${nconf.get('relative_path') || '/'}?noScriptMessage=[[success:post-queued]]`); + } + user.updateOnlineUsers(req.uid); + let path = nconf.get('relative_path'); + if (result.pid) { + path += `/post/${result.pid}`; + } else if (result.topicData) { + path += `/topic/${result.topicData.slug}`; + } + res.redirect(path); + } catch (err) { + helpers.noScriptErrors(req, res, err.message, 400); + } +}; \ No newline at end of file diff --git a/lib/controllers/errors.js b/lib/controllers/errors.js new file mode 100644 index 0000000000..3ba5f21ccc --- /dev/null +++ b/lib/controllers/errors.js @@ -0,0 +1,120 @@ +'use strict'; + +const fs = require('fs'); +const nconf = require('nconf'); +const winston = require('winston'); +const validator = require('validator'); +const path = require('path'); +const translator = require('../translator'); +const plugins = require('../plugins'); +const middleware = require('../middleware'); +const middlewareHelpers = require('../middleware/helpers'); +const helpers = require('./helpers'); +exports.handleURIErrors = async function handleURIErrors(err, req, res, next) { + if (err instanceof URIError) { + const cleanPath = req.path.replace(new RegExp(`^${nconf.get('relative_path')}`), ''); + const tidMatch = cleanPath.match(/^\/topic\/(\d+)\//); + const cidMatch = cleanPath.match(/^\/category\/(\d+)\//); + if (tidMatch) { + res.redirect(nconf.get('relative_path') + tidMatch[0]); + } else if (cidMatch) { + res.redirect(nconf.get('relative_path') + cidMatch[0]); + } else { + winston.warn(`[controller] Bad request: ${req.path}`); + if (req.path.startsWith(`${nconf.get('relative_path')}/api`)) { + res.status(400).json({ + error: '[[global:400.title]]' + }); + } else { + await middleware.buildHeaderAsync(req, res); + res.status(400).render('400', { + error: validator.escape(String(err.message)) + }); + } + } + } else { + next(err); + } +}; +exports.handleErrors = async function handleErrors(err, req, res, next) { + const cases = { + EBADCSRFTOKEN: function () { + winston.error(`${req.method} ${req.originalUrl}\n${err.message}`); + res.sendStatus(403); + }, + 'blacklisted-ip': function () { + res.status(403).type('text/plain').send(err.message); + } + }; + const notFoundHandler = () => { + const controllers = require('.'); + controllers['404'].handle404(req, res); + }; + const notBuiltHandler = async () => { + let file = await fs.promises.readFile(path.join(__dirname, '../../public/500.html'), { + encoding: 'utf-8' + }); + file = file.replace('{message}', 'Failed to lookup view! Did you run `./nodebb build`?'); + return res.type('text/html').send(file); + }; + const defaultHandler = async function () { + if (res.headersSent) { + return; + } + const status = parseInt(err.status, 10); + if ((status === 302 || status === 308) && err.path) { + return res.locals.isAPI ? res.set('X-Redirect', err.path).status(200).json(err.path) : res.redirect(nconf.get('relative_path') + err.path); + } + const path = String(req.path || ''); + if (path.startsWith(`${nconf.get('relative_path')}/api/v3`)) { + let status = 500; + if (err.message.startsWith('[[')) { + status = 400; + err.message = await translator.translate(err.message); + } + return helpers.formatApiResponse(status, res, err); + } + winston.error(`${req.method} ${req.originalUrl}\n${err.stack}`); + res.status(status || 500); + const data = { + path: validator.escape(path), + error: validator.escape(String(err.message)), + bodyClass: middlewareHelpers.buildBodyClass(req, res) + }; + if (res.locals.isAPI) { + res.json(data); + } else { + await middleware.buildHeaderAsync(req, res); + res.render('500', data); + } + }; + const data = await getErrorHandlers(cases); + try { + if (data.cases.hasOwnProperty(err.code)) { + data.cases[err.code](err, req, res, defaultHandler); + } else if (err.message.startsWith('[[error:no-') && err.message !== '[[error:no-privileges]]') { + notFoundHandler(); + } else if (err.message.startsWith('Failed to lookup view')) { + notBuiltHandler(); + } else { + await defaultHandler(); + } + } catch (_err) { + winston.error(`${req.method} ${req.originalUrl}\n${_err.stack}`); + if (!res.headersSent) { + res.status(500).send(_err.message); + } + } +}; +async function getErrorHandlers(cases) { + try { + return await plugins.hooks.fire('filter:error.handle', { + cases: cases + }); + } catch (err) { + winston.warn(`[errors/handle] Unable to retrieve plugin handlers for errors: ${err.message}`); + return { + cases + }; + } +} \ No newline at end of file diff --git a/lib/controllers/globalmods.js b/lib/controllers/globalmods.js new file mode 100644 index 0000000000..b13978c569 --- /dev/null +++ b/lib/controllers/globalmods.js @@ -0,0 +1,30 @@ +'use strict'; + +const user = require('../user'); +const meta = require('../meta'); +const analytics = require('../analytics'); +const usersController = require('./admin/users'); +const helpers = require('./helpers'); +const globalModsController = module.exports; +globalModsController.ipBlacklist = async function (req, res, next) { + const isAdminOrGlobalMod = await user.isAdminOrGlobalMod(req.uid); + if (!isAdminOrGlobalMod) { + return next(); + } + const [rules, analyticsData] = await Promise.all([meta.blacklist.get(), analytics.getBlacklistAnalytics()]); + res.render('ip-blacklist', { + title: '[[pages:ip-blacklist]]', + rules: rules, + analytics: analyticsData, + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[pages:ip-blacklist]]' + }]) + }); +}; +globalModsController.registrationQueue = async function (req, res, next) { + const isAdminOrGlobalMod = await user.isAdminOrGlobalMod(req.uid); + if (!isAdminOrGlobalMod) { + return next(); + } + await usersController.registrationQueue(req, res); +}; \ No newline at end of file diff --git a/lib/controllers/groups.js b/lib/controllers/groups.js new file mode 100644 index 0000000000..8ae05e86cb --- /dev/null +++ b/lib/controllers/groups.js @@ -0,0 +1,101 @@ +'use strict'; + +const validator = require('validator'); +const nconf = require('nconf'); +const meta = require('../meta'); +const groups = require('../groups'); +const user = require('../user'); +const helpers = require('./helpers'); +const pagination = require('../pagination'); +const privileges = require('../privileges'); +const groupsController = module.exports; +groupsController.list = async function (req, res) { + const sort = req.query.sort || 'alpha'; + const [groupData, allowGroupCreation] = await Promise.all([groups.getGroupsBySort(sort, 0, 14), privileges.global.can('group:create', req.uid)]); + res.render('groups/list', { + groups: groupData, + allowGroupCreation: allowGroupCreation, + sort: validator.escape(String(sort)), + nextStart: 15, + title: '[[pages:groups]]', + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[pages:groups]]' + }]) + }); +}; +groupsController.details = async function (req, res, next) { + const lowercaseSlug = req.params.slug.toLowerCase(); + if (req.params.slug !== lowercaseSlug) { + if (res.locals.isAPI) { + req.params.slug = lowercaseSlug; + } else { + return res.redirect(`${nconf.get('relative_path')}/groups/${lowercaseSlug}`); + } + } + const groupName = await groups.getGroupNameByGroupSlug(req.params.slug); + if (!groupName) { + return next(); + } + const [exists, isHidden, isAdmin, isGlobalMod] = await Promise.all([groups.exists(groupName), groups.isHidden(groupName), privileges.admin.can('admin:groups', req.uid), user.isGlobalModerator(req.uid)]); + if (!exists) { + return next(); + } + if (isHidden && !isAdmin && !isGlobalMod) { + const [isMember, isInvited] = await Promise.all([groups.isMember(req.uid, groupName), groups.isInvited(req.uid, groupName)]); + if (!isMember && !isInvited) { + return next(); + } + } + const [groupData, posts] = await Promise.all([groups.get(groupName, { + uid: req.uid, + truncateUserList: true, + userListCount: 20 + }), groups.getLatestMemberPosts(groupName, 10, req.uid)]); + if (!groupData) { + return next(); + } + res.render('groups/details', { + title: `[[pages:group, ${groupData.displayName}]]`, + group: groupData, + posts: posts, + isAdmin: isAdmin, + isGlobalMod: isGlobalMod, + allowPrivateGroups: meta.config.allowPrivateGroups, + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[pages:groups]]', + url: '/groups' + }, { + text: groupData.displayName + }]) + }); +}; +groupsController.members = async function (req, res, next) { + const page = parseInt(req.query.page, 10) || 1; + const usersPerPage = 50; + const start = Math.max(0, (page - 1) * usersPerPage); + const stop = start + usersPerPage - 1; + const groupName = await groups.getGroupNameByGroupSlug(req.params.slug); + if (!groupName) { + return next(); + } + const [groupData, isAdminOrGlobalMod, isMember, isHidden] = await Promise.all([groups.getGroupData(groupName), user.isAdminOrGlobalMod(req.uid), groups.isMember(req.uid, groupName), groups.isHidden(groupName)]); + if (isHidden && !isMember && !isAdminOrGlobalMod) { + return next(); + } + const users = await user.getUsersFromSet(`group:${groupName}:members`, req.uid, start, stop); + const breadcrumbs = helpers.buildBreadcrumbs([{ + text: '[[pages:groups]]', + url: '/groups' + }, { + text: validator.escape(String(groupName)), + url: `/groups/${req.params.slug}` + }, { + text: '[[groups:details.members]]' + }]); + const pageCount = Math.max(1, Math.ceil(groupData.memberCount / usersPerPage)); + res.render('groups/members', { + users: users, + pagination: pagination.create(page, pageCount, req.query), + breadcrumbs: breadcrumbs + }); +}; \ No newline at end of file diff --git a/lib/controllers/helpers.js b/lib/controllers/helpers.js new file mode 100644 index 0000000000..13a598f3a8 --- /dev/null +++ b/lib/controllers/helpers.js @@ -0,0 +1,528 @@ +'use strict'; + +const nconf = require('nconf'); +const validator = require('validator'); +const querystring = require('querystring'); +const _ = require('lodash'); +const chalk = require('chalk'); +const translator = require('../translator'); +const user = require('../user'); +const privileges = require('../privileges'); +const categories = require('../categories'); +const plugins = require('../plugins'); +const meta = require('../meta'); +const middlewareHelpers = require('../middleware/helpers'); +const utils = require('../utils'); +const helpers = module.exports; +const relative_path = nconf.get('relative_path'); +const url = nconf.get('url'); +helpers.noScriptErrors = async function (req, res, error, httpStatus) { + if (req.body.noscript !== 'true') { + if (typeof error === 'string') { + return res.status(httpStatus).send(error); + } + return res.status(httpStatus).json(error); + } + const middleware = require('../middleware'); + const httpStatusString = httpStatus.toString(); + await middleware.buildHeaderAsync(req, res); + res.status(httpStatus).render(httpStatusString, { + path: req.path, + loggedIn: req.loggedIn, + error: error, + returnLink: true, + title: `[[global:${httpStatusString}.title]]` + }); +}; +helpers.terms = { + daily: 'day', + weekly: 'week', + monthly: 'month' +}; +helpers.buildQueryString = function (query, key, value) { + const queryObj = { + ...query + }; + if (value) { + queryObj[key] = value; + } else { + delete queryObj[key]; + } + delete queryObj._; + return Object.keys(queryObj).length ? `?${querystring.stringify(queryObj)}` : ''; +}; +helpers.addLinkTags = function (params) { + params.res.locals.linkTags = params.res.locals.linkTags || []; + const page = params.page > 1 ? `?page=${params.page}` : ''; + params.res.locals.linkTags.push({ + rel: 'canonical', + href: `${url}/${params.url}${page}` + }); + params.tags.forEach(rel => { + rel.href = `${url}/${params.url}${rel.href}`; + params.res.locals.linkTags.push(rel); + }); +}; +helpers.buildFilters = function (url, filter, query) { + return [{ + name: '[[unread:all-topics]]', + url: url + helpers.buildQueryString(query, 'filter', ''), + selected: filter === '', + filter: '', + icon: 'fa-book' + }, { + name: '[[unread:new-topics]]', + url: url + helpers.buildQueryString(query, 'filter', 'new'), + selected: filter === 'new', + filter: 'new', + icon: 'fa-clock-o' + }, { + name: '[[unread:watched-topics]]', + url: url + helpers.buildQueryString(query, 'filter', 'watched'), + selected: filter === 'watched', + filter: 'watched', + icon: 'fa-bell-o' + }, { + name: '[[unread:unreplied-topics]]', + url: url + helpers.buildQueryString(query, 'filter', 'unreplied'), + selected: filter === 'unreplied', + filter: 'unreplied', + icon: 'fa-reply' + }]; +}; +helpers.buildTerms = function (url, term, query) { + return [{ + name: '[[recent:alltime]]', + url: url + helpers.buildQueryString(query, 'term', ''), + selected: term === 'alltime', + term: 'alltime' + }, { + name: '[[recent:day]]', + url: url + helpers.buildQueryString(query, 'term', 'daily'), + selected: term === 'day', + term: 'day' + }, { + name: '[[recent:week]]', + url: url + helpers.buildQueryString(query, 'term', 'weekly'), + selected: term === 'week', + term: 'week' + }, { + name: '[[recent:month]]', + url: url + helpers.buildQueryString(query, 'term', 'monthly'), + selected: term === 'month', + term: 'month' + }]; +}; +helpers.notAllowed = async function (req, res, error) { + ({ + error + } = await plugins.hooks.fire('filter:helpers.notAllowed', { + req, + res, + error + })); + await plugins.hooks.fire('response:helpers.notAllowed', { + req, + res, + error + }); + if (res.headersSent) { + return; + } + if (req.loggedIn || req.uid === -1) { + if (res.locals.isAPI) { + if (req.originalUrl.startsWith(`${relative_path}/api/v3`)) { + helpers.formatApiResponse(403, res, error); + } else { + res.status(403).json({ + path: req.path.replace(/^\/api/, ''), + loggedIn: req.loggedIn, + error: error, + title: '[[global:403.title]]', + bodyClass: middlewareHelpers.buildBodyClass(req, res) + }); + } + } else { + const middleware = require('../middleware'); + await middleware.buildHeaderAsync(req, res); + res.status(403).render('403', { + path: req.path, + loggedIn: req.loggedIn, + error, + title: '[[global:403.title]]' + }); + } + } else if (res.locals.isAPI) { + req.session.returnTo = req.url.replace(/^\/api/, ''); + helpers.formatApiResponse(401, res, error); + } else { + req.session.returnTo = req.url; + res.redirect(`${relative_path}/login${req.path.startsWith('/admin') ? '?local=1' : ''}`); + } +}; +helpers.redirect = function (res, url, permanent) { + if (url.hasOwnProperty('external')) { + const redirectUrl = encodeURI(prependRelativePath(url.external)); + if (res.locals.isAPI) { + res.set('X-Redirect', redirectUrl).status(200).json({ + external: redirectUrl + }); + } else { + res.redirect(permanent ? 308 : 307, redirectUrl); + } + return; + } + if (res.locals.isAPI) { + url = encodeURI(url); + res.set('X-Redirect', url).status(200).json(url); + } else { + res.redirect(permanent ? 308 : 307, encodeURI(prependRelativePath(url))); + } +}; +function prependRelativePath(url) { + return url.startsWith('http://') || url.startsWith('https://') ? url : relative_path + url; +} +helpers.buildCategoryBreadcrumbs = async function (cid) { + const breadcrumbs = []; + while (parseInt(cid, 10)) { + const data = await categories.getCategoryFields(cid, ['name', 'slug', 'parentCid', 'disabled', 'isSection']); + if (!data.disabled && !data.isSection) { + breadcrumbs.unshift({ + text: String(data.name), + url: `${url}/category/${data.slug}`, + cid: cid + }); + } + cid = data.parentCid; + } + if (meta.config.homePageRoute && meta.config.homePageRoute !== 'categories') { + breadcrumbs.unshift({ + text: '[[global:header.categories]]', + url: `${url}/categories` + }); + } + breadcrumbs.unshift({ + text: meta.config.homePageTitle || '[[global:home]]', + url: url + }); + return breadcrumbs; +}; +helpers.buildBreadcrumbs = function (crumbs) { + const breadcrumbs = [{ + text: meta.config.homePageTitle || '[[global:home]]', + url: url + }]; + crumbs.forEach(crumb => { + if (crumb) { + if (crumb.url) { + crumb.url = `${utils.isRelativeUrl(crumb.url) ? `${url}/` : ''}${crumb.url}`; + } + breadcrumbs.push(crumb); + } + }); + return breadcrumbs; +}; +helpers.buildTitle = function (pageTitle) { + pageTitle = pageTitle || ''; + const titleLayout = meta.config.titleLayout || `${pageTitle ? '{pageTitle} | ' : ''}{browserTitle}`; + const browserTitle = validator.escape(String(meta.config.browserTitle || meta.config.title || 'NodeBB')); + const title = titleLayout.replace('{pageTitle}', () => pageTitle).replace('{browserTitle}', () => browserTitle); + return title; +}; +helpers.getCategories = async function (set, uid, privilege, selectedCid) { + const cids = await categories.getCidsByPrivilege(set, uid, privilege); + return await getCategoryData(cids, uid, selectedCid, Object.values(categories.watchStates), privilege); +}; +helpers.getCategoriesByStates = async function (uid, selectedCid, states, privilege = 'topics:read') { + const cids = await categories.getAllCidsFromSet('categories:cid'); + return await getCategoryData(cids, uid, selectedCid, states, privilege); +}; +async function getCategoryData(cids, uid, selectedCid, states, privilege) { + const [visibleCategories, selectData] = await Promise.all([helpers.getVisibleCategories({ + cids, + uid, + states, + privilege, + showLinks: false + }), helpers.getSelectedCategory(selectedCid)]); + const categoriesData = categories.buildForSelectCategories(visibleCategories, ['disabledClass']); + categoriesData.forEach(category => { + category.selected = selectData.selectedCids.includes(category.cid); + }); + selectData.selectedCids.sort((a, b) => a - b); + return { + categories: categoriesData, + selectedCategory: selectData.selectedCategory, + selectedCids: selectData.selectedCids + }; +} +helpers.getVisibleCategories = async function (params) { + const { + cids, + uid, + privilege + } = params; + const states = params.states || [categories.watchStates.watching, categories.watchStates.notwatching]; + const showLinks = !!params.showLinks; + let [allowed, watchState, categoriesData, isAdmin, isModerator] = await Promise.all([privileges.categories.isUserAllowedTo(privilege, cids, uid), categories.getWatchState(cids, uid), categories.getCategoriesData(cids), user.isAdministrator(uid), user.isModerator(uid, cids)]); + const filtered = await plugins.hooks.fire('filter:helpers.getVisibleCategories', { + uid: uid, + allowed: allowed, + watchState: watchState, + categoriesData: categoriesData, + isModerator: isModerator, + isAdmin: isAdmin + }); + ({ + allowed, + watchState, + categoriesData, + isModerator, + isAdmin + } = filtered); + categories.getTree(categoriesData, params.parentCid); + const cidToAllowed = _.zipObject(cids, allowed.map((allowed, i) => isAdmin || isModerator[i] || allowed)); + const cidToCategory = _.zipObject(cids, categoriesData); + const cidToWatchState = _.zipObject(cids, watchState); + return categoriesData.filter(c => { + if (!c) { + return false; + } + const hasVisibleChildren = checkVisibleChildren(c, cidToAllowed, cidToWatchState, states); + const isCategoryVisible = cidToAllowed[c.cid] && (showLinks || !c.link) && !c.disabled && states.includes(cidToWatchState[c.cid]); + const shouldBeRemoved = !hasVisibleChildren && !isCategoryVisible; + const shouldBeDisaplayedAsDisabled = hasVisibleChildren && !isCategoryVisible; + if (shouldBeDisaplayedAsDisabled) { + c.disabledClass = true; + } + if (shouldBeRemoved && c.parent && c.parent.cid && cidToCategory[c.parent.cid]) { + cidToCategory[c.parent.cid].children = cidToCategory[c.parent.cid].children.filter(child => child.cid !== c.cid); + } + return !shouldBeRemoved; + }); +}; +helpers.getSelectedCategory = async function (cids) { + if (cids && !Array.isArray(cids)) { + cids = [cids]; + } + cids = cids && cids.map(cid => parseInt(cid, 10)); + let selectedCategories = await categories.getCategoriesData(cids); + const selectedCids = selectedCategories.map(c => c && c.cid).filter(Boolean); + if (selectedCategories.length > 1) { + selectedCategories = { + icon: 'fa-plus', + name: '[[unread:multiple-categories-selected]]', + bgColor: '#ddd' + }; + } else if (selectedCategories.length === 1 && selectedCategories[0]) { + selectedCategories = selectedCategories[0]; + } else { + selectedCategories = null; + } + return { + selectedCids: selectedCids, + selectedCategory: selectedCategories + }; +}; +helpers.getSelectedTag = function (tags) { + if (tags && !Array.isArray(tags)) { + tags = [tags]; + } + tags = tags || []; + const tagData = tags.map(t => validator.escape(String(t))); + let selectedTag = null; + if (tagData.length) { + selectedTag = { + label: tagData.join(', ') + }; + } + return { + selectedTags: tagData, + selectedTag: selectedTag + }; +}; +helpers.trimChildren = function (category) { + if (category && Array.isArray(category.children)) { + category.children = category.children.slice(0, category.subCategoriesPerPage); + category.children.forEach(child => { + if (category.isSection) { + helpers.trimChildren(child); + } else { + child.children = undefined; + } + }); + } +}; +helpers.setCategoryTeaser = function (category) { + if (Array.isArray(category.posts) && category.posts.length && category.posts[0]) { + const post = category.posts[0]; + category.teaser = { + url: `${nconf.get('relative_path')}/post/${post.pid}`, + timestampISO: post.timestampISO, + pid: post.pid, + tid: post.tid, + index: post.index, + topic: post.topic, + user: post.user + }; + } +}; +function checkVisibleChildren(c, cidToAllowed, cidToWatchState, states) { + if (!c || !Array.isArray(c.children)) { + return false; + } + return c.children.some(c => !c.disabled && (cidToAllowed[c.cid] && states.includes(cidToWatchState[c.cid]) || checkVisibleChildren(c, cidToAllowed, cidToWatchState, states))); +} +helpers.getHomePageRoutes = async function (uid) { + const routes = [{ + route: 'categories', + name: 'Categories' + }, { + route: 'unread', + name: 'Unread' + }, { + route: 'recent', + name: 'Recent' + }, { + route: 'top', + name: 'Top' + }, { + route: 'popular', + name: 'Popular' + }, { + route: 'custom', + name: 'Custom' + }]; + const data = await plugins.hooks.fire('filter:homepage.get', { + uid: uid, + routes: routes + }); + return data.routes; +}; +helpers.formatApiResponse = async (statusCode, res, payload) => { + if (res.req.method === 'HEAD') { + return res.sendStatus(statusCode); + } + if (String(statusCode).startsWith('2')) { + if (res.req.loggedIn) { + res.set('cache-control', 'private'); + } + let code = 'ok'; + let message = 'OK'; + switch (statusCode) { + case 202: + code = 'accepted'; + message = 'Accepted'; + break; + case 204: + code = 'no-content'; + message = 'No Content'; + break; + } + res.status(statusCode).json({ + status: { + code, + message + }, + response: payload || {} + }); + } else if (payload instanceof Error || typeof payload === 'string') { + const message = payload instanceof Error ? payload.message : payload; + const response = {}; + switch (message) { + case '[[error:user-banned]]': + Object.assign(response, await generateBannedResponse(res)); + case '[[error:no-privileges]]': + statusCode = 403; + break; + case '[[error:invalid-uid]]': + statusCode = 401; + break; + case '[[error:no-topic]]': + statusCode = 404; + break; + } + if (message.startsWith('[[error:required-parameters-missing, ')) { + const params = message.slice('[[error:required-parameters-missing, '.length, -2).split(' '); + Object.assign(response, { + params + }); + } + const returnPayload = await helpers.generateError(statusCode, message, res); + returnPayload.response = response; + if (global.env === 'development') { + returnPayload.stack = payload.stack; + process.stdout.write(`[${chalk.yellow('api')}] Exception caught, error with stack trace follows:\n`); + process.stdout.write(payload.stack); + } + res.status(statusCode).json(returnPayload); + } else { + const message = payload ? String(payload) : null; + const returnPayload = await helpers.generateError(statusCode, message, res); + res.status(statusCode).json(returnPayload); + } +}; +async function generateBannedResponse(res) { + const response = {}; + const [reason, expiry] = await Promise.all([user.bans.getReason(res.req.uid), user.getUserField(res.req.uid, 'banned:expire')]); + response.reason = reason; + if (expiry) { + Object.assign(response, { + expiry, + expiryISO: new Date(expiry).toISOString(), + expiryLocaleString: new Date(expiry).toLocaleString() + }); + } + return response; +} +helpers.generateError = async (statusCode, message, res) => { + async function translateMessage(message) { + const { + req + } = res; + const settings = req.query.lang ? null : await user.getSettings(req.uid); + const language = String(req.query.lang || settings.userLang || meta.config.defaultLang); + return await translator.translate(message, language); + } + if (message && message.startsWith('[[')) { + message = await translateMessage(message); + } + const payload = { + status: { + code: 'internal-server-error', + message: message || (await translateMessage(`[[error:api.${statusCode}]]`)) + }, + response: {} + }; + switch (statusCode) { + case 400: + payload.status.code = 'bad-request'; + break; + case 401: + payload.status.code = 'not-authorised'; + break; + case 403: + payload.status.code = 'forbidden'; + break; + case 404: + payload.status.code = 'not-found'; + break; + case 426: + payload.status.code = 'upgrade-required'; + break; + case 429: + payload.status.code = 'too-many-requests'; + break; + case 500: + payload.status.code = 'internal-server-error'; + break; + case 501: + payload.status.code = 'not-implemented'; + break; + case 503: + payload.status.code = 'service-unavailable'; + break; + } + return payload; +}; +require('../promisify')(helpers); \ No newline at end of file diff --git a/lib/controllers/home.js b/lib/controllers/home.js new file mode 100644 index 0000000000..e688c7d1db --- /dev/null +++ b/lib/controllers/home.js @@ -0,0 +1,53 @@ +'use strict'; + +const url = require('url'); +const plugins = require('../plugins'); +const meta = require('../meta'); +const user = require('../user'); +function adminHomePageRoute() { + return ((meta.config.homePageRoute === 'custom' ? meta.config.homePageCustom : meta.config.homePageRoute) || 'categories').replace(/^\//, ''); +} +async function getUserHomeRoute(uid) { + const settings = await user.getSettings(uid); + let route = adminHomePageRoute(); + if (settings.homePageRoute !== 'undefined' && settings.homePageRoute !== 'none') { + route = (settings.homePageRoute || route).replace(/^\/+/, ''); + } + return route; +} +async function rewrite(req, res, next) { + if (req.path !== '/' && req.path !== '/api/' && req.path !== '/api') { + return next(); + } + let route = adminHomePageRoute(); + if (meta.config.allowUserHomePage) { + route = await getUserHomeRoute(req.uid, next); + } + let parsedUrl; + try { + parsedUrl = url.parse(route, true); + } catch (err) { + return next(err); + } + const { + pathname + } = parsedUrl; + const hook = `action:homepage.get:${pathname}`; + if (!plugins.hooks.hasListeners(hook)) { + req.url = req.path + (!req.path.endsWith('/') ? '/' : '') + pathname; + } else { + res.locals.homePageRoute = pathname; + } + req.query = Object.assign(parsedUrl.query, req.query); + next(); +} +exports.rewrite = rewrite; +function pluginHook(req, res, next) { + const hook = `action:homepage.get:${res.locals.homePageRoute}`; + plugins.hooks.fire(hook, { + req: req, + res: res, + next: next + }); +} +exports.pluginHook = pluginHook; \ No newline at end of file diff --git a/lib/controllers/index.js b/lib/controllers/index.js new file mode 100644 index 0000000000..b1b5b7d426 --- /dev/null +++ b/lib/controllers/index.js @@ -0,0 +1,314 @@ +'use strict'; + +const nconf = require('nconf'); +const validator = require('validator'); +const meta = require('../meta'); +const user = require('../user'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const helpers = require('./helpers'); +const Controllers = module.exports; +Controllers.ping = require('./ping'); +Controllers.home = require('./home'); +Controllers.topics = require('./topics'); +Controllers.posts = require('./posts'); +Controllers.categories = require('./categories'); +Controllers.category = require('./category'); +Controllers.unread = require('./unread'); +Controllers.recent = require('./recent'); +Controllers.popular = require('./popular'); +Controllers.top = require('./top'); +Controllers.tags = require('./tags'); +Controllers.search = require('./search'); +Controllers.user = require('./user'); +Controllers.users = require('./users'); +Controllers.groups = require('./groups'); +Controllers.accounts = require('./accounts'); +Controllers.authentication = require('./authentication'); +Controllers.api = require('./api'); +Controllers.admin = require('./admin'); +Controllers.globalMods = require('./globalmods'); +Controllers.mods = require('./mods'); +Controllers.sitemap = require('./sitemap'); +Controllers.osd = require('./osd'); +Controllers['404'] = require('./404'); +Controllers.errors = require('./errors'); +Controllers.composer = require('./composer'); +Controllers.write = require('./write'); +Controllers.reset = async function (req, res) { + if (meta.config['password:disableEdit']) { + return helpers.notAllowed(req, res); + } + res.locals.metaTags = { + ...res.locals.metaTags, + name: 'robots', + content: 'noindex' + }; + const renderReset = function (code, valid) { + res.render('reset_code', { + valid: valid, + displayExpiryNotice: req.session.passwordExpired, + code: code, + minimumPasswordLength: meta.config.minimumPasswordLength, + minimumPasswordStrength: meta.config.minimumPasswordStrength, + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[reset_password:reset-password]]', + url: '/reset' + }, { + text: '[[reset_password:update-password]]' + }]), + title: '[[pages:reset]]' + }); + delete req.session.passwordExpired; + }; + if (req.params.code) { + req.session.reset_code = req.params.code; + } + if (req.session.reset_code) { + const valid = await user.reset.validate(req.session.reset_code); + renderReset(req.session.reset_code, valid); + delete req.session.reset_code; + } else { + res.render('reset', { + code: null, + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[reset_password:reset-password]]' + }]), + title: '[[pages:reset]]' + }); + } +}; +Controllers.login = async function (req, res) { + const data = { + loginFormEntry: [] + }; + const loginStrategies = require('../routes/authentication').getLoginStrategies(); + const registrationType = meta.config.registrationType || 'normal'; + const allowLoginWith = meta.config.allowLoginWith || 'username-email'; + let errorText; + if (req.query.error === 'csrf-invalid') { + errorText = '[[error:csrf-invalid]]'; + } else if (req.query.error) { + errorText = validator.escape(String(req.query.error)); + } + if (req.headers['x-return-to']) { + req.session.returnTo = req.headers['x-return-to']; + } + req.session.returnTo = req.session.returnTo && req.session.returnTo.replace(nconf.get('base_url'), '').replace(nconf.get('relative_path'), ''); + data.alternate_logins = loginStrategies.length > 0; + data.authentication = loginStrategies; + data.allowRegistration = registrationType === 'normal'; + data.allowLoginWith = `[[login:${allowLoginWith}]]`; + data.breadcrumbs = helpers.buildBreadcrumbs([{ + text: '[[global:login]]' + }]); + data.error = req.flash('error')[0] || errorText; + data.title = '[[pages:login]]'; + data.allowPasswordReset = !meta.config['password:disableEdit']; + const hasLoginPrivilege = await privileges.global.canGroup('local:login', 'registered-users'); + data.allowLocalLogin = hasLoginPrivilege || parseInt(req.query.local, 10) === 1; + if (!data.allowLocalLogin && !data.allowRegistration && data.alternate_logins && data.authentication.length === 1) { + return helpers.redirect(res, { + external: data.authentication[0].url + }); + } + if (req.loggedIn) { + const userData = await user.getUserFields(req.uid, ['username']); + data.username = userData.username; + data.alternate_logins = false; + } + res.render('login', data); +}; +Controllers.register = async function (req, res, next) { + const registrationType = meta.config.registrationType || 'normal'; + if (registrationType === 'disabled') { + return setImmediate(next); + } + let errorText; + const returnTo = (req.headers['x-return-to'] || '').replace(nconf.get('base_url') + nconf.get('relative_path'), ''); + if (req.query.error === 'csrf-invalid') { + errorText = '[[error:csrf-invalid]]'; + } + try { + if (registrationType === 'invite-only' || registrationType === 'admin-invite-only') { + try { + await user.verifyInvitation(req.query); + } catch (e) { + return res.render('400', { + error: e.message + }); + } + } + if (returnTo) { + req.session.returnTo = returnTo; + } + const loginStrategies = require('../routes/authentication').getLoginStrategies(); + res.render('register', { + 'register_window:spansize': loginStrategies.length ? 'col-md-6' : 'col-md-12', + alternate_logins: !!loginStrategies.length, + authentication: loginStrategies, + minimumUsernameLength: meta.config.minimumUsernameLength, + maximumUsernameLength: meta.config.maximumUsernameLength, + minimumPasswordLength: meta.config.minimumPasswordLength, + minimumPasswordStrength: meta.config.minimumPasswordStrength, + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[register:register]]' + }]), + regFormEntry: [], + error: req.flash('error')[0] || errorText, + title: '[[pages:register]]' + }); + } catch (err) { + next(err); + } +}; +Controllers.registerInterstitial = async function (req, res, next) { + if (!req.session.hasOwnProperty('registration')) { + return res.redirect(`${nconf.get('relative_path')}/register`); + } + try { + const data = await user.interstitials.get(req, req.session.registration); + if (!data.interstitials.length) { + const returnTo = req.session.returnTo || req.session.registration.returnTo; + delete req.session.registration; + return helpers.redirect(res, returnTo || '/'); + } + const errors = req.flash('errors'); + const renders = data.interstitials.map(interstitial => req.app.renderAsync(interstitial.template, { + ...(interstitial.data || {}), + errors + })); + const sections = await Promise.all(renders); + res.render('registerComplete', { + title: '[[pages:registration-complete]]', + register: data.userData.register, + sections, + errors + }); + } catch (err) { + next(err); + } +}; +Controllers.confirmEmail = async (req, res, next) => { + try { + await user.email.confirmByCode(req.params.code, req.session.id); + if (req.session.registration) { + delete req.session.registration.updateEmail; + } + res.render('confirm', { + title: '[[pages:confirm]]' + }); + } catch (e) { + if (e.message === '[[error:invalid-data]]') { + return next(); + } + throw e; + } +}; +Controllers.robots = function (req, res) { + res.set('Content-Type', 'text/plain'); + if (meta.config['robots:txt']) { + res.send(meta.config['robots:txt']); + } else { + res.send(`${'User-agent: *\n' + 'Disallow: '}${nconf.get('relative_path')}/admin/\n` + `Disallow: ${nconf.get('relative_path')}/reset/\n` + `Disallow: ${nconf.get('relative_path')}/compose\n` + `Sitemap: ${nconf.get('url')}/sitemap.xml`); + } +}; +Controllers.manifest = async function (req, res) { + const manifest = { + name: meta.config.title || 'NodeBB', + short_name: meta.config['title:short'] || meta.config.title || 'NodeBB', + start_url: nconf.get('url'), + display: 'standalone', + orientation: 'portrait', + theme_color: meta.config.themeColor || '#ffffff', + background_color: meta.config.backgroundColor || '#ffffff', + icons: [] + }; + if (meta.config['brand:touchIcon']) { + manifest.icons.push({ + src: `${nconf.get('relative_path')}/assets/uploads/system/touchicon-36.png`, + sizes: '36x36', + type: 'image/png', + density: 0.75 + }, { + src: `${nconf.get('relative_path')}/assets/uploads/system/touchicon-48.png`, + sizes: '48x48', + type: 'image/png', + density: 1.0 + }, { + src: `${nconf.get('relative_path')}/assets/uploads/system/touchicon-72.png`, + sizes: '72x72', + type: 'image/png', + density: 1.5 + }, { + src: `${nconf.get('relative_path')}/assets/uploads/system/touchicon-96.png`, + sizes: '96x96', + type: 'image/png', + density: 2.0 + }, { + src: `${nconf.get('relative_path')}/assets/uploads/system/touchicon-144.png`, + sizes: '144x144', + type: 'image/png', + density: 3.0 + }, { + src: `${nconf.get('relative_path')}/assets/uploads/system/touchicon-192.png`, + sizes: '192x192', + type: 'image/png', + density: 4.0 + }, { + src: `${nconf.get('relative_path')}/assets/uploads/system/touchicon-512.png`, + sizes: '512x512', + type: 'image/png', + density: 10.0 + }); + } + if (meta.config['brand:maskableIcon']) { + manifest.icons.push({ + src: `${nconf.get('relative_path')}/assets/uploads/system/maskableicon-orig.png`, + sizes: '512x512', + type: 'image/png', + purpose: 'maskable' + }); + } else if (meta.config['brand:touchIcon']) { + manifest.icons.push({ + src: `${nconf.get('relative_path')}/assets/uploads/system/touchicon-orig.png`, + sizes: '512x512', + type: 'image/png', + purpose: 'maskable' + }); + } + const data = await plugins.hooks.fire('filter:manifest.build', { + req: req, + res: res, + manifest: manifest + }); + res.status(200).json(data.manifest); +}; +Controllers.outgoing = function (req, res, next) { + const url = req.query.url || ''; + const allowedProtocols = ['http', 'https', 'ftp', 'ftps', 'mailto', 'news', 'irc', 'gopher', 'nntp', 'feed', 'telnet', 'mms', 'rtsp', 'svn', 'tel', 'fax', 'xmpp', 'webcal']; + const parsed = require('url').parse(url); + if (!url || !parsed.protocol || !allowedProtocols.includes(parsed.protocol.slice(0, -1))) { + return next(); + } + res.render('outgoing', { + outgoing: validator.escape(String(url)), + title: meta.config.title, + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[notifications:outgoing-link]]' + }]) + }); +}; +Controllers.termsOfUse = async function (req, res, next) { + if (!meta.config.termsOfUse) { + return next(); + } + const termsOfUse = await plugins.hooks.fire('filter:parse.post', { + postData: { + content: meta.config.termsOfUse || '' + } + }); + res.render('tos', { + termsOfUse: termsOfUse.postData.content + }); +}; \ No newline at end of file diff --git a/lib/controllers/mods.js b/lib/controllers/mods.js new file mode 100644 index 0000000000..bcf15e0b61 --- /dev/null +++ b/lib/controllers/mods.js @@ -0,0 +1,233 @@ +'use strict'; + +const _ = require('lodash'); +const user = require('../user'); +const groups = require('../groups'); +const meta = require('../meta'); +const posts = require('../posts'); +const db = require('../database'); +const flags = require('../flags'); +const analytics = require('../analytics'); +const plugins = require('../plugins'); +const pagination = require('../pagination'); +const privileges = require('../privileges'); +const utils = require('../utils'); +const helpers = require('./helpers'); +const modsController = module.exports; +modsController.flags = {}; +modsController.flags.list = async function (req, res) { + const validFilters = ['assignee', 'state', 'reporterId', 'type', 'targetUid', 'cid', 'quick', 'page', 'perPage']; + const validSorts = ['newest', 'oldest', 'reports', 'upvotes', 'downvotes', 'replies']; + const results = await Promise.all([user.isAdminOrGlobalMod(req.uid), user.getModeratedCids(req.uid), plugins.hooks.fire('filter:flags.validateFilters', { + filters: validFilters + }), plugins.hooks.fire('filter:flags.validateSort', { + sorts: validSorts + })]); + const [isAdminOrGlobalMod, moderatedCids,, { + sorts + }] = results; + let [,, { + filters + }] = results; + if (!(isAdminOrGlobalMod || !!moderatedCids.length)) { + return helpers.notAllowed(req, res); + } + if (!isAdminOrGlobalMod && moderatedCids.length) { + res.locals.cids = moderatedCids.map(cid => String(cid)); + } + filters = filters.reduce((memo, cur) => { + if (req.query.hasOwnProperty(cur)) { + if (typeof req.query[cur] === 'string' && req.query[cur].trim() !== '') { + memo[cur] = req.query[cur].trim(); + } else if (Array.isArray(req.query[cur]) && req.query[cur].length) { + memo[cur] = req.query[cur]; + } + } + return memo; + }, {}); + let hasFilter = !!Object.keys(filters).length; + if (res.locals.cids) { + if (!filters.cid) { + filters.cid = res.locals.cids; + } else if (Array.isArray(filters.cid)) { + filters.cid = filters.cid.filter(cid => res.locals.cids.includes(String(cid))); + } else if (!res.locals.cids.includes(String(filters.cid))) { + filters.cid = res.locals.cids; + hasFilter = false; + } + } + if (Object.keys(filters).length === 1 && filters.hasOwnProperty('page') || Object.keys(filters).length === 2 && filters.hasOwnProperty('page') && filters.hasOwnProperty('perPage')) { + hasFilter = false; + } + let sort; + if (req.query.sort) { + sort = sorts.includes(req.query.sort) ? req.query.sort : null; + } + if (sort === 'newest') { + sort = undefined; + } + hasFilter = hasFilter || !!sort; + const [flagsData, analyticsData, selectData] = await Promise.all([flags.list({ + filters: filters, + sort: sort, + uid: req.uid, + query: req.query + }), analytics.getDailyStatsForSet('analytics:flags', Date.now(), 30), helpers.getSelectedCategory(filters.cid)]); + const selected = {}; + await Promise.all(['assignee', 'reporterId', 'targetUid'].map(async filter => { + let uids = filters[filter]; + if (!uids) { + selected[filter] = []; + return; + } + if (!Array.isArray(uids)) { + uids = [uids]; + } + selected[filter] = await user.getUsersFields(uids, ['username', 'userslug', 'picture']); + })); + res.render('flags/list', { + flags: flagsData.flags, + count: flagsData.count, + analytics: analyticsData, + selectedCategory: selectData.selectedCategory, + selected, + hasFilter: hasFilter, + filters: filters, + expanded: !!(filters.assignee || filters.reporterId || filters.targetUid), + sort: sort || 'newest', + title: '[[pages:flags]]', + pagination: pagination.create(flagsData.page, flagsData.pageCount, req.query), + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[pages:flags]]' + }]) + }); +}; +modsController.flags.detail = async function (req, res, next) { + const results = await utils.promiseParallel({ + isAdminOrGlobalMod: user.isAdminOrGlobalMod(req.uid), + moderatedCids: user.getModeratedCids(req.uid), + flagData: flags.get(req.params.flagId), + privileges: Promise.all(['global', 'admin'].map(async type => privileges[type].get(req.uid))) + }); + results.privileges = { + ...results.privileges[0], + ...results.privileges[1] + }; + if (!results.flagData || !(results.isAdminOrGlobalMod || !!results.moderatedCids.length)) { + return next(); + } + if (!results.isAdminOrGlobalMod) { + if (results.flagData.type === 'user') { + return next(); + } + if (results.flagData.type === 'post') { + const isFlagInModeratedCids = await db.isMemberOfSortedSets(results.moderatedCids.map(cid => `flags:byCid:${cid}`), results.flagData.flagId); + if (!isFlagInModeratedCids.includes(true)) { + return next(); + } + } + } + async function getAssignees(flagData) { + let uids = []; + const [admins, globalMods] = await Promise.all([groups.getMembers('administrators', 0, -1), groups.getMembers('Global Moderators', 0, -1)]); + if (flagData.type === 'user') { + uids = await privileges.admin.getUidsWithPrivilege('admin:users'); + uids = _.uniq(admins.concat(uids)); + } else if (flagData.type === 'post') { + const cid = await posts.getCidByPid(flagData.targetId); + uids = _.uniq(admins.concat(globalMods)); + if (cid) { + const modUids = (await privileges.categories.getUidsWithPrivilege([cid], 'moderate'))[0]; + uids = _.uniq(uids.concat(modUids)); + } + } + const userData = await user.getUsersData(uids); + return userData.filter(u => u && u.userslug); + } + const assignees = await getAssignees(results.flagData); + results.flagData.history = await flags.getHistory(req.params.flagId); + if (results.flagData.type === 'user') { + results.flagData.type_path = 'uid'; + } else if (results.flagData.type === 'post') { + results.flagData.type_path = 'post'; + } + res.render('flags/detail', Object.assign(results.flagData, { + assignees: assignees, + type_bool: ['post', 'user', 'empty'].reduce((memo, cur) => { + if (cur !== 'empty') { + memo[cur] = results.flagData.type === cur && (!results.flagData.target || !!Object.keys(results.flagData.target).length); + } else { + memo[cur] = !Object.keys(results.flagData.target).length; + } + return memo; + }, {}), + states: Object.fromEntries(flags._states), + title: `[[pages:flag-details, ${req.params.flagId}]]`, + privileges: results.privileges, + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[pages:flags]]', + url: '/flags' + }, { + text: `[[pages:flag-details, ${req.params.flagId}]]` + }]) + })); +}; +modsController.postQueue = async function (req, res, next) { + if (!req.loggedIn) { + return next(); + } + const { + id + } = req.params; + const { + cid + } = req.query; + const page = parseInt(req.query.page, 10) || 1; + const postsPerPage = 20; + let postData = await posts.getQueuedPosts({ + id: id + }); + let [isAdmin, isGlobalMod, moderatedCids, categoriesData, _privileges] = await Promise.all([user.isAdministrator(req.uid), user.isGlobalModerator(req.uid), user.getModeratedCids(req.uid), helpers.getSelectedCategory(cid), Promise.all(['global', 'admin'].map(async type => privileges[type].get(req.uid)))]); + _privileges = { + ..._privileges[0], + ..._privileges[1] + }; + postData = postData.filter(p => p && (!categoriesData.selectedCids.length || categoriesData.selectedCids.includes(p.category.cid)) && (isAdmin || isGlobalMod || moderatedCids.includes(Number(p.category.cid)) || req.uid === p.user.uid)).map(post => { + const isSelf = post.user.uid === req.uid; + post.canAccept = !isSelf && (isAdmin || isGlobalMod || !!moderatedCids.length); + return post; + }); + ({ + posts: postData + } = await plugins.hooks.fire('filter:post-queue.get', { + posts: postData, + req: req + })); + const pageCount = Math.max(1, Math.ceil(postData.length / postsPerPage)); + const start = (page - 1) * postsPerPage; + const stop = start + postsPerPage - 1; + postData = postData.slice(start, stop + 1); + const crumbs = [{ + text: '[[pages:post-queue]]', + url: id ? '/post-queue' : undefined + }]; + if (id && postData.length) { + const text = postData[0].data.tid ? '[[post-queue:reply]]' : '[[post-queue:topic]]'; + crumbs.push({ + text: text + }); + } + res.render('post-queue', { + title: '[[pages:post-queue]]', + posts: postData, + isAdmin: isAdmin, + canAccept: isAdmin || isGlobalMod, + ...categoriesData, + allCategoriesUrl: `post-queue${helpers.buildQueryString(req.query, 'cid', '')}`, + pagination: pagination.create(page, pageCount), + breadcrumbs: helpers.buildBreadcrumbs(crumbs), + enabled: meta.config.postQueue, + singlePost: !!id, + privileges: _privileges + }); +}; \ No newline at end of file diff --git a/lib/controllers/osd.js b/lib/controllers/osd.js new file mode 100644 index 0000000000..2f9f6d133d --- /dev/null +++ b/lib/controllers/osd.js @@ -0,0 +1,53 @@ +'use strict'; + +const xml = require('xml'); +const nconf = require('nconf'); +const plugins = require('../plugins'); +const meta = require('../meta'); +module.exports.handle = function (req, res, next) { + if (plugins.hooks.hasListeners('filter:search.query')) { + res.type('application/opensearchdescription+xml').send(generateXML()); + } else { + next(); + } +}; +function generateXML() { + return xml([{ + OpenSearchDescription: [{ + _attr: { + xmlns: 'http://a9.com/-/spec/opensearch/1.1/', + 'xmlns:moz': 'http://www.mozilla.org/2006/browser/search/' + } + }, { + ShortName: trimToLength(String(meta.config.title || meta.config.browserTitle || 'NodeBB'), 16) + }, { + Description: trimToLength(String(meta.config.description || ''), 1024) + }, { + InputEncoding: 'UTF-8' + }, { + Image: [{ + _attr: { + width: '16', + height: '16', + type: 'image/x-icon' + } + }, `${nconf.get('url')}/favicon.ico`] + }, { + Url: { + _attr: { + type: 'text/html', + method: 'get', + template: `${nconf.get('url')}/search?term={searchTerms}&in=titlesposts` + } + } + }, { + 'moz:SearchForm': `${nconf.get('url')}/search` + }] + }], { + declaration: true, + indent: '\t' + }); +} +function trimToLength(string, length) { + return string.trim().substring(0, length).trim(); +} \ No newline at end of file diff --git a/lib/controllers/ping.js b/lib/controllers/ping.js new file mode 100644 index 0000000000..a555d4c63a --- /dev/null +++ b/lib/controllers/ping.js @@ -0,0 +1,12 @@ +'use strict'; + +const nconf = require('nconf'); +const db = require('../database'); +module.exports.ping = async function (req, res, next) { + try { + await db.getObject('config'); + res.status(200).send(req.path === `${nconf.get('relative_path')}/sping` ? 'healthy' : '200'); + } catch (err) { + next(err); + } +}; \ No newline at end of file diff --git a/lib/controllers/popular.js b/lib/controllers/popular.js new file mode 100644 index 0000000000..ce7f5091c0 --- /dev/null +++ b/lib/controllers/popular.js @@ -0,0 +1,29 @@ +'use strict'; + +const nconf = require('nconf'); +const validator = require('validator'); +const helpers = require('./helpers'); +const recentController = require('./recent'); +const popularController = module.exports; +popularController.get = async function (req, res, next) { + const data = await recentController.getData(req, 'popular', 'posts'); + if (!data) { + return next(); + } + const term = helpers.terms[req.query.term] || 'alltime'; + if (req.originalUrl.startsWith(`${nconf.get('relative_path')}/api/popular`) || req.originalUrl.startsWith(`${nconf.get('relative_path')}/popular`)) { + data.title = `[[pages:popular-${term}]]`; + const breadcrumbs = [{ + text: '[[global:header.popular]]' + }]; + data.breadcrumbs = helpers.buildBreadcrumbs(breadcrumbs); + } + if (!data['feeds:disableRSS'] && data.rssFeedUrl) { + const feedQs = data.rssFeedUrl.split('?')[1]; + data.rssFeedUrl = `${nconf.get('relative_path')}/popular/${validator.escape(String(req.query.term || 'alltime'))}.rss`; + if (req.loggedIn) { + data.rssFeedUrl += `?${feedQs}`; + } + } + res.render('popular', data); +}; \ No newline at end of file diff --git a/lib/controllers/posts.js b/lib/controllers/posts.js new file mode 100644 index 0000000000..be1447c796 --- /dev/null +++ b/lib/controllers/posts.js @@ -0,0 +1,30 @@ +'use strict'; + +const querystring = require('querystring'); +const posts = require('../posts'); +const privileges = require('../privileges'); +const helpers = require('./helpers'); +const postsController = module.exports; +postsController.redirectToPost = async function (req, res, next) { + const pid = parseInt(req.params.pid, 10); + if (!pid) { + return next(); + } + const [canRead, path] = await Promise.all([privileges.posts.can('topics:read', pid, req.uid), posts.generatePostPath(pid, req.uid)]); + if (!path) { + return next(); + } + if (!canRead) { + return helpers.notAllowed(req, res); + } + const qs = querystring.stringify(req.query); + helpers.redirect(res, qs ? `${path}?${qs}` : path, true); +}; +postsController.getRecentPosts = async function (req, res) { + const page = parseInt(req.query.page, 10) || 1; + const postsPerPage = 20; + const start = Math.max(0, (page - 1) * postsPerPage); + const stop = start + postsPerPage - 1; + const data = await posts.getRecentPosts(req.uid, start, stop, req.params.term); + res.json(data); +}; \ No newline at end of file diff --git a/lib/controllers/recent.js b/lib/controllers/recent.js new file mode 100644 index 0000000000..a1305a9351 --- /dev/null +++ b/lib/controllers/recent.js @@ -0,0 +1,89 @@ +'use strict'; + +const nconf = require('nconf'); +const user = require('../user'); +const topics = require('../topics'); +const meta = require('../meta'); +const helpers = require('./helpers'); +const pagination = require('../pagination'); +const privileges = require('../privileges'); +const recentController = module.exports; +const relative_path = nconf.get('relative_path'); +recentController.get = async function (req, res, next) { + const data = await recentController.getData(req, 'recent', 'recent'); + if (!data) { + return next(); + } + res.render('recent', data); +}; +recentController.getData = async function (req, url, sort) { + const page = parseInt(req.query.page, 10) || 1; + let term = helpers.terms[req.query.term]; + const { + cid, + tag + } = req.query; + const filter = req.query.filter || ''; + if (!term && req.query.term) { + return null; + } + term = term || 'alltime'; + const [settings, categoryData, tagData, rssToken, canPost, isPrivileged] = await Promise.all([user.getSettings(req.uid), helpers.getSelectedCategory(cid), helpers.getSelectedTag(tag), user.auth.getFeedToken(req.uid), privileges.categories.canPostTopic(req.uid), user.isPrivileged(req.uid)]); + const start = Math.max(0, (page - 1) * settings.topicsPerPage); + const stop = start + settings.topicsPerPage - 1; + const data = await topics.getSortedTopics({ + cids: cid, + tags: tag, + uid: req.uid, + start: start, + stop: stop, + filter: filter, + term: term, + sort: sort, + floatPinned: req.query.pinned, + query: req.query + }); + const isDisplayedAsHome = !(req.originalUrl.startsWith(`${relative_path}/api/${url}`) || req.originalUrl.startsWith(`${relative_path}/${url}`)); + const baseUrl = isDisplayedAsHome ? '' : url; + if (isDisplayedAsHome) { + data.title = meta.config.homePageTitle || '[[pages:home]]'; + } else { + data.title = `[[pages:${url}]]`; + data.breadcrumbs = helpers.buildBreadcrumbs([{ + text: `[[${url}:title]]` + }]); + } + const query = { + ...req.query + }; + delete query.page; + data.canPost = canPost; + data.showSelect = isPrivileged; + data.showTopicTools = isPrivileged; + data.allCategoriesUrl = baseUrl + helpers.buildQueryString(query, 'cid', ''); + data.selectedCategory = categoryData.selectedCategory; + data.selectedCids = categoryData.selectedCids; + data.selectedTag = tagData.selectedTag; + data.selectedTags = tagData.selectedTags; + data['feeds:disableRSS'] = meta.config['feeds:disableRSS'] || 0; + if (!meta.config['feeds:disableRSS']) { + data.rssFeedUrl = `${relative_path}/${url}.rss`; + if (req.loggedIn) { + data.rssFeedUrl += `?uid=${req.uid}&token=${rssToken}`; + } + } + data.filters = helpers.buildFilters(baseUrl, filter, query); + data.selectedFilter = data.filters.find(filter => filter && filter.selected); + data.terms = helpers.buildTerms(baseUrl, term, query); + data.selectedTerm = data.terms.find(term => term && term.selected); + const pageCount = Math.max(1, Math.ceil(data.topicCount / settings.topicsPerPage)); + data.pagination = pagination.create(page, pageCount, req.query); + helpers.addLinkTags({ + url: url, + res: req.res, + tags: data.pagination.rel, + page: page + }); + return data; +}; +require('../promisify')(recentController, ['get']); \ No newline at end of file diff --git a/lib/controllers/search.js b/lib/controllers/search.js new file mode 100644 index 0000000000..b8ef108a48 --- /dev/null +++ b/lib/controllers/search.js @@ -0,0 +1,186 @@ +'use strict'; + +const validator = require('validator'); +const _ = require('lodash'); +const db = require('../database'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const search = require('../search'); +const categories = require('../categories'); +const user = require('../user'); +const topics = require('../topics'); +const pagination = require('../pagination'); +const privileges = require('../privileges'); +const translator = require('../translator'); +const utils = require('../utils'); +const helpers = require('./helpers'); +const searchController = module.exports; +searchController.search = async function (req, res, next) { + if (!plugins.hooks.hasListeners('filter:search.query')) { + return next(); + } + const page = Math.max(1, parseInt(req.query.page, 10)) || 1; + const searchOnly = parseInt(req.query.searchOnly, 10) === 1; + const userPrivileges = await utils.promiseParallel({ + 'search:users': privileges.global.can('search:users', req.uid), + 'search:content': privileges.global.can('search:content', req.uid), + 'search:tags': privileges.global.can('search:tags', req.uid) + }); + req.query.in = req.query.in || meta.config.searchDefaultIn || 'titlesposts'; + let allowed = req.query.in === 'users' && userPrivileges['search:users'] || req.query.in === 'tags' && userPrivileges['search:tags'] || req.query.in === 'categories' || ['titles', 'titlesposts', 'posts', 'bookmarks'].includes(req.query.in) && userPrivileges['search:content']; + ({ + allowed + } = await plugins.hooks.fire('filter:search.isAllowed', { + uid: req.uid, + query: req.query, + allowed + })); + if (!allowed) { + return helpers.notAllowed(req, res); + } + if (req.query.categories && !Array.isArray(req.query.categories)) { + req.query.categories = [req.query.categories]; + } + if (req.query.hasTags && !Array.isArray(req.query.hasTags)) { + req.query.hasTags = [req.query.hasTags]; + } + const data = { + query: req.query.term, + searchIn: req.query.in, + matchWords: req.query.matchWords || 'all', + postedBy: req.query.by, + categories: req.query.categories, + searchChildren: req.query.searchChildren, + hasTags: req.query.hasTags, + replies: validator.escape(String(req.query.replies || '')), + repliesFilter: validator.escape(String(req.query.repliesFilter || '')), + timeRange: validator.escape(String(req.query.timeRange || '')), + timeFilter: validator.escape(String(req.query.timeFilter || '')), + sortBy: validator.escape(String(req.query.sortBy || '')) || meta.config.searchDefaultSortBy || '', + sortDirection: validator.escape(String(req.query.sortDirection || '')), + page: page, + itemsPerPage: req.query.itemsPerPage, + uid: req.uid, + qs: req.query + }; + const [searchData] = await Promise.all([search.search(data), recordSearch(data)]); + searchData.pagination = pagination.create(page, searchData.pageCount, req.query); + searchData.multiplePages = searchData.pageCount > 1; + searchData.search_query = validator.escape(String(req.query.term || '')); + searchData.term = req.query.term; + if (searchOnly) { + return res.json(searchData); + } + searchData.breadcrumbs = helpers.buildBreadcrumbs([{ + text: '[[global:search]]' + }]); + searchData.showAsPosts = !req.query.showAs || req.query.showAs === 'posts'; + searchData.showAsTopics = req.query.showAs === 'topics'; + searchData.title = '[[global:header.search]]'; + if (Array.isArray(data.categories)) { + searchData.selectedCids = data.categories.map(cid => validator.escape(String(cid))); + if (!searchData.selectedCids.includes('all') && searchData.selectedCids.length) { + searchData.selectedCategory = { + cid: 0 + }; + } + } + searchData.filters = { + replies: { + active: !!data.repliesFilter, + label: `[[search:replies-${data.repliesFilter}-count, ${data.replies}]]` + }, + time: { + active: !!(data.timeFilter && data.timeRange), + label: `[[search:time-${data.timeFilter}-than-${data.timeRange}]]` + }, + sort: { + active: !!(data.sortBy && data.sortBy !== 'relevance'), + label: `[[search:sort-by-${data.sortBy}-${data.sortDirection}]]` + }, + users: { + active: !!data.postedBy, + label: translator.compile('search:posted-by-usernames', (Array.isArray(data.postedBy) ? data.postedBy : []).map(u => validator.escape(String(u))).join(', ')) + }, + tags: { + active: !!(Array.isArray(data.hasTags) && data.hasTags.length), + label: translator.compile('search:tags-x', (Array.isArray(data.hasTags) ? data.hasTags : []).map(u => validator.escape(String(u))).join(', ')) + }, + categories: { + active: !!(Array.isArray(data.categories) && data.categories.length && (data.categories.length > 1 || data.categories[0] !== 'all')), + label: await buildSelectedCategoryLabel(searchData.selectedCids) + } + }; + searchData.userFilterSelected = await getSelectedUsers(data.postedBy); + searchData.tagFilterSelected = getSelectedTags(data.hasTags); + searchData.searchDefaultSortBy = meta.config.searchDefaultSortBy || ''; + searchData.searchDefaultIn = meta.config.searchDefaultIn || 'titlesposts'; + searchData.privileges = userPrivileges; + res.render('search', searchData); +}; +const searches = {}; +async function recordSearch(data) { + const { + query, + searchIn + } = data; + if (!query || parseInt(data.qs.composer, 10) === 1) { + return; + } + const cleanedQuery = String(query).trim().toLowerCase().slice(0, 255); + if (['titles', 'titlesposts', 'posts'].includes(searchIn) && cleanedQuery.length > 2) { + searches[data.uid] = searches[data.uid] || { + timeoutId: 0, + queries: [] + }; + searches[data.uid].queries.push(cleanedQuery); + if (searches[data.uid].timeoutId) { + clearTimeout(searches[data.uid].timeoutId); + } + searches[data.uid].timeoutId = setTimeout(async () => { + if (searches[data.uid] && searches[data.uid].queries) { + const copy = searches[data.uid].queries.slice(); + const filtered = searches[data.uid].queries.filter(q => !copy.find(query => query.startsWith(q) && query.length > q.length)); + delete searches[data.uid]; + const dayTimestamp = new Date(); + dayTimestamp.setHours(0, 0, 0, 0); + await Promise.all(_.uniq(filtered).map(async query => { + await db.sortedSetIncrBy('searches:all', 1, query); + await db.sortedSetIncrBy(`searches:${dayTimestamp.getTime()}`, 1, query); + })); + } + }, 5000); + } +} +async function getSelectedUsers(postedBy) { + if (!Array.isArray(postedBy) || !postedBy.length) { + return []; + } + const uids = await user.getUidsByUsernames(postedBy); + return await user.getUsersFields(uids, ['username', 'userslug', 'picture']); +} +function getSelectedTags(hasTags) { + if (!Array.isArray(hasTags) || !hasTags.length) { + return []; + } + const tags = hasTags.map(tag => ({ + value: tag + })); + return topics.getTagData(tags); +} +async function buildSelectedCategoryLabel(selectedCids) { + let label = '[[search:categories]]'; + if (Array.isArray(selectedCids)) { + if (selectedCids.length > 1) { + label = `[[search:categories-x, ${selectedCids.length}]]`; + } else if (selectedCids.length === 1 && selectedCids[0] === 'watched') { + label = `[[search:categories-watched-categories]]`; + } else if (selectedCids.length === 1 && parseInt(selectedCids[0], 10)) { + const categoryData = await categories.getCategoryData(selectedCids[0]); + if (categoryData && categoryData.name) { + label = `[[search:categories-x, ${categoryData.name}]]`; + } + } + } + return label; +} \ No newline at end of file diff --git a/lib/controllers/sitemap.js b/lib/controllers/sitemap.js new file mode 100644 index 0000000000..cacacbfc60 --- /dev/null +++ b/lib/controllers/sitemap.js @@ -0,0 +1,34 @@ +'use strict'; + +const sitemap = require('../sitemap'); +const meta = require('../meta'); +const sitemapController = module.exports; +sitemapController.render = async function (req, res, next) { + if (meta.config['feeds:disableSitemap']) { + return setImmediate(next); + } + const tplData = await sitemap.render(); + const xml = await req.app.renderAsync('sitemap', tplData); + res.header('Content-Type', 'application/xml'); + res.send(xml); +}; +sitemapController.getPages = function (req, res, next) { + sendSitemap(sitemap.getPages, res, next); +}; +sitemapController.getCategories = function (req, res, next) { + sendSitemap(sitemap.getCategories, res, next); +}; +sitemapController.getTopicPage = function (req, res, next) { + sendSitemap(async () => await sitemap.getTopicPage(parseInt(req.params[0], 10)), res, next); +}; +async function sendSitemap(method, res, callback) { + if (meta.config['feeds:disableSitemap']) { + return setImmediate(callback); + } + const xml = await method(); + if (!xml) { + return callback(); + } + res.header('Content-Type', 'application/xml'); + res.send(xml); +} \ No newline at end of file diff --git a/lib/controllers/tags.js b/lib/controllers/tags.js new file mode 100644 index 0000000000..0ca966c933 --- /dev/null +++ b/lib/controllers/tags.js @@ -0,0 +1,78 @@ +'use strict'; + +const validator = require('validator'); +const nconf = require('nconf'); +const meta = require('../meta'); +const user = require('../user'); +const categories = require('../categories'); +const topics = require('../topics'); +const privileges = require('../privileges'); +const pagination = require('../pagination'); +const utils = require('../utils'); +const helpers = require('./helpers'); +const tagsController = module.exports; +tagsController.getTag = async function (req, res) { + const tag = validator.escape(utils.cleanUpTag(req.params.tag, meta.config.maximumTagLength)); + const page = parseInt(req.query.page, 10) || 1; + const cid = Array.isArray(req.query.cid) || !req.query.cid ? req.query.cid : [req.query.cid]; + const templateData = { + topics: [], + tag: tag, + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[tags:tags]]', + url: '/tags' + }, { + text: tag + }]), + title: `[[pages:tag, ${tag}]]` + }; + const [settings, cids, categoryData, canPost, isPrivileged, rssToken, isFollowing] = await Promise.all([user.getSettings(req.uid), cid || categories.getCidsByPrivilege('categories:cid', req.uid, 'topics:read'), helpers.getSelectedCategory(cid), privileges.categories.canPostTopic(req.uid), user.isPrivileged(req.uid), user.auth.getFeedToken(req.uid), topics.isFollowingTag(req.params.tag, req.uid)]); + const start = Math.max(0, (page - 1) * settings.topicsPerPage); + const stop = start + settings.topicsPerPage - 1; + const [topicCount, tids] = await Promise.all([topics.getTagTopicCount(tag, cids), topics.getTagTidsByCids(tag, cids, start, stop)]); + templateData.topics = await topics.getTopics(tids, req.uid); + templateData.canPost = canPost; + templateData.showSelect = isPrivileged; + templateData.showTopicTools = isPrivileged; + templateData.isFollowing = isFollowing; + templateData.allCategoriesUrl = `tags/${tag}${helpers.buildQueryString(req.query, 'cid', '')}`; + templateData.selectedCategory = categoryData.selectedCategory; + templateData.selectedCids = categoryData.selectedCids; + topics.calculateTopicIndices(templateData.topics, start); + res.locals.metaTags = [{ + name: 'title', + content: tag + }, { + property: 'og:title', + content: tag + }]; + const pageCount = Math.max(1, Math.ceil(topicCount / settings.topicsPerPage)); + templateData.pagination = pagination.create(page, pageCount, req.query); + helpers.addLinkTags({ + url: `tags/${tag}`, + res: req.res, + tags: templateData.pagination.rel, + page: page + }); + templateData['feeds:disableRSS'] = meta.config['feeds:disableRSS']; + if (!meta.config['feeds:disableRSS']) { + templateData.rssFeedUrl = `${nconf.get('relative_path')}/tags/${tag}.rss`; + if (req.loggedIn) { + templateData.rssFeedUrl += `?uid=${req.uid}&token=${rssToken}`; + } + } + res.render('tag', templateData); +}; +tagsController.getTags = async function (req, res) { + const cids = await categories.getCidsByPrivilege('categories:cid', req.uid, 'topics:read'); + const [canSearch, tags] = await Promise.all([privileges.global.can('search:tags', req.uid), topics.getCategoryTagsData(cids, 0, 99)]); + res.render('tags', { + tags: tags.filter(Boolean), + displayTagSearch: canSearch, + nextStart: 100, + breadcrumbs: helpers.buildBreadcrumbs([{ + text: '[[tags:tags]]' + }]), + title: '[[pages:tags]]' + }); +}; \ No newline at end of file diff --git a/lib/controllers/top.js b/lib/controllers/top.js new file mode 100644 index 0000000000..e48d61711d --- /dev/null +++ b/lib/controllers/top.js @@ -0,0 +1,25 @@ +'use strict'; + +const nconf = require('nconf'); +const validator = require('validator'); +const helpers = require('./helpers'); +const recentController = require('./recent'); +const topController = module.exports; +topController.get = async function (req, res, next) { + const data = await recentController.getData(req, 'top', 'votes'); + if (!data) { + return next(); + } + const term = helpers.terms[req.query.term] || 'alltime'; + if (req.originalUrl.startsWith(`${nconf.get('relative_path')}/api/top`) || req.originalUrl.startsWith(`${nconf.get('relative_path')}/top`)) { + data.title = `[[pages:top-${term}]]`; + } + if (!data['feeds:disableRSS'] && data.rssFeedUrl) { + const feedQs = data.rssFeedUrl.split('?')[1]; + data.rssFeedUrl = `${nconf.get('relative_path')}/top/${validator.escape(String(req.query.term || 'alltime'))}.rss`; + if (req.loggedIn) { + data.rssFeedUrl += `?${feedQs}`; + } + } + res.render('top', data); +}; \ No newline at end of file diff --git a/lib/controllers/topics.js b/lib/controllers/topics.js new file mode 100644 index 0000000000..870c86a8da --- /dev/null +++ b/lib/controllers/topics.js @@ -0,0 +1,328 @@ +'use strict'; + +const nconf = require('nconf'); +const qs = require('querystring'); +const validator = require('validator'); +const user = require('../user'); +const meta = require('../meta'); +const topics = require('../topics'); +const categories = require('../categories'); +const posts = require('../posts'); +const privileges = require('../privileges'); +const helpers = require('./helpers'); +const pagination = require('../pagination'); +const utils = require('../utils'); +const analytics = require('../analytics'); +const topicsController = module.exports; +const url = nconf.get('url'); +const relative_path = nconf.get('relative_path'); +const upload_url = nconf.get('upload_url'); +const validSorts = ['oldest_to_newest', 'newest_to_oldest', 'most_votes']; +topicsController.get = async function getTopic(req, res, next) { + const tid = req.params.topic_id; + if (req.params.post_index && !utils.isNumber(req.params.post_index) && req.params.post_index !== 'unread' || !utils.isNumber(tid)) { + return next(); + } + let postIndex = parseInt(req.params.post_index, 10) || 1; + const topicData = await topics.getTopicData(tid); + if (!topicData) { + return next(); + } + const [userPrivileges, settings, rssToken] = await Promise.all([privileges.topics.get(tid, req.uid), user.getSettings(req.uid), user.auth.getFeedToken(req.uid)]); + let currentPage = parseInt(req.query.page, 10) || 1; + const pageCount = Math.max(1, Math.ceil((topicData && topicData.postcount) / settings.postsPerPage)); + const invalidPagination = settings.usePagination && (currentPage < 1 || currentPage > pageCount); + if (userPrivileges.disabled || invalidPagination || topicData.scheduled && !userPrivileges.view_scheduled) { + return next(); + } + if (!userPrivileges['topics:read'] || !topicData.scheduled && topicData.deleted && !userPrivileges.view_deleted) { + return helpers.notAllowed(req, res); + } + if (req.params.post_index === 'unread') { + postIndex = await topics.getUserBookmark(tid, req.uid); + } + if (!res.locals.isAPI && (!req.params.slug || topicData.slug !== `${tid}/${req.params.slug}`) && topicData.slug && topicData.slug !== `${tid}/`) { + return helpers.redirect(res, `/topic/${topicData.slug}${postIndex ? `/${postIndex}` : ''}${generateQueryString(req.query)}`, true); + } + if (utils.isNumber(postIndex) && topicData.postcount > 0 && (postIndex < 1 || postIndex > topicData.postcount)) { + return helpers.redirect(res, `/topic/${tid}/${req.params.slug}${postIndex > topicData.postcount ? `/${topicData.postcount}` : ''}${generateQueryString(req.query)}`); + } + postIndex = Math.max(1, postIndex); + const sort = validSorts.includes(req.query.sort) ? req.query.sort : settings.topicPostSort; + const set = sort === 'most_votes' ? `tid:${tid}:posts:votes` : `tid:${tid}:posts`; + const reverse = sort === 'newest_to_oldest' || sort === 'most_votes'; + if (!req.query.page) { + currentPage = calculatePageFromIndex(postIndex, settings); + } + if (settings.usePagination && req.query.page) { + const top = (currentPage - 1) * settings.postsPerPage + 1; + const bottom = top + settings.postsPerPage; + if (!req.params.post_index || postIndex < top || postIndex > bottom) { + postIndex = top; + } + } + const { + start, + stop + } = calculateStartStop(currentPage, postIndex, settings); + await topics.getTopicWithPosts(topicData, set, req.uid, start, stop, reverse); + topics.modifyPostsByPrivilege(topicData, userPrivileges); + topicData.tagWhitelist = categories.filterTagWhitelist(topicData.tagWhitelist, userPrivileges.isAdminOrMod); + topicData.privileges = userPrivileges; + topicData.topicStaleDays = meta.config.topicStaleDays; + topicData['reputation:disabled'] = meta.config['reputation:disabled']; + topicData['downvote:disabled'] = meta.config['downvote:disabled']; + topicData.upvoteVisibility = meta.config.upvoteVisibility; + topicData.downvoteVisibility = meta.config.downvoteVisibility; + topicData['feeds:disableRSS'] = meta.config['feeds:disableRSS'] || 0; + topicData['signatures:hideDuplicates'] = meta.config['signatures:hideDuplicates']; + topicData.bookmarkThreshold = meta.config.bookmarkThreshold; + topicData.necroThreshold = meta.config.necroThreshold; + topicData.postEditDuration = meta.config.postEditDuration; + topicData.postDeleteDuration = meta.config.postDeleteDuration; + topicData.scrollToMyPost = settings.scrollToMyPost; + topicData.updateUrlWithPostIndex = settings.updateUrlWithPostIndex; + topicData.allowMultipleBadges = meta.config.allowMultipleBadges === 1; + topicData.privateUploads = meta.config.privateUploads === 1; + topicData.showPostPreviewsOnHover = meta.config.showPostPreviewsOnHover === 1; + topicData.sortOptionLabel = `[[topic:${validator.escape(String(sort)).replace(/_/g, '-')}]]`; + if (!meta.config['feeds:disableRSS']) { + topicData.rssFeedUrl = `${relative_path}/topic/${topicData.tid}.rss`; + if (req.loggedIn) { + topicData.rssFeedUrl += `?uid=${req.uid}&token=${rssToken}`; + } + } + topicData.postIndex = postIndex; + const [author] = await Promise.all([user.getUserFields(topicData.uid, ['username', 'userslug']), buildBreadcrumbs(topicData), addOldCategory(topicData, userPrivileges), addTags(topicData, req, res, currentPage), incrementViewCount(req, tid), markAsRead(req, tid), analytics.increment([`pageviews:byCid:${topicData.category.cid}`])]); + topicData.author = author; + topicData.pagination = pagination.create(currentPage, pageCount, req.query); + topicData.pagination.rel.forEach(rel => { + rel.href = `${url}/topic/${topicData.slug}${rel.href}`; + res.locals.linkTags.push(rel); + }); + res.render('topic', topicData); +}; +function generateQueryString(query) { + const qString = qs.stringify(query); + return qString.length ? `?${qString}` : ''; +} +function calculatePageFromIndex(postIndex, settings) { + return 1 + Math.floor((postIndex - 1) / settings.postsPerPage); +} +function calculateStartStop(page, postIndex, settings) { + let startSkip = 0; + if (!settings.usePagination) { + if (postIndex > 1) { + page = 1; + } + startSkip = Math.max(0, postIndex - Math.ceil(settings.postsPerPage / 2)); + } + const start = (page - 1) * settings.postsPerPage + startSkip; + const stop = start + settings.postsPerPage - 1; + return { + start: Math.max(0, start), + stop: Math.max(0, stop) + }; +} +async function incrementViewCount(req, tid) { + const allow = req.uid > 0 || meta.config.guestsIncrementTopicViews && req.uid === 0; + if (allow) { + req.session.tids_viewed = req.session.tids_viewed || {}; + const now = Date.now(); + const interval = meta.config.incrementTopicViewsInterval * 60000; + if (!req.session.tids_viewed[tid] || req.session.tids_viewed[tid] < now - interval) { + await topics.increaseViewCount(tid); + req.session.tids_viewed[tid] = now; + } + } +} +async function markAsRead(req, tid) { + if (req.loggedIn) { + const markedRead = await topics.markAsRead([tid], req.uid); + const promises = [topics.markTopicNotificationsRead([tid], req.uid)]; + if (markedRead) { + promises.push(topics.pushUnreadCount(req.uid)); + } + await Promise.all(promises); + } +} +async function buildBreadcrumbs(topicData) { + const breadcrumbs = [{ + text: topicData.category.name, + url: `${url}/category/${topicData.category.slug}`, + cid: topicData.category.cid + }, { + text: topicData.title + }]; + const parentCrumbs = await helpers.buildCategoryBreadcrumbs(topicData.category.parentCid); + topicData.breadcrumbs = parentCrumbs.concat(breadcrumbs); +} +async function addOldCategory(topicData, userPrivileges) { + if (userPrivileges.isAdminOrMod && topicData.oldCid) { + topicData.oldCategory = await categories.getCategoryFields(topicData.oldCid, ['cid', 'name', 'icon', 'bgColor', 'color', 'slug']); + } +} +async function addTags(topicData, req, res, currentPage) { + const postIndex = parseInt(req.params.post_index, 10) || 0; + const postAtIndex = topicData.posts.find(p => parseInt(p.index, 10) === parseInt(Math.max(0, postIndex - 1), 10)); + let description = ''; + if (postAtIndex && postAtIndex.content) { + description = utils.stripHTMLTags(utils.decodeHTMLEntities(postAtIndex.content)).trim(); + } + if (description.length > 160) { + description = `${description.slice(0, 157)}...`; + } + description = description.replace(/\n/g, ' ').trim(); + let mainPost = topicData.posts.find(p => parseInt(p.index, 10) === 0); + if (!mainPost) { + mainPost = await posts.getPostData(topicData.mainPid); + } + res.locals.metaTags = [{ + name: 'title', + content: topicData.titleRaw + }, { + property: 'og:title', + content: topicData.titleRaw + }, { + property: 'og:type', + content: 'article' + }, { + property: 'article:published_time', + content: utils.toISOString(topicData.timestamp) + }, { + property: 'article:modified_time', + content: utils.toISOString(Math.max(topicData.lastposttime, mainPost && mainPost.edited)) + }, { + property: 'article:section', + content: topicData.category ? topicData.category.name : '' + }]; + if (description && description.length) { + res.locals.metaTags.push({ + name: 'description', + content: description + }, { + property: 'og:description', + content: description + }); + } + await addOGImageTags(res, topicData, postAtIndex); + const page = currentPage > 1 ? `?page=${currentPage}` : ''; + res.locals.linkTags = [{ + rel: 'canonical', + href: `${url}/topic/${topicData.slug}${page}`, + noEscape: true + }]; + if (!topicData['feeds:disableRSS']) { + res.locals.linkTags.push({ + rel: 'alternate', + type: 'application/rss+xml', + href: topicData.rssFeedUrl + }); + } + if (topicData.category) { + res.locals.linkTags.push({ + rel: 'up', + href: `${url}/category/${topicData.category.slug}` + }); + } + if (postAtIndex) { + res.locals.linkTags.push({ + rel: 'author', + href: `${url}/user/${postAtIndex.user.userslug}` + }); + } +} +async function addOGImageTags(res, topicData, postAtIndex) { + const uploads = postAtIndex ? await posts.uploads.listWithSizes(postAtIndex.pid) : []; + const images = uploads.map(upload => { + upload.name = `${url + upload_url}/${upload.name}`; + return upload; + }); + if (topicData.thumbs) { + const path = require('path'); + const thumbs = topicData.thumbs.filter(t => t && images.every(img => path.normalize(img.name) !== path.normalize(url + t.url))); + images.push(...thumbs.map(thumbObj => ({ + name: url + thumbObj.url + }))); + } + if (topicData.category.backgroundImage && (!postAtIndex || !postAtIndex.index)) { + images.push(topicData.category.backgroundImage); + } + if (postAtIndex && postAtIndex.user && postAtIndex.user.picture) { + images.push(postAtIndex.user.picture); + } + images.forEach(path => addOGImageTag(res, path)); +} +function addOGImageTag(res, image) { + let imageUrl; + if (typeof image === 'string' && !image.startsWith('http')) { + imageUrl = url + image.replace(new RegExp(`^${relative_path}`), ''); + } else if (typeof image === 'object') { + imageUrl = image.name; + } else { + imageUrl = image; + } + res.locals.metaTags.push({ + property: 'og:image', + content: imageUrl, + noEscape: true + }, { + property: 'og:image:url', + content: imageUrl, + noEscape: true + }); + if (typeof image === 'object' && image.width && image.height) { + res.locals.metaTags.push({ + property: 'og:image:width', + content: String(image.width) + }, { + property: 'og:image:height', + content: String(image.height) + }); + } +} +topicsController.teaser = async function (req, res, next) { + const tid = req.params.topic_id; + if (!utils.isNumber(tid)) { + return next(); + } + const canRead = await privileges.topics.can('topics:read', tid, req.uid); + if (!canRead) { + return res.status(403).json('[[error:no-privileges]]'); + } + const pid = await topics.getLatestUndeletedPid(tid); + if (!pid) { + return res.status(404).json('not-found'); + } + const postData = await posts.getPostSummaryByPids([pid], req.uid, { + stripTags: false + }); + if (!postData.length) { + return res.status(404).json('not-found'); + } + res.json(postData[0]); +}; +topicsController.pagination = async function (req, res, next) { + const tid = req.params.topic_id; + const currentPage = parseInt(req.query.page, 10) || 1; + if (!utils.isNumber(tid)) { + return next(); + } + const topic = await topics.getTopicData(tid); + if (!topic) { + return next(); + } + const [userPrivileges, settings] = await Promise.all([privileges.topics.get(tid, req.uid), user.getSettings(req.uid)]); + if (!userPrivileges.read || !privileges.topics.canViewDeletedScheduled(topic, userPrivileges)) { + return helpers.notAllowed(req, res); + } + const postCount = topic.postcount; + const pageCount = Math.max(1, Math.ceil(postCount / settings.postsPerPage)); + const paginationData = pagination.create(currentPage, pageCount); + paginationData.rel.forEach(rel => { + rel.href = `${url}/topic/${topic.slug}${rel.href}`; + }); + res.json({ + pagination: paginationData + }); +}; \ No newline at end of file diff --git a/lib/controllers/unread.js b/lib/controllers/unread.js new file mode 100644 index 0000000000..43bcc49fde --- /dev/null +++ b/lib/controllers/unread.js @@ -0,0 +1,77 @@ +'use strict'; + +const nconf = require('nconf'); +const querystring = require('querystring'); +const meta = require('../meta'); +const pagination = require('../pagination'); +const user = require('../user'); +const topics = require('../topics'); +const helpers = require('./helpers'); +const privileges = require('../privileges'); +const unreadController = module.exports; +const relative_path = nconf.get('relative_path'); +unreadController.get = async function (req, res) { + const { + cid, + tag + } = req.query; + const filter = req.query.filter || ''; + const [categoryData, tagData, userSettings, canPost, isPrivileged] = await Promise.all([helpers.getSelectedCategory(cid), helpers.getSelectedTag(tag), user.getSettings(req.uid), privileges.categories.canPostTopic(req.uid), user.isPrivileged(req.uid)]); + const page = parseInt(req.query.page, 10) || 1; + const start = Math.max(0, (page - 1) * userSettings.topicsPerPage); + const stop = start + userSettings.topicsPerPage - 1; + const data = await topics.getUnreadTopics({ + cid: cid, + tag: tag, + uid: req.uid, + start: start, + stop: stop, + filter: filter, + query: req.query + }); + const isDisplayedAsHome = !(req.originalUrl.startsWith(`${relative_path}/api/unread`) || req.originalUrl.startsWith(`${relative_path}/unread`)); + const baseUrl = isDisplayedAsHome ? '' : 'unread'; + if (isDisplayedAsHome) { + data.title = meta.config.homePageTitle || '[[pages:home]]'; + } else { + data.title = '[[pages:unread]]'; + data.breadcrumbs = helpers.buildBreadcrumbs([{ + text: '[[unread:title]]' + }]); + } + data.pageCount = Math.max(1, Math.ceil(data.topicCount / userSettings.topicsPerPage)); + data.pagination = pagination.create(page, data.pageCount, req.query); + helpers.addLinkTags({ + url: 'unread', + res: req.res, + tags: data.pagination.rel, + page: page + }); + if (userSettings.usePagination && (page < 1 || page > data.pageCount)) { + req.query.page = Math.max(1, Math.min(data.pageCount, page)); + return helpers.redirect(res, `/unread?${querystring.stringify(req.query)}`); + } + data.canPost = canPost; + data.showSelect = true; + data.showTopicTools = isPrivileged; + data.allCategoriesUrl = `${baseUrl}${helpers.buildQueryString(req.query, 'cid', '')}`; + data.selectedCategory = categoryData.selectedCategory; + data.selectedCids = categoryData.selectedCids; + data.selectCategoryLabel = '[[unread:mark-as-read]]'; + data.selectCategoryIcon = 'fa-inbox'; + data.showCategorySelectLabel = true; + data.selectedTag = tagData.selectedTag; + data.selectedTags = tagData.selectedTags; + data.filters = helpers.buildFilters(baseUrl, filter, req.query); + data.selectedFilter = data.filters.find(filter => filter && filter.selected); + res.render('unread', data); +}; +unreadController.unreadTotal = async function (req, res, next) { + const filter = req.query.filter || ''; + try { + const unreadCount = await topics.getTotalUnread(req.uid, filter); + res.json(unreadCount); + } catch (err) { + next(err); + } +}; \ No newline at end of file diff --git a/lib/controllers/uploads.js b/lib/controllers/uploads.js new file mode 100644 index 0000000000..2289210a69 --- /dev/null +++ b/lib/controllers/uploads.js @@ -0,0 +1,171 @@ +'use strict'; + +const path = require('path'); +const nconf = require('nconf'); +const validator = require('validator'); +const user = require('../user'); +const meta = require('../meta'); +const file = require('../file'); +const plugins = require('../plugins'); +const image = require('../image'); +const privileges = require('../privileges'); +const helpers = require('./helpers'); +const uploadsController = module.exports; +uploadsController.upload = async function (req, res, filesIterator) { + let files; + try { + files = req.files.files; + } catch (e) { + return helpers.formatApiResponse(400, res); + } + if (!Array.isArray(files)) { + return helpers.formatApiResponse(500, res, new Error('[[error:invalid-file]]')); + } + if (Array.isArray(files[0])) { + files = files[0]; + } + try { + const images = []; + for (const fileObj of files) { + images.push(await filesIterator(fileObj)); + } + helpers.formatApiResponse(200, res, { + images + }); + return images; + } catch (err) { + return helpers.formatApiResponse(500, res, err); + } finally { + deleteTempFiles(files); + } +}; +uploadsController.uploadPost = async function (req, res) { + await uploadsController.upload(req, res, async uploadedFile => { + const isImage = uploadedFile.type.match(/image./); + if (isImage) { + return await uploadAsImage(req, uploadedFile); + } + return await uploadAsFile(req, uploadedFile); + }); +}; +async function uploadAsImage(req, uploadedFile) { + const canUpload = await privileges.global.can('upload:post:image', req.uid); + if (!canUpload) { + throw new Error('[[error:no-privileges]]'); + } + await image.checkDimensions(uploadedFile.path); + await image.stripEXIF(uploadedFile.path); + if (plugins.hooks.hasListeners('filter:uploadImage')) { + return await plugins.hooks.fire('filter:uploadImage', { + image: uploadedFile, + uid: req.uid, + folder: 'files' + }); + } + await image.isFileTypeAllowed(uploadedFile.path); + let fileObj = await uploadsController.uploadFile(req.uid, uploadedFile); + const isSVG = uploadedFile.type === 'image/svg+xml'; + if (isSVG || meta.config.resizeImageWidth === 0 || meta.config.resizeImageWidthThreshold === 0) { + return fileObj; + } + fileObj = await resizeImage(fileObj); + return { + url: fileObj.url + }; +} +async function uploadAsFile(req, uploadedFile) { + const canUpload = await privileges.global.can('upload:post:file', req.uid); + if (!canUpload) { + throw new Error('[[error:no-privileges]]'); + } + const fileObj = await uploadsController.uploadFile(req.uid, uploadedFile); + return { + url: fileObj.url, + name: fileObj.name + }; +} +async function resizeImage(fileObj) { + const imageData = await image.size(fileObj.path); + if (imageData.width < meta.config.resizeImageWidthThreshold || meta.config.resizeImageWidth > meta.config.resizeImageWidthThreshold) { + return fileObj; + } + await image.resizeImage({ + path: fileObj.path, + target: file.appendToFileName(fileObj.path, '-resized'), + width: meta.config.resizeImageWidth, + quality: meta.config.resizeImageQuality + }); + fileObj.url = file.appendToFileName(fileObj.url, '-resized'); + return fileObj; +} +uploadsController.uploadThumb = async function (req, res) { + if (!meta.config.allowTopicsThumbnail) { + deleteTempFiles(req.files.files); + return helpers.formatApiResponse(503, res, new Error('[[error:topic-thumbnails-are-disabled]]')); + } + return await uploadsController.upload(req, res, async uploadedFile => { + if (!uploadedFile.type.match(/image./)) { + throw new Error('[[error:invalid-file]]'); + } + await image.isFileTypeAllowed(uploadedFile.path); + const dimensions = await image.checkDimensions(uploadedFile.path); + if (dimensions.width > parseInt(meta.config.topicThumbSize, 10)) { + await image.resizeImage({ + path: uploadedFile.path, + width: meta.config.topicThumbSize + }); + } + if (plugins.hooks.hasListeners('filter:uploadImage')) { + return await plugins.hooks.fire('filter:uploadImage', { + image: uploadedFile, + uid: req.uid, + folder: 'files' + }); + } + return await uploadsController.uploadFile(req.uid, uploadedFile); + }); +}; +uploadsController.uploadFile = async function (uid, uploadedFile) { + if (plugins.hooks.hasListeners('filter:uploadFile')) { + return await plugins.hooks.fire('filter:uploadFile', { + file: uploadedFile, + uid: uid, + folder: 'files' + }); + } + if (!uploadedFile) { + throw new Error('[[error:invalid-file]]'); + } + const isAdmin = await user.isAdministrator(uid); + if (!isAdmin && uploadedFile.size > meta.config.maximumFileSize * 1024) { + throw new Error(`[[error:file-too-big, ${meta.config.maximumFileSize}]]`); + } + const allowed = file.allowedExtensions(); + const extension = path.extname(uploadedFile.name).toLowerCase(); + if (allowed.length > 0 && (!extension || extension === '.' || !allowed.includes(extension))) { + throw new Error(`[[error:invalid-file-type, ${allowed.join(', ')}]]`); + } + return await saveFileToLocal(uid, 'files', uploadedFile); +}; +async function saveFileToLocal(uid, folder, uploadedFile) { + const name = uploadedFile.name || 'upload'; + const extension = path.extname(name) || ''; + const filename = `${Date.now()}-${validator.escape(name.slice(0, -extension.length)).slice(0, 255)}${extension}`; + const upload = await file.saveFileToLocal(filename, folder, uploadedFile.path); + const storedFile = { + url: nconf.get('relative_path') + upload.url, + path: upload.path, + name: uploadedFile.name + }; + await user.associateUpload(uid, upload.url.replace(`${nconf.get('upload_url')}/`, '')); + const data = await plugins.hooks.fire('filter:uploadStored', { + uid: uid, + uploadedFile: uploadedFile, + storedFile: storedFile + }); + return data.storedFile; +} +function deleteTempFiles(files) { + files.forEach(fileObj => file.delete(fileObj.path)); +} +require('../promisify')(uploadsController, ['upload', 'uploadPost', 'uploadThumb']); \ No newline at end of file diff --git a/lib/controllers/user.js b/lib/controllers/user.js new file mode 100644 index 0000000000..5c62b21cce --- /dev/null +++ b/lib/controllers/user.js @@ -0,0 +1,67 @@ +'use strict'; + +const user = require('../user'); +const privileges = require('../privileges'); +const accountHelpers = require('./accounts/helpers'); +const userController = module.exports; +userController.getCurrentUser = async function (req, res) { + if (!req.loggedIn) { + return res.status(401).json('not-authorized'); + } + const userslug = await user.getUserField(req.uid, 'userslug'); + const userData = await accountHelpers.getUserDataByUserSlug(userslug, req.uid, req.query); + res.json(userData); +}; +userController.getUserByUID = async function (req, res, next) { + await byType('uid', req, res, next); +}; +userController.getUserByUsername = async function (req, res, next) { + await byType('username', req, res, next); +}; +userController.getUserByEmail = async function (req, res, next) { + await byType('email', req, res, next); +}; +async function byType(type, req, res, next) { + const userData = await userController.getUserDataByField(req.uid, type, req.params[type]); + if (!userData) { + return next(); + } + res.json(userData); +} +userController.getUserDataByField = async function (callerUid, field, fieldValue) { + let uid = null; + if (field === 'uid') { + uid = fieldValue; + } else if (field === 'username') { + uid = await user.getUidByUsername(fieldValue); + } else if (field === 'email') { + uid = await user.getUidByEmail(fieldValue); + if (uid) { + const isPrivileged = await user.isAdminOrGlobalMod(callerUid); + const settings = await user.getSettings(uid); + if (!isPrivileged && settings && !settings.showemail) { + uid = 0; + } + } + } + if (!uid) { + return null; + } + return await userController.getUserDataByUID(callerUid, uid); +}; +userController.getUserDataByUID = async function (callerUid, uid) { + if (!parseInt(uid, 10)) { + throw new Error('[[error:no-user]]'); + } + const canView = await privileges.global.can('view:users', callerUid); + if (!canView) { + throw new Error('[[error:no-privileges]]'); + } + let userData = await user.getUserData(uid); + if (!userData) { + throw new Error('[[error:no-user]]'); + } + userData = await user.hidePrivateData(userData, callerUid); + return userData; +}; +require('../promisify')(userController, ['getCurrentUser', 'getUserByUID', 'getUserByUsername', 'getUserByEmail']); \ No newline at end of file diff --git a/lib/controllers/users.js b/lib/controllers/users.js new file mode 100644 index 0000000000..05ef042090 --- /dev/null +++ b/lib/controllers/users.js @@ -0,0 +1,194 @@ +'use strict'; + +const user = require('../user'); +const meta = require('../meta'); +const db = require('../database'); +const pagination = require('../pagination'); +const privileges = require('../privileges'); +const helpers = require('./helpers'); +const api = require('../api'); +const utils = require('../utils'); +const usersController = module.exports; +usersController.index = async function (req, res, next) { + const section = req.query.section || 'joindate'; + const sectionToController = { + joindate: usersController.getUsersSortedByJoinDate, + online: usersController.getOnlineUsers, + 'sort-posts': usersController.getUsersSortedByPosts, + 'sort-reputation': usersController.getUsersSortedByReputation, + banned: usersController.getBannedUsers, + flagged: usersController.getFlaggedUsers + }; + if (req.query.query) { + await usersController.search(req, res, next); + } else if (sectionToController.hasOwnProperty(section) && sectionToController[section]) { + await sectionToController[section](req, res, next); + } else { + await usersController.getUsersSortedByJoinDate(req, res, next); + } +}; +usersController.search = async function (req, res) { + const searchData = await api.users.search(req, req.query); + const section = req.query.section || 'joindate'; + searchData.pagination = pagination.create(req.query.page, searchData.pageCount, req.query); + searchData[`section_${section}`] = true; + searchData.displayUserSearch = true; + await render(req, res, searchData); +}; +usersController.getOnlineUsers = async function (req, res) { + const [userData, guests] = await Promise.all([usersController.getUsers('users:online', req.uid, req.query), require('../socket.io/admin/rooms').getTotalGuestCount()]); + let hiddenCount = 0; + if (!userData.isAdminOrGlobalMod) { + userData.users = userData.users.filter(user => { + const showUser = user && (user.uid === req.uid || user.userStatus !== 'offline'); + if (!showUser) { + hiddenCount += 1; + } + return showUser; + }); + } + userData.anonymousUserCount = guests + hiddenCount; + await render(req, res, userData); +}; +usersController.getUsersSortedByPosts = async function (req, res) { + await usersController.renderUsersPage('users:postcount', req, res); +}; +usersController.getUsersSortedByReputation = async function (req, res, next) { + if (meta.config['reputation:disabled']) { + return next(); + } + await usersController.renderUsersPage('users:reputation', req, res); +}; +usersController.getUsersSortedByJoinDate = async function (req, res) { + await usersController.renderUsersPage('users:joindate', req, res); +}; +usersController.getBannedUsers = async function (req, res) { + await renderIfAdminOrGlobalMod('users:banned', req, res); +}; +usersController.getFlaggedUsers = async function (req, res) { + await renderIfAdminOrGlobalMod('users:flags', req, res); +}; +async function renderIfAdminOrGlobalMod(set, req, res) { + const isAdminOrGlobalMod = await user.isAdminOrGlobalMod(req.uid); + if (!isAdminOrGlobalMod) { + return helpers.notAllowed(req, res); + } + await usersController.renderUsersPage(set, req, res); +} +usersController.renderUsersPage = async function (set, req, res) { + const userData = await usersController.getUsers(set, req.uid, req.query); + await render(req, res, userData); +}; +usersController.getUsers = async function (set, uid, query) { + const setToData = { + 'users:postcount': { + title: '[[pages:users/sort-posts]]', + crumb: '[[users:top-posters]]' + }, + 'users:reputation': { + title: '[[pages:users/sort-reputation]]', + crumb: '[[users:most-reputation]]' + }, + 'users:joindate': { + title: '[[pages:users/latest]]', + crumb: '[[global:users]]' + }, + 'users:online': { + title: '[[pages:users/online]]', + crumb: '[[global:online]]' + }, + 'users:banned': { + title: '[[pages:users/banned]]', + crumb: '[[user:banned]]' + }, + 'users:flags': { + title: '[[pages:users/most-flags]]', + crumb: '[[users:most-flags]]' + } + }; + if (!setToData[set]) { + setToData[set] = { + title: '', + crumb: '' + }; + } + const breadcrumbs = [{ + text: setToData[set].crumb + }]; + if (set !== 'users:joindate') { + breadcrumbs.unshift({ + text: '[[global:users]]', + url: '/users' + }); + } + const page = parseInt(query.page, 10) || 1; + const resultsPerPage = meta.config.userSearchResultsPerPage; + const start = Math.max(0, page - 1) * resultsPerPage; + const stop = start + resultsPerPage - 1; + const [isAdmin, isGlobalMod, canSearch, usersData] = await Promise.all([user.isAdministrator(uid), user.isGlobalModerator(uid), privileges.global.can('search:users', uid), usersController.getUsersAndCount(set, uid, start, stop)]); + const pageCount = Math.ceil(usersData.count / resultsPerPage); + return { + users: usersData.users, + pagination: pagination.create(page, pageCount, query), + userCount: usersData.count, + title: setToData[set].title || '[[pages:users/latest]]', + breadcrumbs: helpers.buildBreadcrumbs(breadcrumbs), + isAdminOrGlobalMod: isAdmin || isGlobalMod, + isAdmin: isAdmin, + isGlobalMod: isGlobalMod, + displayUserSearch: canSearch, + [`section_${query.section || 'joindate'}`]: true + }; +}; +usersController.getUsersAndCount = async function (set, uid, start, stop) { + async function getCount() { + if (set === 'users:online') { + return await db.sortedSetCount('users:online', Date.now() - 86400000, '+inf'); + } else if (set === 'users:banned' || set === 'users:flags') { + return await db.sortedSetCard(set); + } + return await db.getObjectField('global', 'userCount'); + } + async function getUsers() { + if (set === 'users:online') { + const count = parseInt(stop, 10) === -1 ? stop : stop - start + 1; + const data = await db.getSortedSetRevRangeByScoreWithScores(set, start, count, '+inf', Date.now() - 86400000); + const uids = data.map(d => d.value); + const scores = data.map(d => d.score); + const [userStatus, userData] = await Promise.all([db.getObjectsFields(uids.map(uid => `user:${uid}`), ['status']), user.getUsers(uids, uid)]); + userData.forEach((user, i) => { + if (user) { + user.lastonline = scores[i]; + user.lastonlineISO = utils.toISOString(user.lastonline); + user.userStatus = userStatus[i].status || 'online'; + } + }); + return userData; + } + return await user.getUsersFromSet(set, uid, start, stop); + } + const [usersData, count] = await Promise.all([getUsers(), getCount()]); + return { + users: usersData.filter(user => user && parseInt(user.uid, 10)), + count: count + }; +}; +async function render(req, res, data) { + const { + registrationType + } = meta.config; + data.maximumInvites = meta.config.maximumInvites; + data.inviteOnly = registrationType === 'invite-only' || registrationType === 'admin-invite-only'; + data.adminInviteOnly = registrationType === 'admin-invite-only'; + data.invites = await user.getInvitesNumber(req.uid); + data.showInviteButton = false; + if (data.adminInviteOnly) { + data.showInviteButton = await privileges.users.isAdministrator(req.uid); + } else if (req.loggedIn) { + const canInvite = await privileges.users.hasInvitePrivilege(req.uid); + data.showInviteButton = canInvite && (!data.maximumInvites || data.invites < data.maximumInvites); + } + data['reputation:disabled'] = meta.config['reputation:disabled']; + res.append('X-Total-Count', data.userCount); + res.render('users', data); +} \ No newline at end of file diff --git a/lib/controllers/write/admin.js b/lib/controllers/write/admin.js new file mode 100644 index 0000000000..6174e102bf --- /dev/null +++ b/lib/controllers/write/admin.js @@ -0,0 +1,86 @@ +'use strict'; + +const api = require('../../api'); +const helpers = require('../helpers'); +const messaging = require('../../messaging'); +const events = require('../../events'); +const Admin = module.exports; +Admin.updateSetting = async (req, res) => { + await api.admin.updateSetting(req, { + setting: req.params.setting, + value: req.body.value + }); + helpers.formatApiResponse(200, res); +}; +Admin.getAnalyticsKeys = async (req, res) => { + helpers.formatApiResponse(200, res, { + keys: await api.admin.getAnalyticsKeys() + }); +}; +Admin.getAnalyticsData = async (req, res) => { + helpers.formatApiResponse(200, res, await api.admin.getAnalyticsData(req, { + set: req.params.set, + until: parseInt(req.query.until, 10) || Date.now(), + amount: req.query.amount, + units: req.query.units + })); +}; +Admin.generateToken = async (req, res) => { + const { + uid, + description + } = req.body; + const token = await api.utils.tokens.generate({ + uid, + description + }); + helpers.formatApiResponse(200, res, await api.utils.tokens.get(token)); +}; +Admin.getToken = async (req, res) => { + helpers.formatApiResponse(200, res, await api.utils.tokens.get(req.params.token)); +}; +Admin.updateToken = async (req, res) => { + const { + uid, + description + } = req.body; + const { + token + } = req.params; + helpers.formatApiResponse(200, res, await api.utils.tokens.update(token, { + uid, + description + })); +}; +Admin.rollToken = async (req, res) => { + let { + token + } = req.params; + token = await api.utils.tokens.roll(token); + helpers.formatApiResponse(200, res, await api.utils.tokens.get(token)); +}; +Admin.deleteToken = async (req, res) => { + const { + token + } = req.params; + helpers.formatApiResponse(200, res, await api.utils.tokens.delete(token)); +}; +Admin.chats = {}; +Admin.chats.deleteRoom = async (req, res) => { + const roomData = await messaging.getRoomData(req.params.roomId); + if (!roomData) { + throw new Error('[[error:no-room]]'); + } + await messaging.deleteRooms([req.params.roomId]); + events.log({ + type: 'chat-room-deleted', + roomId: req.params.roomId, + roomName: roomData.roomName ? roomData.roomName : `No room name`, + uid: req.uid, + ip: req.ip + }); + helpers.formatApiResponse(200, res); +}; +Admin.listGroups = async (req, res) => { + helpers.formatApiResponse(200, res, await api.admin.listGroups()); +}; \ No newline at end of file diff --git a/lib/controllers/write/categories.js b/lib/controllers/write/categories.js new file mode 100644 index 0000000000..37f6146dee --- /dev/null +++ b/lib/controllers/write/categories.js @@ -0,0 +1,125 @@ +'use strict'; + +const categories = require('../../categories'); +const meta = require('../../meta'); +const api = require('../../api'); +const helpers = require('../helpers'); +const Categories = module.exports; +Categories.list = async (req, res) => { + helpers.formatApiResponse(200, res, await api.categories.list(req)); +}; +Categories.get = async (req, res) => { + helpers.formatApiResponse(200, res, await api.categories.get(req, req.params)); +}; +Categories.create = async (req, res) => { + const response = await api.categories.create(req, req.body); + helpers.formatApiResponse(200, res, response); +}; +Categories.update = async (req, res) => { + await api.categories.update(req, { + cid: req.params.cid, + values: req.body + }); + const categoryObjs = await categories.getCategories([req.params.cid]); + helpers.formatApiResponse(200, res, categoryObjs[0]); +}; +Categories.delete = async (req, res) => { + await api.categories.delete(req, { + cid: req.params.cid + }); + helpers.formatApiResponse(200, res); +}; +Categories.getTopicCount = async (req, res) => { + helpers.formatApiResponse(200, res, await api.categories.getTopicCount(req, { + ...req.params + })); +}; +Categories.getPosts = async (req, res) => { + const posts = await api.categories.getPosts(req, { + ...req.params + }); + helpers.formatApiResponse(200, res, { + posts + }); +}; +Categories.getChildren = async (req, res) => { + const { + cid + } = req.params; + const { + start + } = req.query; + helpers.formatApiResponse(200, res, await api.categories.getChildren(req, { + cid, + start + })); +}; +Categories.getTopics = async (req, res) => { + const { + cid + } = req.params; + const result = await api.categories.getTopics(req, { + ...req.query, + cid + }); + helpers.formatApiResponse(200, res, result); +}; +Categories.setWatchState = async (req, res) => { + const { + cid + } = req.params; + let { + uid, + state + } = req.body; + if (req.method === 'DELETE') { + state = categories.watchStates[meta.config.categoryWatchState]; + } else if (Object.keys(categories.watchStates).includes(state)) { + state = categories.watchStates[state]; + } else { + throw new Error('[[error:invalid-data]]'); + } + const { + cids: modified + } = await api.categories.setWatchState(req, { + cid, + state, + uid + }); + helpers.formatApiResponse(200, res, { + modified + }); +}; +Categories.getPrivileges = async (req, res) => { + const privilegeSet = await api.categories.getPrivileges(req, { + cid: req.params.cid + }); + helpers.formatApiResponse(200, res, privilegeSet); +}; +Categories.setPrivilege = async (req, res) => { + const { + cid, + privilege + } = req.params; + await api.categories.setPrivilege(req, { + cid, + privilege, + member: req.body.member, + set: req.method === 'PUT' + }); + const privilegeSet = await api.categories.getPrivileges(req, { + cid: req.params.cid + }); + helpers.formatApiResponse(200, res, privilegeSet); +}; +Categories.setModerator = async (req, res) => { + await api.categories.setModerator(req, { + cid: req.params.cid, + member: req.params.uid, + set: req.method === 'PUT' + }); + const privilegeSet = await api.categories.getPrivileges(req, { + cid: req.params.cid + }); + helpers.formatApiResponse(200, res, privilegeSet); +}; \ No newline at end of file diff --git a/lib/controllers/write/chats.js b/lib/controllers/write/chats.js new file mode 100644 index 0000000000..b3e18f0d48 --- /dev/null +++ b/lib/controllers/write/chats.js @@ -0,0 +1,263 @@ +'use strict'; + +const api = require('../../api'); +const helpers = require('../helpers'); +const Chats = module.exports; +Chats.list = async (req, res) => { + let stop; + let { + page, + perPage, + start, + uid + } = req.query; + [page, perPage, start, uid] = [page, perPage, start, uid].map(value => isFinite(value) && parseInt(value, 10)); + page = page || 1; + perPage = Math.min(100, perPage || 20); + if (start) { + stop = start + perPage - 1; + } else { + start = Math.max(0, page - 1) * perPage; + stop = start + perPage - 1; + } + const { + rooms, + nextStart + } = await api.chats.list(req, { + start, + stop, + uid + }); + helpers.formatApiResponse(200, res, { + rooms, + nextStart + }); +}; +Chats.create = async (req, res) => { + const roomObj = await api.chats.create(req, req.body); + helpers.formatApiResponse(200, res, roomObj); +}; +Chats.getUnread = async (req, res) => helpers.formatApiResponse(200, res, await api.chats.getUnread(req)); +Chats.sortPublicRooms = async (req, res) => { + const { + roomIds, + scores + } = req.body; + await api.chats.sortPublicRooms(req, { + roomIds, + scores + }); + helpers.formatApiResponse(200, res); +}; +Chats.exists = async (req, res) => { + helpers.formatApiResponse(200, res); +}; +Chats.get = async (req, res) => { + helpers.formatApiResponse(200, res, await api.chats.get(req, { + uid: req.query.uid || req.uid, + roomId: req.params.roomId + })); +}; +Chats.post = async (req, res) => { + const messageObj = await api.chats.post(req, { + message: req.body.message, + toMid: req.body.toMid, + roomId: req.params.roomId + }); + helpers.formatApiResponse(200, res, messageObj); +}; +Chats.update = async (req, res) => { + const payload = { + ...req.body + }; + payload.roomId = req.params.roomId; + const roomObj = await api.chats.update(req, payload); + helpers.formatApiResponse(200, res, roomObj); +}; +Chats.rename = async (req, res) => { + const roomObj = await api.chats.rename(req, { + name: req.body.name, + roomId: req.params.roomId + }); + helpers.formatApiResponse(200, res, roomObj); +}; +Chats.mark = async (req, res) => { + const state = req.method === 'PUT' ? 1 : 0; + await api.chats.mark(req, { + roomId: req.params.roomId, + state + }); + helpers.formatApiResponse(200, res); +}; +Chats.watch = async (req, res) => { + const state = req.method === 'DELETE' ? -1 : parseInt(req.body.value, 10) || -1; + await api.chats.watch(req, { + state, + ...req.params + }); + helpers.formatApiResponse(200, res); +}; +Chats.toggleTyping = async (req, res) => { + const { + typing + } = req.body; + await api.chats.toggleTyping(req, { + typing, + ...req.params + }); + helpers.formatApiResponse(200, res); +}; +Chats.users = async (req, res) => { + const { + roomId + } = req.params; + const start = parseInt(req.query.start, 10) || 0; + const users = await api.chats.users(req, { + roomId, + start + }); + helpers.formatApiResponse(200, res, users); +}; +Chats.invite = async (req, res) => { + const { + uids + } = req.body; + const users = await api.chats.invite(req, { + uids, + roomId: req.params.roomId + }); + helpers.formatApiResponse(200, res, users); +}; +Chats.kick = async (req, res) => { + const { + uids + } = req.body; + const users = await api.chats.kick(req, { + uids, + roomId: req.params.roomId + }); + helpers.formatApiResponse(200, res, users); +}; +Chats.kickUser = async (req, res) => { + const uids = [req.params.uid]; + const users = await api.chats.kick(req, { + uids, + roomId: req.params.roomId + }); + helpers.formatApiResponse(200, res, users); +}; +Chats.toggleOwner = async (req, res) => { + const state = req.method === 'PUT'; + await api.chats.toggleOwner(req, { + state, + ...req.params + }); + helpers.formatApiResponse(200, res); +}; +Chats.messages = {}; +Chats.messages.list = async (req, res) => { + const uid = req.query.uid || req.uid; + const { + roomId + } = req.params; + const start = parseInt(req.query.start, 10) || 0; + const direction = parseInt(req.query.direction, 10) || null; + const { + messages + } = await api.chats.listMessages(req, { + uid, + roomId, + start, + direction + }); + helpers.formatApiResponse(200, res, { + messages + }); +}; +Chats.messages.getPinned = async (req, res) => { + const { + start + } = req.query; + helpers.formatApiResponse(200, res, await api.chats.getPinnedMessages(req, { + start, + ...req.params + })); +}; +Chats.messages.get = async (req, res) => { + const { + mid, + roomId + } = req.params; + helpers.formatApiResponse(200, res, await api.chats.getMessage(req, { + mid, + roomId + })); +}; +Chats.messages.getRaw = async (req, res) => { + helpers.formatApiResponse(200, res, await api.chats.getRawMessage(req, { + ...req.params + })); +}; +Chats.messages.getIpAddress = async (req, res) => { + helpers.formatApiResponse(200, res, await api.chats.getIpAddress(req, { + ...req.params + })); +}; +Chats.messages.edit = async (req, res) => { + const { + mid, + roomId + } = req.params; + const { + message + } = req.body; + await api.chats.editMessage(req, { + mid, + roomId, + message + }); + helpers.formatApiResponse(200, res, await api.chats.getMessage(req, { + mid, + roomId + })); +}; +Chats.messages.delete = async (req, res) => { + const { + mid + } = req.params; + await api.chats.deleteMessage(req, { + mid + }); + helpers.formatApiResponse(200, res); +}; +Chats.messages.restore = async (req, res) => { + const { + mid + } = req.params; + await api.chats.restoreMessage(req, { + mid + }); + helpers.formatApiResponse(200, res); +}; +Chats.messages.pin = async (req, res) => { + const { + mid, + roomId + } = req.params; + await api.chats.pinMessage(req, { + mid, + roomId + }); + helpers.formatApiResponse(200, res); +}; +Chats.messages.unpin = async (req, res) => { + const { + mid, + roomId + } = req.params; + await api.chats.unpinMessage(req, { + mid, + roomId + }); + helpers.formatApiResponse(200, res); +}; \ No newline at end of file diff --git a/lib/controllers/write/files.js b/lib/controllers/write/files.js new file mode 100644 index 0000000000..6f29a60659 --- /dev/null +++ b/lib/controllers/write/files.js @@ -0,0 +1,17 @@ +'use strict'; + +const helpers = require('../helpers'); +const api = require('../../api'); +const Files = module.exports; +Files.delete = async (req, res) => { + await api.files.delete(req, { + path: res.locals.cleanedPath + }); + helpers.formatApiResponse(200, res); +}; +Files.createFolder = async (req, res) => { + await api.files.createFolder(req, { + path: res.locals.folderPath + }); + helpers.formatApiResponse(200, res); +}; \ No newline at end of file diff --git a/lib/controllers/write/flags.js b/lib/controllers/write/flags.js new file mode 100644 index 0000000000..b77fc7b16a --- /dev/null +++ b/lib/controllers/write/flags.js @@ -0,0 +1,63 @@ +'use strict'; + +const user = require('../../user'); +const api = require('../../api'); +const helpers = require('../helpers'); +const Flags = module.exports; +Flags.create = async (req, res) => { + const { + type, + id, + reason + } = req.body; + const flagObj = await api.flags.create(req, { + type, + id, + reason + }); + helpers.formatApiResponse(200, res, (await user.isPrivileged(req.uid)) ? flagObj : undefined); +}; +Flags.get = async (req, res) => { + helpers.formatApiResponse(200, res, await api.flags.get(req, req.params)); +}; +Flags.update = async (req, res) => { + const { + state, + assignee + } = req.body; + const history = await api.flags.update(req, { + flagId: req.params.flagId, + state, + assignee + }); + helpers.formatApiResponse(200, res, { + history + }); +}; +Flags.delete = async (req, res) => { + await api.flags.delete(req, { + flagId: req.params.flagId + }); + helpers.formatApiResponse(200, res); +}; +Flags.rescind = async (req, res) => { + await api.flags.rescind(req, { + flagId: req.params.flagId + }); + helpers.formatApiResponse(200, res); +}; +Flags.appendNote = async (req, res) => { + const { + note, + datetime + } = req.body; + const payload = await api.flags.appendNote(req, { + flagId: req.params.flagId, + note, + datetime + }); + helpers.formatApiResponse(200, res, payload); +}; +Flags.deleteNote = async (req, res) => { + helpers.formatApiResponse(200, res, await api.flags.deleteNote(req, req.params)); +}; \ No newline at end of file diff --git a/lib/controllers/write/groups.js b/lib/controllers/write/groups.js new file mode 100644 index 0000000000..28a2571322 --- /dev/null +++ b/lib/controllers/write/groups.js @@ -0,0 +1,85 @@ +'use strict'; + +const api = require('../../api'); +const helpers = require('../helpers'); +const Groups = module.exports; +Groups.list = async (req, res) => { + helpers.formatApiResponse(200, res, await api.groups.list(req, { + ...req.query + })); +}; +Groups.exists = async (req, res) => { + helpers.formatApiResponse(200, res); +}; +Groups.create = async (req, res) => { + const groupObj = await api.groups.create(req, req.body); + helpers.formatApiResponse(200, res, groupObj); +}; +Groups.update = async (req, res) => { + const groupObj = await api.groups.update(req, { + ...req.body, + slug: req.params.slug + }); + helpers.formatApiResponse(200, res, groupObj); +}; +Groups.delete = async (req, res) => { + await api.groups.delete(req, req.params); + helpers.formatApiResponse(200, res); +}; +Groups.listMembers = async (req, res) => { + const { + slug + } = req.params; + helpers.formatApiResponse(200, res, await api.groups.listMembers(req, { + ...req.query, + slug + })); +}; +Groups.join = async (req, res) => { + await api.groups.join(req, req.params); + helpers.formatApiResponse(200, res); +}; +Groups.leave = async (req, res) => { + await api.groups.leave(req, req.params); + helpers.formatApiResponse(200, res); +}; +Groups.grant = async (req, res) => { + await api.groups.grant(req, req.params); + helpers.formatApiResponse(200, res); +}; +Groups.rescind = async (req, res) => { + await api.groups.rescind(req, req.params); + helpers.formatApiResponse(200, res); +}; +Groups.getPending = async (req, res) => { + const pending = await api.groups.getPending(req, req.params); + helpers.formatApiResponse(200, res, { + pending + }); +}; +Groups.accept = async (req, res) => { + await api.groups.accept(req, req.params); + helpers.formatApiResponse(200, res); +}; +Groups.reject = async (req, res) => { + await api.groups.reject(req, req.params); + helpers.formatApiResponse(200, res); +}; +Groups.getInvites = async (req, res) => { + const invites = await api.groups.getInvites(req, req.params); + helpers.formatApiResponse(200, res, { + invites + }); +}; +Groups.issueInvite = async (req, res) => { + await api.groups.issueInvite(req, req.params); + helpers.formatApiResponse(200, res); +}; +Groups.acceptInvite = async (req, res) => { + await api.groups.acceptInvite(req, req.params); + helpers.formatApiResponse(200, res); +}; +Groups.rejectInvite = async (req, res) => { + await api.groups.rejectInvite(req, req.params); + helpers.formatApiResponse(200, res); +}; \ No newline at end of file diff --git a/lib/controllers/write/index.js b/lib/controllers/write/index.js new file mode 100644 index 0000000000..26252de618 --- /dev/null +++ b/lib/controllers/write/index.js @@ -0,0 +1,15 @@ +'use strict'; + +const Write = module.exports; +Write.users = require('./users'); +Write.groups = require('./groups'); +Write.categories = require('./categories'); +Write.topics = require('./topics'); +Write.tags = require('./tags'); +Write.posts = require('./posts'); +Write.chats = require('./chats'); +Write.flags = require('./flags'); +Write.search = require('./search'); +Write.admin = require('./admin'); +Write.files = require('./files'); +Write.utilities = require('./utilities'); \ No newline at end of file diff --git a/lib/controllers/write/posts.js b/lib/controllers/write/posts.js new file mode 100644 index 0000000000..749356de4b --- /dev/null +++ b/lib/controllers/write/posts.js @@ -0,0 +1,193 @@ +'use strict'; + +const nconf = require('nconf'); +const db = require('../../database'); +const topics = require('../../topics'); +const posts = require('../../posts'); +const api = require('../../api'); +const helpers = require('../helpers'); +const Posts = module.exports; +Posts.redirectByIndex = async (req, res, next) => { + const { + tid + } = req.query || req.body; + let { + index + } = req.params; + if (index < 0 || !isFinite(index)) { + index = 0; + } + index = parseInt(index, 10); + let pid; + if (index === 0) { + pid = await topics.getTopicField(tid, 'mainPid'); + } else { + pid = await db.getSortedSetRange(`tid:${tid}:posts`, index - 1, index - 1); + } + pid = Array.isArray(pid) ? pid[0] : pid; + if (!pid) { + return next('route'); + } + const path = req.path.split('/').slice(3).join('/'); + const urlObj = new URL(nconf.get('url') + req.url); + res.redirect(308, nconf.get('relative_path') + encodeURI(`/api/v3/posts/${pid}/${path}${urlObj.search}`)); +}; +Posts.get = async (req, res) => { + const post = await api.posts.get(req, { + pid: req.params.pid + }); + if (!post) { + return helpers.formatApiResponse(404, res, new Error('[[error:no-post]]')); + } + helpers.formatApiResponse(200, res, post); +}; +Posts.getIndex = async (req, res) => { + const { + pid + } = req.params; + const { + sort + } = req.body; + const index = await api.posts.getIndex(req, { + pid, + sort + }); + if (index === null) { + return helpers.formatApiResponse(404, res, new Error('[[error:no-post]]')); + } + helpers.formatApiResponse(200, res, { + index + }); +}; +Posts.getSummary = async (req, res) => { + const post = await api.posts.getSummary(req, { + pid: req.params.pid + }); + if (!post) { + return helpers.formatApiResponse(404, res, new Error('[[error:no-post]]')); + } + helpers.formatApiResponse(200, res, post); +}; +Posts.getRaw = async (req, res) => { + const content = await api.posts.getRaw(req, { + pid: req.params.pid + }); + if (content === null) { + return helpers.formatApiResponse(404, res, new Error('[[error:no-post]]')); + } + helpers.formatApiResponse(200, res, { + content + }); +}; +Posts.edit = async (req, res) => { + const editResult = await api.posts.edit(req, { + ...req.body, + pid: req.params.pid, + uid: req.uid + }); + helpers.formatApiResponse(200, res, editResult); +}; +Posts.purge = async (req, res) => { + await api.posts.purge(req, { + pid: req.params.pid + }); + helpers.formatApiResponse(200, res); +}; +Posts.restore = async (req, res) => { + await api.posts.restore(req, { + pid: req.params.pid + }); + helpers.formatApiResponse(200, res); +}; +Posts.delete = async (req, res) => { + await api.posts.delete(req, { + pid: req.params.pid + }); + helpers.formatApiResponse(200, res); +}; +Posts.move = async (req, res) => { + await api.posts.move(req, { + pid: req.params.pid, + tid: req.body.tid + }); + helpers.formatApiResponse(200, res); +}; +async function mock(req) { + const tid = await posts.getPostField(req.params.pid, 'tid'); + return { + pid: req.params.pid, + room_id: `topic_${tid}` + }; +} +Posts.vote = async (req, res) => { + const data = await mock(req); + if (req.body.delta > 0) { + await api.posts.upvote(req, data); + } else if (req.body.delta < 0) { + await api.posts.downvote(req, data); + } else { + await api.posts.unvote(req, data); + } + helpers.formatApiResponse(200, res); +}; +Posts.unvote = async (req, res) => { + const data = await mock(req); + await api.posts.unvote(req, data); + helpers.formatApiResponse(200, res); +}; +Posts.getVoters = async (req, res) => { + const data = await api.posts.getVoters(req, { + pid: req.params.pid + }); + helpers.formatApiResponse(200, res, data); +}; +Posts.getUpvoters = async (req, res) => { + const data = await api.posts.getUpvoters(req, { + pid: req.params.pid + }); + helpers.formatApiResponse(200, res, data); +}; +Posts.bookmark = async (req, res) => { + const data = await mock(req); + await api.posts.bookmark(req, data); + helpers.formatApiResponse(200, res); +}; +Posts.unbookmark = async (req, res) => { + const data = await mock(req); + await api.posts.unbookmark(req, data); + helpers.formatApiResponse(200, res); +}; +Posts.getDiffs = async (req, res) => { + helpers.formatApiResponse(200, res, await api.posts.getDiffs(req, { + ...req.params + })); +}; +Posts.loadDiff = async (req, res) => { + helpers.formatApiResponse(200, res, await api.posts.loadDiff(req, { + ...req.params + })); +}; +Posts.restoreDiff = async (req, res) => { + helpers.formatApiResponse(200, res, await api.posts.restoreDiff(req, { + ...req.params + })); +}; +Posts.deleteDiff = async (req, res) => { + await api.posts.deleteDiff(req, { + ...req.params + }); + helpers.formatApiResponse(200, res, await api.posts.getDiffs(req, { + ...req.params + })); +}; +Posts.getReplies = async (req, res) => { + const replies = await api.posts.getReplies(req, { + ...req.params + }); + if (replies === null) { + return helpers.formatApiResponse(404, res, new Error('[[error:no-post]]')); + } + helpers.formatApiResponse(200, res, { + replies + }); +}; \ No newline at end of file diff --git a/lib/controllers/write/search.js b/lib/controllers/write/search.js new file mode 100644 index 0000000000..f38666b70e --- /dev/null +++ b/lib/controllers/write/search.js @@ -0,0 +1,28 @@ +'use strict'; + +const api = require('../../api'); +const helpers = require('../helpers'); +const Search = module.exports; +Search.categories = async (req, res) => { + helpers.formatApiResponse(200, res, await api.search.categories(req, req.query)); +}; +Search.roomUsers = async (req, res) => { + const { + query, + uid + } = req.query; + helpers.formatApiResponse(200, res, await api.search.roomUsers(req, { + query, + uid, + ...req.params + })); +}; +Search.roomMessages = async (req, res) => { + const { + query + } = req.query; + helpers.formatApiResponse(200, res, await api.search.roomMessages(req, { + query, + ...req.params + })); +}; \ No newline at end of file diff --git a/lib/controllers/write/tags.js b/lib/controllers/write/tags.js new file mode 100644 index 0000000000..b1595217b7 --- /dev/null +++ b/lib/controllers/write/tags.js @@ -0,0 +1,13 @@ +'use strict'; + +const api = require('../../api'); +const helpers = require('../helpers'); +const Tags = module.exports; +Tags.follow = async (req, res) => { + await api.tags.follow(req, req.params); + helpers.formatApiResponse(200, res); +}; +Tags.unfollow = async (req, res) => { + await api.tags.unfollow(req, req.params); + helpers.formatApiResponse(200, res); +}; \ No newline at end of file diff --git a/lib/controllers/write/topics.js b/lib/controllers/write/topics.js new file mode 100644 index 0000000000..57f8d2fe0b --- /dev/null +++ b/lib/controllers/write/topics.js @@ -0,0 +1,210 @@ +'use strict'; + +const db = require('../../database'); +const api = require('../../api'); +const topics = require('../../topics'); +const helpers = require('../helpers'); +const middleware = require('../../middleware'); +const uploadsController = require('../uploads'); +const Topics = module.exports; +Topics.get = async (req, res) => { + helpers.formatApiResponse(200, res, await api.topics.get(req, req.params)); +}; +Topics.create = async (req, res) => { + const id = await lockPosting(req, '[[error:already-posting]]'); + try { + const payload = await api.topics.create(req, req.body); + if (payload.queued) { + helpers.formatApiResponse(202, res, payload); + } else { + helpers.formatApiResponse(200, res, payload); + } + } finally { + await db.deleteObjectField('locks', id); + } +}; +Topics.reply = async (req, res) => { + const id = await lockPosting(req, '[[error:already-posting]]'); + try { + const payload = await api.topics.reply(req, { + ...req.body, + tid: req.params.tid + }); + helpers.formatApiResponse(200, res, payload); + } finally { + await db.deleteObjectField('locks', id); + } +}; +async function lockPosting(req, error) { + const id = req.uid > 0 ? req.uid : req.sessionID; + const value = `posting${id}`; + const count = await db.incrObjectField('locks', value); + if (count > 1) { + throw new Error(error); + } + return value; +} +Topics.delete = async (req, res) => { + await api.topics.delete(req, { + tids: [req.params.tid] + }); + helpers.formatApiResponse(200, res); +}; +Topics.restore = async (req, res) => { + await api.topics.restore(req, { + tids: [req.params.tid] + }); + helpers.formatApiResponse(200, res); +}; +Topics.purge = async (req, res) => { + await api.topics.purge(req, { + tids: [req.params.tid] + }); + helpers.formatApiResponse(200, res); +}; +Topics.pin = async (req, res) => { + const { + expiry + } = req.body; + await api.topics.pin(req, { + tids: [req.params.tid], + expiry + }); + helpers.formatApiResponse(200, res); +}; +Topics.unpin = async (req, res) => { + await api.topics.unpin(req, { + tids: [req.params.tid] + }); + helpers.formatApiResponse(200, res); +}; +Topics.lock = async (req, res) => { + await api.topics.lock(req, { + tids: [req.params.tid] + }); + helpers.formatApiResponse(200, res); +}; +Topics.unlock = async (req, res) => { + await api.topics.unlock(req, { + tids: [req.params.tid] + }); + helpers.formatApiResponse(200, res); +}; +Topics.follow = async (req, res) => { + await api.topics.follow(req, req.params); + helpers.formatApiResponse(200, res); +}; +Topics.ignore = async (req, res) => { + await api.topics.ignore(req, req.params); + helpers.formatApiResponse(200, res); +}; +Topics.unfollow = async (req, res) => { + await api.topics.unfollow(req, req.params); + helpers.formatApiResponse(200, res); +}; +Topics.updateTags = async (req, res) => { + const payload = await api.topics.updateTags(req, { + tid: req.params.tid, + tags: req.body.tags + }); + helpers.formatApiResponse(200, res, payload); +}; +Topics.addTags = async (req, res) => { + const payload = await api.topics.addTags(req, { + tid: req.params.tid, + tags: req.body.tags + }); + helpers.formatApiResponse(200, res, payload); +}; +Topics.deleteTags = async (req, res) => { + await api.topics.deleteTags(req, { + tid: req.params.tid + }); + helpers.formatApiResponse(200, res); +}; +Topics.getThumbs = async (req, res) => { + helpers.formatApiResponse(200, res, await api.topics.getThumbs(req, { + ...req.params + })); +}; +Topics.addThumb = async (req, res) => { + await api.topics._checkThumbPrivileges({ + tid: req.params.tid, + uid: req.user.uid + }); + const files = await uploadsController.uploadThumb(req, res); + if (files && files.length) { + await Promise.all(files.map(async fileObj => { + await topics.thumbs.associate({ + id: req.params.tid, + path: fileObj.path || fileObj.url + }); + })); + } +}; +Topics.migrateThumbs = async (req, res) => { + await api.topics.migrateThumbs(req, { + from: req.params.tid, + to: req.body.tid + }); + helpers.formatApiResponse(200, res, await api.topics.getThumbs(req, { + tid: req.body.tid + })); +}; +Topics.deleteThumb = async (req, res) => { + if (!req.body.path.startsWith('http')) { + await middleware.assert.path(req, res, () => {}); + if (res.headersSent) { + return; + } + } + await api.topics.deleteThumb(req, { + tid: req.params.tid, + path: req.body.path + }); + helpers.formatApiResponse(200, res, await topics.thumbs.get(req.params.tid)); +}; +Topics.reorderThumbs = async (req, res) => { + const { + path, + order + } = req.body; + await api.topics.reorderThumbs(req, { + path, + order, + ...req.params + }); + helpers.formatApiResponse(200, res, await topics.thumbs.get(req.params.tid)); +}; +Topics.getEvents = async (req, res) => { + const events = await api.topics.getEvents(req, { + ...req.params + }); + helpers.formatApiResponse(200, res, { + events + }); +}; +Topics.deleteEvent = async (req, res) => { + await api.topics.deleteEvent(req, { + ...req.params + }); + helpers.formatApiResponse(200, res); +}; +Topics.markRead = async (req, res) => { + await api.topics.markRead(req, { + ...req.params + }); + helpers.formatApiResponse(200, res); +}; +Topics.markUnread = async (req, res) => { + await api.topics.markUnread(req, { + ...req.params + }); + helpers.formatApiResponse(200, res); +}; +Topics.bump = async (req, res) => { + await api.topics.bump(req, { + ...req.params + }); + helpers.formatApiResponse(200, res); +}; \ No newline at end of file diff --git a/lib/controllers/write/users.js b/lib/controllers/write/users.js new file mode 100644 index 0000000000..a012414ea7 --- /dev/null +++ b/lib/controllers/write/users.js @@ -0,0 +1,260 @@ +'use strict'; + +const nconf = require('nconf'); +const path = require('path'); +const crypto = require('crypto'); +const api = require('../../api'); +const user = require('../../user'); +const helpers = require('../helpers'); +const Users = module.exports; +Users.redirectBySlug = async (req, res) => { + const uid = await user.getUidByUserslug(req.params.userslug); + if (uid) { + const path = req.path.split('/').slice(3).join('/'); + const urlObj = new URL(nconf.get('url') + req.url); + res.redirect(308, nconf.get('relative_path') + encodeURI(`/api/v3/users/${uid}/${path}${urlObj.search}`)); + } else { + helpers.formatApiResponse(404, res); + } +}; +Users.create = async (req, res) => { + const userObj = await api.users.create(req, req.body); + helpers.formatApiResponse(200, res, userObj); +}; +Users.exists = async (req, res) => { + helpers.formatApiResponse(200, res); +}; +Users.get = async (req, res) => { + helpers.formatApiResponse(200, res, await api.users.get(req, { + ...req.params + })); +}; +Users.update = async (req, res) => { + const userObj = await api.users.update(req, { + ...req.body, + uid: req.params.uid + }); + helpers.formatApiResponse(200, res, userObj); +}; +Users.delete = async (req, res) => { + await api.users.delete(req, { + ...req.params, + password: req.body.password + }); + helpers.formatApiResponse(200, res); +}; +Users.deleteContent = async (req, res) => { + await api.users.deleteContent(req, { + ...req.params, + password: req.body.password + }); + helpers.formatApiResponse(200, res); +}; +Users.deleteAccount = async (req, res) => { + await api.users.deleteAccount(req, { + ...req.params, + password: req.body.password + }); + helpers.formatApiResponse(200, res); +}; +Users.deleteMany = async (req, res) => { + await api.users.deleteMany(req, req.body); + helpers.formatApiResponse(200, res); +}; +Users.changePicture = async (req, res) => { + await api.users.changePicture(req, { + ...req.body, + uid: req.params.uid + }); + helpers.formatApiResponse(200, res); +}; +Users.getStatus = async (req, res) => { + helpers.formatApiResponse(200, res, await api.users.getStatus(req, { + ...req.params + })); +}; +Users.checkStatus = async (req, res) => { + const { + uid, + status + } = req.params; + const { + status: current + } = await api.users.getStatus(req, { + uid + }); + helpers.formatApiResponse(current === status ? 200 : 404, res); +}; +Users.getPrivateRoomId = async (req, res) => { + helpers.formatApiResponse(200, res, await api.users.getPrivateRoomId(req, { + ...req.params + })); +}; +Users.updateSettings = async (req, res) => { + const settings = await api.users.updateSettings(req, { + ...req.body, + uid: req.params.uid + }); + helpers.formatApiResponse(200, res, settings); +}; +Users.changePassword = async (req, res) => { + await api.users.changePassword(req, { + ...req.body, + uid: req.params.uid + }); + helpers.formatApiResponse(200, res); +}; +Users.follow = async (req, res) => { + await api.users.follow(req, req.params); + helpers.formatApiResponse(200, res); +}; +Users.unfollow = async (req, res) => { + await api.users.unfollow(req, req.params); + helpers.formatApiResponse(200, res); +}; +Users.ban = async (req, res) => { + await api.users.ban(req, { + ...req.body, + uid: req.params.uid + }); + helpers.formatApiResponse(200, res); +}; +Users.unban = async (req, res) => { + await api.users.unban(req, { + ...req.body, + uid: req.params.uid + }); + helpers.formatApiResponse(200, res); +}; +Users.mute = async (req, res) => { + await api.users.mute(req, { + ...req.body, + uid: req.params.uid + }); + helpers.formatApiResponse(200, res); +}; +Users.unmute = async (req, res) => { + await api.users.unmute(req, { + ...req.body, + uid: req.params.uid + }); + helpers.formatApiResponse(200, res); +}; +Users.generateToken = async (req, res) => { + const { + description + } = req.body; + const token = await api.users.generateToken(req, { + description, + ...req.params + }); + helpers.formatApiResponse(200, res, token); +}; +Users.deleteToken = async (req, res) => { + const ok = await api.users.deleteToken(req, { + ...req.params + }); + helpers.formatApiResponse(ok ? 200 : 404, res); +}; +Users.revokeSession = async (req, res) => { + await api.users.revokeSession(req, { + ...req.params + }); + helpers.formatApiResponse(200, res); +}; +Users.invite = async (req, res) => { + const { + emails, + groupsToJoin = [] + } = req.body; + try { + await api.users.invite(req, { + emails, + groupsToJoin, + ...req.params + }); + helpers.formatApiResponse(200, res); + } catch (e) { + if (e.message.startsWith('[[error:invite-maximum-met')) { + return helpers.formatApiResponse(403, res, e); + } + throw e; + } +}; +Users.getInviteGroups = async function (req, res) { + return helpers.formatApiResponse(200, res, await api.users.getInviteGroups(req, { + ...req.params + })); +}; +Users.addEmail = async (req, res) => { + const { + email, + skipConfirmation + } = req.body; + const emails = await api.users.addEmail(req, { + email, + skipConfirmation, + ...req.params + }); + helpers.formatApiResponse(200, res, { + emails + }); +}; +Users.listEmails = async (req, res) => { + const emails = await api.users.listEmails(req, { + ...req.params + }); + if (emails) { + helpers.formatApiResponse(200, res, { + emails + }); + } else { + helpers.formatApiResponse(204, res); + } +}; +Users.getEmail = async (req, res) => { + const ok = await api.users.getEmail(req, { + ...req.params + }); + helpers.formatApiResponse(ok ? 204 : 404, res); +}; +Users.confirmEmail = async (req, res) => { + const ok = await api.users.confirmEmail(req, { + sessionId: req.session.id, + ...req.params + }); + helpers.formatApiResponse(ok ? 200 : 404, res); +}; +Users.checkExportByType = async (req, res) => { + const stat = await api.users.checkExportByType(req, { + ...req.params + }); + const modified = new Date(stat.mtimeMs); + res.set('Last-Modified', modified.toUTCString()); + res.set('ETag', `"${crypto.createHash('md5').update(String(stat.mtimeMs)).digest('hex')}"`); + res.sendStatus(204); +}; +Users.getExportByType = async (req, res, next) => { + const data = await api.users.getExportByType(req, { + ...req.params + }); + if (!data) { + return next(); + } + res.status(200); + res.sendFile(data.filename, { + root: path.join(__dirname, '../../../build/export'), + headers: { + 'Content-Type': data.mime, + 'Content-Disposition': `attachment; filename=${data.filename}` + } + }, err => { + if (err) { + throw err; + } + }); +}; +Users.generateExportsByType = async (req, res) => { + await api.users.generateExport(req, req.params); + helpers.formatApiResponse(202, res); +}; \ No newline at end of file diff --git a/lib/controllers/write/utilities.js b/lib/controllers/write/utilities.js new file mode 100644 index 0000000000..dafd11f278 --- /dev/null +++ b/lib/controllers/write/utilities.js @@ -0,0 +1,28 @@ +'use strict'; + +const user = require('../../user'); +const authenticationController = require('../authentication'); +const helpers = require('../helpers'); +const Utilities = module.exports; +Utilities.ping = {}; +Utilities.ping.get = (req, res) => { + helpers.formatApiResponse(200, res, { + pong: true + }); +}; +Utilities.ping.post = (req, res) => { + helpers.formatApiResponse(200, res, { + uid: req.user.uid, + received: req.body + }); +}; +Utilities.login = (req, res) => { + res.locals.redirectAfterLogin = async (req, res) => { + const userData = (await user.getUsers([req.uid], req.uid)).pop(); + helpers.formatApiResponse(200, res, userData); + }; + res.locals.noScriptErrors = (req, res, err, statusCode) => { + helpers.formatApiResponse(statusCode, res, new Error(err)); + }; + authenticationController.login(req, res); +}; \ No newline at end of file diff --git a/lib/coverPhoto.js b/lib/coverPhoto.js new file mode 100644 index 0000000000..8e105c7854 --- /dev/null +++ b/lib/coverPhoto.js @@ -0,0 +1,32 @@ +'use strict'; + +const nconf = require('nconf'); +const meta = require('./meta'); +const relative_path = nconf.get('relative_path'); +const coverPhoto = module.exports; +coverPhoto.getDefaultGroupCover = function (groupName) { + return getCover('groups', groupName); +}; +coverPhoto.getDefaultProfileCover = function (uid) { + return getCover('profile', parseInt(uid, 10)); +}; +function getCover(type, id) { + const defaultCover = `${relative_path}/assets/images/cover-default.png`; + if (meta.config[`${type}:defaultCovers`]) { + const covers = String(meta.config[`${type}:defaultCovers`]).trim().split(/[\s,]+/g); + let coverPhoto = defaultCover; + if (!covers.length) { + return coverPhoto; + } + if (typeof id === 'string') { + id = (id.charCodeAt(0) + id.charCodeAt(1)) % covers.length; + } else { + id %= covers.length; + } + if (covers[id]) { + coverPhoto = covers[id].startsWith('http') ? covers[id] : relative_path + covers[id]; + } + return coverPhoto; + } + return defaultCover; +} \ No newline at end of file diff --git a/lib/database/cache.js b/lib/database/cache.js new file mode 100644 index 0000000000..0d8f6d1aeb --- /dev/null +++ b/lib/database/cache.js @@ -0,0 +1,10 @@ +'use strict'; + +module.exports.create = function (name) { + const cacheCreate = require('../cache/lru'); + return cacheCreate({ + name: `${name}-object`, + max: 40000, + ttl: 0 + }); +}; \ No newline at end of file diff --git a/lib/database/helpers.js b/lib/database/helpers.js new file mode 100644 index 0000000000..220d147fd4 --- /dev/null +++ b/lib/database/helpers.js @@ -0,0 +1,23 @@ +'use strict'; + +const helpers = module.exports; +helpers.mergeBatch = function (batchData, start, stop, sort) { + function getFirst() { + let selectedArray = batchData[0]; + for (let i = 1; i < batchData.length; i++) { + if (batchData[i].length && (!selectedArray.length || sort === 1 && batchData[i][0].score < selectedArray[0].score || sort === -1 && batchData[i][0].score > selectedArray[0].score)) { + selectedArray = batchData[i]; + } + } + return selectedArray.length ? selectedArray.shift() : null; + } + let item = null; + const result = []; + do { + item = getFirst(batchData); + if (item) { + result.push(item); + } + } while (item && (result.length < stop - start + 1 || stop === -1)); + return result; +}; \ No newline at end of file diff --git a/lib/database/index.js b/lib/database/index.js new file mode 100644 index 0000000000..6fcedeaabe --- /dev/null +++ b/lib/database/index.js @@ -0,0 +1,45 @@ +'use strict'; + +const nconf = require('nconf'); +const databaseName = nconf.get('database'); +const winston = require('winston'); +if (!databaseName) { + winston.error(new Error('Database type not set! Run ./nodebb setup')); + process.exit(); +} +const primaryDB = require(`./${databaseName}`); +primaryDB.parseIntFields = function (data, intFields, requestedFields) { + intFields.forEach(field => { + if (!requestedFields || !requestedFields.length || requestedFields.includes(field)) { + data[field] = parseInt(data[field], 10) || 0; + } + }); +}; +primaryDB.initSessionStore = async function () { + const sessionStoreConfig = nconf.get('session_store') || nconf.get('redis') || nconf.get(databaseName); + let sessionStoreDB = primaryDB; + if (nconf.get('session_store')) { + sessionStoreDB = require(`./${sessionStoreConfig.name}`); + } else if (nconf.get('redis')) { + sessionStoreDB = require('./redis'); + } + primaryDB.sessionStore = await sessionStoreDB.createSessionStore(sessionStoreConfig); +}; +function promisifySessionStoreMethod(method, sid) { + return new Promise((resolve, reject) => { + if (!primaryDB.sessionStore) { + resolve(method === 'get' ? null : undefined); + return; + } + primaryDB.sessionStore[method](sid, (err, result) => { + if (err) reject(err);else resolve(method === 'get' ? result || null : undefined); + }); + }); +} +primaryDB.sessionStoreGet = function (sid) { + return promisifySessionStoreMethod('get', sid); +}; +primaryDB.sessionStoreDestroy = function (sid) { + return promisifySessionStoreMethod('destroy', sid); +}; +module.exports = primaryDB; \ No newline at end of file diff --git a/lib/database/mongo.js b/lib/database/mongo.js new file mode 100644 index 0000000000..0f3274f7c9 --- /dev/null +++ b/lib/database/mongo.js @@ -0,0 +1,189 @@ +'use strict'; + +const winston = require('winston'); +const nconf = require('nconf'); +const semver = require('semver'); +const prompt = require('prompt'); +const utils = require('../utils'); +let client; +const connection = require('./mongo/connection'); +const mongoModule = module.exports; +function isUriNotSpecified() { + return !prompt.history('mongo:uri').value; +} +mongoModule.questions = [{ + name: 'mongo:uri', + description: 'MongoDB connection URI: (leave blank if you wish to specify host, port, username/password and database individually)\nFormat: mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]', + default: nconf.get('mongo:uri') || nconf.get('defaults:mongo:uri') || '', + hideOnWebInstall: true +}, { + name: 'mongo:host', + description: 'Host IP or address of your MongoDB instance', + default: nconf.get('mongo:host') || nconf.get('defaults:mongo:host') || '127.0.0.1', + ask: isUriNotSpecified +}, { + name: 'mongo:port', + description: 'Host port of your MongoDB instance', + default: nconf.get('mongo:port') || nconf.get('defaults:mongo:port') || 27017, + ask: isUriNotSpecified +}, { + name: 'mongo:username', + description: 'MongoDB username', + default: nconf.get('mongo:username') || nconf.get('defaults:mongo:username') || '', + ask: isUriNotSpecified +}, { + name: 'mongo:password', + description: 'Password of your MongoDB database', + default: nconf.get('mongo:password') || nconf.get('defaults:mongo:password') || '', + hidden: true, + ask: isUriNotSpecified, + before: function (value) { + value = value || nconf.get('mongo:password') || ''; + return value; + } +}, { + name: 'mongo:database', + description: 'MongoDB database name', + default: nconf.get('mongo:database') || nconf.get('defaults:mongo:database') || 'nodebb', + ask: isUriNotSpecified +}]; +mongoModule.init = async function (opts) { + client = await connection.connect(opts || nconf.get('mongo')); + mongoModule.client = client.db(); +}; +mongoModule.createSessionStore = async function (options) { + const MongoStore = require('connect-mongo'); + const meta = require('../meta'); + const store = MongoStore.create({ + clientPromise: connection.connect(options), + ttl: meta.getSessionTTLSeconds() + }); + return store; +}; +mongoModule.createIndices = async function () { + if (!mongoModule.client) { + winston.warn('[database/createIndices] database not initialized'); + return; + } + winston.info('[database] Checking database indices.'); + const collection = mongoModule.client.collection('objects'); + await collection.createIndex({ + _key: 1, + score: -1 + }, { + background: true + }); + await collection.createIndex({ + _key: 1, + value: -1 + }, { + background: true, + unique: true, + sparse: true + }); + await collection.createIndex({ + expireAt: 1 + }, { + expireAfterSeconds: 0, + background: true + }); + winston.info('[database] Checking database indices done!'); +}; +mongoModule.checkCompatibility = function (callback) { + const mongoPkg = require('mongodb/package.json'); + mongoModule.checkCompatibilityVersion(mongoPkg.version, callback); +}; +mongoModule.checkCompatibilityVersion = function (version, callback) { + if (semver.lt(version, '2.0.0')) { + return callback(new Error('The `mongodb` package is out-of-date, please run `./nodebb setup` again.')); + } + callback(); +}; +mongoModule.info = async function (db) { + if (!db) { + const client = await connection.connect(nconf.get('mongo')); + db = client.db(); + } + mongoModule.client = mongoModule.client || db; + let serverStatusError = ''; + async function getServerStatus() { + try { + return await db.command({ + serverStatus: 1 + }); + } catch (err) { + serverStatusError = err.message; + if (err.name === 'MongoError' && err.codeName === 'Unauthorized') { + serverStatusError = '[[admin/advanced/database:mongo.unauthorized]]'; + } + winston.error(err.stack); + } + } + let [serverStatus, stats, listCollections] = await Promise.all([getServerStatus(), db.command({ + dbStats: 1 + }), getCollectionStats(db)]); + stats = stats || {}; + serverStatus = serverStatus || {}; + stats.serverStatusError = serverStatusError; + const scale = 1024 * 1024 * 1024; + listCollections = listCollections.map(collectionInfo => ({ + name: collectionInfo.ns, + count: collectionInfo.count, + size: collectionInfo.storageStats && collectionInfo.storageStats.size, + avgObjSize: collectionInfo.storageStats && collectionInfo.storageStats.avgObjSize, + storageSize: collectionInfo.storageStats && collectionInfo.storageStats.storageSize, + totalIndexSize: collectionInfo.storageStats && collectionInfo.storageStats.totalIndexSize, + indexSizes: collectionInfo.storageStats && collectionInfo.storageStats.indexSizes + })); + stats.mem = serverStatus.mem || { + resident: 0, + virtual: 0 + }; + stats.mem.resident = (stats.mem.resident / 1024).toFixed(3); + stats.mem.virtual = (stats.mem.virtual / 1024).toFixed(3); + stats.collectionData = listCollections; + stats.network = serverStatus.network || { + bytesIn: 0, + bytesOut: 0, + numRequests: 0 + }; + stats.network.bytesIn = (stats.network.bytesIn / scale).toFixed(3); + stats.network.bytesOut = (stats.network.bytesOut / scale).toFixed(3); + stats.network.numRequests = utils.addCommas(stats.network.numRequests); + stats.raw = JSON.stringify(stats, null, 4); + stats.avgObjSize = stats.avgObjSize.toFixed(2); + stats.dataSize = (stats.dataSize / scale).toFixed(3); + stats.storageSize = (stats.storageSize / scale).toFixed(3); + stats.fileSize = stats.fileSize ? (stats.fileSize / scale).toFixed(3) : 0; + stats.indexSize = (stats.indexSize / scale).toFixed(3); + stats.storageEngine = serverStatus.storageEngine ? serverStatus.storageEngine.name : 'mmapv1'; + stats.host = serverStatus.host; + stats.version = serverStatus.version; + stats.uptime = serverStatus.uptime; + stats.mongo = true; + return stats; +}; +async function getCollectionStats(db) { + const items = await db.listCollections().toArray(); + const cols = await Promise.all(items.map(collection => db.collection(collection.name).aggregate([{ + $collStats: { + latencyStats: {}, + storageStats: {}, + count: {} + } + }]).toArray())); + return cols.map(col => col[0]); +} +mongoModule.close = async function () { + await client.close(); + if (mongoModule.objectCache) { + mongoModule.objectCache.reset(); + } +}; +require('./mongo/main')(mongoModule); +require('./mongo/hash')(mongoModule); +require('./mongo/sets')(mongoModule); +require('./mongo/sorted')(mongoModule); +require('./mongo/list')(mongoModule); +require('./mongo/transaction')(mongoModule); +require('../promisify')(mongoModule, ['client', 'sessionStore']); \ No newline at end of file diff --git a/lib/database/mongo/connection.js b/lib/database/mongo/connection.js new file mode 100644 index 0000000000..9447c4f1c2 --- /dev/null +++ b/lib/database/mongo/connection.js @@ -0,0 +1,49 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const _ = require('lodash'); +const connection = module.exports; +connection.getConnectionString = function (mongo) { + mongo = mongo || nconf.get('mongo'); + let usernamePassword = ''; + const uri = mongo.uri || ''; + if (mongo.username && mongo.password) { + usernamePassword = `${mongo.username}:${encodeURIComponent(mongo.password)}@`; + } else if (!uri.includes('@') || !uri.slice(uri.indexOf('://') + 3, uri.indexOf('@'))) { + winston.warn('You have no mongo username/password setup!'); + } + if (!mongo.host) { + mongo.host = '127.0.0.1'; + } + if (!mongo.port) { + mongo.port = 27017; + } + const dbName = mongo.database; + if (dbName === undefined || dbName === '') { + winston.warn('You have no database name, using "nodebb"'); + mongo.database = 'nodebb'; + } + const hosts = mongo.host.split(','); + const ports = mongo.port.toString().split(','); + const servers = []; + for (let i = 0; i < hosts.length; i += 1) { + servers.push(`${hosts[i]}:${ports[i]}`); + } + return uri || `mongodb://${usernamePassword}${servers.join()}/${mongo.database}`; +}; +connection.getConnectionOptions = function (mongo) { + mongo = mongo || nconf.get('mongo'); + const connOptions = { + maxPoolSize: 20, + minPoolSize: 3, + connectTimeoutMS: 90000 + }; + return _.merge(connOptions, mongo.options || {}); +}; +connection.connect = async function (options) { + const mongoClient = require('mongodb').MongoClient; + const connString = connection.getConnectionString(options); + const connOptions = connection.getConnectionOptions(options); + return await mongoClient.connect(connString, connOptions); +}; \ No newline at end of file diff --git a/lib/database/mongo/hash.js b/lib/database/mongo/hash.js new file mode 100644 index 0000000000..79a89070e3 --- /dev/null +++ b/lib/database/mongo/hash.js @@ -0,0 +1,295 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + const cache = require('../cache').create('mongo'); + module.objectCache = cache; + module.setObject = async function (key, data) { + const isArray = Array.isArray(key); + if (!key || !data || isArray && !key.length) { + return; + } + const writeData = helpers.serializeData(data); + if (!Object.keys(writeData).length) { + return; + } + try { + if (isArray) { + const bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + key.forEach(key => bulk.find({ + _key: key + }).upsert().updateOne({ + $set: writeData + })); + await bulk.execute(); + } else { + await module.client.collection('objects').updateOne({ + _key: key + }, { + $set: writeData + }, { + upsert: true + }); + } + } catch (err) { + if (err && err.message.includes('E11000 duplicate key error')) { + console.log(new Error('e11000').stack, key, data); + return await module.setObject(key, data); + } + throw err; + } + cache.del(key); + }; + module.setObjectBulk = async function (...args) { + let data = args[0]; + if (!Array.isArray(data) || !data.length) { + return; + } + if (Array.isArray(args[1])) { + console.warn('[deprecated] db.setObjectBulk(keys, data) usage is deprecated, please use db.setObjectBulk(data)'); + data = args[0].map((key, i) => [key, args[1][i]]); + } + try { + let bulk; + data.forEach(item => { + const writeData = helpers.serializeData(item[1]); + if (Object.keys(writeData).length) { + if (!bulk) { + bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + } + bulk.find({ + _key: item[0] + }).upsert().updateOne({ + $set: writeData + }); + } + }); + if (bulk) { + await bulk.execute(); + } + } catch (err) { + if (err && err.message.includes('E11000 duplicate key error')) { + console.log(new Error('e11000').stack, data); + return await module.setObjectBulk(data); + } + throw err; + } + cache.del(data.map(item => item[0])); + }; + module.setObjectField = async function (key, field, value) { + if (!field) { + return; + } + const data = {}; + data[field] = value; + await module.setObject(key, data); + }; + module.getObject = async function (key, fields = []) { + if (!key) { + return null; + } + const data = await module.getObjects([key], fields); + return data && data.length ? data[0] : null; + }; + module.getObjects = async function (keys, fields = []) { + return await module.getObjectsFields(keys, fields); + }; + module.getObjectField = async function (key, field) { + if (!key) { + return null; + } + const cachedData = {}; + cache.getUnCachedKeys([key], cachedData); + if (cachedData[key]) { + return cachedData[key].hasOwnProperty(field) ? cachedData[key][field] : null; + } + field = helpers.fieldToString(field); + const item = await module.client.collection('objects').findOne({ + _key: key + }, { + projection: { + _id: 0, + [field]: 1 + } + }); + if (!item) { + return null; + } + return item.hasOwnProperty(field) ? item[field] : null; + }; + module.getObjectFields = async function (key, fields) { + if (!key) { + return null; + } + const data = await module.getObjectsFields([key], fields); + return data ? data[0] : null; + }; + module.getObjectsFields = async function (keys, fields) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const cachedData = {}; + const unCachedKeys = cache.getUnCachedKeys(keys, cachedData); + if (unCachedKeys.length >= 1) { + let data = await module.client.collection('objects').find({ + _key: unCachedKeys.length === 1 ? unCachedKeys[0] : { + $in: unCachedKeys + } + }, { + projection: { + _id: 0 + } + }).toArray(); + data = data.map(helpers.deserializeData); + const map = helpers.toMap(data); + unCachedKeys.forEach(key => { + cachedData[key] = map[key] || null; + cache.set(key, cachedData[key]); + }); + } + if (!Array.isArray(fields) || !fields.length) { + return keys.map(key => cachedData[key] ? { + ...cachedData[key] + } : null); + } + return keys.map(key => { + const item = cachedData[key] || {}; + const result = {}; + fields.forEach(field => { + result[field] = item[field] !== undefined ? item[field] : null; + }); + return result; + }); + }; + module.getObjectKeys = async function (key) { + const data = await module.getObject(key); + return data ? Object.keys(data) : []; + }; + module.getObjectValues = async function (key) { + const data = await module.getObject(key); + return data ? Object.values(data) : []; + }; + module.isObjectField = async function (key, field) { + const data = await module.isObjectFields(key, [field]); + return Array.isArray(data) && data.length ? data[0] : false; + }; + module.isObjectFields = async function (key, fields) { + if (!key) { + return; + } + const data = {}; + fields.forEach(field => { + field = helpers.fieldToString(field); + if (field) { + data[field] = 1; + } + }); + const item = await module.client.collection('objects').findOne({ + _key: key + }, { + projection: data + }); + const results = fields.map(f => !!item && item[f] !== undefined && item[f] !== null); + return results; + }; + module.deleteObjectField = async function (key, field) { + await module.deleteObjectFields(key, [field]); + }; + module.deleteObjectFields = async function (key, fields) { + if (!key || Array.isArray(key) && !key.length || !Array.isArray(fields) || !fields.length) { + return; + } + fields = fields.filter(Boolean); + if (!fields.length) { + return; + } + const data = {}; + fields.forEach(field => { + field = helpers.fieldToString(field); + data[field] = ''; + }); + if (Array.isArray(key)) { + await module.client.collection('objects').updateMany({ + _key: { + $in: key + } + }, { + $unset: data + }); + } else { + await module.client.collection('objects').updateOne({ + _key: key + }, { + $unset: data + }); + } + cache.del(key); + }; + module.incrObjectField = async function (key, field) { + return await module.incrObjectFieldBy(key, field, 1); + }; + module.decrObjectField = async function (key, field) { + return await module.incrObjectFieldBy(key, field, -1); + }; + module.incrObjectFieldBy = async function (key, field, value) { + value = parseInt(value, 10); + if (!key || isNaN(value)) { + return null; + } + const increment = {}; + field = helpers.fieldToString(field); + increment[field] = value; + if (Array.isArray(key)) { + const bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + key.forEach(key => { + bulk.find({ + _key: key + }).upsert().update({ + $inc: increment + }); + }); + await bulk.execute(); + cache.del(key); + const result = await module.getObjectsFields(key, [field]); + return result.map(data => data && data[field]); + } + try { + const result = await module.client.collection('objects').findOneAndUpdate({ + _key: key + }, { + $inc: increment + }, { + returnDocument: 'after', + includeResultMetadata: true, + upsert: true + }); + cache.del(key); + return result && result.value ? result.value[field] : null; + } catch (err) { + if (err && err.message.includes('E11000 duplicate key error')) { + console.log(new Error('e11000').stack, key, field, value); + return await module.incrObjectFieldBy(key, field, value); + } + throw err; + } + }; + module.incrObjectFieldByBulk = async function (data) { + if (!Array.isArray(data) || !data.length) { + return; + } + const bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + data.forEach(item => { + const increment = {}; + for (const [field, value] of Object.entries(item[1])) { + increment[helpers.fieldToString(field)] = value; + } + bulk.find({ + _key: item[0] + }).upsert().update({ + $inc: increment + }); + }); + await bulk.execute(); + cache.del(data.map(item => item[0])); + }; +}; \ No newline at end of file diff --git a/lib/database/mongo/helpers.js b/lib/database/mongo/helpers.js new file mode 100644 index 0000000000..be5426c239 --- /dev/null +++ b/lib/database/mongo/helpers.js @@ -0,0 +1,58 @@ +'use strict'; + +const helpers = module.exports; +const utils = require('../../utils'); +helpers.noop = function () {}; +helpers.toMap = function (data) { + const map = {}; + for (let i = 0; i < data.length; i += 1) { + map[data[i]._key] = data[i]; + delete data[i]._key; + } + return map; +}; +helpers.fieldToString = function (field) { + if (field === null || field === undefined) { + return field; + } + if (typeof field !== 'string') { + field = field.toString(); + } + return field.replace(/\./g, '\uff0E'); +}; +helpers.serializeData = function (data) { + const serialized = {}; + for (const [field, value] of Object.entries(data)) { + if (field !== '') { + serialized[helpers.fieldToString(field)] = value; + } + } + return serialized; +}; +helpers.deserializeData = function (data) { + const deserialized = {}; + for (const [field, value] of Object.entries(data)) { + deserialized[field.replace(/\uff0E/g, '.')] = value; + } + return deserialized; +}; +helpers.valueToString = function (value) { + return String(value); +}; +helpers.buildMatchQuery = function (match) { + let _match = match; + if (match.startsWith('*')) { + _match = _match.substring(1); + } + if (match.endsWith('*')) { + _match = _match.substring(0, _match.length - 1); + } + _match = utils.escapeRegexChars(_match); + if (!match.startsWith('*')) { + _match = `^${_match}`; + } + if (!match.endsWith('*')) { + _match += '$'; + } + return _match; +}; \ No newline at end of file diff --git a/lib/database/mongo/list.js b/lib/database/mongo/list.js new file mode 100644 index 0000000000..a1fc0e24a3 --- /dev/null +++ b/lib/database/mongo/list.js @@ -0,0 +1,117 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + module.listPrepend = async function (key, value) { + if (!key) { + return; + } + value = Array.isArray(value) ? value : [value]; + value.reverse(); + const exists = await module.isObjectField(key, 'array'); + if (exists) { + await listPush(key, value, { + $position: 0 + }); + } else { + await module.listAppend(key, value); + } + }; + module.listAppend = async function (key, value) { + if (!key) { + return; + } + value = Array.isArray(value) ? value : [value]; + await listPush(key, value); + }; + async function listPush(key, values, position) { + values = values.map(helpers.valueToString); + await module.client.collection('objects').updateOne({ + _key: key + }, { + $push: { + array: { + $each: values, + ...(position || {}) + } + } + }, { + upsert: true + }); + } + module.listRemoveLast = async function (key) { + if (!key) { + return; + } + const value = await module.getListRange(key, -1, -1); + module.client.collection('objects').updateOne({ + _key: key + }, { + $pop: { + array: 1 + } + }); + return value && value.length ? value[0] : null; + }; + module.listRemoveAll = async function (key, value) { + if (!key) { + return; + } + const isArray = Array.isArray(value); + if (isArray) { + value = value.map(helpers.valueToString); + } else { + value = helpers.valueToString(value); + } + await module.client.collection('objects').updateOne({ + _key: key + }, { + $pull: { + array: isArray ? { + $in: value + } : value + } + }); + }; + module.listTrim = async function (key, start, stop) { + if (!key) { + return; + } + const value = await module.getListRange(key, start, stop); + await module.client.collection('objects').updateOne({ + _key: key + }, { + $set: { + array: value + } + }); + }; + module.getListRange = async function (key, start, stop) { + if (!key) { + return; + } + const data = await module.client.collection('objects').findOne({ + _key: key + }, { + array: 1 + }); + if (!(data && data.array)) { + return []; + } + return data.array.slice(start, stop !== -1 ? stop + 1 : undefined); + }; + module.listLength = async function (key) { + const result = await module.client.collection('objects').aggregate([{ + $match: { + _key: key + } + }, { + $project: { + count: { + $size: '$array' + } + } + }]).toArray(); + return Array.isArray(result) && result.length && result[0].count; + }; +}; \ No newline at end of file diff --git a/lib/database/mongo/main.js b/lib/database/mongo/main.js new file mode 100644 index 0000000000..8f882627dc --- /dev/null +++ b/lib/database/mongo/main.js @@ -0,0 +1,184 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + module.flushdb = async function () { + await module.client.dropDatabase(); + }; + module.emptydb = async function () { + await module.client.collection('objects').deleteMany({}); + module.objectCache.reset(); + }; + module.exists = async function (key) { + if (!key) { + return; + } + if (Array.isArray(key)) { + if (!key.length) { + return []; + } + const data = await module.client.collection('objects').find({ + _key: { + $in: key + } + }, { + _id: 0, + _key: 1 + }).toArray(); + const map = Object.create(null); + data.forEach(item => { + map[item._key] = true; + }); + return key.map(key => !!map[key]); + } + const item = await module.client.collection('objects').findOne({ + _key: key + }, { + _id: 0, + _key: 1 + }); + return item !== undefined && item !== null; + }; + module.scan = async function (params) { + const match = helpers.buildMatchQuery(params.match); + return await module.client.collection('objects').distinct('_key', { + _key: { + $regex: new RegExp(match) + } + }); + }; + module.delete = async function (key) { + if (!key) { + return; + } + await module.client.collection('objects').deleteMany({ + _key: key + }); + module.objectCache.del(key); + }; + module.deleteAll = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + await module.client.collection('objects').deleteMany({ + _key: { + $in: keys + } + }); + module.objectCache.del(keys); + }; + module.get = async function (key) { + if (!key) { + return; + } + const objectData = await module.client.collection('objects').findOne({ + _key: key + }, { + projection: { + _id: 0 + } + }); + let value = null; + if (objectData) { + if (objectData.hasOwnProperty('data')) { + value = objectData.data; + } else if (objectData.hasOwnProperty('value')) { + value = objectData.value; + } + } + return value; + }; + module.mget = async function (keys) { + if (!keys || !Array.isArray(keys) || !keys.length) { + return []; + } + const data = await module.client.collection('objects').find({ + _key: { + $in: keys + } + }, { + projection: { + _id: 0 + } + }).toArray(); + const map = {}; + data.forEach(d => { + map[d._key] = d.data; + }); + return keys.map(k => map.hasOwnProperty(k) ? map[k] : null); + }; + module.set = async function (key, value) { + if (!key) { + return; + } + await module.setObject(key, { + data: value + }); + }; + module.increment = async function (key) { + if (!key) { + return; + } + const result = await module.client.collection('objects').findOneAndUpdate({ + _key: key + }, { + $inc: { + data: 1 + } + }, { + returnDocument: 'after', + includeResultMetadata: true, + upsert: true + }); + return result && result.value ? result.value.data : null; + }; + module.rename = async function (oldKey, newKey) { + await module.client.collection('objects').updateMany({ + _key: oldKey + }, { + $set: { + _key: newKey + } + }); + module.objectCache.del([oldKey, newKey]); + }; + module.type = async function (key) { + const data = await module.client.collection('objects').findOne({ + _key: key + }); + if (!data) { + return null; + } + delete data.expireAt; + const keys = Object.keys(data); + if (keys.length === 4 && data.hasOwnProperty('_key') && data.hasOwnProperty('score') && data.hasOwnProperty('value')) { + return 'zset'; + } else if (keys.length === 3 && data.hasOwnProperty('_key') && data.hasOwnProperty('members')) { + return 'set'; + } else if (keys.length === 3 && data.hasOwnProperty('_key') && data.hasOwnProperty('array')) { + return 'list'; + } else if (keys.length === 3 && data.hasOwnProperty('_key') && data.hasOwnProperty('data')) { + return 'string'; + } + return 'hash'; + }; + module.expire = async function (key, seconds) { + await module.expireAt(key, Math.round(Date.now() / 1000) + seconds); + }; + module.expireAt = async function (key, timestamp) { + await module.setObjectField(key, 'expireAt', new Date(timestamp * 1000)); + }; + module.pexpire = async function (key, ms) { + await module.pexpireAt(key, Date.now() + parseInt(ms, 10)); + }; + module.pexpireAt = async function (key, timestamp) { + timestamp = Math.min(timestamp, 8640000000000000); + await module.setObjectField(key, 'expireAt', new Date(timestamp)); + }; + module.ttl = async function (key) { + return Math.round(((await module.getObjectField(key, 'expireAt')) - Date.now()) / 1000); + }; + module.pttl = async function (key) { + return (await module.getObjectField(key, 'expireAt')) - Date.now(); + }; +}; \ No newline at end of file diff --git a/lib/database/mongo/sets.js b/lib/database/mongo/sets.js new file mode 100644 index 0000000000..4aa3432633 --- /dev/null +++ b/lib/database/mongo/sets.js @@ -0,0 +1,230 @@ +'use strict'; + +module.exports = function (module) { + const _ = require('lodash'); + const helpers = require('./helpers'); + module.setAdd = async function (key, value) { + if (!Array.isArray(value)) { + value = [value]; + } + if (!value.length) { + return; + } + value = value.map(v => helpers.valueToString(v)); + try { + await module.client.collection('objects').updateOne({ + _key: key + }, { + $addToSet: { + members: { + $each: value + } + } + }, { + upsert: true + }); + } catch (err) { + if (err && err.message.includes('E11000 duplicate key error')) { + console.log(new Error('e11000').stack, key, value); + return await module.setAdd(key, value); + } + throw err; + } + }; + module.setsAdd = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + if (!Array.isArray(value)) { + value = [value]; + } + value = value.map(v => helpers.valueToString(v)); + const bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + for (let i = 0; i < keys.length; i += 1) { + bulk.find({ + _key: keys[i] + }).upsert().updateOne({ + $addToSet: { + members: { + $each: value + } + } + }); + } + try { + await bulk.execute(); + } catch (err) { + if (err && err.message.includes('E11000 duplicate key error')) { + console.log(new Error('e11000').stack, keys, value); + return await module.setsAdd(keys, value); + } + throw err; + } + }; + module.setRemove = async function (key, value) { + if (!Array.isArray(value)) { + value = [value]; + } + value = value.map(v => helpers.valueToString(v)); + await module.client.collection('objects').updateMany({ + _key: Array.isArray(key) ? { + $in: key + } : key + }, { + $pullAll: { + members: value + } + }); + }; + module.setsRemove = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + value = helpers.valueToString(value); + await module.client.collection('objects').updateMany({ + _key: { + $in: keys + } + }, { + $pull: { + members: value + } + }); + }; + module.isSetMember = async function (key, value) { + if (!key) { + return false; + } + value = helpers.valueToString(value); + const item = await module.client.collection('objects').findOne({ + _key: key, + members: value + }, { + projection: { + _id: 0, + members: 0 + } + }); + return item !== null && item !== undefined; + }; + module.isSetMembers = async function (key, values) { + if (!key || !Array.isArray(values) || !values.length) { + return []; + } + values = values.map(v => helpers.valueToString(v)); + const result = await module.client.collection('objects').findOne({ + _key: key + }, { + projection: { + _id: 0, + _key: 0 + } + }); + const membersSet = new Set(result && Array.isArray(result.members) ? result.members : []); + return values.map(v => membersSet.has(v)); + }; + module.isMemberOfSets = async function (sets, value) { + if (!Array.isArray(sets) || !sets.length) { + return []; + } + value = helpers.valueToString(value); + const result = await module.client.collection('objects').find({ + _key: { + $in: sets + }, + members: value + }, { + projection: { + _id: 0, + members: 0 + } + }).toArray(); + const map = {}; + result.forEach(item => { + map[item._key] = true; + }); + return sets.map(set => !!map[set]); + }; + module.getSetMembers = async function (key) { + if (!key) { + return []; + } + const data = await module.client.collection('objects').findOne({ + _key: key + }, { + projection: { + _id: 0, + _key: 0 + } + }); + return data ? data.members : []; + }; + module.getSetsMembers = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const data = await module.client.collection('objects').find({ + _key: { + $in: keys + } + }, { + projection: { + _id: 0 + } + }).toArray(); + const sets = {}; + data.forEach(set => { + sets[set._key] = set.members || []; + }); + return keys.map(k => sets[k] || []); + }; + module.setCount = async function (key) { + if (!key) { + return 0; + } + const data = await module.client.collection('objects').aggregate([{ + $match: { + _key: key + } + }, { + $project: { + _id: 0, + count: { + $size: '$members' + } + } + }]).toArray(); + return Array.isArray(data) && data.length ? data[0].count : 0; + }; + module.setsCount = async function (keys) { + const data = await module.client.collection('objects').aggregate([{ + $match: { + _key: { + $in: keys + } + } + }, { + $project: { + _id: 0, + _key: 1, + count: { + $size: '$members' + } + } + }]).toArray(); + const map = _.keyBy(data, '_key'); + return keys.map(key => map.hasOwnProperty(key) ? map[key].count : 0); + }; + module.setRemoveRandom = async function (key) { + const data = await module.client.collection('objects').findOne({ + _key: key + }); + if (!data) { + return; + } + const randomIndex = Math.floor(Math.random() * data.members.length); + const value = data.members[randomIndex]; + await module.setRemove(data._key, value); + return value; + }; +}; \ No newline at end of file diff --git a/lib/database/mongo/sorted.js b/lib/database/mongo/sorted.js new file mode 100644 index 0000000000..b3b2d6532f --- /dev/null +++ b/lib/database/mongo/sorted.js @@ -0,0 +1,663 @@ +'use strict'; + +const _ = require('lodash'); +const utils = require('../../utils'); +module.exports = function (module) { + const helpers = require('./helpers'); + const dbHelpers = require('../helpers'); + const util = require('util'); + const sleep = util.promisify(setTimeout); + require('./sorted/add')(module); + require('./sorted/remove')(module); + require('./sorted/union')(module); + require('./sorted/intersect')(module); + module.getSortedSetRange = async function (key, start, stop) { + return await getSortedSetRange(key, start, stop, '-inf', '+inf', 1, false); + }; + module.getSortedSetRevRange = async function (key, start, stop) { + return await getSortedSetRange(key, start, stop, '-inf', '+inf', -1, false); + }; + module.getSortedSetRangeWithScores = async function (key, start, stop) { + return await getSortedSetRange(key, start, stop, '-inf', '+inf', 1, true); + }; + module.getSortedSetRevRangeWithScores = async function (key, start, stop) { + return await getSortedSetRange(key, start, stop, '-inf', '+inf', -1, true); + }; + async function getSortedSetRange(key, start, stop, min, max, sort, withScores) { + if (!key) { + return; + } + const isArray = Array.isArray(key); + if (start < 0 && start > stop || isArray && !key.length) { + return []; + } + const query = { + _key: key + }; + if (isArray) { + if (key.length > 1) { + query._key = { + $in: key + }; + } else { + query._key = key[0]; + } + } + if (min !== '-inf') { + query.score = { + $gte: parseFloat(min) + }; + } + if (max !== '+inf') { + query.score = query.score || {}; + query.score.$lte = parseFloat(max); + } + if (max === min) { + query.score = parseFloat(max); + } + const fields = { + _id: 0, + _key: 0 + }; + if (!withScores) { + fields.score = 0; + } + let reverse = false; + if (start === 0 && stop < -1) { + reverse = true; + sort *= -1; + start = Math.abs(stop + 1); + stop = -1; + } else if (start < 0 && stop > start) { + const tmp1 = Math.abs(stop + 1); + stop = Math.abs(start + 1); + start = tmp1; + } + let limit = stop - start + 1; + if (limit <= 0) { + limit = 0; + } + let result = []; + async function doQuery(_key, fields, skip, limit) { + return await module.client.collection('objects').find({ + ...query, + ...{ + _key: _key + } + }, { + projection: fields + }).sort({ + score: sort + }).skip(skip).limit(limit).toArray(); + } + if (isArray && key.length > 100) { + const batches = []; + const batch = require('../../batch'); + const batchSize = Math.ceil(key.length / Math.ceil(key.length / 100)); + await batch.processArray(key, async currentBatch => batches.push(currentBatch), { + batch: batchSize + }); + const batchData = await Promise.all(batches.map(batch => doQuery({ + $in: batch + }, { + _id: 0, + _key: 0 + }, 0, stop + 1))); + result = dbHelpers.mergeBatch(batchData, 0, stop, sort); + if (start > 0) { + result = result.slice(start, stop !== -1 ? stop + 1 : undefined); + } + } else { + result = await doQuery(query._key, fields, start, limit); + } + if (reverse) { + result.reverse(); + } + if (!withScores) { + result = result.map(item => item.value); + } + return result; + } + module.getSortedSetRangeByScore = async function (key, start, count, min, max) { + return await getSortedSetRangeByScore(key, start, count, min, max, 1, false); + }; + module.getSortedSetRevRangeByScore = async function (key, start, count, max, min) { + return await getSortedSetRangeByScore(key, start, count, min, max, -1, false); + }; + module.getSortedSetRangeByScoreWithScores = async function (key, start, count, min, max) { + return await getSortedSetRangeByScore(key, start, count, min, max, 1, true); + }; + module.getSortedSetRevRangeByScoreWithScores = async function (key, start, count, max, min) { + return await getSortedSetRangeByScore(key, start, count, min, max, -1, true); + }; + async function getSortedSetRangeByScore(key, start, count, min, max, sort, withScores) { + if (parseInt(count, 10) === 0) { + return []; + } + const stop = parseInt(count, 10) === -1 ? -1 : start + count - 1; + return await getSortedSetRange(key, start, stop, min, max, sort, withScores); + } + module.sortedSetCount = async function (key, min, max) { + if (!key) { + return; + } + const query = { + _key: key + }; + if (min !== '-inf') { + query.score = { + $gte: min + }; + } + if (max !== '+inf') { + query.score = query.score || {}; + query.score.$lte = max; + } + return await module.client.collection('objects').countDocuments(query); + }; + module.sortedSetCard = async function (key) { + if (!key) { + return 0; + } + return await module.client.collection('objects').countDocuments({ + _key: key + }); + }; + module.sortedSetsCard = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + return await Promise.all(keys.map(module.sortedSetCard)); + }; + module.sortedSetsCardSum = async function (keys, min = '-inf', max = '+inf') { + const isArray = Array.isArray(keys); + if (!keys || isArray && !keys.length) { + return 0; + } + const query = { + _key: isArray ? { + $in: keys + } : keys + }; + if (min !== '-inf') { + query.score = { + $gte: min + }; + } + if (max !== '+inf') { + query.score = query.score || {}; + query.score.$lte = max; + } + return await module.client.collection('objects').countDocuments(query); + }; + module.sortedSetRank = async function (key, value) { + return await getSortedSetRank(false, key, value); + }; + module.sortedSetRevRank = async function (key, value) { + return await getSortedSetRank(true, key, value); + }; + async function getSortedSetRank(reverse, key, value) { + if (!key) { + return; + } + value = helpers.valueToString(value); + const score = await module.sortedSetScore(key, value); + if (score === null) { + return null; + } + return await module.client.collection('objects').countDocuments({ + $or: [{ + _key: key, + score: reverse ? { + $gt: score + } : { + $lt: score + } + }, { + _key: key, + score: score, + value: reverse ? { + $gt: value + } : { + $lt: value + } + }] + }); + } + module.sortedSetsRanks = async function (keys, values) { + return await sortedSetsRanks(module.sortedSetRank, keys, values); + }; + module.sortedSetsRevRanks = async function (keys, values) { + return await sortedSetsRanks(module.sortedSetRevRank, keys, values); + }; + async function sortedSetsRanks(method, keys, values) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const data = new Array(values.length); + for (let i = 0; i < values.length; i += 1) { + data[i] = { + key: keys[i], + value: values[i] + }; + } + const promises = data.map(item => method(item.key, item.value)); + return await Promise.all(promises); + } + module.sortedSetRanks = async function (key, values) { + return await sortedSetRanks(false, key, values); + }; + module.sortedSetRevRanks = async function (key, values) { + return await sortedSetRanks(true, key, values); + }; + async function sortedSetRanks(reverse, key, values) { + if (values.length === 1) { + return [await getSortedSetRank(reverse, key, values[0])]; + } + const sortedSet = await module[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](key, 0, -1); + return values.map(value => { + if (!value) { + return null; + } + const index = sortedSet.indexOf(value.toString()); + return index !== -1 ? index : null; + }); + } + module.sortedSetScore = async function (key, value) { + if (!key) { + return null; + } + value = helpers.valueToString(value); + const result = await module.client.collection('objects').findOne({ + _key: key, + value: value + }, { + projection: { + _id: 0, + _key: 0, + value: 0 + } + }); + return result ? result.score : null; + }; + module.sortedSetsScore = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + value = helpers.valueToString(value); + const result = await module.client.collection('objects').find({ + _key: { + $in: keys + }, + value: value + }, { + projection: { + _id: 0, + value: 0 + } + }).toArray(); + const map = {}; + result.forEach(item => { + if (item) { + map[item._key] = item; + } + }); + return keys.map(key => map[key] ? map[key].score : null); + }; + module.sortedSetScores = async function (key, values) { + if (!key) { + return null; + } + if (!values.length) { + return []; + } + values = values.map(helpers.valueToString); + const result = await module.client.collection('objects').find({ + _key: key, + value: { + $in: values + } + }, { + projection: { + _id: 0, + _key: 0 + } + }).toArray(); + const valueToScore = {}; + result.forEach(item => { + if (item) { + valueToScore[item.value] = item.score; + } + }); + return values.map(v => utils.isNumber(valueToScore[v]) ? valueToScore[v] : null); + }; + module.isSortedSetMember = async function (key, value) { + if (!key) { + return; + } + value = helpers.valueToString(value); + const result = await module.client.collection('objects').findOne({ + _key: key, + value: value + }, { + projection: { + _id: 0, + value: 1 + } + }); + return !!result; + }; + module.isSortedSetMembers = async function (key, values) { + if (!key) { + return; + } + if (!values.length) { + return []; + } + values = values.map(helpers.valueToString); + const results = await module.client.collection('objects').find({ + _key: key, + value: { + $in: values + } + }, { + projection: { + _id: 0, + value: 1 + } + }).toArray(); + const isMember = {}; + results.forEach(item => { + if (item) { + isMember[item.value] = true; + } + }); + return values.map(value => !!isMember[value]); + }; + module.isMemberOfSortedSets = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + value = helpers.valueToString(value); + const results = await module.client.collection('objects').find({ + _key: { + $in: keys + }, + value: value + }, { + projection: { + _id: 0, + _key: 1, + value: 1 + } + }).toArray(); + const isMember = {}; + results.forEach(item => { + if (item) { + isMember[item._key] = true; + } + }); + return keys.map(key => !!isMember[key]); + }; + module.getSortedSetMembers = async function (key) { + const data = await getSortedSetsMembersWithScores([key], false); + return data && data[0]; + }; + module.getSortedSetMembersWithScores = async function (key) { + const data = await getSortedSetsMembersWithScores([key], true); + return data && data[0]; + }; + module.getSortedSetsMembers = async function (keys) { + return await getSortedSetsMembersWithScores(keys, false); + }; + module.getSortedSetsMembersWithScores = async function (keys) { + return await getSortedSetsMembersWithScores(keys, true); + }; + async function getSortedSetsMembersWithScores(keys, withScores) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const arrayOfKeys = keys.length > 1; + const projection = { + _id: 0, + value: 1 + }; + if (withScores) { + projection.score = 1; + } + if (arrayOfKeys) { + projection._key = 1; + } + const data = await module.client.collection('objects').find({ + _key: arrayOfKeys ? { + $in: keys + } : keys[0] + }, { + projection: projection + }).sort({ + score: 1 + }).toArray(); + if (!arrayOfKeys) { + return [withScores ? data.map(i => ({ + value: i.value, + score: i.score + })) : data.map(item => item.value)]; + } + const sets = {}; + data.forEach(item => { + sets[item._key] = sets[item._key] || []; + if (withScores) { + sets[item._key].push({ + value: item.value, + score: item.score + }); + } else { + sets[item._key].push(item.value); + } + }); + return keys.map(k => sets[k] || []); + } + module.sortedSetIncrBy = async function (key, increment, value) { + if (!key) { + return; + } + const data = {}; + value = helpers.valueToString(value); + data.score = parseFloat(increment); + try { + const result = await module.client.collection('objects').findOneAndUpdate({ + _key: key, + value: value + }, { + $inc: data + }, { + returnDocument: 'after', + includeResultMetadata: true, + upsert: true + }); + return result && result.value ? result.value.score : null; + } catch (err) { + if (err && err.message.includes('E11000 duplicate key error')) { + console.log(new Error('e11000').stack, key, increment, value); + return await module.sortedSetIncrBy(key, increment, value); + } + throw err; + } + }; + module.sortedSetIncrByBulk = async function (data) { + const bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + data.forEach(item => { + bulk.find({ + _key: item[0], + value: helpers.valueToString(item[2]) + }).upsert().update({ + $inc: { + score: parseFloat(item[1]) + } + }); + }); + await bulk.execute(); + const result = await module.client.collection('objects').find({ + _key: { + $in: _.uniq(data.map(i => i[0])) + }, + value: { + $in: _.uniq(data.map(i => i[2])) + } + }, { + projection: { + _id: 0, + _key: 1, + value: 1, + score: 1 + } + }).toArray(); + const map = {}; + result.forEach(item => { + map[`${item._key}:${item.value}`] = item.score; + }); + return data.map(item => map[`${item[0]}:${item[2]}`]); + }; + module.getSortedSetRangeByLex = async function (key, min, max, start, count) { + return await sortedSetLex(key, min, max, 1, start, count); + }; + module.getSortedSetRevRangeByLex = async function (key, max, min, start, count) { + return await sortedSetLex(key, min, max, -1, start, count); + }; + module.sortedSetLexCount = async function (key, min, max) { + const data = await sortedSetLex(key, min, max, 1, 0, 0); + return data ? data.length : null; + }; + async function sortedSetLex(key, min, max, sort, start, count) { + const query = { + _key: key + }; + start = start !== undefined ? start : 0; + count = count !== undefined ? count : 0; + buildLexQuery(query, min, max); + const data = await module.client.collection('objects').find(query, { + projection: { + _id: 0, + value: 1 + } + }).sort({ + value: sort + }).skip(start).limit(count === -1 ? 0 : count).toArray(); + return data.map(item => item && item.value); + } + module.sortedSetRemoveRangeByLex = async function (key, min, max) { + const query = { + _key: key + }; + buildLexQuery(query, min, max); + await module.client.collection('objects').deleteMany(query); + }; + function buildLexQuery(query, min, max) { + if (min !== '-') { + if (min.match(/^\(/)) { + query.value = { + $gt: min.slice(1) + }; + } else if (min.match(/^\[/)) { + query.value = { + $gte: min.slice(1) + }; + } else { + query.value = { + $gte: min + }; + } + } + if (max !== '+') { + query.value = query.value || {}; + if (max.match(/^\(/)) { + query.value.$lt = max.slice(1); + } else if (max.match(/^\[/)) { + query.value.$lte = max.slice(1); + } else { + query.value.$lte = max; + } + } + } + module.getSortedSetScan = async function (params) { + const project = { + _id: 0, + value: 1 + }; + if (params.withScores) { + project.score = 1; + } + const match = helpers.buildMatchQuery(params.match); + let regex; + try { + regex = new RegExp(match); + } catch (err) { + return []; + } + const cursor = module.client.collection('objects').find({ + _key: params.key, + value: { + $regex: regex + } + }, { + projection: project + }); + if (params.limit) { + cursor.limit(params.limit); + } + const data = await cursor.toArray(); + if (!params.withScores) { + return data.map(d => d.value); + } + return data; + }; + module.processSortedSet = async function (setKey, processFn, options) { + let done = false; + const ids = []; + const project = { + _id: 0, + _key: 0 + }; + const sort = options.reverse ? -1 : 1; + if (!options.withScores) { + project.score = 0; + } + const query = { + _key: setKey + }; + if (options.min && options.min !== '-inf') { + query.score = { + $gte: parseFloat(options.min) + }; + } + if (options.max && options.max !== '+inf') { + query.score = query.score || {}; + query.score.$lte = parseFloat(options.max); + } + const cursor = await module.client.collection('objects').find(query, { + projection: project + }).sort({ + score: sort + }).batchSize(options.batch); + if (processFn && processFn.constructor && processFn.constructor.name !== 'AsyncFunction') { + processFn = util.promisify(processFn); + } + let iteration = 1; + while (!done) { + const item = await cursor.next(); + if (item === null) { + done = true; + } else { + ids.push(options.withScores ? item : item.value); + } + if (ids.length >= options.batch || done && ids.length !== 0) { + if (iteration > 1 && options.interval) { + await sleep(options.interval); + } + await processFn(ids); + iteration += 1; + ids.length = 0; + } + } + }; +}; \ No newline at end of file diff --git a/lib/database/mongo/sorted/add.js b/lib/database/mongo/sorted/add.js new file mode 100644 index 0000000000..6e31ce94a6 --- /dev/null +++ b/lib/database/mongo/sorted/add.js @@ -0,0 +1,107 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('../helpers'); + const utils = require('../../../utils'); + module.sortedSetAdd = async function (key, score, value) { + if (!key) { + return; + } + if (Array.isArray(score) && Array.isArray(value)) { + return await sortedSetAddBulk(key, score, value); + } + if (!utils.isNumber(score)) { + throw new Error(`[[error:invalid-score, ${score}]]`); + } + value = helpers.valueToString(value); + try { + await module.client.collection('objects').updateOne({ + _key: key, + value: value + }, { + $set: { + score: parseFloat(score) + } + }, { + upsert: true + }); + } catch (err) { + if (err && err.message.includes('E11000 duplicate key error')) { + console.log(new Error('e11000').stack, key, score, value); + return await module.sortedSetAdd(key, score, value); + } + throw err; + } + }; + async function sortedSetAddBulk(key, scores, values) { + if (!scores.length || !values.length) { + return; + } + if (scores.length !== values.length) { + throw new Error('[[error:invalid-data]]'); + } + for (let i = 0; i < scores.length; i += 1) { + if (!utils.isNumber(scores[i])) { + throw new Error(`[[error:invalid-score, ${scores[i]}]]`); + } + } + values = values.map(helpers.valueToString); + const bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + for (let i = 0; i < scores.length; i += 1) { + bulk.find({ + _key: key, + value: values[i] + }).upsert().updateOne({ + $set: { + score: parseFloat(scores[i]) + } + }); + } + await bulk.execute(); + } + module.sortedSetsAdd = async function (keys, scores, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + const isArrayOfScores = Array.isArray(scores); + if (!isArrayOfScores && !utils.isNumber(scores) || isArrayOfScores && scores.map(s => utils.isNumber(s)).includes(false)) { + throw new Error(`[[error:invalid-score, ${scores}]]`); + } + if (isArrayOfScores && scores.length !== keys.length) { + throw new Error('[[error:invalid-data]]'); + } + value = helpers.valueToString(value); + const bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + for (let i = 0; i < keys.length; i += 1) { + bulk.find({ + _key: keys[i], + value: value + }).upsert().updateOne({ + $set: { + score: parseFloat(isArrayOfScores ? scores[i] : scores) + } + }); + } + await bulk.execute(); + }; + module.sortedSetAddBulk = async function (data) { + if (!Array.isArray(data) || !data.length) { + return; + } + const bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + data.forEach(item => { + if (!utils.isNumber(item[1])) { + throw new Error(`[[error:invalid-score, ${item[1]}]]`); + } + bulk.find({ + _key: item[0], + value: String(item[2]) + }).upsert().updateOne({ + $set: { + score: parseFloat(item[1]) + } + }); + }); + await bulk.execute(); + }; +}; \ No newline at end of file diff --git a/lib/database/mongo/sorted/intersect.js b/lib/database/mongo/sorted/intersect.js new file mode 100644 index 0000000000..2455051530 --- /dev/null +++ b/lib/database/mongo/sorted/intersect.js @@ -0,0 +1,271 @@ +'use strict'; + +module.exports = function (module) { + module.sortedSetIntersectCard = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return 0; + } + const objects = module.client.collection('objects'); + const counts = await countSets(keys, 50000); + if (counts.minCount === 0) { + return 0; + } + let items = await objects.find({ + _key: counts.smallestSet + }, { + projection: { + _id: 0, + value: 1 + } + }).batchSize(counts.minCount + 1).toArray(); + const otherSets = keys.filter(s => s !== counts.smallestSet); + for (let i = 0; i < otherSets.length; i++) { + const query = { + _key: otherSets[i], + value: { + $in: items.map(i => i.value) + } + }; + if (i === otherSets.length - 1) { + return await objects.countDocuments(query); + } + items = await objects.find(query, { + projection: { + _id: 0, + value: 1 + } + }).batchSize(items.length + 1).toArray(); + } + }; + async function countSets(sets, limit) { + const objects = module.client.collection('objects'); + const counts = await Promise.all(sets.map(s => objects.countDocuments({ + _key: s + }, { + limit: limit || 25000 + }))); + const minCount = Math.min(...counts); + const index = counts.indexOf(minCount); + const smallestSet = sets[index]; + return { + minCount: minCount, + smallestSet: smallestSet + }; + } + module.getSortedSetIntersect = async function (params) { + params.sort = 1; + return await getSortedSetRevIntersect(params); + }; + module.getSortedSetRevIntersect = async function (params) { + params.sort = -1; + return await getSortedSetRevIntersect(params); + }; + async function getSortedSetRevIntersect(params) { + params.start = params.hasOwnProperty('start') ? params.start : 0; + params.stop = params.hasOwnProperty('stop') ? params.stop : -1; + params.weights = params.weights || []; + params.limit = params.stop - params.start + 1; + if (params.limit <= 0) { + params.limit = 0; + } + params.counts = await countSets(params.sets); + if (params.counts.minCount === 0) { + return []; + } + const simple = params.weights.filter(w => w === 1).length === 1 && params.limit !== 0; + if (params.counts.minCount < 25000 && simple) { + return await intersectSingle(params); + } else if (simple) { + return await intersectBatch(params); + } + return await intersectAggregate(params); + } + async function intersectSingle(params) { + const objects = module.client.collection('objects'); + const sortSet = params.sets[params.weights.indexOf(1)]; + if (sortSet === params.counts.smallestSet) { + return await intersectBatch(params); + } + const cursorSmall = objects.find({ + _key: params.counts.smallestSet + }, { + projection: { + _id: 0, + value: 1 + } + }); + if (params.counts.minCount > 1) { + cursorSmall.batchSize(params.counts.minCount + 1); + } + let items = await cursorSmall.toArray(); + const project = { + _id: 0, + value: 1 + }; + if (params.withScores) { + project.score = 1; + } + const otherSets = params.sets.filter(s => s !== params.counts.smallestSet); + otherSets.push(otherSets.splice(otherSets.indexOf(sortSet), 1)[0]); + for (let i = 0; i < otherSets.length; i++) { + const cursor = objects.find({ + _key: otherSets[i], + value: { + $in: items.map(i => i.value) + } + }); + cursor.batchSize(items.length + 1); + if (i === otherSets.length - 1) { + cursor.project(project).sort({ + score: params.sort + }).skip(params.start).limit(params.limit); + } else { + cursor.project({ + _id: 0, + value: 1 + }); + } + items = await cursor.toArray(); + } + if (!params.withScores) { + items = items.map(i => i.value); + } + return items; + } + async function intersectBatch(params) { + const project = { + _id: 0, + value: 1 + }; + if (params.withScores) { + project.score = 1; + } + const sortSet = params.sets[params.weights.indexOf(1)]; + const batchSize = 10000; + const cursor = await module.client.collection('objects').find({ + _key: sortSet + }, { + projection: project + }).sort({ + score: params.sort + }).batchSize(batchSize); + const otherSets = params.sets.filter(s => s !== sortSet); + let inters = []; + let done = false; + while (!done) { + const items = []; + while (items.length < batchSize) { + const nextItem = await cursor.next(); + if (!nextItem) { + done = true; + break; + } + items.push(nextItem); + } + const members = await Promise.all(otherSets.map(async s => { + const data = await module.client.collection('objects').find({ + _key: s, + value: { + $in: items.map(i => i.value) + } + }, { + projection: { + _id: 0, + value: 1 + } + }).batchSize(items.length + 1).toArray(); + return new Set(data.map(i => i.value)); + })); + inters = inters.concat(items.filter(item => members.every(arr => arr.has(item.value)))); + if (inters.length >= params.stop) { + done = true; + inters = inters.slice(params.start, params.stop + 1); + } + } + if (!params.withScores) { + inters = inters.map(item => item.value); + } + return inters; + } + async function intersectAggregate(params) { + const aggregate = {}; + if (params.aggregate) { + aggregate[`$${params.aggregate.toLowerCase()}`] = '$score'; + } else { + aggregate.$sum = '$score'; + } + const pipeline = [{ + $match: { + _key: { + $in: params.sets + } + } + }]; + params.weights.forEach((weight, index) => { + if (weight !== 1) { + pipeline.push({ + $project: { + value: 1, + score: { + $cond: { + if: { + $eq: ['$_key', params.sets[index]] + }, + then: { + $multiply: ['$score', weight] + }, + else: '$score' + } + } + } + }); + } + }); + pipeline.push({ + $group: { + _id: { + value: '$value' + }, + totalScore: aggregate, + count: { + $sum: 1 + } + } + }); + pipeline.push({ + $match: { + count: params.sets.length + } + }); + pipeline.push({ + $sort: { + totalScore: params.sort + } + }); + if (params.start) { + pipeline.push({ + $skip: params.start + }); + } + if (params.limit > 0) { + pipeline.push({ + $limit: params.limit + }); + } + const project = { + _id: 0, + value: '$_id.value' + }; + if (params.withScores) { + project.score = '$totalScore'; + } + pipeline.push({ + $project: project + }); + let data = await module.client.collection('objects').aggregate(pipeline).toArray(); + if (!params.withScores) { + data = data.map(item => item.value); + } + return data; + } +}; \ No newline at end of file diff --git a/lib/database/mongo/sorted/remove.js b/lib/database/mongo/sorted/remove.js new file mode 100644 index 0000000000..43183b2215 --- /dev/null +++ b/lib/database/mongo/sorted/remove.js @@ -0,0 +1,73 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('../helpers'); + module.sortedSetRemove = async function (key, value) { + if (!key) { + return; + } + const isValueArray = Array.isArray(value); + if (!value || isValueArray && !value.length) { + return; + } + if (isValueArray) { + value = value.map(helpers.valueToString); + } else { + value = helpers.valueToString(value); + } + await module.client.collection('objects').deleteMany({ + _key: Array.isArray(key) ? { + $in: key + } : key, + value: isValueArray ? { + $in: value + } : value + }); + }; + module.sortedSetsRemove = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + value = helpers.valueToString(value); + await module.client.collection('objects').deleteMany({ + _key: { + $in: keys + }, + value: value + }); + }; + module.sortedSetsRemoveRangeByScore = async function (keys, min, max) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + const query = { + _key: { + $in: keys + } + }; + if (keys.length === 1) { + query._key = keys[0]; + } + if (min !== '-inf') { + query.score = { + $gte: parseFloat(min) + }; + } + if (max !== '+inf') { + query.score = query.score || {}; + query.score.$lte = parseFloat(max); + } + await module.client.collection('objects').deleteMany(query); + }; + module.sortedSetRemoveBulk = async function (data) { + if (!Array.isArray(data) || !data.length) { + return; + } + const bulk = module.client.collection('objects').initializeUnorderedBulkOp(); + data.forEach(item => bulk.find({ + _key: item[0], + value: String(item[1]) + }).delete()); + await bulk.execute(); + }; +}; \ No newline at end of file diff --git a/lib/database/mongo/sorted/union.js b/lib/database/mongo/sorted/union.js new file mode 100644 index 0000000000..be47a58349 --- /dev/null +++ b/lib/database/mongo/sorted/union.js @@ -0,0 +1,97 @@ +'use strict'; + +module.exports = function (module) { + module.sortedSetUnionCard = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return 0; + } + const data = await module.client.collection('objects').aggregate([{ + $match: { + _key: { + $in: keys + } + } + }, { + $group: { + _id: { + value: '$value' + } + } + }, { + $group: { + _id: null, + count: { + $sum: 1 + } + } + }]).toArray(); + return Array.isArray(data) && data.length ? data[0].count : 0; + }; + module.getSortedSetUnion = async function (params) { + params.sort = 1; + return await getSortedSetUnion(params); + }; + module.getSortedSetRevUnion = async function (params) { + params.sort = -1; + return await getSortedSetUnion(params); + }; + async function getSortedSetUnion(params) { + if (!Array.isArray(params.sets) || !params.sets.length) { + return []; + } + let limit = params.stop - params.start + 1; + if (limit <= 0) { + limit = 0; + } + const aggregate = {}; + if (params.aggregate) { + aggregate[`$${params.aggregate.toLowerCase()}`] = '$score'; + } else { + aggregate.$sum = '$score'; + } + const pipeline = [{ + $match: { + _key: { + $in: params.sets + } + } + }, { + $group: { + _id: { + value: '$value' + }, + totalScore: aggregate + } + }, { + $sort: { + totalScore: params.sort, + _id: 1 + } + }]; + if (params.start) { + pipeline.push({ + $skip: params.start + }); + } + if (limit > 0) { + pipeline.push({ + $limit: limit + }); + } + const project = { + _id: 0, + value: '$_id.value' + }; + if (params.withScores) { + project.score = '$totalScore'; + } + pipeline.push({ + $project: project + }); + let data = await module.client.collection('objects').aggregate(pipeline).toArray(); + if (!params.withScores) { + data = data.map(item => item.value); + } + return data; + } +}; \ No newline at end of file diff --git a/lib/database/mongo/transaction.js b/lib/database/mongo/transaction.js new file mode 100644 index 0000000000..e7492c3d65 --- /dev/null +++ b/lib/database/mongo/transaction.js @@ -0,0 +1,7 @@ +'use strict'; + +module.exports = function (module) { + module.transaction = function (perform, callback) { + perform(module.client, callback); + }; +}; \ No newline at end of file diff --git a/lib/database/postgres.js b/lib/database/postgres.js new file mode 100644 index 0000000000..8006cf50e0 --- /dev/null +++ b/lib/database/postgres.js @@ -0,0 +1,375 @@ +'use strict'; + +const winston = require('winston'); +const nconf = require('nconf'); +const session = require('express-session'); +const semver = require('semver'); +const connection = require('./postgres/connection'); +const postgresModule = module.exports; +postgresModule.questions = [{ + name: 'postgres:host', + description: 'Host IP or address of your PostgreSQL instance', + default: nconf.get('postgres:host') || nconf.get('defaults:postgres:host') || '127.0.0.1' +}, { + name: 'postgres:port', + description: 'Host port of your PostgreSQL instance', + default: nconf.get('postgres:port') || nconf.get('defaults:postgres:port') || 5432 +}, { + name: 'postgres:username', + description: 'PostgreSQL username', + default: nconf.get('postgres:username') || nconf.get('defaults:postgres:username') || '' +}, { + name: 'postgres:password', + description: 'Password of your PostgreSQL database', + hidden: true, + default: nconf.get('postgres:password') || nconf.get('defaults:postgres:password') || '', + before: function (value) { + value = value || nconf.get('postgres:password') || ''; + return value; + } +}, { + name: 'postgres:database', + description: 'PostgreSQL database name', + default: nconf.get('postgres:database') || nconf.get('defaults:postgres:database') || 'nodebb' +}, { + name: 'postgres:ssl', + description: 'Enable SSL for PostgreSQL database access', + default: nconf.get('postgres:ssl') || nconf.get('defaults:postgres:ssl') || false +}]; +postgresModule.init = async function (opts) { + const { + Pool + } = require('pg'); + const connOptions = connection.getConnectionOptions(opts); + const pool = new Pool(connOptions); + postgresModule.pool = pool; + postgresModule.client = pool; + const client = await pool.connect(); + try { + await checkUpgrade(client); + } catch (err) { + winston.error(`NodeBB could not connect to your PostgreSQL database. PostgreSQL returned the following error: ${err.message}`); + throw err; + } finally { + client.release(); + } +}; +async function checkUpgrade(client) { + const res = await client.query(` +SELECT EXISTS(SELECT * + FROM "information_schema"."columns" + WHERE "table_schema" = 'public' + AND "table_name" = 'objects' + AND "column_name" = 'data') a, + EXISTS(SELECT * + FROM "information_schema"."columns" + WHERE "table_schema" = 'public' + AND "table_name" = 'legacy_hash' + AND "column_name" = '_key') b, + EXISTS(SELECT * + FROM "information_schema"."routines" + WHERE "routine_schema" = 'public' + AND "routine_name" = 'nodebb_get_sorted_set_members') c, + EXISTS(SELECT * + FROM "information_schema"."routines" + WHERE "routine_schema" = 'public' + AND "routine_name" = 'nodebb_get_sorted_set_members_withscores') d`); + if (res.rows[0].a && res.rows[0].b && res.rows[0].c && res.rows[0].d) { + return; + } + await client.query(`BEGIN`); + try { + if (!res.rows[0].b) { + await client.query(` +CREATE TYPE LEGACY_OBJECT_TYPE AS ENUM ( + 'hash', 'zset', 'set', 'list', 'string' +)`); + await client.query(` +CREATE TABLE "legacy_object" ( + "_key" TEXT NOT NULL + PRIMARY KEY, + "type" LEGACY_OBJECT_TYPE NOT NULL, + "expireAt" TIMESTAMPTZ DEFAULT NULL, + UNIQUE ( "_key", "type" ) +)`); + await client.query(` +CREATE TABLE "legacy_hash" ( + "_key" TEXT NOT NULL + PRIMARY KEY, + "data" JSONB NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'hash'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'hash' ), + CONSTRAINT "fk__legacy_hash__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`); + await client.query(` +CREATE TABLE "legacy_zset" ( + "_key" TEXT NOT NULL, + "value" TEXT NOT NULL, + "score" NUMERIC NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'zset'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'zset' ), + PRIMARY KEY ("_key", "value"), + CONSTRAINT "fk__legacy_zset__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`); + await client.query(` +CREATE TABLE "legacy_set" ( + "_key" TEXT NOT NULL, + "member" TEXT NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'set'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'set' ), + PRIMARY KEY ("_key", "member"), + CONSTRAINT "fk__legacy_set__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`); + await client.query(` +CREATE TABLE "legacy_list" ( + "_key" TEXT NOT NULL + PRIMARY KEY, + "array" TEXT[] NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'list'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'list' ), + CONSTRAINT "fk__legacy_list__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`); + await client.query(` +CREATE TABLE "legacy_string" ( + "_key" TEXT NOT NULL + PRIMARY KEY, + "data" TEXT NOT NULL, + "type" LEGACY_OBJECT_TYPE NOT NULL + DEFAULT 'string'::LEGACY_OBJECT_TYPE + CHECK ( "type" = 'string' ), + CONSTRAINT "fk__legacy_string__key" + FOREIGN KEY ("_key", "type") + REFERENCES "legacy_object"("_key", "type") + ON UPDATE CASCADE + ON DELETE CASCADE +)`); + if (res.rows[0].a) { + await client.query(` +INSERT INTO "legacy_object" ("_key", "type", "expireAt") +SELECT DISTINCT "data"->>'_key', + CASE WHEN (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + THEN CASE WHEN ("data" ? 'value') + OR ("data" ? 'data') + THEN 'string' + WHEN "data" ? 'array' + THEN 'list' + WHEN "data" ? 'members' + THEN 'set' + ELSE 'hash' + END + WHEN (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 3 + THEN CASE WHEN ("data" ? 'value') + AND ("data" ? 'score') + THEN 'zset' + ELSE 'hash' + END + ELSE 'hash' + END::LEGACY_OBJECT_TYPE, + CASE WHEN ("data" ? 'expireAt') + THEN to_timestamp(("data"->>'expireAt')::double precision / 1000) + ELSE NULL + END + FROM "objects"`); + await client.query(` +INSERT INTO "legacy_hash" ("_key", "data") +SELECT "data"->>'_key', + "data" - '_key' - 'expireAt' + FROM "objects" + WHERE CASE WHEN (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + THEN NOT (("data" ? 'value') + OR ("data" ? 'data') + OR ("data" ? 'members') + OR ("data" ? 'array')) + WHEN (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 3 + THEN NOT (("data" ? 'value') + AND ("data" ? 'score')) + ELSE TRUE + END`); + await client.query(` +INSERT INTO "legacy_zset" ("_key", "value", "score") +SELECT "data"->>'_key', + "data"->>'value', + ("data"->>'score')::NUMERIC + FROM "objects" + WHERE (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 3 + AND ("data" ? 'value') + AND ("data" ? 'score')`); + await client.query(` +INSERT INTO "legacy_set" ("_key", "member") +SELECT "data"->>'_key', + jsonb_array_elements_text("data"->'members') + FROM "objects" + WHERE (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + AND ("data" ? 'members')`); + await client.query(` +INSERT INTO "legacy_list" ("_key", "array") +SELECT "data"->>'_key', + ARRAY(SELECT t + FROM jsonb_array_elements_text("data"->'list') WITH ORDINALITY l(t, i) + ORDER BY i ASC) + FROM "objects" + WHERE (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + AND ("data" ? 'array')`); + await client.query(` +INSERT INTO "legacy_string" ("_key", "data") +SELECT "data"->>'_key', + CASE WHEN "data" ? 'value' + THEN "data"->>'value' + ELSE "data"->>'data' + END + FROM "objects" + WHERE (SELECT COUNT(*) + FROM jsonb_object_keys("data" - 'expireAt')) = 2 + AND (("data" ? 'value') + OR ("data" ? 'data'))`); + await client.query(`DROP TABLE "objects" CASCADE`); + await client.query(`DROP FUNCTION "fun__objects__expireAt"() CASCADE`); + } + await client.query(` +CREATE VIEW "legacy_object_live" AS +SELECT "_key", "type" + FROM "legacy_object" + WHERE "expireAt" IS NULL + OR "expireAt" > CURRENT_TIMESTAMP`); + } + if (!res.rows[0].c) { + await client.query(` +CREATE FUNCTION "nodebb_get_sorted_set_members"(TEXT) RETURNS TEXT[] AS $$ + SELECT array_agg(z."value" ORDER BY z."score" ASC) + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1 +$$ LANGUAGE sql +STABLE +STRICT +PARALLEL SAFE`); + } + if (!res.rows[0].d) { + await client.query(` + CREATE FUNCTION "nodebb_get_sorted_set_members_withscores"(TEXT) RETURNS JSON AS $$ + SELECT json_agg(json_build_object('value', z."value", 'score', z."score") ORDER BY z."score" ASC) as item + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1 + $$ LANGUAGE sql + STABLE + STRICT + PARALLEL SAFE`); + } + } catch (ex) { + await client.query(`ROLLBACK`); + throw ex; + } + await client.query(`COMMIT`); +} +postgresModule.createSessionStore = async function (options) { + const meta = require('../meta'); + function done(db) { + const sessionStore = require('connect-pg-simple')(session); + return new sessionStore({ + pool: db, + ttl: meta.getSessionTTLSeconds(), + pruneSessionInterval: nconf.get('isPrimary') ? 60 : false + }); + } + const db = await connection.connect(options); + if (!nconf.get('isPrimary')) { + return done(db); + } + await db.query(` +CREATE TABLE IF NOT EXISTS "session" ( + "sid" CHAR(32) NOT NULL + COLLATE "C" + PRIMARY KEY, + "sess" JSONB NOT NULL, + "expire" TIMESTAMPTZ NOT NULL +) WITHOUT OIDS; + +CREATE INDEX IF NOT EXISTS "session_expire_idx" ON "session"("expire"); + +ALTER TABLE "session" + ALTER "sid" SET STORAGE MAIN, + CLUSTER ON "session_expire_idx";`); + return done(db); +}; +postgresModule.createIndices = async function () { + if (!postgresModule.pool) { + winston.warn('[database/createIndices] database not initialized'); + return; + } + winston.info('[database] Checking database indices.'); + try { + await postgresModule.pool.query(`CREATE INDEX IF NOT EXISTS "idx__legacy_zset__key__score" ON "legacy_zset"("_key" ASC, "score" DESC)`); + await postgresModule.pool.query(`CREATE INDEX IF NOT EXISTS "idx__legacy_object__expireAt" ON "legacy_object"("expireAt" ASC)`); + winston.info('[database] Checking database indices done!'); + } catch (err) { + winston.error(`Error creating index ${err.message}`); + throw err; + } +}; +postgresModule.checkCompatibility = function (callback) { + const postgresPkg = require('pg/package.json'); + postgresModule.checkCompatibilityVersion(postgresPkg.version, callback); +}; +postgresModule.checkCompatibilityVersion = function (version, callback) { + if (semver.lt(version, '7.0.0')) { + return callback(new Error('The `pg` package is out-of-date, please run `./nodebb setup` again.')); + } + callback(); +}; +postgresModule.info = async function (db) { + if (!db) { + db = await connection.connect(nconf.get('postgres')); + } + postgresModule.pool = postgresModule.pool || db; + const res = await db.query(` + SELECT true "postgres", + current_setting('server_version') "version", + EXTRACT(EPOCH FROM NOW() - pg_postmaster_start_time()) * 1000 "uptime" + `); + return { + ...res.rows[0], + raw: JSON.stringify(res.rows[0], null, 4) + }; +}; +postgresModule.close = async function () { + await postgresModule.pool.end(); +}; +require('./postgres/main')(postgresModule); +require('./postgres/hash')(postgresModule); +require('./postgres/sets')(postgresModule); +require('./postgres/sorted')(postgresModule); +require('./postgres/list')(postgresModule); +require('./postgres/transaction')(postgresModule); +require('../promisify')(postgresModule, ['client', 'sessionStore', 'pool', 'transaction']); \ No newline at end of file diff --git a/lib/database/postgres/connection.js b/lib/database/postgres/connection.js new file mode 100644 index 0000000000..1e8c484892 --- /dev/null +++ b/lib/database/postgres/connection.js @@ -0,0 +1,41 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const _ = require('lodash'); +const connection = module.exports; +connection.getConnectionOptions = function (postgres) { + postgres = postgres || nconf.get('postgres'); + if (!postgres.host) { + postgres.host = '127.0.0.1'; + } + if (!postgres.port) { + postgres.port = 5432; + } + const dbName = postgres.database; + if (dbName === undefined || dbName === '') { + winston.warn('You have no database name, using "nodebb"'); + postgres.database = 'nodebb'; + } + const connOptions = { + host: postgres.host, + port: postgres.port, + user: postgres.username, + password: postgres.password, + database: postgres.database, + ssl: String(postgres.ssl) === 'true', + max: 20, + connectionTimeoutMillis: 90000 + }; + return _.merge(connOptions, postgres.options || {}); +}; +connection.connect = async function (options) { + const { + Pool + } = require('pg'); + const connOptions = connection.getConnectionOptions(options); + const db = new Pool(connOptions); + await db.connect(); + return db; +}; +require('../../promisify')(connection); \ No newline at end of file diff --git a/lib/database/postgres/hash.js b/lib/database/postgres/hash.js new file mode 100644 index 0000000000..cb77fad823 --- /dev/null +++ b/lib/database/postgres/hash.js @@ -0,0 +1,347 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + module.setObject = async function (key, data) { + if (!key || !data) { + return; + } + if (data.hasOwnProperty('')) { + delete data['']; + } + if (!Object.keys(data).length) { + return; + } + await module.transaction(async client => { + const dataString = JSON.stringify(data); + if (Array.isArray(key)) { + await helpers.ensureLegacyObjectsType(client, key, 'hash'); + await client.query({ + name: 'setObjectKeys', + text: ` + INSERT INTO "legacy_hash" ("_key", "data") + SELECT k, $2::TEXT::JSONB + FROM UNNEST($1::TEXT[]) vs(k) + ON CONFLICT ("_key") + DO UPDATE SET "data" = "legacy_hash"."data" || $2::TEXT::JSONB`, + values: [key, dataString] + }); + } else { + await helpers.ensureLegacyObjectType(client, key, 'hash'); + await client.query({ + name: 'setObject', + text: ` + INSERT INTO "legacy_hash" ("_key", "data") + VALUES ($1::TEXT, $2::TEXT::JSONB) + ON CONFLICT ("_key") + DO UPDATE SET "data" = "legacy_hash"."data" || $2::TEXT::JSONB`, + values: [key, dataString] + }); + } + }); + }; + module.setObjectBulk = async function (...args) { + let data = args[0]; + if (!Array.isArray(data) || !data.length) { + return; + } + if (Array.isArray(args[1])) { + console.warn('[deprecated] db.setObjectBulk(keys, data) usage is deprecated, please use db.setObjectBulk(data)'); + data = args[0].map((key, i) => [key, args[1][i]]); + } + await module.transaction(async client => { + data = data.filter(item => { + if (item[1].hasOwnProperty('')) { + delete item[1]['']; + } + return !!Object.keys(item[1]).length; + }); + const keys = data.map(item => item[0]); + if (!keys.length) { + return; + } + await helpers.ensureLegacyObjectsType(client, keys, 'hash'); + const dataStrings = data.map(item => JSON.stringify(item[1])); + await client.query({ + name: 'setObjectBulk', + text: ` + INSERT INTO "legacy_hash" ("_key", "data") + SELECT k, d + FROM UNNEST($1::TEXT[], $2::TEXT::JSONB[]) vs(k, d) + ON CONFLICT ("_key") + DO UPDATE SET "data" = "legacy_hash"."data" || EXCLUDED.data`, + values: [keys, dataStrings] + }); + }); + }; + module.setObjectField = async function (key, field, value) { + if (!field) { + return; + } + await module.transaction(async client => { + const valueString = JSON.stringify(value); + if (Array.isArray(key)) { + await module.setObject(key, { + [field]: value + }); + } else { + await helpers.ensureLegacyObjectType(client, key, 'hash'); + await client.query({ + name: 'setObjectField', + text: ` + INSERT INTO "legacy_hash" ("_key", "data") + VALUES ($1::TEXT, jsonb_build_object($2::TEXT, $3::TEXT::JSONB)) + ON CONFLICT ("_key") + DO UPDATE SET "data" = jsonb_set("legacy_hash"."data", ARRAY[$2::TEXT], $3::TEXT::JSONB)`, + values: [key, field, valueString] + }); + } + }); + }; + module.getObject = async function (key, fields = []) { + if (!key) { + return null; + } + if (fields.length) { + return await module.getObjectFields(key, fields); + } + const res = await module.pool.query({ + name: 'getObject', + text: ` +SELECT h."data" + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key] + }); + return res.rows.length ? res.rows[0].data : null; + }; + module.getObjects = async function (keys, fields = []) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + if (fields.length) { + return await module.getObjectsFields(keys, fields); + } + const res = await module.pool.query({ + name: 'getObjects', + text: ` +SELECT h."data" + FROM UNNEST($1::TEXT[]) WITH ORDINALITY k("_key", i) + LEFT OUTER JOIN "legacy_object_live" o + ON o."_key" = k."_key" + LEFT OUTER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + ORDER BY k.i ASC`, + values: [keys] + }); + return res.rows.map(row => row.data); + }; + module.getObjectField = async function (key, field) { + if (!key) { + return null; + } + const res = await module.pool.query({ + name: 'getObjectField', + text: ` +SELECT h."data"->>$2::TEXT f + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key, field] + }); + return res.rows.length ? res.rows[0].f : null; + }; + module.getObjectFields = async function (key, fields) { + if (!key) { + return null; + } + if (!Array.isArray(fields) || !fields.length) { + return await module.getObject(key); + } + const res = await module.pool.query({ + name: 'getObjectFields', + text: ` +SELECT (SELECT jsonb_object_agg(f, d."value") + FROM UNNEST($2::TEXT[]) f + LEFT OUTER JOIN jsonb_each(h."data") d + ON d."key" = f) d + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT`, + values: [key, fields] + }); + if (res.rows.length) { + return res.rows[0].d; + } + const obj = {}; + fields.forEach(f => { + obj[f] = null; + }); + return obj; + }; + module.getObjectsFields = async function (keys, fields) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + if (!Array.isArray(fields) || !fields.length) { + return await module.getObjects(keys); + } + const res = await module.pool.query({ + name: 'getObjectsFields', + text: ` +SELECT (SELECT jsonb_object_agg(f, d."value") + FROM UNNEST($2::TEXT[]) f + LEFT OUTER JOIN jsonb_each(h."data") d + ON d."key" = f) d + FROM UNNEST($1::text[]) WITH ORDINALITY k("_key", i) + LEFT OUTER JOIN "legacy_object_live" o + ON o."_key" = k."_key" + LEFT OUTER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + ORDER BY k.i ASC`, + values: [keys, fields] + }); + return res.rows.map(row => row.d); + }; + module.getObjectKeys = async function (key) { + if (!key) { + return; + } + const res = await module.pool.query({ + name: 'getObjectKeys', + text: ` +SELECT ARRAY(SELECT jsonb_object_keys(h."data")) k + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key] + }); + return res.rows.length ? res.rows[0].k : []; + }; + module.getObjectValues = async function (key) { + const data = await module.getObject(key); + return data ? Object.values(data) : []; + }; + module.isObjectField = async function (key, field) { + if (!key) { + return; + } + const res = await module.pool.query({ + name: 'isObjectField', + text: ` +SELECT (h."data" ? $2::TEXT AND h."data"->>$2::TEXT IS NOT NULL) b + FROM "legacy_object_live" o + INNER JOIN "legacy_hash" h + ON o."_key" = h."_key" + AND o."type" = h."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key, field] + }); + return res.rows.length ? res.rows[0].b : false; + }; + module.isObjectFields = async function (key, fields) { + if (!key) { + return; + } + const data = await module.getObjectFields(key, fields); + if (!data) { + return fields.map(() => false); + } + return fields.map(field => data.hasOwnProperty(field) && data[field] !== null); + }; + module.deleteObjectField = async function (key, field) { + await module.deleteObjectFields(key, [field]); + }; + module.deleteObjectFields = async function (key, fields) { + if (!key || Array.isArray(key) && !key.length || !Array.isArray(fields) || !fields.length) { + return; + } + if (Array.isArray(key)) { + await module.pool.query({ + name: 'deleteObjectFieldsKeys', + text: ` + UPDATE "legacy_hash" + SET "data" = COALESCE((SELECT jsonb_object_agg("key", "value") + FROM jsonb_each("data") + WHERE "key" <> ALL ($2::TEXT[])), '{}') + WHERE "_key" = ANY($1::TEXT[])`, + values: [key, fields] + }); + } else { + await module.pool.query({ + name: 'deleteObjectFields', + text: ` + UPDATE "legacy_hash" + SET "data" = COALESCE((SELECT jsonb_object_agg("key", "value") + FROM jsonb_each("data") + WHERE "key" <> ALL ($2::TEXT[])), '{}') + WHERE "_key" = $1::TEXT`, + values: [key, fields] + }); + } + }; + module.incrObjectField = async function (key, field) { + return await module.incrObjectFieldBy(key, field, 1); + }; + module.decrObjectField = async function (key, field) { + return await module.incrObjectFieldBy(key, field, -1); + }; + module.incrObjectFieldBy = async function (key, field, value) { + value = parseInt(value, 10); + if (!key || isNaN(value)) { + return null; + } + return await module.transaction(async client => { + if (Array.isArray(key)) { + await helpers.ensureLegacyObjectsType(client, key, 'hash'); + } else { + await helpers.ensureLegacyObjectType(client, key, 'hash'); + } + const res = await client.query(Array.isArray(key) ? { + name: 'incrObjectFieldByMulti', + text: ` +INSERT INTO "legacy_hash" ("_key", "data") +SELECT UNNEST($1::TEXT[]), jsonb_build_object($2::TEXT, $3::NUMERIC) +ON CONFLICT ("_key") +DO UPDATE SET "data" = jsonb_set("legacy_hash"."data", ARRAY[$2::TEXT], to_jsonb(COALESCE(("legacy_hash"."data"->>$2::TEXT)::NUMERIC, 0) + $3::NUMERIC)) +RETURNING ("data"->>$2::TEXT)::NUMERIC v`, + values: [key, field, value] + } : { + name: 'incrObjectFieldBy', + text: ` +INSERT INTO "legacy_hash" ("_key", "data") +VALUES ($1::TEXT, jsonb_build_object($2::TEXT, $3::NUMERIC)) +ON CONFLICT ("_key") +DO UPDATE SET "data" = jsonb_set("legacy_hash"."data", ARRAY[$2::TEXT], to_jsonb(COALESCE(("legacy_hash"."data"->>$2::TEXT)::NUMERIC, 0) + $3::NUMERIC)) +RETURNING ("data"->>$2::TEXT)::NUMERIC v`, + values: [key, field, value] + }); + return Array.isArray(key) ? res.rows.map(r => parseFloat(r.v)) : parseFloat(res.rows[0].v); + }); + }; + module.incrObjectFieldByBulk = async function (data) { + if (!Array.isArray(data) || !data.length) { + return; + } + await Promise.all(data.map(async item => { + for (const [field, value] of Object.entries(item[1])) { + await module.incrObjectFieldBy(item[0], field, value); + } + })); + }; +}; \ No newline at end of file diff --git a/lib/database/postgres/helpers.js b/lib/database/postgres/helpers.js new file mode 100644 index 0000000000..882bdb1df4 --- /dev/null +++ b/lib/database/postgres/helpers.js @@ -0,0 +1,83 @@ +'use strict'; + +const helpers = module.exports; +helpers.valueToString = function (value) { + return String(value); +}; +helpers.removeDuplicateValues = function (values, ...others) { + for (let i = 0; i < values.length; i++) { + if (values.lastIndexOf(values[i]) !== i) { + values.splice(i, 1); + for (let j = 0; j < others.length; j++) { + others[j].splice(i, 1); + } + i -= 1; + } + } +}; +helpers.ensureLegacyObjectType = async function (db, key, type) { + await db.query({ + name: 'ensureLegacyObjectTypeBefore', + text: ` +DELETE FROM "legacy_object" + WHERE "expireAt" IS NOT NULL + AND "expireAt" <= CURRENT_TIMESTAMP` + }); + await db.query({ + name: 'ensureLegacyObjectType1', + text: ` +INSERT INTO "legacy_object" ("_key", "type") +VALUES ($1::TEXT, $2::TEXT::LEGACY_OBJECT_TYPE) + ON CONFLICT + DO NOTHING`, + values: [key, type] + }); + const res = await db.query({ + name: 'ensureLegacyObjectType2', + text: ` +SELECT "type" + FROM "legacy_object_live" + WHERE "_key" = $1::TEXT`, + values: [key] + }); + if (res.rows[0].type !== type) { + throw new Error(`database: cannot insert ${JSON.stringify(key)} as ${type} because it already exists as ${res.rows[0].type}`); + } +}; +helpers.ensureLegacyObjectsType = async function (db, keys, type) { + await db.query({ + name: 'ensureLegacyObjectTypeBefore', + text: ` +DELETE FROM "legacy_object" + WHERE "expireAt" IS NOT NULL + AND "expireAt" <= CURRENT_TIMESTAMP` + }); + await db.query({ + name: 'ensureLegacyObjectsType1', + text: ` +INSERT INTO "legacy_object" ("_key", "type") +SELECT k, $2::TEXT::LEGACY_OBJECT_TYPE + FROM UNNEST($1::TEXT[]) k + ON CONFLICT + DO NOTHING`, + values: [keys, type] + }); + const res = await db.query({ + name: 'ensureLegacyObjectsType2', + text: ` +SELECT "_key", "type" + FROM "legacy_object_live" + WHERE "_key" = ANY($1::TEXT[])`, + values: [keys] + }); + const invalid = res.rows.filter(r => r.type !== type); + if (invalid.length) { + const parts = invalid.map(r => `${JSON.stringify(r._key)} is ${r.type}`); + throw new Error(`database: cannot insert multiple objects as ${type} because they already exist: ${parts.join(', ')}`); + } + const missing = keys.filter(k => !res.rows.some(r => r._key === k)); + if (missing.length) { + throw new Error(`database: failed to insert keys for objects: ${JSON.stringify(missing)}`); + } +}; +helpers.noop = function () {}; \ No newline at end of file diff --git a/lib/database/postgres/list.js b/lib/database/postgres/list.js new file mode 100644 index 0000000000..2dca9ca667 --- /dev/null +++ b/lib/database/postgres/list.js @@ -0,0 +1,189 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + module.listPrepend = async function (key, value) { + if (!key) { + return; + } + await module.transaction(async client => { + await helpers.ensureLegacyObjectType(client, key, 'list'); + value = Array.isArray(value) ? value : [value]; + value.reverse(); + await client.query({ + name: 'listPrependValues', + text: ` +INSERT INTO "legacy_list" ("_key", "array") +VALUES ($1::TEXT, $2::TEXT[]) +ON CONFLICT ("_key") +DO UPDATE SET "array" = EXCLUDED.array || "legacy_list"."array"`, + values: [key, value] + }); + }); + }; + module.listAppend = async function (key, value) { + if (!key) { + return; + } + await module.transaction(async client => { + value = Array.isArray(value) ? value : [value]; + await helpers.ensureLegacyObjectType(client, key, 'list'); + await client.query({ + name: 'listAppend', + text: ` +INSERT INTO "legacy_list" ("_key", "array") +VALUES ($1::TEXT, $2::TEXT[]) +ON CONFLICT ("_key") +DO UPDATE SET "array" = "legacy_list"."array" || EXCLUDED.array`, + values: [key, value] + }); + }); + }; + module.listRemoveLast = async function (key) { + if (!key) { + return; + } + const res = await module.pool.query({ + name: 'listRemoveLast', + text: ` +WITH A AS ( + SELECT l.* + FROM "legacy_object_live" o + INNER JOIN "legacy_list" l + ON o."_key" = l."_key" + AND o."type" = l."type" + WHERE o."_key" = $1::TEXT + FOR UPDATE) +UPDATE "legacy_list" l + SET "array" = A."array"[1 : array_length(A."array", 1) - 1] + FROM A + WHERE A."_key" = l."_key" +RETURNING A."array"[array_length(A."array", 1)] v`, + values: [key] + }); + return res.rows.length ? res.rows[0].v : null; + }; + module.listRemoveAll = async function (key, value) { + if (!key) { + return; + } + if (Array.isArray(value)) { + await Promise.all(value.map(v => module.listRemoveAll(key, v))); + return; + } + await module.pool.query({ + name: 'listRemoveAll', + text: ` +UPDATE "legacy_list" l + SET "array" = array_remove(l."array", $2::TEXT) + FROM "legacy_object_live" o + WHERE o."_key" = l."_key" + AND o."type" = l."type" + AND o."_key" = $1::TEXT`, + values: [key, value] + }); + }; + module.listTrim = async function (key, start, stop) { + if (!key) { + return; + } + stop += 1; + await module.pool.query(stop > 0 ? { + name: 'listTrim', + text: ` +UPDATE "legacy_list" l + SET "array" = ARRAY(SELECT m.m + FROM UNNEST(l."array") WITH ORDINALITY m(m, i) + ORDER BY m.i ASC + LIMIT ($3::INTEGER - $2::INTEGER) + OFFSET $2::INTEGER) + FROM "legacy_object_live" o + WHERE o."_key" = l."_key" + AND o."type" = l."type" + AND o."_key" = $1::TEXT`, + values: [key, start, stop] + } : { + name: 'listTrimBack', + text: ` +UPDATE "legacy_list" l + SET "array" = ARRAY(SELECT m.m + FROM UNNEST(l."array") WITH ORDINALITY m(m, i) + ORDER BY m.i ASC + LIMIT ($3::INTEGER - $2::INTEGER + array_length(l."array", 1)) + OFFSET $2::INTEGER) + FROM "legacy_object_live" o + WHERE o."_key" = l."_key" + AND o."type" = l."type" + AND o."_key" = $1::TEXT`, + values: [key, start, stop] + }); + }; + module.getListRange = async function (key, start, stop) { + if (!key) { + return; + } + if (start < 0 && stop < 0) { + const res = await module.pool.query({ + name: 'getListRangeReverse', + text: ` + SELECT ARRAY(SELECT m.m + FROM UNNEST(l."array") WITH ORDINALITY m(m, i) + ORDER BY m.i ASC + LIMIT ($3::INTEGER - $2::INTEGER + 1) + OFFSET (array_length(l."array", 1) + $2::INTEGER)) l + FROM "legacy_object_live" o + INNER JOIN "legacy_list" l + ON o."_key" = l."_key" + AND o."type" = l."type" + WHERE o."_key" = $1::TEXT`, + values: [key, start, stop] + }); + return res.rows.length ? res.rows[0].l : []; + } + stop += 1; + const res = await module.pool.query(stop > 0 ? { + name: 'getListRange', + text: ` +SELECT ARRAY(SELECT m.m + FROM UNNEST(l."array") WITH ORDINALITY m(m, i) + ORDER BY m.i ASC + LIMIT ($3::INTEGER - $2::INTEGER) + OFFSET $2::INTEGER) l + FROM "legacy_object_live" o + INNER JOIN "legacy_list" l + ON o."_key" = l."_key" + AND o."type" = l."type" + WHERE o."_key" = $1::TEXT`, + values: [key, start, stop] + } : { + name: 'getListRangeBack', + text: ` +SELECT ARRAY(SELECT m.m + FROM UNNEST(l."array") WITH ORDINALITY m(m, i) + ORDER BY m.i ASC + LIMIT ($3::INTEGER - $2::INTEGER + array_length(l."array", 1)) + OFFSET $2::INTEGER) l + FROM "legacy_object_live" o + INNER JOIN "legacy_list" l + ON o."_key" = l."_key" + AND o."type" = l."type" + WHERE o."_key" = $1::TEXT`, + values: [key, start, stop] + }); + return res.rows.length ? res.rows[0].l : []; + }; + module.listLength = async function (key) { + const res = await module.pool.query({ + name: 'listLength', + text: ` +SELECT array_length(l."array", 1) l + FROM "legacy_object_live" o + INNER JOIN "legacy_list" l + ON o."_key" = l."_key" + AND o."type" = l."type" + WHERE o."_key" = $1::TEXT`, + values: [key] + }); + return res.rows.length ? res.rows[0].l : 0; + }; +}; \ No newline at end of file diff --git a/lib/database/postgres/main.js b/lib/database/postgres/main.js new file mode 100644 index 0000000000..15b3cfeb60 --- /dev/null +++ b/lib/database/postgres/main.js @@ -0,0 +1,254 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + module.flushdb = async function () { + await module.pool.query(`DROP SCHEMA "public" CASCADE`); + await module.pool.query(`CREATE SCHEMA "public"`); + }; + module.emptydb = async function () { + await module.pool.query(`DELETE FROM "legacy_object"`); + }; + module.exists = async function (key) { + if (!key) { + return; + } + const isArray = Array.isArray(key); + if (isArray && !key.length) { + return []; + } + async function checkIfzSetsExist(keys) { + const members = await Promise.all(keys.map(key => module.getSortedSetRange(key, 0, 0))); + return members.map(member => member.length > 0); + } + async function checkIfKeysExist(keys) { + const res = await module.pool.query({ + name: 'existsArray', + text: ` + SELECT o."_key" k + FROM "legacy_object_live" o + WHERE o."_key" = ANY($1::TEXT[])`, + values: [keys] + }); + return keys.map(k => res.rows.some(r => r.k === k)); + } + if (isArray) { + const types = await Promise.all(key.map(module.type)); + const zsetKeys = key.filter((_key, i) => types[i] === 'zset'); + const otherKeys = key.filter((_key, i) => types[i] !== 'zset'); + const [zsetExits, otherExists] = await Promise.all([checkIfzSetsExist(zsetKeys), checkIfKeysExist(otherKeys)]); + const existsMap = Object.create(null); + zsetKeys.forEach((k, i) => { + existsMap[k] = zsetExits[i]; + }); + otherKeys.forEach((k, i) => { + existsMap[k] = otherExists[i]; + }); + return key.map(k => existsMap[k]); + } + const type = await module.type(key); + if (type === 'zset') { + const members = await module.getSortedSetRange(key, 0, 0); + return members.length > 0; + } + const res = await module.pool.query({ + name: 'exists', + text: ` + SELECT EXISTS(SELECT * + FROM "legacy_object_live" + WHERE "_key" = $1::TEXT + LIMIT 1) e`, + values: [key] + }); + return res.rows[0].e; + }; + module.scan = async function (params) { + let { + match + } = params; + if (match.startsWith('*')) { + match = `%${match.substring(1)}`; + } + if (match.endsWith('*')) { + match = `${match.substring(0, match.length - 1)}%`; + } + const res = await module.pool.query({ + text: ` + SELECT o."_key" + FROM "legacy_object_live" o + WHERE o."_key" LIKE '${match}'` + }); + return res.rows.map(r => r._key); + }; + module.delete = async function (key) { + if (!key) { + return; + } + await module.pool.query({ + name: 'delete', + text: ` +DELETE FROM "legacy_object" + WHERE "_key" = $1::TEXT`, + values: [key] + }); + }; + module.deleteAll = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + await module.pool.query({ + name: 'deleteAll', + text: ` +DELETE FROM "legacy_object" + WHERE "_key" = ANY($1::TEXT[])`, + values: [keys] + }); + }; + module.get = async function (key) { + if (!key) { + return; + } + const res = await module.pool.query({ + name: 'get', + text: ` +SELECT s."data" t + FROM "legacy_object_live" o + INNER JOIN "legacy_string" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT + LIMIT 1`, + values: [key] + }); + return res.rows.length ? res.rows[0].t : null; + }; + module.mget = async function (keys) { + if (!keys || !Array.isArray(keys) || !keys.length) { + return []; + } + const res = await module.pool.query({ + name: 'mget', + text: ` +SELECT s."data", s."_key" + FROM "legacy_object_live" o + INNER JOIN "legacy_string" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = ANY($1::TEXT[]) + LIMIT 1`, + values: [keys] + }); + const map = {}; + res.rows.forEach(d => { + map[d._key] = d.data; + }); + return keys.map(k => map.hasOwnProperty(k) ? map[k] : null); + }; + module.set = async function (key, value) { + if (!key) { + return; + } + await module.transaction(async client => { + await helpers.ensureLegacyObjectType(client, key, 'string'); + await client.query({ + name: 'set', + text: ` +INSERT INTO "legacy_string" ("_key", "data") +VALUES ($1::TEXT, $2::TEXT) +ON CONFLICT ("_key") +DO UPDATE SET "data" = $2::TEXT`, + values: [key, value] + }); + }); + }; + module.increment = async function (key) { + if (!key) { + return; + } + return await module.transaction(async client => { + await helpers.ensureLegacyObjectType(client, key, 'string'); + const res = await client.query({ + name: 'increment', + text: ` +INSERT INTO "legacy_string" ("_key", "data") +VALUES ($1::TEXT, '1') +ON CONFLICT ("_key") +DO UPDATE SET "data" = ("legacy_string"."data"::NUMERIC + 1)::TEXT +RETURNING "data" d`, + values: [key] + }); + return parseFloat(res.rows[0].d); + }); + }; + module.rename = async function (oldKey, newKey) { + await module.transaction(async client => { + await client.query({ + name: 'deleteRename', + text: ` + DELETE FROM "legacy_object" + WHERE "_key" = $1::TEXT`, + values: [newKey] + }); + await client.query({ + name: 'rename', + text: ` +UPDATE "legacy_object" +SET "_key" = $2::TEXT +WHERE "_key" = $1::TEXT`, + values: [oldKey, newKey] + }); + }); + }; + module.type = async function (key) { + const res = await module.pool.query({ + name: 'type', + text: ` +SELECT "type"::TEXT t + FROM "legacy_object_live" + WHERE "_key" = $1::TEXT + LIMIT 1`, + values: [key] + }); + return res.rows.length ? res.rows[0].t : null; + }; + async function doExpire(key, date) { + await module.pool.query({ + name: 'expire', + text: ` +UPDATE "legacy_object" + SET "expireAt" = $2::TIMESTAMPTZ + WHERE "_key" = $1::TEXT`, + values: [key, date] + }); + } + module.expire = async function (key, seconds) { + await doExpire(key, new Date((Date.now() / 1000 + seconds) * 1000)); + }; + module.expireAt = async function (key, timestamp) { + await doExpire(key, new Date(timestamp * 1000)); + }; + module.pexpire = async function (key, ms) { + await doExpire(key, new Date(Date.now() + parseInt(ms, 10))); + }; + module.pexpireAt = async function (key, timestamp) { + await doExpire(key, new Date(timestamp)); + }; + async function getExpire(key) { + const res = await module.pool.query({ + name: 'ttl', + text: ` +SELECT "expireAt"::TEXT + FROM "legacy_object" + WHERE "_key" = $1::TEXT + LIMIT 1`, + values: [key] + }); + return res.rows.length ? new Date(res.rows[0].expireAt).getTime() : null; + } + module.ttl = async function (key) { + return Math.round(((await getExpire(key)) - Date.now()) / 1000); + }; + module.pttl = async function (key) { + return (await getExpire(key)) - Date.now(); + }; +}; \ No newline at end of file diff --git a/lib/database/postgres/sets.js b/lib/database/postgres/sets.js new file mode 100644 index 0000000000..9a6aa24421 --- /dev/null +++ b/lib/database/postgres/sets.js @@ -0,0 +1,231 @@ +'use strict'; + +const _ = require('lodash'); +module.exports = function (module) { + const helpers = require('./helpers'); + module.setAdd = async function (key, value) { + if (!Array.isArray(value)) { + value = [value]; + } + if (!value.length) { + return; + } + await module.transaction(async client => { + await helpers.ensureLegacyObjectType(client, key, 'set'); + await client.query({ + name: 'setAdd', + text: ` +INSERT INTO "legacy_set" ("_key", "member") +SELECT $1::TEXT, m +FROM UNNEST($2::TEXT[]) m +ON CONFLICT ("_key", "member") +DO NOTHING`, + values: [key, value] + }); + }); + }; + module.setsAdd = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + if (!Array.isArray(value)) { + value = [value]; + } + keys = _.uniq(keys); + await module.transaction(async client => { + await helpers.ensureLegacyObjectsType(client, keys, 'set'); + await client.query({ + name: 'setsAdd', + text: ` +INSERT INTO "legacy_set" ("_key", "member") +SELECT k, m +FROM UNNEST($1::TEXT[]) k +CROSS JOIN UNNEST($2::TEXT[]) m +ON CONFLICT ("_key", "member") +DO NOTHING`, + values: [keys, value] + }); + }); + }; + module.setRemove = async function (key, value) { + if (!Array.isArray(key)) { + key = [key]; + } + if (!Array.isArray(value)) { + value = [value]; + } + await module.pool.query({ + name: 'setRemove', + text: ` +DELETE FROM "legacy_set" + WHERE "_key" = ANY($1::TEXT[]) + AND "member" = ANY($2::TEXT[])`, + values: [key, value] + }); + }; + module.setsRemove = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + await module.pool.query({ + name: 'setsRemove', + text: ` +DELETE FROM "legacy_set" + WHERE "_key" = ANY($1::TEXT[]) + AND "member" = $2::TEXT`, + values: [keys, value] + }); + }; + module.isSetMember = async function (key, value) { + if (!key) { + return false; + } + const res = await module.pool.query({ + name: 'isSetMember', + text: ` +SELECT 1 + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT + AND s."member" = $2::TEXT`, + values: [key, value] + }); + return !!res.rows.length; + }; + module.isSetMembers = async function (key, values) { + if (!key || !Array.isArray(values) || !values.length) { + return []; + } + values = values.map(helpers.valueToString); + const res = await module.pool.query({ + name: 'isSetMembers', + text: ` +SELECT s."member" m + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT + AND s."member" = ANY($2::TEXT[])`, + values: [key, values] + }); + return values.map(v => res.rows.some(r => r.m === v)); + }; + module.isMemberOfSets = async function (sets, value) { + if (!Array.isArray(sets) || !sets.length) { + return []; + } + value = helpers.valueToString(value); + const res = await module.pool.query({ + name: 'isMemberOfSets', + text: ` +SELECT o."_key" k + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = ANY($1::TEXT[]) + AND s."member" = $2::TEXT`, + values: [sets, value] + }); + return sets.map(s => res.rows.some(r => r.k === s)); + }; + module.getSetMembers = async function (key) { + if (!key) { + return []; + } + const res = await module.pool.query({ + name: 'getSetMembers', + text: ` +SELECT s."member" m + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT`, + values: [key] + }); + return res.rows.map(r => r.m); + }; + module.getSetsMembers = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const res = await module.pool.query({ + name: 'getSetsMembers', + text: ` +SELECT o."_key" k, + array_agg(s."member") m + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = ANY($1::TEXT[]) + GROUP BY o."_key"`, + values: [keys] + }); + return keys.map(k => (res.rows.find(r => r.k === k) || { + m: [] + }).m); + }; + module.setCount = async function (key) { + if (!key) { + return 0; + } + const res = await module.pool.query({ + name: 'setCount', + text: ` +SELECT COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT`, + values: [key] + }); + return parseInt(res.rows[0].c, 10); + }; + module.setsCount = async function (keys) { + const res = await module.pool.query({ + name: 'setsCount', + text: ` +SELECT o."_key" k, + COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = ANY($1::TEXT[]) + GROUP BY o."_key"`, + values: [keys] + }); + return keys.map(k => (res.rows.find(r => r.k === k) || { + c: 0 + }).c); + }; + module.setRemoveRandom = async function (key) { + const res = await module.pool.query({ + name: 'setRemoveRandom', + text: ` +WITH A AS ( + SELECT s."member" + FROM "legacy_object_live" o + INNER JOIN "legacy_set" s + ON o."_key" = s."_key" + AND o."type" = s."type" + WHERE o."_key" = $1::TEXT + ORDER BY RANDOM() + LIMIT 1 + FOR UPDATE) +DELETE FROM "legacy_set" s + USING A + WHERE s."_key" = $1::TEXT + AND s."member" = A."member" +RETURNING A."member" m`, + values: [key] + }); + return res.rows.length ? res.rows[0].m : null; + }; +}; \ No newline at end of file diff --git a/lib/database/postgres/sorted.js b/lib/database/postgres/sorted.js new file mode 100644 index 0000000000..b77e326888 --- /dev/null +++ b/lib/database/postgres/sorted.js @@ -0,0 +1,650 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + const util = require('util'); + const Cursor = require('pg-cursor'); + Cursor.prototype.readAsync = util.promisify(Cursor.prototype.read); + const sleep = util.promisify(setTimeout); + require('./sorted/add')(module); + require('./sorted/remove')(module); + require('./sorted/union')(module); + require('./sorted/intersect')(module); + module.getSortedSetRange = async function (key, start, stop) { + return await getSortedSetRange(key, start, stop, 1, false); + }; + module.getSortedSetRevRange = async function (key, start, stop) { + return await getSortedSetRange(key, start, stop, -1, false); + }; + module.getSortedSetRangeWithScores = async function (key, start, stop) { + return await getSortedSetRange(key, start, stop, 1, true); + }; + module.getSortedSetRevRangeWithScores = async function (key, start, stop) { + return await getSortedSetRange(key, start, stop, -1, true); + }; + async function getSortedSetRange(key, start, stop, sort, withScores) { + if (!key) { + return; + } + if (!Array.isArray(key)) { + key = [key]; + } + if (start < 0 && start > stop) { + return []; + } + let reverse = false; + if (start === 0 && stop < -1) { + reverse = true; + sort *= -1; + start = Math.abs(stop + 1); + stop = -1; + } else if (start < 0 && stop > start) { + const tmp1 = Math.abs(stop + 1); + stop = Math.abs(start + 1); + start = tmp1; + } + let limit = stop - start + 1; + if (limit <= 0) { + limit = null; + } + const res = await module.pool.query({ + name: `getSortedSetRangeWithScores${sort > 0 ? 'Asc' : 'Desc'}`, + text: ` +SELECT z."value", + z."score" + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + ORDER BY z."score" ${sort > 0 ? 'ASC' : 'DESC'} + LIMIT $3::INTEGER +OFFSET $2::INTEGER`, + values: [key, start, limit] + }); + if (reverse) { + res.rows.reverse(); + } + if (withScores) { + res.rows = res.rows.map(r => ({ + value: r.value, + score: parseFloat(r.score) + })); + } else { + res.rows = res.rows.map(r => r.value); + } + return res.rows; + } + module.getSortedSetRangeByScore = async function (key, start, count, min, max) { + return await getSortedSetRangeByScore(key, start, count, min, max, 1, false); + }; + module.getSortedSetRevRangeByScore = async function (key, start, count, max, min) { + return await getSortedSetRangeByScore(key, start, count, min, max, -1, false); + }; + module.getSortedSetRangeByScoreWithScores = async function (key, start, count, min, max) { + return await getSortedSetRangeByScore(key, start, count, min, max, 1, true); + }; + module.getSortedSetRevRangeByScoreWithScores = async function (key, start, count, max, min) { + return await getSortedSetRangeByScore(key, start, count, min, max, -1, true); + }; + async function getSortedSetRangeByScore(key, start, count, min, max, sort, withScores) { + if (!key) { + return; + } + if (!Array.isArray(key)) { + key = [key]; + } + if (parseInt(count, 10) === -1) { + count = null; + } + if (min === '-inf') { + min = null; + } + if (max === '+inf') { + max = null; + } + const res = await module.pool.query({ + name: `getSortedSetRangeByScoreWithScores${sort > 0 ? 'Asc' : 'Desc'}`, + text: ` +SELECT z."value", + z."score" + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + AND (z."score" >= $4::NUMERIC OR $4::NUMERIC IS NULL) + AND (z."score" <= $5::NUMERIC OR $5::NUMERIC IS NULL) + ORDER BY z."score" ${sort > 0 ? 'ASC' : 'DESC'} + LIMIT $3::INTEGER +OFFSET $2::INTEGER`, + values: [key, start, count, min, max] + }); + if (withScores) { + res.rows = res.rows.map(r => ({ + value: r.value, + score: parseFloat(r.score) + })); + } else { + res.rows = res.rows.map(r => r.value); + } + return res.rows; + } + module.sortedSetCount = async function (key, min, max) { + if (!key) { + return; + } + if (min === '-inf') { + min = null; + } + if (max === '+inf') { + max = null; + } + const res = await module.pool.query({ + name: 'sortedSetCount', + text: ` +SELECT COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND (z."score" >= $2::NUMERIC OR $2::NUMERIC IS NULL) + AND (z."score" <= $3::NUMERIC OR $3::NUMERIC IS NULL)`, + values: [key, min, max] + }); + return parseInt(res.rows[0].c, 10); + }; + module.sortedSetCard = async function (key) { + if (!key) { + return 0; + } + const res = await module.pool.query({ + name: 'sortedSetCard', + text: ` +SELECT COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT`, + values: [key] + }); + return parseInt(res.rows[0].c, 10); + }; + module.sortedSetsCard = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const res = await module.pool.query({ + name: 'sortedSetsCard', + text: ` +SELECT o."_key" k, + COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + GROUP BY o."_key"`, + values: [keys] + }); + return keys.map(k => parseInt((res.rows.find(r => r.k === k) || { + c: 0 + }).c, 10)); + }; + module.sortedSetsCardSum = async function (keys, min = '-inf', max = '+inf') { + if (!keys || Array.isArray(keys) && !keys.length) { + return 0; + } + if (!Array.isArray(keys)) { + keys = [keys]; + } + let counts = []; + if (min !== '-inf' || max !== '+inf') { + if (min === '-inf') { + min = null; + } + if (max === '+inf') { + max = null; + } + const res = await module.pool.query({ + name: 'sortedSetsCardSum', + text: ` + SELECT o."_key" k, + COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + AND (z."score" >= $2::NUMERIC OR $2::NUMERIC IS NULL) + AND (z."score" <= $3::NUMERIC OR $3::NUMERIC IS NULL) + GROUP BY o."_key"`, + values: [keys, min, max] + }); + counts = keys.map(k => parseInt((res.rows.find(r => r.k === k) || { + c: 0 + }).c, 10)); + } else { + counts = await module.sortedSetsCard(keys); + } + return counts.reduce((acc, val) => acc + val, 0); + }; + module.sortedSetRank = async function (key, value) { + const result = await getSortedSetRank('ASC', [key], [value]); + return result ? result[0] : null; + }; + module.sortedSetRevRank = async function (key, value) { + const result = await getSortedSetRank('DESC', [key], [value]); + return result ? result[0] : null; + }; + async function getSortedSetRank(sort, keys, values) { + values = values.map(helpers.valueToString); + const res = await module.pool.query({ + name: `getSortedSetRank${sort}`, + text: ` +SELECT (SELECT r + FROM (SELECT z."value" v, + RANK() OVER (PARTITION BY o."_key" + ORDER BY z."score" ${sort}, + z."value" ${sort}) - 1 r + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = kvi.k) r + WHERE v = kvi.v) r + FROM UNNEST($1::TEXT[], $2::TEXT[]) WITH ORDINALITY kvi(k, v, i) + ORDER BY kvi.i ASC`, + values: [keys, values] + }); + return res.rows.map(r => r.r === null ? null : parseFloat(r.r)); + } + module.sortedSetsRanks = async function (keys, values) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + return await getSortedSetRank('ASC', keys, values); + }; + module.sortedSetsRevRanks = async function (keys, values) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + return await getSortedSetRank('DESC', keys, values); + }; + module.sortedSetRanks = async function (key, values) { + if (!Array.isArray(values) || !values.length) { + return []; + } + return await getSortedSetRank('ASC', new Array(values.length).fill(key), values); + }; + module.sortedSetRevRanks = async function (key, values) { + if (!Array.isArray(values) || !values.length) { + return []; + } + return await getSortedSetRank('DESC', new Array(values.length).fill(key), values); + }; + module.sortedSetScore = async function (key, value) { + if (!key) { + return null; + } + value = helpers.valueToString(value); + const res = await module.pool.query({ + name: 'sortedSetScore', + text: ` +SELECT z."score" s + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND z."value" = $2::TEXT`, + values: [key, value] + }); + if (res.rows.length) { + return parseFloat(res.rows[0].s); + } + return null; + }; + module.sortedSetsScore = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + value = helpers.valueToString(value); + const res = await module.pool.query({ + name: 'sortedSetsScore', + text: ` +SELECT o."_key" k, + z."score" s + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + AND z."value" = $2::TEXT`, + values: [keys, value] + }); + return keys.map(k => { + const s = res.rows.find(r => r.k === k); + return s ? parseFloat(s.s) : null; + }); + }; + module.sortedSetScores = async function (key, values) { + if (!key) { + return null; + } + if (!values.length) { + return []; + } + values = values.map(helpers.valueToString); + const res = await module.pool.query({ + name: 'sortedSetScores', + text: ` +SELECT z."value" v, + z."score" s + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND z."value" = ANY($2::TEXT[])`, + values: [key, values] + }); + return values.map(v => { + const s = res.rows.find(r => r.v === v); + return s ? parseFloat(s.s) : null; + }); + }; + module.isSortedSetMember = async function (key, value) { + if (!key) { + return; + } + value = helpers.valueToString(value); + const res = await module.pool.query({ + name: 'isSortedSetMember', + text: ` +SELECT 1 + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND z."value" = $2::TEXT`, + values: [key, value] + }); + return !!res.rows.length; + }; + module.isSortedSetMembers = async function (key, values) { + if (!key) { + return; + } + if (!values.length) { + return []; + } + values = values.map(helpers.valueToString); + const res = await module.pool.query({ + name: 'isSortedSetMembers', + text: ` +SELECT z."value" v + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND z."value" = ANY($2::TEXT[])`, + values: [key, values] + }); + return values.map(v => res.rows.some(r => r.v === v)); + }; + module.isMemberOfSortedSets = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + value = helpers.valueToString(value); + const res = await module.pool.query({ + name: 'isMemberOfSortedSets', + text: ` +SELECT o."_key" k + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + AND z."value" = $2::TEXT`, + values: [keys, value] + }); + return keys.map(k => res.rows.some(r => r.k === k)); + }; + module.getSortedSetMembers = async function (key) { + const data = await module.getSortedSetsMembers([key]); + return data && data[0]; + }; + module.getSortedSetMembersWithScores = async function (key) { + const data = await module.getSortedSetsMembersWithScores([key]); + return data && data[0]; + }; + module.getSortedSetsMembers = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const res = await module.pool.query({ + name: 'getSortedSetsMembers', + text: ` +SELECT "_key" k, + "nodebb_get_sorted_set_members"("_key") m + FROM UNNEST($1::TEXT[]) "_key";`, + values: [keys] + }); + return keys.map(k => (res.rows.find(r => r.k === k) || {}).m || []); + }; + module.getSortedSetsMembersWithScores = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const res = await module.pool.query({ + name: 'getSortedSetsMembersWithScores', + text: ` +SELECT "_key" k, + "nodebb_get_sorted_set_members_withscores"("_key") m + FROM UNNEST($1::TEXT[]) "_key";`, + values: [keys] + }); + return keys.map(k => (res.rows.find(r => r.k === k) || {}).m || []); + }; + module.sortedSetIncrBy = async function (key, increment, value) { + if (!key) { + return; + } + value = helpers.valueToString(value); + increment = parseFloat(increment); + return await module.transaction(async client => { + await helpers.ensureLegacyObjectType(client, key, 'zset'); + const res = await client.query({ + name: 'sortedSetIncrBy', + text: ` +INSERT INTO "legacy_zset" ("_key", "value", "score") +VALUES ($1::TEXT, $2::TEXT, $3::NUMERIC) +ON CONFLICT ("_key", "value") +DO UPDATE SET "score" = "legacy_zset"."score" + $3::NUMERIC +RETURNING "score" s`, + values: [key, value, increment] + }); + return parseFloat(res.rows[0].s); + }); + }; + module.sortedSetIncrByBulk = async function (data) { + return await Promise.all(data.map(item => module.sortedSetIncrBy(item[0], item[1], item[2]))); + }; + module.getSortedSetRangeByLex = async function (key, min, max, start, count) { + return await sortedSetLex(key, min, max, 1, start, count); + }; + module.getSortedSetRevRangeByLex = async function (key, max, min, start, count) { + return await sortedSetLex(key, min, max, -1, start, count); + }; + module.sortedSetLexCount = async function (key, min, max) { + const q = buildLexQuery(key, min, max); + const res = await module.pool.query({ + name: `sortedSetLexCount${q.suffix}`, + text: ` +SELECT COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE ${q.where}`, + values: q.values + }); + return parseInt(res.rows[0].c, 10); + }; + async function sortedSetLex(key, min, max, sort, start, count) { + start = start !== undefined ? start : 0; + count = count !== undefined ? count : 0; + const q = buildLexQuery(key, min, max); + q.values.push(start); + q.values.push(count <= 0 ? null : count); + const res = await module.pool.query({ + name: `sortedSetLex${sort > 0 ? 'Asc' : 'Desc'}${q.suffix}`, + text: ` +SELECT z."value" v + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE ${q.where} + ORDER BY z."value" ${sort > 0 ? 'ASC' : 'DESC'} + LIMIT $${q.values.length}::INTEGER +OFFSET $${q.values.length - 1}::INTEGER`, + values: q.values + }); + return res.rows.map(r => r.v); + } + module.sortedSetRemoveRangeByLex = async function (key, min, max) { + const q = buildLexQuery(key, min, max); + await module.pool.query({ + name: `sortedSetRemoveRangeByLex${q.suffix}`, + text: ` +DELETE FROM "legacy_zset" z + USING "legacy_object_live" o + WHERE o."_key" = z."_key" + AND o."type" = z."type" + AND ${q.where}`, + values: q.values + }); + }; + function buildLexQuery(key, min, max) { + const q = { + suffix: '', + where: `o."_key" = $1::TEXT`, + values: [key] + }; + if (min !== '-') { + if (min.match(/^\(/)) { + q.values.push(min.slice(1)); + q.suffix += 'GT'; + q.where += ` AND z."value" > $${q.values.length}::TEXT COLLATE "C"`; + } else if (min.match(/^\[/)) { + q.values.push(min.slice(1)); + q.suffix += 'GE'; + q.where += ` AND z."value" >= $${q.values.length}::TEXT COLLATE "C"`; + } else { + q.values.push(min); + q.suffix += 'GE'; + q.where += ` AND z."value" >= $${q.values.length}::TEXT COLLATE "C"`; + } + } + if (max !== '+') { + if (max.match(/^\(/)) { + q.values.push(max.slice(1)); + q.suffix += 'LT'; + q.where += ` AND z."value" < $${q.values.length}::TEXT COLLATE "C"`; + } else if (max.match(/^\[/)) { + q.values.push(max.slice(1)); + q.suffix += 'LE'; + q.where += ` AND z."value" <= $${q.values.length}::TEXT COLLATE "C"`; + } else { + q.values.push(max); + q.suffix += 'LE'; + q.where += ` AND z."value" <= $${q.values.length}::TEXT COLLATE "C"`; + } + } + return q; + } + module.getSortedSetScan = async function (params) { + let { + match + } = params; + if (match.startsWith('*')) { + match = `%${match.substring(1)}`; + } + if (match.endsWith('*')) { + match = `${match.substring(0, match.length - 1)}%`; + } + const res = await module.pool.query({ + text: ` +SELECT z."value", + z."score" + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND z."value" LIKE '${match}' + LIMIT $2::INTEGER`, + values: [params.key, params.limit] + }); + if (!params.withScores) { + return res.rows.map(r => r.value); + } + return res.rows.map(r => ({ + value: r.value, + score: parseFloat(r.score) + })); + }; + module.processSortedSet = async function (setKey, process, options) { + const client = await module.pool.connect(); + const batchSize = (options || {}).batch || 100; + const sort = options.reverse ? 'DESC' : 'ASC'; + const min = options.min && options.min !== '-inf' ? options.min : null; + const max = options.max && options.max !== '+inf' ? options.max : null; + const cursor = client.query(new Cursor(` +SELECT z."value", z."score" + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = $1::TEXT + AND (z."score" >= $2::NUMERIC OR $2::NUMERIC IS NULL) + AND (z."score" <= $3::NUMERIC OR $3::NUMERIC IS NULL) + ORDER BY z."score" ${sort}, z."value" ${sort}`, [setKey, min, max])); + if (process && process.constructor && process.constructor.name !== 'AsyncFunction') { + process = util.promisify(process); + } + let iteration = 1; + while (true) { + let rows = await cursor.readAsync(batchSize); + if (!rows.length) { + client.release(); + return; + } + if (options.withScores) { + rows = rows.map(r => ({ + value: r.value, + score: parseFloat(r.score) + })); + } else { + rows = rows.map(r => r.value); + } + try { + if (iteration > 1 && options.interval) { + await sleep(options.interval); + } + await process(rows); + iteration += 1; + } catch (err) { + await client.release(); + throw err; + } + } + }; +}; \ No newline at end of file diff --git a/lib/database/postgres/sorted/add.js b/lib/database/postgres/sorted/add.js new file mode 100644 index 0000000000..b15a061be2 --- /dev/null +++ b/lib/database/postgres/sorted/add.js @@ -0,0 +1,121 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('../helpers'); + const utils = require('../../../utils'); + module.sortedSetAdd = async function (key, score, value) { + if (!key) { + return; + } + if (Array.isArray(score) && Array.isArray(value)) { + return await sortedSetAddBulk(key, score, value); + } + if (!utils.isNumber(score)) { + throw new Error(`[[error:invalid-score, ${score}]]`); + } + value = helpers.valueToString(value); + score = parseFloat(score); + await module.transaction(async client => { + await helpers.ensureLegacyObjectType(client, key, 'zset'); + await client.query({ + name: 'sortedSetAdd', + text: ` + INSERT INTO "legacy_zset" ("_key", "value", "score") + VALUES ($1::TEXT, $2::TEXT, $3::NUMERIC) + ON CONFLICT ("_key", "value") + DO UPDATE SET "score" = $3::NUMERIC`, + values: [key, value, score] + }); + }); + }; + async function sortedSetAddBulk(key, scores, values) { + if (!scores.length || !values.length) { + return; + } + if (scores.length !== values.length) { + throw new Error('[[error:invalid-data]]'); + } + for (let i = 0; i < scores.length; i += 1) { + if (!utils.isNumber(scores[i])) { + throw new Error(`[[error:invalid-score, ${scores[i]}]]`); + } + } + values = values.map(helpers.valueToString); + scores = scores.map(score => parseFloat(score)); + helpers.removeDuplicateValues(values, scores); + await module.transaction(async client => { + await helpers.ensureLegacyObjectType(client, key, 'zset'); + await client.query({ + name: 'sortedSetAddBulk', + text: ` +INSERT INTO "legacy_zset" ("_key", "value", "score") +SELECT $1::TEXT, v, s +FROM UNNEST($2::TEXT[], $3::NUMERIC[]) vs(v, s) +ON CONFLICT ("_key", "value") +DO UPDATE SET "score" = EXCLUDED."score"`, + values: [key, values, scores] + }); + }); + } + module.sortedSetsAdd = async function (keys, scores, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + const isArrayOfScores = Array.isArray(scores); + if (!isArrayOfScores && !utils.isNumber(scores) || isArrayOfScores && scores.map(s => utils.isNumber(s)).includes(false)) { + throw new Error(`[[error:invalid-score, ${scores}]]`); + } + if (isArrayOfScores && scores.length !== keys.length) { + throw new Error('[[error:invalid-data]]'); + } + value = helpers.valueToString(value); + scores = isArrayOfScores ? scores.map(score => parseFloat(score)) : parseFloat(scores); + await module.transaction(async client => { + await helpers.ensureLegacyObjectsType(client, keys, 'zset'); + await client.query({ + name: isArrayOfScores ? 'sortedSetsAddScores' : 'sortedSetsAdd', + text: isArrayOfScores ? ` +INSERT INTO "legacy_zset" ("_key", "value", "score") +SELECT k, $2::TEXT, s +FROM UNNEST($1::TEXT[], $3::NUMERIC[]) vs(k, s) +ON CONFLICT ("_key", "value") + DO UPDATE SET "score" = EXCLUDED."score"` : ` +INSERT INTO "legacy_zset" ("_key", "value", "score") + SELECT k, $2::TEXT, $3::NUMERIC + FROM UNNEST($1::TEXT[]) k + ON CONFLICT ("_key", "value") + DO UPDATE SET "score" = $3::NUMERIC`, + values: [keys, value, scores] + }); + }); + }; + module.sortedSetAddBulk = async function (data) { + if (!Array.isArray(data) || !data.length) { + return; + } + const keys = []; + const values = []; + const scores = []; + data.forEach(item => { + if (!utils.isNumber(item[1])) { + throw new Error(`[[error:invalid-score, ${item[1]}]]`); + } + keys.push(item[0]); + scores.push(item[1]); + values.push(item[2]); + }); + await module.transaction(async client => { + await helpers.ensureLegacyObjectsType(client, keys, 'zset'); + await client.query({ + name: 'sortedSetAddBulk2', + text: ` +INSERT INTO "legacy_zset" ("_key", "value", "score") +SELECT k, v, s +FROM UNNEST($1::TEXT[], $2::TEXT[], $3::NUMERIC[]) vs(k, v, s) +ON CONFLICT ("_key", "value") +DO UPDATE SET "score" = EXCLUDED."score"`, + values: [keys, values, scores] + }); + }); + }; +}; \ No newline at end of file diff --git a/lib/database/postgres/sorted/intersect.js b/lib/database/postgres/sorted/intersect.js new file mode 100644 index 0000000000..46215a15e0 --- /dev/null +++ b/lib/database/postgres/sorted/intersect.js @@ -0,0 +1,84 @@ +'use strict'; + +module.exports = function (module) { + module.sortedSetIntersectCard = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return 0; + } + const res = await module.pool.query({ + name: 'sortedSetIntersectCard', + text: ` +WITH A AS (SELECT z."value" v, + COUNT(*) c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[]) + GROUP BY z."value") +SELECT COUNT(*) c + FROM A + WHERE A.c = array_length($1::TEXT[], 1)`, + values: [keys] + }); + return parseInt(res.rows[0].c, 10); + }; + module.getSortedSetIntersect = async function (params) { + params.sort = 1; + return await getSortedSetIntersect(params); + }; + module.getSortedSetRevIntersect = async function (params) { + params.sort = -1; + return await getSortedSetIntersect(params); + }; + async function getSortedSetIntersect(params) { + const { + sets + } = params; + const start = params.hasOwnProperty('start') ? params.start : 0; + const stop = params.hasOwnProperty('stop') ? params.stop : -1; + let weights = params.weights || []; + const aggregate = params.aggregate || 'SUM'; + if (sets.length < weights.length) { + weights = weights.slice(0, sets.length); + } + while (sets.length > weights.length) { + weights.push(1); + } + let limit = stop - start + 1; + if (limit <= 0) { + limit = null; + } + const res = await module.pool.query({ + name: `getSortedSetIntersect${aggregate}${params.sort > 0 ? 'Asc' : 'Desc'}WithScores`, + text: ` +WITH A AS (SELECT z."value", + ${aggregate}(z."score" * k."weight") "score", + COUNT(*) c + FROM UNNEST($1::TEXT[], $2::NUMERIC[]) k("_key", "weight") + INNER JOIN "legacy_object_live" o + ON o."_key" = k."_key" + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + GROUP BY z."value") +SELECT A."value", + A."score" + FROM A + WHERE c = array_length($1::TEXT[], 1) + ORDER BY A."score" ${params.sort > 0 ? 'ASC' : 'DESC'} + LIMIT $4::INTEGER +OFFSET $3::INTEGER`, + values: [sets, weights, start, limit] + }); + if (params.withScores) { + res.rows = res.rows.map(r => ({ + value: r.value, + score: parseFloat(r.score) + })); + } else { + res.rows = res.rows.map(r => r.value); + } + return res.rows; + } +}; \ No newline at end of file diff --git a/lib/database/postgres/sorted/remove.js b/lib/database/postgres/sorted/remove.js new file mode 100644 index 0000000000..e2b05edf93 --- /dev/null +++ b/lib/database/postgres/sorted/remove.js @@ -0,0 +1,80 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('../helpers'); + module.sortedSetRemove = async function (key, value) { + if (!key) { + return; + } + const isValueArray = Array.isArray(value); + if (!value || isValueArray && !value.length) { + return; + } + if (!Array.isArray(key)) { + key = [key]; + } + if (!isValueArray) { + value = [value]; + } + value = value.map(helpers.valueToString); + await module.pool.query({ + name: 'sortedSetRemove', + text: ` +DELETE FROM "legacy_zset" + WHERE "_key" = ANY($1::TEXT[]) + AND "value" = ANY($2::TEXT[])`, + values: [key, value] + }); + }; + module.sortedSetsRemove = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + value = helpers.valueToString(value); + await module.pool.query({ + name: 'sortedSetsRemove', + text: ` +DELETE FROM "legacy_zset" + WHERE "_key" = ANY($1::TEXT[]) + AND "value" = $2::TEXT`, + values: [keys, value] + }); + }; + module.sortedSetsRemoveRangeByScore = async function (keys, min, max) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + if (min === '-inf') { + min = null; + } + if (max === '+inf') { + max = null; + } + await module.pool.query({ + name: 'sortedSetsRemoveRangeByScore', + text: ` +DELETE FROM "legacy_zset" + WHERE "_key" = ANY($1::TEXT[]) + AND ("score" >= $2::NUMERIC OR $2::NUMERIC IS NULL) + AND ("score" <= $3::NUMERIC OR $3::NUMERIC IS NULL)`, + values: [keys, min, max] + }); + }; + module.sortedSetRemoveBulk = async function (data) { + if (!Array.isArray(data) || !data.length) { + return; + } + const keys = data.map(d => d[0]); + const values = data.map(d => d[1]); + await module.pool.query({ + name: 'sortedSetRemoveBulk', + text: ` + DELETE FROM "legacy_zset" + WHERE (_key, value) IN ( + SELECT k, v + FROM UNNEST($1::TEXT[], $2::TEXT[]) vs(k, v) + )`, + values: [keys, values] + }); + }; +}; \ No newline at end of file diff --git a/lib/database/postgres/sorted/union.js b/lib/database/postgres/sorted/union.js new file mode 100644 index 0000000000..e18eb3693a --- /dev/null +++ b/lib/database/postgres/sorted/union.js @@ -0,0 +1,80 @@ +'use strict'; + +module.exports = function (module) { + module.sortedSetUnionCard = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return 0; + } + const res = await module.pool.query({ + name: 'sortedSetUnionCard', + text: ` +SELECT COUNT(DISTINCT z."value") c + FROM "legacy_object_live" o + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + WHERE o."_key" = ANY($1::TEXT[])`, + values: [keys] + }); + return res.rows[0].c; + }; + module.getSortedSetUnion = async function (params) { + params.sort = 1; + return await getSortedSetUnion(params); + }; + module.getSortedSetRevUnion = async function (params) { + params.sort = -1; + return await getSortedSetUnion(params); + }; + async function getSortedSetUnion(params) { + const { + sets + } = params; + if (!sets || !sets.length) { + return []; + } + const start = params.hasOwnProperty('start') ? params.start : 0; + const stop = params.hasOwnProperty('stop') ? params.stop : -1; + let weights = params.weights || []; + const aggregate = params.aggregate || 'SUM'; + if (sets.length < weights.length) { + weights = weights.slice(0, sets.length); + } + while (sets.length > weights.length) { + weights.push(1); + } + let limit = stop - start + 1; + if (limit <= 0) { + limit = null; + } + const res = await module.pool.query({ + name: `getSortedSetUnion${aggregate}${params.sort > 0 ? 'Asc' : 'Desc'}WithScores`, + text: ` +WITH A AS (SELECT z."value", + ${aggregate}(z."score" * k."weight") "score" + FROM UNNEST($1::TEXT[], $2::NUMERIC[]) k("_key", "weight") + INNER JOIN "legacy_object_live" o + ON o."_key" = k."_key" + INNER JOIN "legacy_zset" z + ON o."_key" = z."_key" + AND o."type" = z."type" + GROUP BY z."value") +SELECT A."value", + A."score" + FROM A + ORDER BY A."score" ${params.sort > 0 ? 'ASC' : 'DESC'} + LIMIT $4::INTEGER +OFFSET $3::INTEGER`, + values: [sets, weights, start, limit] + }); + if (params.withScores) { + res.rows = res.rows.map(r => ({ + value: r.value, + score: parseFloat(r.score) + })); + } else { + res.rows = res.rows.map(r => r.value); + } + return res.rows; + } +}; \ No newline at end of file diff --git a/lib/database/postgres/transaction.js b/lib/database/postgres/transaction.js new file mode 100644 index 0000000000..84c21b62b7 --- /dev/null +++ b/lib/database/postgres/transaction.js @@ -0,0 +1,30 @@ +'use strict'; + +module.exports = function (module) { + module.transaction = async function (perform, txClient) { + let res; + if (txClient) { + await txClient.query(`SAVEPOINT nodebb_subtx`); + try { + res = await perform(txClient); + } catch (err) { + await txClient.query(`ROLLBACK TO SAVEPOINT nodebb_subtx`); + throw err; + } + await txClient.query(`RELEASE SAVEPOINT nodebb_subtx`); + return res; + } + const client = await module.pool.connect(); + try { + await client.query('BEGIN'); + res = await perform(client); + await client.query('COMMIT'); + } catch (err) { + await client.query('ROLLBACK'); + throw err; + } finally { + client.release(); + } + return res; + }; +}; \ No newline at end of file diff --git a/lib/database/redis.js b/lib/database/redis.js new file mode 100644 index 0000000000..4e8c7b9e49 --- /dev/null +++ b/lib/database/redis.js @@ -0,0 +1,102 @@ +'use strict'; + +const nconf = require('nconf'); +const semver = require('semver'); +const connection = require('./redis/connection'); +const redisModule = module.exports; +redisModule.questions = [{ + name: 'redis:host', + description: 'Host IP or address of your Redis instance', + default: nconf.get('redis:host') || nconf.get('defaults:redis:host') || '127.0.0.1' +}, { + name: 'redis:port', + description: 'Host port of your Redis instance', + default: nconf.get('redis:port') || nconf.get('defaults:redis:port') || 6379 +}, { + name: 'redis:password', + description: 'Password of your Redis database', + hidden: true, + default: nconf.get('redis:password') || nconf.get('defaults:redis:password') || '', + before: function (value) { + value = value || nconf.get('redis:password') || ''; + return value; + } +}, { + name: 'redis:database', + description: 'Which database to use (0..n)', + default: nconf.get('redis:database') || nconf.get('defaults:redis:database') || 0 +}]; +redisModule.init = async function (opts) { + redisModule.client = await connection.connect(opts || nconf.get('redis')); +}; +redisModule.createSessionStore = async function (options) { + const meta = require('../meta'); + const sessionStore = require('connect-redis').default; + const client = await connection.connect(options); + const store = new sessionStore({ + client: client, + ttl: meta.getSessionTTLSeconds() + }); + return store; +}; +redisModule.checkCompatibility = async function () { + const info = await redisModule.info(redisModule.client); + await redisModule.checkCompatibilityVersion(info.redis_version); +}; +redisModule.checkCompatibilityVersion = function (version, callback) { + if (semver.lt(version, '2.8.9')) { + callback(new Error('Your Redis version is not new enough to support NodeBB, please upgrade Redis to v2.8.9 or higher.')); + } + callback(); +}; +redisModule.close = async function () { + await redisModule.client.quit(); + if (redisModule.objectCache) { + redisModule.objectCache.reset(); + } +}; +redisModule.info = async function (cxn) { + if (!cxn) { + cxn = await connection.connect(nconf.get('redis')); + } + redisModule.client = redisModule.client || cxn; + const data = await cxn.info(); + const lines = data.toString().split('\r\n').sort(); + const redisData = {}; + lines.forEach(line => { + const parts = line.split(':'); + if (parts[1]) { + redisData[parts[0]] = parts[1]; + } + }); + const keyInfo = redisData[`db${nconf.get('redis:database')}`]; + if (keyInfo) { + const split = keyInfo.split(','); + redisData.keys = (split[0] || '').replace('keys=', ''); + redisData.expires = (split[1] || '').replace('expires=', ''); + redisData.avg_ttl = (split[2] || '').replace('avg_ttl=', ''); + } + redisData.instantaneous_input = (redisData.instantaneous_input_kbps / 1024).toFixed(3); + redisData.instantaneous_output = (redisData.instantaneous_output_kbps / 1024).toFixed(3); + redisData.total_net_input = (redisData.total_net_input_bytes / (1024 * 1024 * 1024)).toFixed(3); + redisData.total_net_output = (redisData.total_net_output_bytes / (1024 * 1024 * 1024)).toFixed(3); + redisData.used_memory_human = (redisData.used_memory / (1024 * 1024 * 1024)).toFixed(3); + redisData.raw = JSON.stringify(redisData, null, 4); + redisData.redis = true; + return redisData; +}; +redisModule.socketAdapter = async function () { + const redisAdapter = require('@socket.io/redis-adapter'); + const pub = await connection.connect(nconf.get('redis')); + const sub = await connection.connect(nconf.get('redis')); + return redisAdapter(pub, sub, { + key: `db:${nconf.get('redis:database')}:adapter_key` + }); +}; +require('./redis/main')(redisModule); +require('./redis/hash')(redisModule); +require('./redis/sets')(redisModule); +require('./redis/sorted')(redisModule); +require('./redis/list')(redisModule); +require('./redis/transaction')(redisModule); +require('../promisify')(redisModule, ['client', 'sessionStore']); \ No newline at end of file diff --git a/lib/database/redis/connection.js b/lib/database/redis/connection.js new file mode 100644 index 0000000000..056081f3e1 --- /dev/null +++ b/lib/database/redis/connection.js @@ -0,0 +1,52 @@ +'use strict'; + +const nconf = require('nconf'); +const Redis = require('ioredis'); +const winston = require('winston'); +const connection = module.exports; +connection.connect = async function (options) { + return new Promise((resolve, reject) => { + options = options || nconf.get('redis'); + const redis_socket_or_host = options.host; + let cxn; + if (options.cluster) { + cxn = new Redis.Cluster(options.cluster, options.options); + } else if (options.sentinels) { + cxn = new Redis({ + sentinels: options.sentinels, + ...options.options + }); + } else if (redis_socket_or_host && String(redis_socket_or_host).indexOf('/') >= 0) { + cxn = new Redis({ + ...options.options, + path: redis_socket_or_host, + password: options.password, + db: options.database + }); + } else { + cxn = new Redis({ + ...options.options, + host: redis_socket_or_host, + port: options.port, + password: options.password, + db: options.database + }); + } + const dbIdx = parseInt(options.database, 10); + if (!(dbIdx >= 0)) { + throw new Error('[[error:no-database-selected]]'); + } + cxn.on('error', err => { + winston.error(err.stack); + reject(err); + }); + cxn.on('ready', () => { + cxn.batch = cxn.pipeline; + resolve(cxn); + }); + if (options.password) { + cxn.auth(options.password); + } + }); +}; +require('../../promisify')(connection); \ No newline at end of file diff --git a/lib/database/redis/hash.js b/lib/database/redis/hash.js new file mode 100644 index 0000000000..6b98a553f1 --- /dev/null +++ b/lib/database/redis/hash.js @@ -0,0 +1,203 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + const cache = require('../cache').create('redis'); + module.objectCache = cache; + module.setObject = async function (key, data) { + if (!key || !data) { + return; + } + if (data.hasOwnProperty('')) { + delete data['']; + } + Object.keys(data).forEach(key => { + if (data[key] === undefined || data[key] === null) { + delete data[key]; + } + }); + if (!Object.keys(data).length) { + return; + } + if (Array.isArray(key)) { + const batch = module.client.batch(); + key.forEach(k => batch.hmset(k, data)); + await helpers.execBatch(batch); + } else { + await module.client.hmset(key, data); + } + cache.del(key); + }; + module.setObjectBulk = async function (...args) { + let data = args[0]; + if (!Array.isArray(data) || !data.length) { + return; + } + if (Array.isArray(args[1])) { + console.warn('[deprecated] db.setObjectBulk(keys, data) usage is deprecated, please use db.setObjectBulk(data)'); + data = args[0].map((key, i) => [key, args[1][i]]); + } + const batch = module.client.batch(); + data.forEach(item => { + if (Object.keys(item[1]).length) { + batch.hmset(item[0], item[1]); + } + }); + await helpers.execBatch(batch); + cache.del(data.map(item => item[0])); + }; + module.setObjectField = async function (key, field, value) { + if (!field) { + return; + } + if (Array.isArray(key)) { + const batch = module.client.batch(); + key.forEach(k => batch.hset(k, field, value)); + await helpers.execBatch(batch); + } else { + await module.client.hset(key, field, value); + } + cache.del(key); + }; + module.getObject = async function (key, fields = []) { + if (!key) { + return null; + } + const data = await module.getObjectsFields([key], fields); + return data && data.length ? data[0] : null; + }; + module.getObjects = async function (keys, fields = []) { + return await module.getObjectsFields(keys, fields); + }; + module.getObjectField = async function (key, field) { + if (!key) { + return null; + } + const cachedData = {}; + cache.getUnCachedKeys([key], cachedData); + if (cachedData[key]) { + return cachedData[key].hasOwnProperty(field) ? cachedData[key][field] : null; + } + return await module.client.hget(key, String(field)); + }; + module.getObjectFields = async function (key, fields) { + if (!key) { + return null; + } + const results = await module.getObjectsFields([key], fields); + return results ? results[0] : null; + }; + module.getObjectsFields = async function (keys, fields) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const cachedData = {}; + const unCachedKeys = cache.getUnCachedKeys(keys, cachedData); + let data = []; + if (unCachedKeys.length > 1) { + const batch = module.client.batch(); + unCachedKeys.forEach(k => batch.hgetall(k)); + data = await helpers.execBatch(batch); + } else if (unCachedKeys.length === 1) { + data = [await module.client.hgetall(unCachedKeys[0])]; + } + data = data.map(elem => { + if (!Object.keys(elem).length) { + return null; + } + return elem; + }); + unCachedKeys.forEach((key, i) => { + cachedData[key] = data[i] || null; + cache.set(key, cachedData[key]); + }); + if (!Array.isArray(fields) || !fields.length) { + return keys.map(key => cachedData[key] ? { + ...cachedData[key] + } : null); + } + return keys.map(key => { + const item = cachedData[key] || {}; + const result = {}; + fields.forEach(field => { + result[field] = item[field] !== undefined ? item[field] : null; + }); + return result; + }); + }; + module.getObjectKeys = async function (key) { + return await module.client.hkeys(key); + }; + module.getObjectValues = async function (key) { + return await module.client.hvals(key); + }; + module.isObjectField = async function (key, field) { + const exists = await module.client.hexists(key, field); + return exists === 1; + }; + module.isObjectFields = async function (key, fields) { + const batch = module.client.batch(); + fields.forEach(f => batch.hexists(String(key), String(f))); + const results = await helpers.execBatch(batch); + return Array.isArray(results) ? helpers.resultsToBool(results) : null; + }; + module.deleteObjectField = async function (key, field) { + if (key === undefined || key === null || field === undefined || field === null) { + return; + } + await module.client.hdel(key, field); + cache.del(key); + }; + module.deleteObjectFields = async function (key, fields) { + if (!key || Array.isArray(key) && !key.length || !Array.isArray(fields) || !fields.length) { + return; + } + fields = fields.filter(Boolean); + if (!fields.length) { + return; + } + if (Array.isArray(key)) { + const batch = module.client.batch(); + key.forEach(k => batch.hdel(k, fields)); + await helpers.execBatch(batch); + } else { + await module.client.hdel(key, fields); + } + cache.del(key); + }; + module.incrObjectField = async function (key, field) { + return await module.incrObjectFieldBy(key, field, 1); + }; + module.decrObjectField = async function (key, field) { + return await module.incrObjectFieldBy(key, field, -1); + }; + module.incrObjectFieldBy = async function (key, field, value) { + value = parseInt(value, 10); + if (!key || isNaN(value)) { + return null; + } + let result; + if (Array.isArray(key)) { + const batch = module.client.batch(); + key.forEach(k => batch.hincrby(k, field, value)); + result = await helpers.execBatch(batch); + } else { + result = await module.client.hincrby(key, field, value); + } + cache.del(key); + return Array.isArray(result) ? result.map(value => parseInt(value, 10)) : parseInt(result, 10); + }; + module.incrObjectFieldByBulk = async function (data) { + if (!Array.isArray(data) || !data.length) { + return; + } + const batch = module.client.batch(); + data.forEach(item => { + for (const [field, value] of Object.entries(item[1])) { + batch.hincrby(item[0], field, value); + } + }); + await helpers.execBatch(batch); + cache.del(data.map(item => item[0])); + }; +}; \ No newline at end of file diff --git a/lib/database/redis/helpers.js b/lib/database/redis/helpers.js new file mode 100644 index 0000000000..bcebd4eb45 --- /dev/null +++ b/lib/database/redis/helpers.js @@ -0,0 +1,29 @@ +'use strict'; + +const helpers = module.exports; +helpers.noop = function () {}; +helpers.execBatch = async function (batch) { + const results = await batch.exec(); + return results.map(([err, res]) => { + if (err) { + throw err; + } + return res; + }); +}; +helpers.resultsToBool = function (results) { + for (let i = 0; i < results.length; i += 1) { + results[i] = results[i] === 1; + } + return results; +}; +helpers.zsetToObjectArray = function (data) { + const objects = new Array(data.length / 2); + for (let i = 0, k = 0; i < objects.length; i += 1, k += 2) { + objects[i] = { + value: data[k], + score: parseFloat(data[k + 1]) + }; + } + return objects; +}; \ No newline at end of file diff --git a/lib/database/redis/list.js b/lib/database/redis/list.js new file mode 100644 index 0000000000..aa3f2e53a5 --- /dev/null +++ b/lib/database/redis/list.js @@ -0,0 +1,50 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + module.listPrepend = async function (key, value) { + if (!key) { + return; + } + await module.client.lpush(key, value); + }; + module.listAppend = async function (key, value) { + if (!key) { + return; + } + await module.client.rpush(key, value); + }; + module.listRemoveLast = async function (key) { + if (!key) { + return; + } + return await module.client.rpop(key); + }; + module.listRemoveAll = async function (key, value) { + if (!key) { + return; + } + if (Array.isArray(value)) { + const batch = module.client.batch(); + value.forEach(value => batch.lrem(key, 0, value)); + await helpers.execBatch(batch); + } else { + await module.client.lrem(key, 0, value); + } + }; + module.listTrim = async function (key, start, stop) { + if (!key) { + return; + } + await module.client.ltrim(key, start, stop); + }; + module.getListRange = async function (key, start, stop) { + if (!key) { + return; + } + return await module.client.lrange(key, start, stop); + }; + module.listLength = async function (key) { + return await module.client.llen(key); + }; +}; \ No newline at end of file diff --git a/lib/database/redis/main.js b/lib/database/redis/main.js new file mode 100644 index 0000000000..d88a6dba91 --- /dev/null +++ b/lib/database/redis/main.js @@ -0,0 +1,101 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + module.flushdb = async function () { + await module.client.send_command('flushdb', []); + }; + module.emptydb = async function () { + await module.flushdb(); + module.objectCache.reset(); + }; + module.exists = async function (key) { + if (Array.isArray(key)) { + if (!key.length) { + return []; + } + const batch = module.client.batch(); + key.forEach(key => batch.exists(key)); + const data = await helpers.execBatch(batch); + return data.map(exists => exists === 1); + } + const exists = await module.client.exists(key); + return exists === 1; + }; + module.scan = async function (params) { + let cursor = '0'; + let returnData = []; + const seen = Object.create(null); + do { + const res = await module.client.scan(cursor, 'MATCH', params.match, 'COUNT', 10000); + cursor = res[0]; + const values = res[1].filter(value => { + const isSeen = !!seen[value]; + if (!isSeen) { + seen[value] = 1; + } + return !isSeen; + }); + returnData = returnData.concat(values); + } while (cursor !== '0'); + return returnData; + }; + module.delete = async function (key) { + await module.client.del(key); + module.objectCache.del(key); + }; + module.deleteAll = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + await module.client.del(keys); + module.objectCache.del(keys); + }; + module.get = async function (key) { + return await module.client.get(key); + }; + module.mget = async function (keys) { + if (!keys || !Array.isArray(keys) || !keys.length) { + return []; + } + return await module.client.mget(keys); + }; + module.set = async function (key, value) { + await module.client.set(key, value); + }; + module.increment = async function (key) { + return await module.client.incr(key); + }; + module.rename = async function (oldKey, newKey) { + try { + await module.client.rename(oldKey, newKey); + } catch (err) { + if (err && err.message !== 'ERR no such key') { + throw err; + } + } + module.objectCache.del([oldKey, newKey]); + }; + module.type = async function (key) { + const type = await module.client.type(key); + return type !== 'none' ? type : null; + }; + module.expire = async function (key, seconds) { + await module.client.expire(key, seconds); + }; + module.expireAt = async function (key, timestamp) { + await module.client.expireat(key, timestamp); + }; + module.pexpire = async function (key, ms) { + await module.client.pexpire(key, ms); + }; + module.pexpireAt = async function (key, timestamp) { + await module.client.pexpireat(key, timestamp); + }; + module.ttl = async function (key) { + return await module.client.ttl(key); + }; + module.pttl = async function (key) { + return await module.client.pttl(key); + }; +}; \ No newline at end of file diff --git a/lib/database/redis/pubsub.js b/lib/database/redis/pubsub.js new file mode 100644 index 0000000000..71c25584f4 --- /dev/null +++ b/lib/database/redis/pubsub.js @@ -0,0 +1,48 @@ +'use strict'; + +const nconf = require('nconf'); +const util = require('util'); +const winston = require('winston'); +const { + EventEmitter +} = require('events'); +const connection = require('./connection'); +let channelName; +const PubSub = function () { + const self = this; + channelName = `db:${nconf.get('redis:database')}:pubsub_channel`; + self.queue = []; + connection.connect().then(client => { + self.subClient = client; + self.subClient.subscribe(channelName); + self.subClient.on('message', (channel, message) => { + if (channel !== channelName) { + return; + } + try { + const msg = JSON.parse(message); + self.emit(msg.event, msg.data); + } catch (err) { + winston.error(err.stack); + } + }); + }); + connection.connect().then(client => { + self.pubClient = client; + self.queue.forEach(payload => client.publish(channelName, payload)); + self.queue.length = 0; + }); +}; +util.inherits(PubSub, EventEmitter); +PubSub.prototype.publish = function (event, data) { + const payload = JSON.stringify({ + event: event, + data: data + }); + if (this.pubClient) { + this.pubClient.publish(channelName, payload); + } else { + this.queue.push(payload); + } +}; +module.exports = new PubSub(); \ No newline at end of file diff --git a/lib/database/redis/sets.js b/lib/database/redis/sets.js new file mode 100644 index 0000000000..2e277cc7db --- /dev/null +++ b/lib/database/redis/sets.js @@ -0,0 +1,77 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('./helpers'); + module.setAdd = async function (key, value) { + if (!Array.isArray(value)) { + value = [value]; + } + if (!value.length) { + return; + } + await module.client.sadd(key, value); + }; + module.setsAdd = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + const batch = module.client.batch(); + keys.forEach(k => batch.sadd(String(k), String(value))); + await helpers.execBatch(batch); + }; + module.setRemove = async function (key, value) { + if (!Array.isArray(value)) { + value = [value]; + } + if (!Array.isArray(key)) { + key = [key]; + } + if (!value.length) { + return; + } + const batch = module.client.batch(); + key.forEach(k => batch.srem(String(k), value)); + await helpers.execBatch(batch); + }; + module.setsRemove = async function (keys, value) { + const batch = module.client.batch(); + keys.forEach(k => batch.srem(String(k), value)); + await helpers.execBatch(batch); + }; + module.isSetMember = async function (key, value) { + const result = await module.client.sismember(key, value); + return result === 1; + }; + module.isSetMembers = async function (key, values) { + const batch = module.client.batch(); + values.forEach(v => batch.sismember(String(key), String(v))); + const results = await helpers.execBatch(batch); + return results ? helpers.resultsToBool(results) : null; + }; + module.isMemberOfSets = async function (sets, value) { + const batch = module.client.batch(); + sets.forEach(s => batch.sismember(String(s), String(value))); + const results = await helpers.execBatch(batch); + return results ? helpers.resultsToBool(results) : null; + }; + module.getSetMembers = async function (key) { + return await module.client.smembers(key); + }; + module.getSetsMembers = async function (keys) { + const batch = module.client.batch(); + keys.forEach(k => batch.smembers(String(k))); + return await helpers.execBatch(batch); + }; + module.setCount = async function (key) { + return await module.client.scard(key); + }; + module.setsCount = async function (keys) { + const batch = module.client.batch(); + keys.forEach(k => batch.scard(String(k))); + return await helpers.execBatch(batch); + }; + module.setRemoveRandom = async function (key) { + return await module.client.spop(key); + }; + return module; +}; \ No newline at end of file diff --git a/lib/database/redis/sorted.js b/lib/database/redis/sorted.js new file mode 100644 index 0000000000..93a155394b --- /dev/null +++ b/lib/database/redis/sorted.js @@ -0,0 +1,295 @@ +'use strict'; + +module.exports = function (module) { + const utils = require('../../utils'); + const helpers = require('./helpers'); + const dbHelpers = require('../helpers'); + require('./sorted/add')(module); + require('./sorted/remove')(module); + require('./sorted/union')(module); + require('./sorted/intersect')(module); + module.getSortedSetRange = async function (key, start, stop) { + return await sortedSetRange('zrange', key, start, stop, '-inf', '+inf', false); + }; + module.getSortedSetRevRange = async function (key, start, stop) { + return await sortedSetRange('zrevrange', key, start, stop, '-inf', '+inf', false); + }; + module.getSortedSetRangeWithScores = async function (key, start, stop) { + return await sortedSetRange('zrange', key, start, stop, '-inf', '+inf', true); + }; + module.getSortedSetRevRangeWithScores = async function (key, start, stop) { + return await sortedSetRange('zrevrange', key, start, stop, '-inf', '+inf', true); + }; + async function sortedSetRange(method, key, start, stop, min, max, withScores) { + if (Array.isArray(key)) { + if (!key.length) { + return []; + } + const batch = module.client.batch(); + key.forEach(key => batch[method](genParams(method, key, 0, stop, min, max, true))); + const data = await helpers.execBatch(batch); + const batchData = data.map(setData => helpers.zsetToObjectArray(setData)); + let objects = dbHelpers.mergeBatch(batchData, 0, stop, method === 'zrange' ? 1 : -1); + if (start > 0) { + objects = objects.slice(start, stop !== -1 ? stop + 1 : undefined); + } + if (!withScores) { + objects = objects.map(item => item.value); + } + return objects; + } + const params = genParams(method, key, start, stop, min, max, withScores); + const data = await module.client[method](params); + if (!withScores) { + return data; + } + const objects = helpers.zsetToObjectArray(data); + return objects; + } + function genParams(method, key, start, stop, min, max, withScores) { + const params = { + zrevrange: [key, start, stop], + zrange: [key, start, stop], + zrangebyscore: [key, min, max], + zrevrangebyscore: [key, max, min] + }; + if (withScores) { + params[method].push('WITHSCORES'); + } + if (method === 'zrangebyscore' || method === 'zrevrangebyscore') { + const count = stop !== -1 ? stop - start + 1 : stop; + params[method].push('LIMIT', start, count); + } + return params[method]; + } + module.getSortedSetRangeByScore = async function (key, start, count, min, max) { + return await sortedSetRangeByScore('zrangebyscore', key, start, count, min, max, false); + }; + module.getSortedSetRevRangeByScore = async function (key, start, count, max, min) { + return await sortedSetRangeByScore('zrevrangebyscore', key, start, count, min, max, false); + }; + module.getSortedSetRangeByScoreWithScores = async function (key, start, count, min, max) { + return await sortedSetRangeByScore('zrangebyscore', key, start, count, min, max, true); + }; + module.getSortedSetRevRangeByScoreWithScores = async function (key, start, count, max, min) { + return await sortedSetRangeByScore('zrevrangebyscore', key, start, count, min, max, true); + }; + async function sortedSetRangeByScore(method, key, start, count, min, max, withScores) { + if (parseInt(count, 10) === 0) { + return []; + } + const stop = parseInt(count, 10) === -1 ? -1 : start + count - 1; + return await sortedSetRange(method, key, start, stop, min, max, withScores); + } + module.sortedSetCount = async function (key, min, max) { + return await module.client.zcount(key, min, max); + }; + module.sortedSetCard = async function (key) { + return await module.client.zcard(key); + }; + module.sortedSetsCard = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const batch = module.client.batch(); + keys.forEach(k => batch.zcard(String(k))); + return await helpers.execBatch(batch); + }; + module.sortedSetsCardSum = async function (keys, min = '-inf', max = '+inf') { + if (!keys || Array.isArray(keys) && !keys.length) { + return 0; + } + if (!Array.isArray(keys)) { + keys = [keys]; + } + const batch = module.client.batch(); + if (min !== '-inf' || max !== '+inf') { + keys.forEach(k => batch.zcount(String(k), min, max)); + } else { + keys.forEach(k => batch.zcard(String(k))); + } + const counts = await helpers.execBatch(batch); + return counts.reduce((acc, val) => acc + val, 0); + }; + module.sortedSetRank = async function (key, value) { + return await module.client.zrank(key, value); + }; + module.sortedSetRevRank = async function (key, value) { + return await module.client.zrevrank(key, value); + }; + module.sortedSetsRanks = async function (keys, values) { + const batch = module.client.batch(); + for (let i = 0; i < values.length; i += 1) { + batch.zrank(keys[i], String(values[i])); + } + return await helpers.execBatch(batch); + }; + module.sortedSetsRevRanks = async function (keys, values) { + const batch = module.client.batch(); + for (let i = 0; i < values.length; i += 1) { + batch.zrevrank(keys[i], String(values[i])); + } + return await helpers.execBatch(batch); + }; + module.sortedSetRanks = async function (key, values) { + const batch = module.client.batch(); + for (let i = 0; i < values.length; i += 1) { + batch.zrank(key, String(values[i])); + } + return await helpers.execBatch(batch); + }; + module.sortedSetRevRanks = async function (key, values) { + const batch = module.client.batch(); + for (let i = 0; i < values.length; i += 1) { + batch.zrevrank(key, String(values[i])); + } + return await helpers.execBatch(batch); + }; + module.sortedSetScore = async function (key, value) { + if (!key || value === undefined) { + return null; + } + const score = await module.client.zscore(key, value); + return score === null ? score : parseFloat(score); + }; + module.sortedSetsScore = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const batch = module.client.batch(); + keys.forEach(key => batch.zscore(String(key), String(value))); + const scores = await helpers.execBatch(batch); + return scores.map(d => d === null ? d : parseFloat(d)); + }; + module.sortedSetScores = async function (key, values) { + if (!values.length) { + return []; + } + const batch = module.client.batch(); + values.forEach(value => batch.zscore(String(key), String(value))); + const scores = await helpers.execBatch(batch); + return scores.map(d => d === null ? d : parseFloat(d)); + }; + module.isSortedSetMember = async function (key, value) { + const score = await module.sortedSetScore(key, value); + return utils.isNumber(score); + }; + module.isSortedSetMembers = async function (key, values) { + if (!values.length) { + return []; + } + const batch = module.client.batch(); + values.forEach(v => batch.zscore(key, String(v))); + const results = await helpers.execBatch(batch); + return results.map(utils.isNumber); + }; + module.isMemberOfSortedSets = async function (keys, value) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const batch = module.client.batch(); + keys.forEach(k => batch.zscore(k, String(value))); + const results = await helpers.execBatch(batch); + return results.map(utils.isNumber); + }; + module.getSortedSetMembers = async function (key) { + return await module.client.zrange(key, 0, -1); + }; + module.getSortedSetMembersWithScores = async function (key) { + return helpers.zsetToObjectArray(await module.client.zrange(key, 0, -1, 'WITHSCORES')); + }; + module.getSortedSetsMembers = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const batch = module.client.batch(); + keys.forEach(k => batch.zrange(k, 0, -1)); + return await helpers.execBatch(batch); + }; + module.getSortedSetsMembersWithScores = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return []; + } + const batch = module.client.batch(); + keys.forEach(k => batch.zrange(k, 0, -1, 'WITHSCORES')); + const res = await helpers.execBatch(batch); + return res.map(helpers.zsetToObjectArray); + }; + module.sortedSetIncrBy = async function (key, increment, value) { + const newValue = await module.client.zincrby(key, increment, value); + return parseFloat(newValue); + }; + module.sortedSetIncrByBulk = async function (data) { + const multi = module.client.multi(); + data.forEach(item => { + multi.zincrby(item[0], item[1], item[2]); + }); + const result = await multi.exec(); + return result.map(item => item && parseFloat(item[1])); + }; + module.getSortedSetRangeByLex = async function (key, min, max, start, count) { + return await sortedSetLex('zrangebylex', false, key, min, max, start, count); + }; + module.getSortedSetRevRangeByLex = async function (key, max, min, start, count) { + return await sortedSetLex('zrevrangebylex', true, key, max, min, start, count); + }; + module.sortedSetRemoveRangeByLex = async function (key, min, max) { + await sortedSetLex('zremrangebylex', false, key, min, max); + }; + module.sortedSetLexCount = async function (key, min, max) { + return await sortedSetLex('zlexcount', false, key, min, max); + }; + async function sortedSetLex(method, reverse, key, min, max, start, count) { + let minmin; + let maxmax; + if (reverse) { + minmin = '+'; + maxmax = '-'; + } else { + minmin = '-'; + maxmax = '+'; + } + if (min !== minmin && !min.match(/^[[(]/)) { + min = `[${min}`; + } + if (max !== maxmax && !max.match(/^[[(]/)) { + max = `[${max}`; + } + const args = [key, min, max]; + if (count) { + args.push('LIMIT', start, count); + } + return await module.client[method](args); + } + module.getSortedSetScan = async function (params) { + let cursor = '0'; + const returnData = []; + let done = false; + const seen = Object.create(null); + do { + const res = await module.client.zscan(params.key, cursor, 'MATCH', params.match, 'COUNT', 5000); + cursor = res[0]; + done = cursor === '0'; + const data = res[1]; + for (let i = 0; i < data.length; i += 2) { + const value = data[i]; + if (!seen[value]) { + seen[value] = 1; + if (params.withScores) { + returnData.push({ + value: value, + score: parseFloat(data[i + 1]) + }); + } else { + returnData.push(value); + } + if (params.limit && returnData.length >= params.limit) { + done = true; + break; + } + } + } + } while (!done); + return returnData; + }; +}; \ No newline at end of file diff --git a/lib/database/redis/sorted/add.js b/lib/database/redis/sorted/add.js new file mode 100644 index 0000000000..3664d4dcf3 --- /dev/null +++ b/lib/database/redis/sorted/add.js @@ -0,0 +1,68 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('../helpers'); + const utils = require('../../../utils'); + module.sortedSetAdd = async function (key, score, value) { + if (!key) { + return; + } + if (Array.isArray(score) && Array.isArray(value)) { + return await sortedSetAddMulti(key, score, value); + } + if (!utils.isNumber(score)) { + throw new Error(`[[error:invalid-score, ${score}]]`); + } + await module.client.zadd(key, score, String(value)); + }; + async function sortedSetAddMulti(key, scores, values) { + if (!scores.length || !values.length) { + return; + } + if (scores.length !== values.length) { + throw new Error('[[error:invalid-data]]'); + } + for (let i = 0; i < scores.length; i += 1) { + if (!utils.isNumber(scores[i])) { + throw new Error(`[[error:invalid-score, ${scores[i]}]]`); + } + } + const args = [key]; + for (let i = 0; i < scores.length; i += 1) { + args.push(scores[i], String(values[i])); + } + await module.client.zadd(args); + } + module.sortedSetsAdd = async function (keys, scores, value) { + if (!Array.isArray(keys) || !keys.length) { + return; + } + const isArrayOfScores = Array.isArray(scores); + if (!isArrayOfScores && !utils.isNumber(scores) || isArrayOfScores && scores.map(s => utils.isNumber(s)).includes(false)) { + throw new Error(`[[error:invalid-score, ${scores}]]`); + } + if (isArrayOfScores && scores.length !== keys.length) { + throw new Error('[[error:invalid-data]]'); + } + const batch = module.client.batch(); + for (let i = 0; i < keys.length; i += 1) { + if (keys[i]) { + batch.zadd(keys[i], isArrayOfScores ? scores[i] : scores, String(value)); + } + } + await helpers.execBatch(batch); + }; + module.sortedSetAddBulk = async function (data) { + if (!Array.isArray(data) || !data.length) { + return; + } + const batch = module.client.batch(); + data.forEach(item => { + if (!utils.isNumber(item[1])) { + throw new Error(`[[error:invalid-score, ${item[1]}]]`); + } + batch.zadd(item[0], item[1], item[2]); + }); + await helpers.execBatch(batch); + }; +}; \ No newline at end of file diff --git a/lib/database/redis/sorted/intersect.js b/lib/database/redis/sorted/intersect.js new file mode 100644 index 0000000000..9e78a70106 --- /dev/null +++ b/lib/database/redis/sorted/intersect.js @@ -0,0 +1,56 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('../helpers'); + module.sortedSetIntersectCard = async function (keys) { + if (!Array.isArray(keys) || !keys.length) { + return 0; + } + const tempSetName = `temp_${Date.now()}`; + const interParams = [tempSetName, keys.length].concat(keys); + const multi = module.client.multi(); + multi.zinterstore(interParams); + multi.zcard(tempSetName); + multi.del(tempSetName); + const results = await helpers.execBatch(multi); + return results[1] || 0; + }; + module.getSortedSetIntersect = async function (params) { + params.method = 'zrange'; + return await getSortedSetRevIntersect(params); + }; + module.getSortedSetRevIntersect = async function (params) { + params.method = 'zrevrange'; + return await getSortedSetRevIntersect(params); + }; + async function getSortedSetRevIntersect(params) { + const { + sets + } = params; + const start = params.hasOwnProperty('start') ? params.start : 0; + const stop = params.hasOwnProperty('stop') ? params.stop : -1; + const weights = params.weights || []; + const tempSetName = `temp_${Date.now()}`; + let interParams = [tempSetName, sets.length].concat(sets); + if (weights.length) { + interParams = interParams.concat(['WEIGHTS'].concat(weights)); + } + if (params.aggregate) { + interParams = interParams.concat(['AGGREGATE', params.aggregate]); + } + const rangeParams = [tempSetName, start, stop]; + if (params.withScores) { + rangeParams.push('WITHSCORES'); + } + const multi = module.client.multi(); + multi.zinterstore(interParams); + multi[params.method](rangeParams); + multi.del(tempSetName); + let results = await helpers.execBatch(multi); + if (!params.withScores) { + return results ? results[1] : null; + } + results = results[1] || []; + return helpers.zsetToObjectArray(results); + } +}; \ No newline at end of file diff --git a/lib/database/redis/sorted/remove.js b/lib/database/redis/sorted/remove.js new file mode 100644 index 0000000000..85479bf64a --- /dev/null +++ b/lib/database/redis/sorted/remove.js @@ -0,0 +1,40 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('../helpers'); + module.sortedSetRemove = async function (key, value) { + if (!key) { + return; + } + const isValueArray = Array.isArray(value); + if (!value || isValueArray && !value.length) { + return; + } + if (!isValueArray) { + value = [value]; + } + if (Array.isArray(key)) { + const batch = module.client.batch(); + key.forEach(k => batch.zrem(k, value)); + await helpers.execBatch(batch); + } else { + await module.client.zrem(key, value); + } + }; + module.sortedSetsRemove = async function (keys, value) { + await module.sortedSetRemove(keys, value); + }; + module.sortedSetsRemoveRangeByScore = async function (keys, min, max) { + const batch = module.client.batch(); + keys.forEach(k => batch.zremrangebyscore(k, min, max)); + await helpers.execBatch(batch); + }; + module.sortedSetRemoveBulk = async function (data) { + if (!Array.isArray(data) || !data.length) { + return; + } + const batch = module.client.batch(); + data.forEach(item => batch.zrem(item[0], item[1])); + await helpers.execBatch(batch); + }; +}; \ No newline at end of file diff --git a/lib/database/redis/sorted/union.js b/lib/database/redis/sorted/union.js new file mode 100644 index 0000000000..f11d99cd2c --- /dev/null +++ b/lib/database/redis/sorted/union.js @@ -0,0 +1,45 @@ +'use strict'; + +module.exports = function (module) { + const helpers = require('../helpers'); + module.sortedSetUnionCard = async function (keys) { + const tempSetName = `temp_${Date.now()}`; + if (!keys.length) { + return 0; + } + const multi = module.client.multi(); + multi.zunionstore([tempSetName, keys.length].concat(keys)); + multi.zcard(tempSetName); + multi.del(tempSetName); + const results = await helpers.execBatch(multi); + return Array.isArray(results) && results.length ? results[1] : 0; + }; + module.getSortedSetUnion = async function (params) { + params.method = 'zrange'; + return await module.sortedSetUnion(params); + }; + module.getSortedSetRevUnion = async function (params) { + params.method = 'zrevrange'; + return await module.sortedSetUnion(params); + }; + module.sortedSetUnion = async function (params) { + if (!params.sets.length) { + return []; + } + const tempSetName = `temp_${Date.now()}`; + const rangeParams = [tempSetName, params.start, params.stop]; + if (params.withScores) { + rangeParams.push('WITHSCORES'); + } + const multi = module.client.multi(); + multi.zunionstore([tempSetName, params.sets.length].concat(params.sets)); + multi[params.method](rangeParams); + multi.del(tempSetName); + let results = await helpers.execBatch(multi); + if (!params.withScores) { + return results ? results[1] : null; + } + results = results[1] || []; + return helpers.zsetToObjectArray(results); + }; +}; \ No newline at end of file diff --git a/lib/database/redis/transaction.js b/lib/database/redis/transaction.js new file mode 100644 index 0000000000..e7492c3d65 --- /dev/null +++ b/lib/database/redis/transaction.js @@ -0,0 +1,7 @@ +'use strict'; + +module.exports = function (module) { + module.transaction = function (perform, callback) { + perform(module.client, callback); + }; +}; \ No newline at end of file diff --git a/lib/emailer.js b/lib/emailer.js new file mode 100644 index 0000000000..7d0a4ea6e7 --- /dev/null +++ b/lib/emailer.js @@ -0,0 +1,306 @@ +'use strict'; + +const winston = require('winston'); +const nconf = require('nconf'); +const Benchpress = require('benchpressjs'); +const nodemailer = require('nodemailer'); +const wellKnownServices = require('nodemailer/lib/well-known/services'); +const { + htmlToText +} = require('html-to-text'); +const url = require('url'); +const path = require('path'); +const fs = require('fs'); +const _ = require('lodash'); +const jwt = require('jsonwebtoken'); +const User = require('./user'); +const Plugins = require('./plugins'); +const meta = require('./meta'); +const translator = require('./translator'); +const pubsub = require('./pubsub'); +const file = require('./file'); +const viewsDir = nconf.get('views_dir'); +const Emailer = module.exports; +let prevConfig; +let app; +Emailer.fallbackNotFound = false; +Emailer.transports = { + sendmail: nodemailer.createTransport({ + sendmail: true, + newline: 'unix' + }), + smtp: undefined +}; +Emailer.listServices = () => Object.keys(wellKnownServices); +Emailer._defaultPayload = {}; +const smtpSettingsChanged = config => { + const settings = ['email:smtpTransport:enabled', 'email:smtpTransport:pool', 'email:smtpTransport:user', 'email:smtpTransport:pass', 'email:smtpTransport:service', 'email:smtpTransport:port', 'email:smtpTransport:host', 'email:smtpTransport:security']; + return settings.some(key => config.hasOwnProperty(key) && config[key] !== prevConfig[key]); +}; +const getHostname = () => { + const configUrl = nconf.get('url'); + const parsed = url.parse(configUrl); + return parsed.hostname; +}; +const buildCustomTemplates = async config => { + try { + const toBuild = Object.keys(config).filter(prop => prop.startsWith('email:custom:')).map(key => key.split(':')[2]); + if (!toBuild.length) { + return; + } + const [templates, allPaths] = await Promise.all([Emailer.getTemplates(config), file.walk(viewsDir)]); + const templatesToBuild = templates.filter(template => toBuild.includes(template.path)); + const paths = _.fromPairs(allPaths.map(p => { + const relative = path.relative(viewsDir, p).replace(/\\/g, '/'); + return [relative, p]; + })); + await Promise.all(templatesToBuild.map(async template => { + const source = await meta.templates.processImports(paths, template.path, template.text); + const compiled = await Benchpress.precompile(source, { + filename: template.path + }); + await fs.promises.writeFile(template.fullpath.replace(/\.tpl$/, '.js'), compiled); + })); + Benchpress.flush(); + winston.verbose('[emailer] Built custom email templates'); + } catch (err) { + winston.error(`[emailer] Failed to build custom email templates\n${err.stack}`); + } +}; +Emailer.getTemplates = async config => { + const emailsPath = path.join(viewsDir, 'emails'); + let emails = await file.walk(emailsPath); + emails = emails.filter(email => !email.endsWith('.js')); + const templates = await Promise.all(emails.map(async email => { + const path = email.replace(emailsPath, '').slice(1).replace('.tpl', ''); + const original = await fs.promises.readFile(email, 'utf8'); + return { + path: path, + fullpath: email, + text: config[`email:custom:${path}`] || original, + original: original, + isCustom: !!config[`email:custom:${path}`] + }; + })); + return templates; +}; +Emailer.setupFallbackTransport = config => { + winston.verbose('[emailer] Setting up fallback transport'); + if (parseInt(config['email:smtpTransport:enabled'], 10) === 1) { + const smtpOptions = { + name: getHostname(), + pool: config['email:smtpTransport:pool'] + }; + if (config['email:smtpTransport:user'] || config['email:smtpTransport:pass']) { + smtpOptions.auth = { + user: config['email:smtpTransport:user'], + pass: config['email:smtpTransport:pass'] + }; + } + if (config['email:smtpTransport:service'] === 'nodebb-custom-smtp') { + smtpOptions.port = config['email:smtpTransport:port']; + smtpOptions.host = config['email:smtpTransport:host']; + if (config['email:smtpTransport:security'] === 'NONE') { + smtpOptions.secure = false; + smtpOptions.requireTLS = false; + smtpOptions.ignoreTLS = true; + } else if (config['email:smtpTransport:security'] === 'STARTTLS') { + smtpOptions.secure = false; + smtpOptions.requireTLS = true; + smtpOptions.ignoreTLS = false; + } else { + smtpOptions.secure = true; + smtpOptions.requireTLS = true; + smtpOptions.ignoreTLS = false; + } + } else { + smtpOptions.service = String(config['email:smtpTransport:service']); + } + Emailer.transports.smtp = nodemailer.createTransport(smtpOptions); + Emailer.fallbackTransport = Emailer.transports.smtp; + } else { + Emailer.fallbackTransport = Emailer.transports.sendmail; + } +}; +Emailer.registerApp = expressApp => { + app = expressApp; + let logo = null; + if (meta.config.hasOwnProperty('brand:emailLogo')) { + logo = (!meta.config['brand:emailLogo'].startsWith('http') ? nconf.get('url') : '') + meta.config['brand:emailLogo']; + } + Emailer._defaultPayload = { + url: nconf.get('url'), + site_title: meta.config.title || 'NodeBB', + logo: { + src: logo, + height: meta.config['brand:emailLogo:height'], + width: meta.config['brand:emailLogo:width'] + } + }; + Emailer.setupFallbackTransport(meta.config); + buildCustomTemplates(meta.config); + prevConfig = { + ...meta.config + }; + pubsub.on('config:update', config => { + if (config) { + if (config.hasOwnProperty('brand:emailLogo')) { + Emailer._defaultPayload.logo.src = config['brand:emailLogo']; + } + if (config.hasOwnProperty('brand:emailLogo:height')) { + Emailer._defaultPayload.logo.height = config['brand:emailLogo:height']; + } + if (config.hasOwnProperty('brand:emailLogo:width')) { + Emailer._defaultPayload.logo.width = config['brand:emailLogo:width']; + } + if (smtpSettingsChanged(config)) { + Emailer.setupFallbackTransport(config); + } + buildCustomTemplates(config); + prevConfig = { + ...prevConfig, + ...config + }; + } + }); + return Emailer; +}; +Emailer.send = async (template, uid, params) => { + if (!app) { + throw Error('[emailer] App not ready!'); + } + let userData = await User.getUserFields(uid, ['email', 'username', 'email:confirmed', 'banned']); + if (['welcome', 'verify-email'].includes(template)) { + userData.email = params.email; + } + ({ + template, + userData, + params + } = await Plugins.hooks.fire('filter:email.prepare', { + template, + uid, + userData, + params + })); + if (!meta.config.sendEmailToBanned && template !== 'banned') { + if (userData.banned) { + winston.warn(`[emailer/send] User ${userData.username} (uid: ${uid}) is banned; not sending email due to system config.`); + return; + } + } + if (!userData || !userData.email) { + if (process.env.NODE_ENV === 'development') { + winston.warn(`uid : ${uid} has no email, not sending "${template}" email.`); + } + return; + } + const allowedTpls = ['verify-email', 'welcome', 'registration_accepted', 'reset', 'reset_notify']; + if (!meta.config.includeUnverifiedEmails && !userData['email:confirmed'] && !allowedTpls.includes(template)) { + if (process.env.NODE_ENV === 'development') { + winston.warn(`uid : ${uid} (${userData.email}) has not confirmed email, not sending "${template}" email.`); + } + return; + } + const userSettings = await User.getSettings(uid); + params = { + ...Emailer._defaultPayload, + ...params + }; + params.uid = uid; + params.username = userData.username; + params.rtl = (await translator.translate('[[language:dir]]', userSettings.userLang)) === 'rtl'; + const result = await Plugins.hooks.fire('filter:email.cancel', { + cancel: false, + template: template, + params: params + }); + if (result.cancel) { + return; + } + await Emailer.sendToEmail(template, userData.email, userSettings.userLang, params); +}; +Emailer.sendToEmail = async (template, email, language, params) => { + const lang = language || meta.config.defaultLang || 'en-GB'; + const unsubscribable = ['digest', 'notification']; + let payload = { + template: template, + uid: params.uid + }; + if (unsubscribable.includes(template)) { + if (template === 'notification') { + payload.type = params.notification.type; + } + payload = jwt.sign(payload, nconf.get('secret'), { + expiresIn: '30d' + }); + const unsubUrl = [nconf.get('url'), 'email', 'unsubscribe', payload].join('/'); + params.headers = { + 'List-Id': `<${[template, params.uid, getHostname()].join('.')}>`, + 'List-Unsubscribe': `<${unsubUrl}>`, + 'List-Unsubscribe-Post': 'List-Unsubscribe=One-Click', + ...params.headers + }; + params.unsubUrl = unsubUrl; + } + const result = await Plugins.hooks.fire('filter:email.params', { + template: template, + email: email, + language: lang, + params: params + }); + template = result.template; + email = result.email; + params = result.params; + const [html, subject] = await Promise.all([Emailer.renderAndTranslate(template, params, result.language), translator.translate(params.subject, result.language)]); + const data = await Plugins.hooks.fire('filter:email.modify', { + _raw: params, + to: email, + from: meta.config['email:from'] || `no-reply@${getHostname()}`, + from_name: meta.config['email:from_name'] || 'NodeBB', + subject: `[${meta.config.title}] ${_.unescape(subject)}`, + html: html, + plaintext: htmlToText(html, { + tags: { + img: { + format: 'skip' + } + } + }), + template: template, + uid: params.uid, + pid: params.pid, + fromUid: params.fromUid, + headers: params.headers, + rtl: params.rtl + }); + const usingFallback = !Plugins.hooks.hasListeners('filter:email.send') && !Plugins.hooks.hasListeners('static:email.send'); + try { + if (Plugins.hooks.hasListeners('filter:email.send')) { + await Plugins.hooks.fire('filter:email.send', data); + } else if (Plugins.hooks.hasListeners('static:email.send')) { + await Plugins.hooks.fire('static:email.send', data); + } else { + await Emailer.sendViaFallback(data); + } + } catch (err) { + if (err.code === 'ENOENT' && usingFallback) { + Emailer.fallbackNotFound = true; + throw new Error('[[error:sendmail-not-found]]'); + } else { + throw err; + } + } +}; +Emailer.sendViaFallback = async data => { + data.text = data.plaintext; + delete data.plaintext; + data.from = `${data.from_name}<${data.from}>`; + delete data.from_name; + await Emailer.fallbackTransport.sendMail(data); +}; +Emailer.renderAndTranslate = async (template, params, lang) => { + const html = await app.renderAsync(`emails/${template}`, params); + return await translator.translate(html, lang); +}; +require('./promisify')(Emailer, ['transports']); \ No newline at end of file diff --git a/lib/events.js b/lib/events.js new file mode 100644 index 0000000000..04e83595c0 --- /dev/null +++ b/lib/events.js @@ -0,0 +1,140 @@ +'use strict'; + +const validator = require('validator'); +const _ = require('lodash'); +const db = require('./database'); +const batch = require('./batch'); +const user = require('./user'); +const utils = require('./utils'); +const plugins = require('./plugins'); +const events = module.exports; +events.types = ['plugin-activate', 'plugin-deactivate', 'plugin-install', 'plugin-uninstall', 'restart', 'build', 'config-change', 'settings-change', 'category-purge', 'privilege-change', 'post-delete', 'post-restore', 'post-purge', 'post-edit', 'post-move', 'post-change-owner', 'post-queue-reply-accept', 'post-queue-topic-accept', 'post-queue-reply-reject', 'post-queue-topic-reject', 'topic-delete', 'topic-restore', 'topic-purge', 'topic-rename', 'topic-merge', 'topic-fork', 'topic-move', 'topic-move-all', 'password-reset', 'user-makeAdmin', 'user-removeAdmin', 'user-ban', 'user-unban', 'user-mute', 'user-unmute', 'user-delete', 'user-deleteAccount', 'user-deleteContent', 'password-change', 'email-confirmation-sent', 'email-change', 'username-change', 'ip-blacklist-save', 'ip-blacklist-addRule', 'registration-approved', 'registration-rejected', 'group-join', 'group-request-membership', 'group-add-member', 'group-leave', 'group-owner-grant', 'group-owner-rescind', 'group-accept-membership', 'group-reject-membership', 'group-invite', 'group-invite-accept', 'group-invite-reject', 'group-kick', 'theme-set', 'export:uploads', 'account-locked', 'getUsersCSV', 'chat-room-deleted']; +events.log = async function (data) { + const eid = await db.incrObjectField('global', 'nextEid'); + data.timestamp = Date.now(); + data.eid = eid; + const setKeys = ['events:time', `events:time:${data.type}`]; + if (data.hasOwnProperty('uid') && data.uid) { + setKeys.push(`events:time:uid:${data.uid}`); + } + await Promise.all([db.sortedSetsAdd(setKeys, data.timestamp, eid), db.setObject(`event:${eid}`, data)]); + plugins.hooks.fire('action:events.log', { + data: data + }); +}; +events.getEvents = async function (options) { + if (arguments.length > 1) { + const args = Array.prototype.slice.call(arguments); + options = { + filter: args[0], + start: args[1], + stop: args[2], + from: args[3], + to: args[4] + }; + } + const from = options.hasOwnProperty('from') ? options.from : '-inf'; + const to = options.hasOwnProperty('to') ? options.to : '+inf'; + const { + filter, + start, + stop, + uids + } = options; + let eids = []; + if (Array.isArray(uids)) { + if (filter === '') { + eids = await db.getSortedSetRevRangeByScore(uids.map(uid => `events:time:uid:${uid}`), start, stop === -1 ? -1 : stop - start + 1, to, from); + } else { + eids = await Promise.all(uids.map(uid => db.getSortedSetRevIntersect({ + sets: [`events:time:uid:${uid}`, `events:time:${filter}`], + start: 0, + stop: -1, + weights: [1, 0], + withScores: true + }))); + eids = _.flatten(eids).filter(i => (from === '-inf' || i.score >= from) && (to === '+inf' || i.score <= to)).sort((a, b) => b.score - a.score).slice(start, stop + 1).map(i => i.value); + } + } else { + eids = await db.getSortedSetRevRangeByScore(`events:time${filter ? `:${filter}` : ''}`, start, stop === -1 ? -1 : stop - start + 1, to, from); + } + return await events.getEventsByEventIds(eids); +}; +events.getEventCount = async options => { + const { + filter, + uids, + from, + to + } = options; + if (Array.isArray(uids)) { + if (filter === '') { + const counts = await Promise.all(uids.map(uid => db.sortedSetCount(`events:time:uid:${uid}`, from, to))); + return counts.reduce((prev, cur) => prev + cur, 0); + } + const eids = await Promise.all(uids.map(uid => db.getSortedSetRevIntersect({ + sets: [`events:time:uid:${uid}`, `events:time:${filter}`], + start: 0, + stop: -1, + weights: [1, 0], + withScores: true + }))); + return _.flatten(eids).filter(i => (from === '-inf' || i.score >= from) && (to === '+inf' || i.score <= to)).length; + } + return await db.sortedSetCount(`events:time${filter ? `:${filter}` : ''}`, from || '-inf', to); +}; +events.getEventsByEventIds = async eids => { + let eventsData = await db.getObjects(eids.map(eid => `event:${eid}`)); + eventsData = eventsData.filter(Boolean); + await addUserData(eventsData, 'uid', 'user'); + await addUserData(eventsData, 'targetUid', 'targetUser'); + eventsData.forEach(event => { + Object.keys(event).forEach(key => { + if (typeof event[key] === 'string') { + event[key] = validator.escape(String(event[key] || '')); + } + }); + const e = utils.merge(event); + e.eid = undefined; + e.uid = undefined; + e.type = undefined; + e.ip = undefined; + e.user = undefined; + event.jsonString = JSON.stringify(e, null, 4); + event.timestampISO = new Date(parseInt(event.timestamp, 10)).toUTCString(); + }); + return eventsData; +}; +async function addUserData(eventsData, field, objectName) { + const uids = _.uniq(eventsData.map(event => event && event[field])); + if (!uids.length) { + return eventsData; + } + const [isAdmin, userData] = await Promise.all([user.isAdministrator(uids), user.getUsersFields(uids, ['username', 'userslug', 'picture'])]); + const map = {}; + userData.forEach((user, index) => { + user.isAdmin = isAdmin[index]; + map[user.uid] = user; + }); + eventsData.forEach(event => { + if (map[event[field]]) { + event[objectName] = map[event[field]]; + } + }); + return eventsData; +} +events.deleteEvents = async function (eids) { + const keys = eids.map(eid => `event:${eid}`); + const eventData = await db.getObjectsFields(keys, ['type']); + const sets = _.uniq(['events:time'].concat(eventData.map(e => `events:time:${e.type}`)).concat(eventData.map(e => `events:time:uid:${e.uid}`))); + await Promise.all([db.deleteAll(keys), db.sortedSetRemove(sets, eids)]); +}; +events.deleteAll = async function () { + await batch.processSortedSet('events:time', async eids => { + await events.deleteEvents(eids); + }, { + alwaysStartAt: 0, + batch: 500 + }); +}; +require('./promisify')(events); \ No newline at end of file diff --git a/lib/file.js b/lib/file.js new file mode 100644 index 0000000000..ff4a17b4be --- /dev/null +++ b/lib/file.js @@ -0,0 +1,131 @@ +'use strict'; + +const fs = require('fs'); +const nconf = require('nconf'); +const path = require('path'); +const winston = require('winston'); +const { + mkdirp +} = require('mkdirp'); +const mime = require('mime'); +const graceful = require('graceful-fs'); +const slugify = require('./slugify'); +graceful.gracefulify(fs); +const file = module.exports; +file.saveFileToLocal = async function (filename, folder, tempPath) { + filename = filename.split('.').map(name => slugify(name)).join('.'); + const uploadPath = path.join(nconf.get('upload_path'), folder, filename); + if (!uploadPath.startsWith(nconf.get('upload_path'))) { + throw new Error('[[error:invalid-path]]'); + } + winston.verbose(`Saving file ${filename} to : ${uploadPath}`); + await mkdirp(path.dirname(uploadPath)); + await fs.promises.copyFile(tempPath, uploadPath); + return { + url: `/assets/uploads/${folder ? `${folder}/` : ''}${filename}`, + path: uploadPath + }; +}; +file.base64ToLocal = async function (imageData, uploadPath) { + const buffer = Buffer.from(imageData.slice(imageData.indexOf('base64') + 7), 'base64'); + uploadPath = path.join(nconf.get('upload_path'), uploadPath); + await fs.promises.writeFile(uploadPath, buffer, { + encoding: 'base64' + }); + return uploadPath; +}; +file.appendToFileName = function (filename, string) { + const dotIndex = filename.lastIndexOf('.'); + if (dotIndex === -1) { + return filename + string; + } + return filename.substring(0, dotIndex) + string + filename.substring(dotIndex); +}; +file.allowedExtensions = function () { + const meta = require('./meta'); + let allowedExtensions = (meta.config.allowedFileExtensions || '').trim(); + if (!allowedExtensions) { + return []; + } + allowedExtensions = allowedExtensions.split(','); + allowedExtensions = allowedExtensions.filter(Boolean).map(extension => { + extension = extension.trim(); + if (!extension.startsWith('.')) { + extension = `.${extension}`; + } + return extension.toLowerCase(); + }); + if (allowedExtensions.includes('.jpg') && !allowedExtensions.includes('.jpeg')) { + allowedExtensions.push('.jpeg'); + } + return allowedExtensions; +}; +file.exists = async function (path) { + try { + await fs.promises.stat(path); + } catch (err) { + if (err.code === 'ENOENT') { + return false; + } + throw err; + } + return true; +}; +file.existsSync = function (path) { + try { + fs.statSync(path); + } catch (err) { + if (err.code === 'ENOENT') { + return false; + } + throw err; + } + return true; +}; +file.delete = async function (path) { + if (!path) { + return; + } + try { + await fs.promises.unlink(path); + } catch (err) { + if (err.code === 'ENOENT') { + winston.verbose(`[file] Attempted to delete non-existent file: ${path}`); + return; + } + winston.warn(err); + } +}; +file.link = async function link(filePath, destPath, relative) { + if (relative && process.platform !== 'win32') { + filePath = path.relative(path.dirname(destPath), filePath); + } + if (process.platform === 'win32') { + await fs.promises.link(filePath, destPath); + } else { + await fs.promises.symlink(filePath, destPath, 'file'); + } +}; +file.linkDirs = async function linkDirs(sourceDir, destDir, relative) { + if (relative && process.platform !== 'win32') { + sourceDir = path.relative(path.dirname(destDir), sourceDir); + } + const type = process.platform === 'win32' ? 'junction' : 'dir'; + await fs.promises.symlink(sourceDir, destDir, type); +}; +file.typeToExtension = function (type) { + let extension = ''; + if (type) { + extension = `.${mime.getExtension(type)}`; + } + return extension; +}; +file.walk = async function (dir) { + const subdirs = await fs.promises.readdir(dir); + const files = await Promise.all(subdirs.map(async subdir => { + const res = path.resolve(dir, subdir); + return (await fs.promises.stat(res)).isDirectory() ? file.walk(res) : res; + })); + return files.reduce((a, f) => a.concat(f), []); +}; +require('./promisify')(file); \ No newline at end of file diff --git a/lib/flags.js b/lib/flags.js new file mode 100644 index 0000000000..5efbcfd1af --- /dev/null +++ b/lib/flags.js @@ -0,0 +1,844 @@ +'use strict'; + +const _ = require('lodash'); +const winston = require('winston'); +const validator = require('validator'); +const db = require('./database'); +const user = require('./user'); +const groups = require('./groups'); +const meta = require('./meta'); +const notifications = require('./notifications'); +const analytics = require('./analytics'); +const categories = require('./categories'); +const topics = require('./topics'); +const posts = require('./posts'); +const privileges = require('./privileges'); +const plugins = require('./plugins'); +const utils = require('./utils'); +const batch = require('./batch'); +const Flags = module.exports; +Flags._states = new Map([['open', { + label: '[[flags:state-open]]', + class: 'danger' +}], ['wip', { + label: '[[flags:state-wip]]', + class: 'warning' +}], ['resolved', { + label: '[[flags:state-resolved]]', + class: 'success' +}], ['rejected', { + label: '[[flags:state-rejected]]', + class: 'secondary' +}]]); +Flags.init = async function () { + function prepareSets(sets, orSets, prefix, value) { + if (!Array.isArray(value)) { + sets.push(prefix + value); + } else if (value.length) { + if (value.length === 1) { + sets.push(prefix + value[0]); + } else { + orSets.push(value.map(x => prefix + x)); + } + } + } + const hookData = { + filters: { + type: function (sets, orSets, key) { + prepareSets(sets, orSets, 'flags:byType:', key); + }, + state: function (sets, orSets, key) { + prepareSets(sets, orSets, 'flags:byState:', key); + }, + reporterId: function (sets, orSets, key) { + prepareSets(sets, orSets, 'flags:byReporter:', key); + }, + assignee: function (sets, orSets, key) { + prepareSets(sets, orSets, 'flags:byAssignee:', key); + }, + targetUid: function (sets, orSets, key) { + prepareSets(sets, orSets, 'flags:byTargetUid:', key); + }, + cid: function (sets, orSets, key) { + prepareSets(sets, orSets, 'flags:byCid:', key); + }, + page: function () {}, + perPage: function () {}, + quick: function (sets, orSets, key, uid) { + switch (key) { + case 'mine': + sets.push(`flags:byAssignee:${uid}`); + break; + case 'unresolved': + prepareSets(sets, orSets, 'flags:byState:', ['open', 'wip']); + break; + } + } + }, + states: Flags._states, + helpers: { + prepareSets: prepareSets + } + }; + try { + ({ + filters: Flags._filters + } = await plugins.hooks.fire('filter:flags.getFilters', hookData)); + ({ + filters: Flags._filters, + states: Flags._states + } = await plugins.hooks.fire('filter:flags.init', hookData)); + } catch (err) { + winston.error(`[flags/init] Could not retrieve filters\n${err.stack}`); + Flags._filters = {}; + } +}; +Flags.get = async function (flagId) { + const [base, notes, reports] = await Promise.all([db.getObject(`flag:${flagId}`), Flags.getNotes(flagId), Flags.getReports(flagId)]); + if (!base) { + throw new Error('[[error:no-flag]]'); + } + const flagObj = { + state: 'open', + assignee: null, + ...base, + datetimeISO: utils.toISOString(base.datetime), + target_readable: `${base.type.charAt(0).toUpperCase() + base.type.slice(1)} ${base.targetId}`, + target: await Flags.getTarget(base.type, base.targetId, 0), + notes, + reports + }; + const data = await plugins.hooks.fire('filter:flags.get', { + flag: flagObj + }); + return data.flag; +}; +Flags.getCount = async function ({ + uid, + filters, + query +}) { + filters = filters || {}; + const flagIds = await Flags.getFlagIdsWithFilters({ + filters, + uid, + query + }); + return flagIds.length; +}; +Flags.getFlagIdsWithFilters = async function ({ + filters, + uid, + query +}) { + let sets = []; + const orSets = []; + filters.page = filters.hasOwnProperty('page') ? Math.abs(parseInt(filters.page, 10) || 1) : 1; + filters.perPage = filters.hasOwnProperty('perPage') ? Math.abs(parseInt(filters.perPage, 10) || 20) : 20; + for (const type of Object.keys(filters)) { + if (Flags._filters.hasOwnProperty(type)) { + Flags._filters[type](sets, orSets, filters[type], uid); + } else { + winston.warn(`[flags/list] No flag filter type found: ${type}`); + } + } + sets = sets.length || orSets.length ? sets : ['flags:datetime']; + let flagIds = []; + if (sets.length === 1) { + flagIds = await db.getSortedSetRevRange(sets[0], 0, -1); + } else if (sets.length > 1) { + flagIds = await db.getSortedSetRevIntersect({ + sets: sets, + start: 0, + stop: -1, + aggregate: 'MAX' + }); + } + if (orSets.length) { + let _flagIds = await Promise.all(orSets.map(async orSet => await db.getSortedSetRevUnion({ + sets: orSet, + start: 0, + stop: -1, + aggregate: 'MAX' + }))); + _flagIds = _.intersection(..._flagIds); + if (sets.length) { + flagIds = _.intersection(flagIds, _flagIds); + } else { + flagIds = _.union(flagIds, _flagIds); + } + } + const result = await plugins.hooks.fire('filter:flags.getFlagIdsWithFilters', { + filters, + uid, + query, + flagIds + }); + return result.flagIds; +}; +Flags.list = async function (data) { + const filters = data.filters || {}; + let flagIds = await Flags.getFlagIdsWithFilters({ + filters, + uid: data.uid, + query: data.query + }); + flagIds = await Flags.sort(flagIds, data.sort); + const count = flagIds.length; + const flagsPerPage = Math.abs(parseInt(filters.perPage, 10) || 1); + const pageCount = Math.ceil(flagIds.length / flagsPerPage); + flagIds = flagIds.slice((filters.page - 1) * flagsPerPage, filters.page * flagsPerPage); + const reportCounts = await db.sortedSetsCard(flagIds.map(flagId => `flag:${flagId}:reports`)); + const flags = await Promise.all(flagIds.map(async (flagId, idx) => { + let flagObj = await db.getObject(`flag:${flagId}`); + flagObj = { + state: 'open', + assignee: null, + heat: reportCounts[idx], + ...flagObj + }; + flagObj.labelClass = Flags._states.get(flagObj.state).class; + return Object.assign(flagObj, { + target_readable: `${flagObj.type.charAt(0).toUpperCase() + flagObj.type.slice(1)} ${flagObj.targetId}`, + datetimeISO: utils.toISOString(flagObj.datetime) + }); + })); + const payload = await plugins.hooks.fire('filter:flags.list', { + flags: flags, + page: filters.page, + uid: data.uid + }); + return { + flags: payload.flags, + count, + page: payload.page, + pageCount: pageCount + }; +}; +Flags.sort = async function (flagIds, sort) { + const filterPosts = async flagIds => { + const keys = flagIds.map(id => `flag:${id}`); + const types = await db.getObjectsFields(keys, ['type']); + return flagIds.filter((id, idx) => types[idx].type === 'post'); + }; + switch (sort) { + case 'oldest': + flagIds = flagIds.reverse(); + break; + case 'reports': + { + const keys = flagIds.map(id => `flag:${id}:reports`); + const heat = await db.sortedSetsCard(keys); + const mapped = heat.map((el, i) => ({ + index: i, + heat: el + })); + mapped.sort((a, b) => b.heat - a.heat); + flagIds = mapped.map(obj => flagIds[obj.index]); + break; + } + case 'upvotes': + case 'downvotes': + case 'replies': + { + flagIds = await filterPosts(flagIds); + const keys = flagIds.map(id => `flag:${id}`); + const pids = (await db.getObjectsFields(keys, ['targetId'])).map(obj => obj.targetId); + const votes = (await posts.getPostsFields(pids, [sort])).map(obj => parseInt(obj[sort], 10) || 0); + const sortRef = flagIds.reduce((memo, cur, idx) => { + memo[cur] = votes[idx]; + return memo; + }, {}); + flagIds = flagIds.sort((a, b) => sortRef[b] - sortRef[a]); + } + } + return flagIds; +}; +Flags.validate = async function (payload) { + const [target, reporter] = await Promise.all([Flags.getTarget(payload.type, payload.id, payload.uid), user.getUserData(payload.uid)]); + if (!target) { + throw new Error('[[error:invalid-data]]'); + } else if (target.deleted) { + throw new Error('[[error:post-deleted]]'); + } else if (!reporter || !reporter.userslug) { + throw new Error('[[error:no-user]]'); + } else if (reporter.banned) { + throw new Error('[[error:user-banned]]'); + } + const [targetPrivileged, reporterPrivileged] = await Promise.all([user.isPrivileged(target.uid), user.isPrivileged(reporter.uid)]); + if (targetPrivileged && !reporterPrivileged) { + throw new Error('[[error:cant-flag-privileged]]'); + } + if (payload.type === 'post') { + const editable = await privileges.posts.canEdit(payload.id, payload.uid); + if (!editable.flag && !meta.config['reputation:disabled'] && reporter.reputation < meta.config['min:rep:flag']) { + throw new Error(`[[error:not-enough-reputation-to-flag, ${meta.config['min:rep:flag']}]]`); + } + } else if (payload.type === 'user') { + if (parseInt(payload.id, 10) === parseInt(payload.uid, 10)) { + throw new Error('[[error:cant-flag-self]]'); + } + const editable = await privileges.users.canEdit(payload.uid, payload.id); + if (!editable && !meta.config['reputation:disabled'] && reporter.reputation < meta.config['min:rep:flag']) { + throw new Error(`[[error:not-enough-reputation-to-flag, ${meta.config['min:rep:flag']}]]`); + } + } else { + throw new Error('[[error:invalid-data]]'); + } +}; +Flags.getNotes = async function (flagId) { + let notes = await db.getSortedSetRevRangeWithScores(`flag:${flagId}:notes`, 0, -1); + notes = await modifyNotes(notes); + return notes; +}; +Flags.getNote = async function (flagId, datetime) { + datetime = parseInt(datetime, 10); + if (isNaN(datetime)) { + throw new Error('[[error:invalid-data]]'); + } + let notes = await db.getSortedSetRangeByScoreWithScores(`flag:${flagId}:notes`, 0, 1, datetime, datetime); + if (!notes.length) { + throw new Error('[[error:invalid-data]]'); + } + notes = await modifyNotes(notes); + return notes[0]; +}; +Flags.getFlagIdByTarget = async function (type, id) { + let method; + switch (type) { + case 'post': + method = posts.getPostField; + break; + case 'user': + method = user.getUserField; + break; + default: + throw new Error('[[error:invalid-data]]'); + } + return await method(id, 'flagId'); +}; +async function modifyNotes(notes) { + const uids = []; + notes = notes.map(note => { + const noteObj = JSON.parse(note.value); + uids.push(noteObj[0]); + return { + uid: noteObj[0], + content: noteObj[1], + datetime: note.score, + datetimeISO: utils.toISOString(note.score) + }; + }); + const userData = await user.getUsersFields(uids, ['username', 'userslug', 'picture']); + return notes.map((note, idx) => { + note.user = userData[idx]; + note.content = validator.escape(note.content); + return note; + }); +} +Flags.deleteNote = async function (flagId, datetime) { + datetime = parseInt(datetime, 10); + if (isNaN(datetime)) { + throw new Error('[[error:invalid-data]]'); + } + const note = await db.getSortedSetRangeByScore(`flag:${flagId}:notes`, 0, 1, datetime, datetime); + if (!note.length) { + throw new Error('[[error:invalid-data]]'); + } + await db.sortedSetRemove(`flag:${flagId}:notes`, note[0]); +}; +Flags.create = async function (type, id, uid, reason, timestamp, forceFlag = false) { + let doHistoryAppend = false; + if (!timestamp) { + timestamp = Date.now(); + doHistoryAppend = true; + } + const [flagExists, targetExists,, targetFlagged, targetUid, targetCid] = await Promise.all([Flags.exists(type, id, uid), Flags.targetExists(type, id), Flags.canFlag(type, id, uid, forceFlag), Flags.targetFlagged(type, id), Flags.getTargetUid(type, id), Flags.getTargetCid(type, id)]); + if (!forceFlag && flagExists) { + throw new Error(`[[error:${type}-already-flagged]]`); + } else if (!targetExists) { + throw new Error('[[error:invalid-data]]'); + } + if (targetFlagged) { + const flagId = await Flags.getFlagIdByTarget(type, id); + await Promise.all([Flags.addReport(flagId, type, id, uid, reason, timestamp), Flags.update(flagId, uid, { + state: 'open', + report: 'added' + })]); + return await Flags.get(flagId); + } + const flagId = await db.incrObjectField('global', 'nextFlagId'); + const batched = []; + batched.push(db.setObject(`flag:${flagId}`, { + flagId: flagId, + type: type, + targetId: id, + targetUid: targetUid, + datetime: timestamp + }), Flags.addReport(flagId, type, id, uid, reason, timestamp), db.sortedSetAdd('flags:datetime', timestamp, flagId), db.sortedSetAdd(`flags:byType:${type}`, timestamp, flagId), db.sortedSetIncrBy('flags:byTarget', 1, [type, id].join(':')), analytics.increment('flags')); + if (targetUid) { + batched.push(db.sortedSetAdd(`flags:byTargetUid:${targetUid}`, timestamp, flagId)); + } + if (targetCid) { + batched.push(db.sortedSetAdd(`flags:byCid:${targetCid}`, timestamp, flagId)); + } + if (type === 'post') { + batched.push(db.sortedSetAdd(`flags:byPid:${id}`, timestamp, flagId), posts.setPostField(id, 'flagId', flagId)); + if (targetUid && parseInt(targetUid, 10) !== parseInt(uid, 10)) { + batched.push(user.incrementUserFlagsBy(targetUid, 1)); + } + } else if (type === 'user') { + batched.push(user.setUserField(id, 'flagId', flagId)); + } + await Promise.all(batched); + if (doHistoryAppend) { + await Flags.update(flagId, uid, { + state: 'open' + }); + } + const flagObj = await Flags.get(flagId); + plugins.hooks.fire('action:flags.create', { + flag: flagObj + }); + return flagObj; +}; +Flags.purge = async function (flagIds) { + const flagData = (await db.getObjects(flagIds.map(flagId => `flag:${flagId}`))).filter(Boolean); + const postFlags = flagData.filter(flagObj => flagObj.type === 'post'); + const userFlags = flagData.filter(flagObj => flagObj.type === 'user'); + const assignedFlags = flagData.filter(flagObj => !!flagObj.assignee); + const [allReports, cids] = await Promise.all([db.getSortedSetsMembers(flagData.map(flagObj => `flag:${flagObj.flagId}:reports`)), categories.getAllCidsFromSet('categories:cid')]); + const allReporterUids = allReports.map(flagReports => flagReports.map(report => report && report.split(';')[0])); + const removeReporters = []; + flagData.forEach((flagObj, i) => { + if (Array.isArray(allReporterUids[i])) { + allReporterUids[i].forEach(uid => { + removeReporters.push([`flags:hash`, [flagObj.type, flagObj.targetId, uid].join(':')]); + removeReporters.push([`flags:byReporter:${uid}`, flagObj.flagId]); + }); + } + }); + await Promise.all([db.sortedSetRemoveBulk([...flagData.map(flagObj => [`flags:byType:${flagObj.type}`, flagObj.flagId]), ...flagData.map(flagObj => [`flags:byState:${flagObj.state}`, flagObj.flagId]), ...removeReporters, ...postFlags.map(flagObj => [`flags:byPid:${flagObj.targetId}`, flagObj.flagId]), ...assignedFlags.map(flagObj => [`flags:byAssignee:${flagObj.assignee}`, flagObj.flagId]), ...userFlags.map(flagObj => [`flags:byTargetUid:${flagObj.targetUid}`, flagObj.flagId])]), db.deleteObjectFields(postFlags.map(flagObj => `post:${flagObj.targetId}`, ['flagId'])), db.deleteObjectFields(userFlags.map(flagObj => `user:${flagObj.targetId}`, ['flagId'])), db.deleteAll([...flagIds.map(flagId => `flag:${flagId}`), ...flagIds.map(flagId => `flag:${flagId}:notes`), ...flagIds.map(flagId => `flag:${flagId}:reports`), ...flagIds.map(flagId => `flag:${flagId}:history`)]), db.sortedSetRemove(cids.map(cid => `flags:byCid:${cid}`), flagIds), db.sortedSetRemove('flags:datetime', flagIds), db.sortedSetRemove('flags:byTarget', flagData.map(flagObj => [flagObj.type, flagObj.targetId].join(':')))]); +}; +Flags.getReports = async function (flagId) { + const payload = await db.getSortedSetRevRangeWithScores(`flag:${flagId}:reports`, 0, -1); + const [reports, uids] = payload.reduce((memo, cur) => { + const value = cur.value.split(';'); + memo[1].push(value.shift()); + cur.value = validator.escape(String(value.join(';'))); + memo[0].push(cur); + return memo; + }, [[], []]); + await Promise.all(reports.map(async (report, idx) => { + report.timestamp = report.score; + report.timestampISO = new Date(report.score).toISOString(); + delete report.score; + report.reporter = await user.getUserFields(uids[idx], ['username', 'userslug', 'picture', 'reputation']); + })); + return reports; +}; +Flags.addReport = async function (flagId, type, id, uid, reason, timestamp) { + await db.sortedSetAddBulk([[`flags:byReporter:${uid}`, timestamp, flagId], [`flag:${flagId}:reports`, timestamp, [uid, reason].join(';')], ['flags:hash', flagId, [type, id, uid].join(':')]]); + plugins.hooks.fire('action:flags.addReport', { + flagId, + type, + id, + uid, + reason, + timestamp + }); +}; +Flags.rescindReport = async (type, id, uid) => { + const exists = await Flags.exists(type, id, uid); + if (!exists) { + return true; + } + const flagId = await db.sortedSetScore('flags:hash', [type, id, uid].join(':')); + const reports = await db.getSortedSetMembers(`flag:${flagId}:reports`); + let reason; + reports.forEach(payload => { + if (!reason) { + const [payloadUid, payloadReason] = payload.split(';'); + if (parseInt(payloadUid, 10) === parseInt(uid, 10)) { + reason = payloadReason; + } + } + }); + if (!reason) { + throw new Error('[[error:cant-locate-flag-report]]'); + } + await db.sortedSetRemoveBulk([[`flags:byReporter:${uid}`, flagId], [`flag:${flagId}:reports`, [uid, reason].join(';')], ['flags:hash', [type, id, uid].join(':')]]); + const reportCount = await db.sortedSetCard(`flag:${flagId}:reports`); + if (reportCount < 1) { + await Flags.update(flagId, uid, { + state: 'resolved', + report: 'rescinded' + }); + } +}; +Flags.exists = async function (type, id, uid) { + return await db.isSortedSetMember('flags:hash', [type, id, uid].join(':')); +}; +Flags.canView = async (flagId, uid) => { + const exists = await db.isSortedSetMember('flags:datetime', flagId); + if (!exists) { + return false; + } + const [{ + type, + targetId + }, isAdminOrGlobalMod] = await Promise.all([db.getObject(`flag:${flagId}`), user.isAdminOrGlobalMod(uid)]); + if (type === 'post') { + const cid = await Flags.getTargetCid(type, targetId); + const isModerator = await user.isModerator(uid, cid); + return isAdminOrGlobalMod || isModerator; + } + return isAdminOrGlobalMod; +}; +Flags.canFlag = async function (type, id, uid, skipLimitCheck = false) { + const limit = meta.config['flags:limitPerTarget']; + if (!skipLimitCheck && limit > 0) { + const score = await db.sortedSetScore('flags:byTarget', `${type}:${id}`); + if (score >= limit) { + throw new Error(`[[error:${type}-flagged-too-many-times]]`); + } + } + const oneday = 24 * 60 * 60 * 1000; + const now = Date.now(); + const [flagIds, canRead, isPrivileged] = await Promise.all([db.getSortedSetRangeByScore(`flags:byReporter:${uid}`, 0, -1, now - oneday, '+inf'), privileges.posts.can('topics:read', id, uid), user.isPrivileged(uid)]); + const allowedFlagsPerDay = meta.config[`flags:${type}FlagsPerDay`]; + if (!isPrivileged && allowedFlagsPerDay > 0) { + const flagData = await db.getObjects(flagIds.map(id => `flag:${id}`)); + const flagsOfType = flagData.filter(f => f && f.type === type); + if (allowedFlagsPerDay > 0 && flagsOfType.length > allowedFlagsPerDay) { + throw new Error(`[[error:too-many-${type}-flags-per-day, ${allowedFlagsPerDay}]]`); + } + } + switch (type) { + case 'user': + return true; + case 'post': + if (!canRead) { + throw new Error('[[error:no-privileges]]'); + } + break; + default: + throw new Error('[[error:invalid-data]]'); + } +}; +Flags.getTarget = async function (type, id, uid) { + if (type === 'user') { + const userData = await user.getUserData(id); + return userData && userData.uid ? userData : {}; + } + if (type === 'post') { + let postData = await posts.getPostData(id); + if (!postData) { + return {}; + } + postData = await posts.parsePost(postData); + postData = await topics.addPostData([postData], uid); + return postData[0]; + } + throw new Error('[[error:invalid-data]]'); +}; +Flags.targetExists = async function (type, id) { + if (type === 'post') { + return await posts.exists(id); + } else if (type === 'user') { + return await user.exists(id); + } + throw new Error('[[error:invalid-data]]'); +}; +Flags.targetFlagged = async function (type, id) { + return (await db.sortedSetScore('flags:byTarget', [type, id].join(':'))) >= 1; +}; +Flags.getTargetUid = async function (type, id) { + if (type === 'post') { + return await posts.getPostField(id, 'uid'); + } + return id; +}; +Flags.getTargetCid = async function (type, id) { + if (type === 'post') { + return await posts.getCidByPid(id); + } + return null; +}; +Flags.update = async function (flagId, uid, changeset) { + const current = await db.getObjectFields(`flag:${flagId}`, ['uid', 'state', 'assignee', 'type', 'targetId']); + if (!current.type) { + return; + } + const now = changeset.datetime || Date.now(); + const notifyAssignee = async function (assigneeId) { + if (assigneeId === '' || parseInt(uid, 10) === parseInt(assigneeId, 10)) { + return; + } + const notifObj = await notifications.create({ + type: 'my-flags', + bodyShort: `[[notifications:flag-assigned-to-you, ${flagId}]]`, + bodyLong: '', + path: `/flags/${flagId}`, + nid: `flags:assign:${flagId}:uid:${assigneeId}`, + from: uid + }); + await notifications.push(notifObj, [assigneeId]); + }; + const isAssignable = async function (assigneeId) { + let allowed = false; + allowed = await user.isAdminOrGlobalMod(assigneeId); + if (!allowed && current.type === 'post') { + const cid = await posts.getCidByPid(current.targetId); + allowed = await user.isModerator(assigneeId, cid); + } + return allowed; + }; + async function rescindNotifications(match) { + const nids = await db.getSortedSetScan({ + key: 'notifications', + match: `${match}*` + }); + return notifications.rescind(nids); + } + const tasks = []; + for (const prop of Object.keys(changeset)) { + if (current[prop] === changeset[prop]) { + delete changeset[prop]; + } else if (prop === 'state') { + if (!Flags._states.has(changeset[prop])) { + delete changeset[prop]; + } else { + tasks.push(db.sortedSetAdd(`flags:byState:${changeset[prop]}`, now, flagId)); + tasks.push(db.sortedSetRemove(`flags:byState:${current[prop]}`, flagId)); + if (changeset[prop] === 'resolved' && meta.config['flags:actionOnResolve'] === 'rescind') { + tasks.push(rescindNotifications(`flag:${current.type}:${current.targetId}`)); + } + if (changeset[prop] === 'rejected' && meta.config['flags:actionOnReject'] === 'rescind') { + tasks.push(rescindNotifications(`flag:${current.type}:${current.targetId}`)); + } + } + } else if (prop === 'assignee') { + if (changeset[prop] === '') { + tasks.push(db.sortedSetRemove(`flags:byAssignee:${changeset[prop]}`, flagId)); + } else if (!(await isAssignable(parseInt(changeset[prop], 10)))) { + delete changeset[prop]; + } else { + tasks.push(db.sortedSetAdd(`flags:byAssignee:${changeset[prop]}`, now, flagId)); + tasks.push(notifyAssignee(changeset[prop])); + } + } + } + if (!Object.keys(changeset).length) { + return; + } + tasks.push(db.setObject(`flag:${flagId}`, changeset)); + tasks.push(Flags.appendHistory(flagId, uid, changeset)); + await Promise.all(tasks); + plugins.hooks.fire('action:flags.update', { + flagId: flagId, + changeset: changeset, + uid: uid + }); +}; +Flags.resolveFlag = async function (type, id, uid) { + const flagId = await Flags.getFlagIdByTarget(type, id); + if (parseInt(flagId, 10)) { + await Flags.update(flagId, uid, { + state: 'resolved' + }); + } +}; +Flags.resolveUserPostFlags = async function (uid, callerUid) { + if (meta.config['flags:autoResolveOnBan']) { + await batch.processSortedSet(`uid:${uid}:posts`, async pids => { + let postData = await posts.getPostsFields(pids, ['pid', 'flagId']); + postData = postData.filter(p => p && p.flagId && parseInt(p.flagId, 10)); + for (const postObj of postData) { + await Flags.update(postObj.flagId, callerUid, { + state: 'resolved' + }); + } + }, { + batch: 500 + }); + } +}; +Flags.getHistory = async function (flagId) { + const uids = []; + let history = await db.getSortedSetRevRangeWithScores(`flag:${flagId}:history`, 0, -1); + const targetUid = await db.getObjectField(`flag:${flagId}`, 'targetUid'); + history = history.map(entry => { + entry.value = JSON.parse(entry.value); + uids.push(entry.value[0]); + const changeset = entry.value[1]; + if (changeset.hasOwnProperty('state')) { + changeset.state = changeset.state === undefined ? '' : `[[flags:state-${changeset.state}]]`; + } + if (changeset.hasOwnProperty('report')) { + changeset.report = `[[flags:report-${changeset.report}]]`; + } + return { + uid: entry.value[0], + fields: changeset, + datetime: entry.score, + datetimeISO: utils.toISOString(entry.score) + }; + }); + await Promise.all(history.map(async entry => { + if (entry.fields.hasOwnProperty('assignee')) { + entry.fields.assignee = await user.getUserField(entry.fields.assignee, 'username'); + } + })); + history = await mergeBanHistory(history, targetUid, uids); + history = await mergeMuteHistory(history, targetUid, uids); + history = await mergeUsernameEmailChanges(history, targetUid, uids); + const userData = await user.getUsersFields(uids, ['username', 'userslug', 'picture']); + history.forEach((event, idx) => { + event.user = userData[idx]; + }); + history = history.sort((a, b) => b.datetime - a.datetime); + return history; +}; +Flags.appendHistory = async function (flagId, uid, changeset) { + const datetime = changeset.datetime || Date.now(); + delete changeset.datetime; + const payload = JSON.stringify([uid, changeset, datetime]); + await db.sortedSetAdd(`flag:${flagId}:history`, datetime, payload); +}; +Flags.appendNote = async function (flagId, uid, note, datetime) { + if (datetime) { + try { + await Flags.deleteNote(flagId, datetime); + } catch (e) { + if (!e.message === '[[error:invalid-data]]') { + throw e; + } + } + } + datetime = datetime || Date.now(); + const payload = JSON.stringify([uid, note]); + await db.sortedSetAdd(`flag:${flagId}:notes`, datetime, payload); + await Flags.appendHistory(flagId, uid, { + notes: null, + datetime: datetime + }); +}; +Flags.notify = async function (flagObj, uid, notifySelf = false) { + const [admins, globalMods] = await Promise.all([groups.getMembers('administrators', 0, -1), groups.getMembers('Global Moderators', 0, -1)]); + let uids = admins.concat(globalMods); + let notifObj = null; + const { + displayname + } = flagObj.reports[flagObj.reports.length - 1].reporter; + if (flagObj.type === 'post') { + const [title, cid] = await Promise.all([topics.getTitleByPid(flagObj.targetId), posts.getCidByPid(flagObj.targetId)]); + const modUids = await categories.getModeratorUids([cid]); + const titleEscaped = utils.decodeHTMLEntities(title).replace(/%/g, '%').replace(/,/g, ','); + notifObj = await notifications.create({ + type: 'new-post-flag', + bodyShort: `[[notifications:user-flagged-post-in, ${displayname}, ${titleEscaped}]]`, + bodyLong: await plugins.hooks.fire('filter:parse.raw', String(flagObj.description || '')), + pid: flagObj.targetId, + path: `/flags/${flagObj.flagId}`, + nid: `flag:post:${flagObj.targetId}:${uid}`, + from: uid, + mergeId: `notifications:user-flagged-post-in|${flagObj.targetId}`, + topicTitle: title + }); + uids = uids.concat(modUids[0]); + } else if (flagObj.type === 'user') { + const targetDisplayname = flagObj.target && flagObj.target.displayname ? flagObj.target.displayname : '[[global:guest]]'; + notifObj = await notifications.create({ + type: 'new-user-flag', + bodyShort: `[[notifications:user-flagged-user, ${displayname}, ${targetDisplayname}]]`, + bodyLong: await plugins.hooks.fire('filter:parse.raw', String(flagObj.description || '')), + path: `/flags/${flagObj.flagId}`, + nid: `flag:user:${flagObj.targetId}:${uid}`, + from: uid, + mergeId: `notifications:user-flagged-user|${flagObj.targetId}` + }); + } else { + throw new Error('[[error:invalid-data]]'); + } + plugins.hooks.fire('action:flags.notify', { + flag: flagObj, + notification: notifObj, + from: uid, + to: uids + }); + if (!notifySelf) { + uids = uids.filter(_uid => parseInt(_uid, 10) !== parseInt(uid, 10)); + } + await notifications.push(notifObj, uids); +}; +async function mergeBanHistory(history, targetUid, uids) { + return await mergeBanMuteHistory(history, uids, { + set: `uid:${targetUid}:bans:timestamp`, + label: '[[user:banned]]', + reasonDefault: '[[user:info.banned-no-reason]]', + expiryKey: '[[user:info.banned-expiry]]' + }); +} +async function mergeMuteHistory(history, targetUid, uids) { + return await mergeBanMuteHistory(history, uids, { + set: `uid:${targetUid}:mutes:timestamp`, + label: '[[user:muted]]', + reasonDefault: '[[user:info.muted-no-reason]]', + expiryKey: '[[user:info.muted-expiry]]' + }); +} +async function mergeBanMuteHistory(history, uids, params) { + let recentObjs = await db.getSortedSetRevRange(params.set, 0, 19); + recentObjs = await db.getObjects(recentObjs); + return history.concat(recentObjs.reduce((memo, cur) => { + uids.push(cur.fromUid); + memo.push({ + uid: cur.fromUid, + meta: [{ + key: params.label, + value: validator.escape(String(cur.reason || params.reasonDefault)), + labelClass: 'danger' + }, { + key: params.expiryKey, + value: new Date(parseInt(cur.expire, 10)).toISOString(), + labelClass: 'default' + }], + datetime: parseInt(cur.timestamp, 10), + datetimeISO: utils.toISOString(parseInt(cur.timestamp, 10)) + }); + return memo; + }, [])); +} +async function mergeUsernameEmailChanges(history, targetUid, uids) { + const usernameChanges = await user.getHistory(`user:${targetUid}:usernames`); + const emailChanges = await user.getHistory(`user:${targetUid}:emails`); + return history.concat(usernameChanges.reduce((memo, changeObj) => { + uids.push(targetUid); + memo.push({ + uid: targetUid, + meta: [{ + key: '[[user:change-username]]', + value: changeObj.value, + labelClass: 'primary' + }], + datetime: changeObj.timestamp, + datetimeISO: changeObj.timestampISO + }); + return memo; + }, [])).concat(emailChanges.reduce((memo, changeObj) => { + uids.push(targetUid); + memo.push({ + uid: targetUid, + meta: [{ + key: '[[user:change-email]]', + value: changeObj.value, + labelClass: 'primary' + }], + datetime: changeObj.timestamp, + datetimeISO: changeObj.timestampISO + }); + return memo; + }, [])); +} +require('./promisify')(Flags); \ No newline at end of file diff --git a/lib/groups/cache.js b/lib/groups/cache.js new file mode 100644 index 0000000000..3534c45174 --- /dev/null +++ b/lib/groups/cache.js @@ -0,0 +1,17 @@ +'use strict'; + +const cacheCreate = require('../cache/lru'); +module.exports = function (Groups) { + Groups.cache = cacheCreate({ + name: 'group', + max: 40000, + ttl: 0 + }); + Groups.clearCache = function (uid, groupNames) { + if (!Array.isArray(groupNames)) { + groupNames = [groupNames]; + } + const keys = groupNames.map(name => `${uid}:${name}`); + Groups.cache.del(keys); + }; +}; \ No newline at end of file diff --git a/lib/groups/cover.js b/lib/groups/cover.js new file mode 100644 index 0000000000..1b81d97355 --- /dev/null +++ b/lib/groups/cover.js @@ -0,0 +1,72 @@ +'use strict'; + +const path = require('path'); +const nconf = require('nconf'); +const db = require('../database'); +const image = require('../image'); +const file = require('../file'); +module.exports = function (Groups) { + const allowedTypes = ['image/png', 'image/jpeg', 'image/bmp']; + Groups.updateCoverPosition = async function (groupName, position) { + if (!groupName) { + throw new Error('[[error:invalid-data]]'); + } + await Groups.setGroupField(groupName, 'cover:position', position); + }; + Groups.updateCover = async function (uid, data) { + let tempPath = data.file ? data.file.path : ''; + try { + if (!data.imageData && !data.file && data.position) { + return await Groups.updateCoverPosition(data.groupName, data.position); + } + const type = data.file ? data.file.type : image.mimeFromBase64(data.imageData); + if (!type || !allowedTypes.includes(type)) { + throw new Error('[[error:invalid-image]]'); + } + if (!tempPath) { + tempPath = await image.writeImageDataToTempFile(data.imageData); + } + const filename = `groupCover-${data.groupName}${path.extname(tempPath)}`; + const uploadData = await image.uploadImage(filename, 'files', { + path: tempPath, + uid: uid, + name: 'groupCover' + }); + const { + url + } = uploadData; + await Groups.setGroupField(data.groupName, 'cover:url', url); + await image.resizeImage({ + path: tempPath, + width: 358 + }); + const thumbUploadData = await image.uploadImage(`groupCoverThumb-${data.groupName}${path.extname(tempPath)}`, 'files', { + path: tempPath, + uid: uid, + name: 'groupCover' + }); + await Groups.setGroupField(data.groupName, 'cover:thumb:url', thumbUploadData.url); + if (data.position) { + await Groups.updateCoverPosition(data.groupName, data.position); + } + return { + url: url + }; + } finally { + file.delete(tempPath); + } + }; + Groups.removeCover = async function (data) { + const fields = ['cover:url', 'cover:thumb:url']; + const values = await Groups.getGroupFields(data.groupName, fields); + await Promise.all(fields.map(field => { + if (!values[field] || !values[field].startsWith(`${nconf.get('relative_path')}/assets/uploads/files/`)) { + return; + } + const filename = values[field].split('/').pop(); + const filePath = path.join(nconf.get('upload_path'), 'files', filename); + return file.delete(filePath); + })); + await db.deleteObjectFields(`group:${data.groupName}`, ['cover:url', 'cover:thumb:url', 'cover:position']); + }; +}; \ No newline at end of file diff --git a/lib/groups/create.js b/lib/groups/create.js new file mode 100644 index 0000000000..53acbd2a13 --- /dev/null +++ b/lib/groups/create.js @@ -0,0 +1,83 @@ +'use strict'; + +const meta = require('../meta'); +const plugins = require('../plugins'); +const slugify = require('../slugify'); +const db = require('../database'); +module.exports = function (Groups) { + Groups.create = async function (data) { + const isSystem = isSystemGroup(data); + const timestamp = data.timestamp || Date.now(); + let disableJoinRequests = parseInt(data.disableJoinRequests, 10) === 1 ? 1 : 0; + if (data.name === 'administrators') { + disableJoinRequests = 1; + } + const disableLeave = parseInt(data.disableLeave, 10) === 1 ? 1 : 0; + const isHidden = parseInt(data.hidden, 10) === 1; + Groups.validateGroupName(data.name); + const [exists, privGroupExists] = await Promise.all([meta.userOrGroupExists(data.name), privilegeGroupExists(data.name)]); + if (exists || privGroupExists) { + throw new Error('[[error:group-already-exists]]'); + } + const memberCount = data.hasOwnProperty('ownerUid') ? 1 : 0; + const isPrivate = data.hasOwnProperty('private') && data.private !== undefined ? parseInt(data.private, 10) === 1 : true; + let groupData = { + name: data.name, + slug: slugify(data.name), + createtime: timestamp, + userTitle: data.userTitle || data.name, + userTitleEnabled: parseInt(data.userTitleEnabled, 10) === 1 ? 1 : 0, + description: data.description || '', + memberCount: memberCount, + hidden: isHidden ? 1 : 0, + system: isSystem ? 1 : 0, + private: isPrivate ? 1 : 0, + disableJoinRequests: disableJoinRequests, + disableLeave: disableLeave + }; + await plugins.hooks.fire('filter:group.create', { + group: groupData, + data: data + }); + await db.sortedSetAdd('groups:createtime', groupData.createtime, groupData.name); + await db.setObject(`group:${groupData.name}`, groupData); + if (data.hasOwnProperty('ownerUid')) { + await db.setAdd(`group:${groupData.name}:owners`, data.ownerUid); + await db.sortedSetAdd(`group:${groupData.name}:members`, timestamp, data.ownerUid); + } + if (!isHidden && !isSystem) { + await db.sortedSetAddBulk([['groups:visible:createtime', timestamp, groupData.name], ['groups:visible:memberCount', groupData.memberCount, groupData.name], ['groups:visible:name', 0, `${groupData.name.toLowerCase()}:${groupData.name}`]]); + } + if (!Groups.isPrivilegeGroup(groupData.name)) { + await db.setObjectField('groupslug:groupname', groupData.slug, groupData.name); + } + groupData = await Groups.getGroupData(groupData.name); + plugins.hooks.fire('action:group.create', { + group: groupData + }); + return groupData; + }; + function isSystemGroup(data) { + return data.system === true || parseInt(data.system, 10) === 1 || Groups.systemGroups.includes(data.name) || Groups.isPrivilegeGroup(data.name); + } + async function privilegeGroupExists(name) { + return Groups.isPrivilegeGroup(name) && (await db.isSortedSetMember('groups:createtime', name)); + } + Groups.validateGroupName = function (name) { + if (!name) { + throw new Error('[[error:group-name-too-short]]'); + } + if (typeof name !== 'string') { + throw new Error('[[error:invalid-group-name]]'); + } + if (!Groups.isPrivilegeGroup(name) && name.length > meta.config.maximumGroupNameLength) { + throw new Error('[[error:group-name-too-long]]'); + } + if (name === 'guests' || !Groups.isPrivilegeGroup(name) && name.includes(':')) { + throw new Error('[[error:invalid-group-name]]'); + } + if (name.includes('/') || !slugify(name)) { + throw new Error('[[error:invalid-group-name]]'); + } + }; +}; \ No newline at end of file diff --git a/lib/groups/data.js b/lib/groups/data.js new file mode 100644 index 0000000000..0583f180b3 --- /dev/null +++ b/lib/groups/data.js @@ -0,0 +1,92 @@ +'use strict'; + +const validator = require('validator'); +const nconf = require('nconf'); +const db = require('../database'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const translator = require('../translator'); +const intFields = ['createtime', 'memberCount', 'hidden', 'system', 'private', 'userTitleEnabled', 'disableJoinRequests', 'disableLeave']; +module.exports = function (Groups) { + Groups.getGroupsFields = async function (groupNames, fields) { + if (!Array.isArray(groupNames) || !groupNames.length) { + return []; + } + const ephemeralIdx = groupNames.reduce((memo, cur, idx) => { + if (Groups.ephemeralGroups.includes(cur)) { + memo.push(idx); + } + return memo; + }, []); + const keys = groupNames.map(groupName => `group:${groupName}`); + const groupData = await db.getObjects(keys, fields); + if (ephemeralIdx.length) { + ephemeralIdx.forEach(idx => { + groupData[idx] = Groups.getEphemeralGroup(groupNames[idx]); + }); + } + groupData.forEach(group => modifyGroup(group, fields)); + const results = await plugins.hooks.fire('filter:groups.get', { + groups: groupData + }); + return results.groups; + }; + Groups.getGroupsData = async function (groupNames) { + return await Groups.getGroupsFields(groupNames, []); + }; + Groups.getGroupData = async function (groupName) { + const groupsData = await Groups.getGroupsData([groupName]); + return Array.isArray(groupsData) && groupsData[0] ? groupsData[0] : null; + }; + Groups.getGroupField = async function (groupName, field) { + const groupData = await Groups.getGroupFields(groupName, [field]); + return groupData ? groupData[field] : null; + }; + Groups.getGroupFields = async function (groupName, fields) { + const groups = await Groups.getGroupsFields([groupName], fields); + return groups ? groups[0] : null; + }; + Groups.setGroupField = async function (groupName, field, value) { + await db.setObjectField(`group:${groupName}`, field, value); + plugins.hooks.fire('action:group.set', { + field: field, + value: value, + type: 'set' + }); + }; +}; +function modifyGroup(group, fields) { + if (group) { + db.parseIntFields(group, intFields, fields); + escapeGroupData(group); + group.userTitleEnabled = [null, undefined].includes(group.userTitleEnabled) ? 1 : group.userTitleEnabled; + group.labelColor = validator.escape(String(group.labelColor || '#000000')); + group.textColor = validator.escape(String(group.textColor || '#ffffff')); + group.icon = validator.escape(String(group.icon || '')); + group.createtimeISO = utils.toISOString(group.createtime); + group.private = [null, undefined].includes(group.private) ? 1 : group.private; + group.memberPostCids = group.memberPostCids || ''; + group.memberPostCidsArray = group.memberPostCids.split(',').map(cid => parseInt(cid, 10)).filter(Boolean); + group['cover:thumb:url'] = group['cover:thumb:url'] || group['cover:url']; + if (group['cover:url']) { + group['cover:url'] = group['cover:url'].startsWith('http') ? group['cover:url'] : nconf.get('relative_path') + group['cover:url']; + } else { + group['cover:url'] = require('../coverPhoto').getDefaultGroupCover(group.name); + } + if (group['cover:thumb:url']) { + group['cover:thumb:url'] = group['cover:thumb:url'].startsWith('http') ? group['cover:thumb:url'] : nconf.get('relative_path') + group['cover:thumb:url']; + } else { + group['cover:thumb:url'] = require('../coverPhoto').getDefaultGroupCover(group.name); + } + group['cover:position'] = validator.escape(String(group['cover:position'] || '50% 50%')); + } +} +function escapeGroupData(group) { + if (group) { + group.nameEncoded = encodeURIComponent(group.name); + group.displayName = validator.escape(String(group.name)); + group.description = validator.escape(String(group.description || '')); + group.userTitle = validator.escape(String(group.userTitle || '')); + group.userTitleEscaped = translator.escape(group.userTitle); + } +} \ No newline at end of file diff --git a/lib/groups/delete.js b/lib/groups/delete.js new file mode 100644 index 0000000000..09cd27cd02 --- /dev/null +++ b/lib/groups/delete.js @@ -0,0 +1,38 @@ +'use strict'; + +const plugins = require('../plugins'); +const slugify = require('../slugify'); +const db = require('../database'); +const batch = require('../batch'); +module.exports = function (Groups) { + Groups.destroy = async function (groupNames) { + if (!Array.isArray(groupNames)) { + groupNames = [groupNames]; + } + let groupsData = await Groups.getGroupsData(groupNames); + groupsData = groupsData.filter(Boolean); + if (!groupsData.length) { + return; + } + const keys = []; + groupNames.forEach(groupName => { + keys.push(`group:${groupName}`, `group:${groupName}:members`, `group:${groupName}:pending`, `group:${groupName}:invited`, `group:${groupName}:owners`, `group:${groupName}:member:pids`); + }); + const sets = groupNames.map(groupName => `${groupName.toLowerCase()}:${groupName}`); + const groupSlugs = groupNames.filter(groupName => !Groups.isPrivilegeGroup(groupName)).map(groupName => slugify(groupName)); + await Promise.all([db.deleteAll(keys), db.sortedSetRemove(['groups:createtime', 'groups:visible:createtime', 'groups:visible:memberCount'], groupNames), db.sortedSetRemove('groups:visible:name', sets), db.deleteObjectFields('groupslug:groupname', groupSlugs), removeGroupsFromPrivilegeGroups(groupNames)]); + Groups.cache.reset(); + plugins.hooks.fire('action:groups.destroy', { + groups: groupsData + }); + }; + async function removeGroupsFromPrivilegeGroups(groupNames) { + await batch.processSortedSet('groups:createtime', async otherGroups => { + const privilegeGroups = otherGroups.filter(group => Groups.isPrivilegeGroup(group)); + const keys = privilegeGroups.map(group => `group:${group}:members`); + await db.sortedSetRemove(keys, groupNames); + }, { + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/groups/index.js b/lib/groups/index.js new file mode 100644 index 0000000000..7f008f1953 --- /dev/null +++ b/lib/groups/index.js @@ -0,0 +1,207 @@ +'use strict'; + +const user = require('../user'); +const db = require('../database'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const slugify = require('../slugify'); +const Groups = module.exports; +require('./data')(Groups); +require('./create')(Groups); +require('./delete')(Groups); +require('./update')(Groups); +require('./invite')(Groups); +require('./membership')(Groups); +require('./ownership')(Groups); +require('./search')(Groups); +require('./cover')(Groups); +require('./posts')(Groups); +require('./user')(Groups); +require('./join')(Groups); +require('./leave')(Groups); +require('./cache')(Groups); +Groups.BANNED_USERS = 'banned-users'; +Groups.ephemeralGroups = ['guests', 'spiders']; +Groups.systemGroups = ['registered-users', 'verified-users', 'unverified-users', Groups.BANNED_USERS, 'administrators', 'Global Moderators']; +Groups.getEphemeralGroup = function (groupName) { + return { + name: groupName, + slug: slugify(groupName), + description: '', + hidden: 0, + system: 1 + }; +}; +Groups.removeEphemeralGroups = function (groups) { + for (let x = groups.length; x >= 0; x -= 1) { + if (Groups.ephemeralGroups.includes(groups[x])) { + groups.splice(x, 1); + } + } + return groups; +}; +const isPrivilegeGroupRegex = /^cid:(?:-?\d+|admin):privileges:[\w\-:]+$/; +Groups.isPrivilegeGroup = function (groupName) { + return isPrivilegeGroupRegex.test(groupName); +}; +Groups.getGroupsFromSet = async function (set, start, stop) { + let groupNames; + if (set === 'groups:visible:name') { + groupNames = await db.getSortedSetRangeByLex(set, '-', '+', start, stop - start + 1); + } else { + groupNames = await db.getSortedSetRevRange(set, start, stop); + } + if (set === 'groups:visible:name') { + groupNames = groupNames.map(name => name.split(':')[1]); + } + return await Groups.getGroupsAndMembers(groupNames); +}; +Groups.getGroupsBySort = async function (sort, start, stop) { + let set = 'groups:visible:name'; + if (sort === 'count') { + set = 'groups:visible:memberCount'; + } else if (sort === 'date') { + set = 'groups:visible:createtime'; + } + return await Groups.getGroupsFromSet(set, start, stop); +}; +Groups.getNonPrivilegeGroups = async function (set, start, stop, flags) { + if (!flags) { + flags = { + ephemeral: true + }; + } + let groupNames = await db.getSortedSetRevRange(set, start, stop); + groupNames = groupNames.filter(groupName => !Groups.isPrivilegeGroup(groupName)); + if (flags.ephemeral) { + groupNames = groupNames.concat(Groups.ephemeralGroups); + } + const groupsData = await Groups.getGroupsData(groupNames); + return groupsData.filter(Boolean); +}; +Groups.getGroups = async function (set, start, stop) { + return await db.getSortedSetRevRange(set, start, stop); +}; +Groups.getGroupsAndMembers = async function (groupNames) { + const [groups, members] = await Promise.all([Groups.getGroupsData(groupNames), Groups.getMemberUsers(groupNames, 0, 9)]); + groups.forEach((group, index) => { + if (group) { + group.members = members[index] || []; + group.truncated = group.memberCount > group.members.length; + } + }); + return groups; +}; +Groups.get = async function (groupName, options) { + if (!groupName) { + throw new Error('[[error:invalid-group]]'); + } + let stop = -1; + if (options.truncateUserList) { + stop = (parseInt(options.userListCount, 10) || 4) - 1; + } + const [groupData, members, isMember, isPending, isInvited, isOwner, isAdmin, isGlobalMod] = await Promise.all([Groups.getGroupData(groupName), Groups.getOwnersAndMembers(groupName, options.uid, 0, stop), Groups.isMember(options.uid, groupName), Groups.isPending(options.uid, groupName), Groups.isInvited(options.uid, groupName), Groups.ownership.isOwner(options.uid, groupName), privileges.admin.can('admin:groups', options.uid), user.isGlobalModerator(options.uid)]); + if (!groupData) { + return null; + } + groupData.isOwner = isOwner || isAdmin || isGlobalMod && !groupData.system; + if (groupData.isOwner) { + [groupData.pending, groupData.invited] = await Promise.all([Groups.getPending(groupName), Groups.getInvites(groupName)]); + } + const descriptionParsed = await plugins.hooks.fire('filter:parse.raw', String(groupData.description || '')); + groupData.descriptionParsed = descriptionParsed; + groupData.members = members; + groupData.membersNextStart = stop + 1; + groupData.isMember = isMember; + groupData.isPending = isPending; + groupData.isInvited = isInvited; + const results = await plugins.hooks.fire('filter:group.get', { + group: groupData + }); + return results.group; +}; +Groups.getOwners = async function (groupName) { + return await db.getSetMembers(`group:${groupName}:owners`); +}; +Groups.getOwnersAndMembers = async function (groupName, uid, start, stop) { + const ownerUids = await db.getSetMembers(`group:${groupName}:owners`); + const countToReturn = stop - start + 1; + const ownerUidsOnPage = ownerUids.slice(start, stop !== -1 ? stop + 1 : undefined); + const owners = await user.getUsers(ownerUidsOnPage, uid); + owners.forEach(user => { + if (user) { + user.isOwner = true; + } + }); + let done = false; + let returnUsers = owners; + let memberStart = start - ownerUids.length; + let memberStop = memberStart + countToReturn - 1; + memberStart = Math.max(0, memberStart); + memberStop = Math.max(0, memberStop); + async function addMembers(start, stop) { + let batch = await user.getUsersFromSet(`group:${groupName}:members`, uid, start, stop); + if (!batch.length) { + done = true; + } + batch = batch.filter(user => user && user.uid && !ownerUids.includes(user.uid.toString())); + returnUsers = returnUsers.concat(batch); + } + if (stop === -1) { + await addMembers(memberStart, -1); + } else { + while (returnUsers.length < countToReturn && !done) { + await addMembers(memberStart, memberStop); + memberStart = memberStop + 1; + memberStop = memberStart + countToReturn - 1; + } + } + returnUsers = countToReturn > 0 ? returnUsers.slice(0, countToReturn) : returnUsers; + const result = await plugins.hooks.fire('filter:group.getOwnersAndMembers', { + users: returnUsers, + uid: uid, + start: start, + stop: stop + }); + return result.users; +}; +Groups.getByGroupslug = async function (slug, options) { + options = options || {}; + const groupName = await db.getObjectField('groupslug:groupname', slug); + if (!groupName) { + throw new Error('[[error:no-group]]'); + } + return await Groups.get(groupName, options); +}; +Groups.getGroupNameByGroupSlug = async function (slug) { + return await db.getObjectField('groupslug:groupname', slug); +}; +Groups.isPrivate = async function (groupName) { + return await isFieldOn(groupName, 'private'); +}; +Groups.isHidden = async function (groupName) { + return await isFieldOn(groupName, 'hidden'); +}; +async function isFieldOn(groupName, field) { + const value = await db.getObjectField(`group:${groupName}`, field); + return parseInt(value, 10) === 1; +} +Groups.exists = async function (name) { + if (Array.isArray(name)) { + const slugs = name.map(groupName => slugify(groupName)); + const isMembersOfRealGroups = await db.isSortedSetMembers('groups:createtime', name); + const isMembersOfEphemeralGroups = slugs.map(slug => Groups.ephemeralGroups.includes(slug)); + return name.map((n, index) => isMembersOfRealGroups[index] || isMembersOfEphemeralGroups[index]); + } + const slug = slugify(name); + const isMemberOfRealGroups = await db.isSortedSetMember('groups:createtime', name); + const isMemberOfEphemeralGroups = Groups.ephemeralGroups.includes(slug); + return isMemberOfRealGroups || isMemberOfEphemeralGroups; +}; +Groups.existsBySlug = async function (slug) { + if (Array.isArray(slug)) { + return await db.isObjectFields('groupslug:groupname', slug); + } + return await db.isObjectField('groupslug:groupname', slug); +}; +require('../promisify')(Groups); \ No newline at end of file diff --git a/lib/groups/invite.js b/lib/groups/invite.js new file mode 100644 index 0000000000..8096bf2930 --- /dev/null +++ b/lib/groups/invite.js @@ -0,0 +1,95 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const user = require('../user'); +const slugify = require('../slugify'); +const plugins = require('../plugins'); +const notifications = require('../notifications'); +module.exports = function (Groups) { + Groups.getPending = async function (groupName) { + return await Groups.getUsersFromSet(`group:${groupName}:pending`, ['username', 'userslug', 'picture']); + }; + Groups.getInvites = async function (groupName) { + return await Groups.getUsersFromSet(`group:${groupName}:invited`, ['username', 'userslug', 'picture']); + }; + Groups.requestMembership = async function (groupName, uid) { + await inviteOrRequestMembership(groupName, uid, 'request'); + const { + displayname + } = await user.getUserFields(uid, ['username']); + const [notification, owners] = await Promise.all([notifications.create({ + type: 'group-request-membership', + bodyShort: `[[groups:request.notification-title, ${displayname}]]`, + bodyLong: `[[groups:request.notification-text, ${displayname}, ${groupName}]]`, + nid: `group:${groupName}:uid:${uid}:request`, + path: `/groups/${slugify(groupName)}`, + from: uid + }), Groups.getOwners(groupName)]); + await notifications.push(notification, owners); + }; + Groups.acceptMembership = async function (groupName, uid) { + await db.setsRemove([`group:${groupName}:pending`, `group:${groupName}:invited`], uid); + await Groups.join(groupName, uid); + const notification = await notifications.create({ + type: 'group-invite', + bodyShort: `[[groups:membership.accept.notification-title, ${groupName}]]`, + nid: `group:${groupName}:uid:${uid}:invite-accepted`, + path: `/groups/${slugify(groupName)}`, + icon: 'fa-users' + }); + await notifications.push(notification, [uid]); + }; + Groups.rejectMembership = async function (groupNames, uid) { + if (!Array.isArray(groupNames)) { + groupNames = [groupNames]; + } + const sets = []; + groupNames.forEach(groupName => sets.push(`group:${groupName}:pending`, `group:${groupName}:invited`)); + await db.setsRemove(sets, uid); + }; + Groups.invite = async function (groupName, uids) { + uids = Array.isArray(uids) ? uids : [uids]; + uids = await inviteOrRequestMembership(groupName, uids, 'invite'); + const notificationData = await Promise.all(uids.map(uid => notifications.create({ + type: 'group-invite', + bodyShort: `[[groups:invited.notification-title, ${groupName}]]`, + bodyLong: '', + nid: `group:${groupName}:uid:${uid}:invite`, + path: `/groups/${slugify(groupName)}`, + icon: 'fa-users' + }))); + await Promise.all(uids.map((uid, index) => notifications.push(notificationData[index], uid))); + }; + async function inviteOrRequestMembership(groupName, uids, type) { + uids = Array.isArray(uids) ? uids : [uids]; + uids = uids.filter(uid => parseInt(uid, 10) > 0); + const [exists, isMember, isPending, isInvited] = await Promise.all([Groups.exists(groupName), Groups.isMembers(uids, groupName), Groups.isPending(uids, groupName), Groups.isInvited(uids, groupName)]); + if (!exists) { + throw new Error('[[error:no-group]]'); + } + uids = uids.filter((uid, i) => !isMember[i] && (type === 'invite' && !isInvited[i] || type === 'request' && !isPending[i])); + const set = type === 'invite' ? `group:${groupName}:invited` : `group:${groupName}:pending`; + await db.setAdd(set, uids); + const hookName = type === 'invite' ? 'inviteMember' : 'requestMembership'; + plugins.hooks.fire(`action:group.${hookName}`, { + groupName: groupName, + uids: uids + }); + return uids; + } + Groups.isInvited = async function (uids, groupName) { + return await checkInvitePending(uids, `group:${groupName}:invited`); + }; + Groups.isPending = async function (uids, groupName) { + return await checkInvitePending(uids, `group:${groupName}:pending`); + }; + async function checkInvitePending(uids, set) { + const isArray = Array.isArray(uids); + uids = isArray ? uids : [uids]; + const checkUids = uids.filter(uid => parseInt(uid, 10) > 0); + const isMembers = await db.isSetMembers(set, checkUids); + const map = _.zipObject(checkUids, isMembers); + return isArray ? uids.map(uid => !!map[uid]) : !!map[uids[0]]; + } +}; \ No newline at end of file diff --git a/lib/groups/join.js b/lib/groups/join.js new file mode 100644 index 0000000000..524d222abc --- /dev/null +++ b/lib/groups/join.js @@ -0,0 +1,77 @@ +'use strict'; + +const winston = require('winston'); +const db = require('../database'); +const user = require('../user'); +const plugins = require('../plugins'); +const cache = require('../cache'); +module.exports = function (Groups) { + Groups.join = async function (groupNames, uid) { + if (!groupNames) { + throw new Error('[[error:invalid-data]]'); + } + if (Array.isArray(groupNames) && !groupNames.length) { + return; + } + if (!Array.isArray(groupNames)) { + groupNames = [groupNames]; + } + if (!uid) { + throw new Error('[[error:invalid-uid]]'); + } + const [isMembers, exists, isAdmin] = await Promise.all([Groups.isMemberOfGroups(uid, groupNames), Groups.exists(groupNames), user.isAdministrator(uid)]); + const groupsToCreate = groupNames.filter((groupName, index) => groupName && !exists[index]); + const groupsToJoin = groupNames.filter((groupName, index) => !isMembers[index]); + if (!groupsToJoin.length) { + return; + } + await createNonExistingGroups(groupsToCreate); + const promises = [db.sortedSetsAdd(groupsToJoin.map(groupName => `group:${groupName}:members`), Date.now(), uid), db.incrObjectField(groupsToJoin.map(groupName => `group:${groupName}`), 'memberCount')]; + if (isAdmin) { + promises.push(db.setsAdd(groupsToJoin.map(groupName => `group:${groupName}:owners`), uid)); + } + await Promise.all(promises); + Groups.clearCache(uid, groupsToJoin); + cache.del(groupsToJoin.map(name => `group:${name}:members`)); + const groupData = await Groups.getGroupsFields(groupsToJoin, ['name', 'hidden', 'memberCount']); + const visibleGroups = groupData.filter(groupData => groupData && !groupData.hidden); + if (visibleGroups.length) { + await db.sortedSetAdd('groups:visible:memberCount', visibleGroups.map(groupData => groupData.memberCount), visibleGroups.map(groupData => groupData.name)); + } + await setGroupTitleIfNotSet(groupsToJoin, uid); + plugins.hooks.fire('action:group.join', { + groupNames: groupsToJoin, + uid: uid + }); + }; + async function createNonExistingGroups(groupsToCreate) { + if (!groupsToCreate.length) { + return; + } + for (const groupName of groupsToCreate) { + try { + await Groups.create({ + name: groupName, + hidden: 1 + }); + } catch (err) { + if (err && err.message !== '[[error:group-already-exists]]') { + winston.error(`[groups.join] Could not create new hidden group (${groupName})\n${err.stack}`); + throw err; + } + } + } + } + async function setGroupTitleIfNotSet(groupNames, uid) { + const ignore = ['registered-users', 'verified-users', 'unverified-users', Groups.BANNED_USERS]; + groupNames = groupNames.filter(groupName => !ignore.includes(groupName) && !Groups.isPrivilegeGroup(groupName)); + if (!groupNames.length) { + return; + } + const currentTitle = await db.getObjectField(`user:${uid}`, 'groupTitle'); + if (currentTitle || currentTitle === '') { + return; + } + await user.setUserField(uid, 'groupTitle', JSON.stringify(groupNames)); + } +}; \ No newline at end of file diff --git a/lib/groups/leave.js b/lib/groups/leave.js new file mode 100644 index 0000000000..e9be84675f --- /dev/null +++ b/lib/groups/leave.js @@ -0,0 +1,82 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const user = require('../user'); +const plugins = require('../plugins'); +const cache = require('../cache'); +const messaging = require('../messaging'); +module.exports = function (Groups) { + Groups.leave = async function (groupNames, uid) { + if (Array.isArray(groupNames) && !groupNames.length) { + return; + } + if (!Array.isArray(groupNames)) { + groupNames = [groupNames]; + } + const isMembers = await Groups.isMemberOfGroups(uid, groupNames); + const groupsToLeave = groupNames.filter((groupName, index) => isMembers[index]); + if (!groupsToLeave.length) { + return; + } + await Promise.all([db.sortedSetRemove(groupsToLeave.map(groupName => `group:${groupName}:members`), uid), db.setRemove(groupsToLeave.map(groupName => `group:${groupName}:owners`), uid), db.decrObjectField(groupsToLeave.map(groupName => `group:${groupName}`), 'memberCount')]); + Groups.clearCache(uid, groupsToLeave); + cache.del(groupsToLeave.map(name => `group:${name}:members`)); + const groupData = await Groups.getGroupsFields(groupsToLeave, ['name', 'hidden', 'memberCount']); + if (!groupData) { + return; + } + const emptyPrivilegeGroups = groupData.filter(g => g && Groups.isPrivilegeGroup(g.name) && g.memberCount === 0); + const visibleGroups = groupData.filter(g => g && !g.hidden); + const promises = []; + if (emptyPrivilegeGroups.length) { + promises.push(Groups.destroy, emptyPrivilegeGroups); + } + if (visibleGroups.length) { + promises.push(db.sortedSetAdd, 'groups:visible:memberCount', visibleGroups.map(groupData => groupData.memberCount), visibleGroups.map(groupData => groupData.name)); + } + await Promise.all(promises); + await Promise.all([clearGroupTitleIfSet(groupsToLeave, uid), leavePublicRooms(groupsToLeave, uid)]); + plugins.hooks.fire('action:group.leave', { + groupNames: groupsToLeave, + uid: uid + }); + }; + async function leavePublicRooms(groupNames, uid) { + const allRoomIds = await messaging.getPublicRoomIdsFromSet('chat:rooms:public:order'); + const allRoomData = await messaging.getRoomsData(allRoomIds); + const roomData = allRoomData.filter(room => room && room.groups.some(group => groupNames.includes(group))); + const isMemberOfAny = _.zipObject(roomData.map(r => r.roomId), await Promise.all(roomData.map(r => Groups.isMemberOfAny(uid, r.groups)))); + const roomIds = roomData.filter(r => isMemberOfAny[r.roomId]).map(r => r.roomId); + await messaging.leaveRooms(uid, roomIds); + } + async function clearGroupTitleIfSet(groupNames, uid) { + groupNames = groupNames.filter(groupName => groupName !== 'registered-users' && !Groups.isPrivilegeGroup(groupName)); + if (!groupNames.length) { + return; + } + const userData = await user.getUserData(uid); + if (!userData) { + return; + } + const newTitleArray = userData.groupTitleArray.filter(groupTitle => !groupNames.includes(groupTitle)); + if (newTitleArray.length) { + await db.setObjectField(`user:${uid}`, 'groupTitle', JSON.stringify(newTitleArray)); + } else { + await db.deleteObjectField(`user:${uid}`, 'groupTitle'); + } + } + Groups.leaveAllGroups = async function (uid) { + const groups = await db.getSortedSetRange('groups:createtime', 0, -1); + await Promise.all([Groups.leave(groups, uid), Groups.rejectMembership(groups, uid)]); + }; + Groups.kick = async function (uid, groupName, isOwner) { + if (isOwner) { + const numOwners = await db.setCount(`group:${groupName}:owners`); + if (numOwners <= 1) { + throw new Error('[[error:group-needs-owner]]'); + } + } + await Groups.leave(groupName, uid); + }; +}; \ No newline at end of file diff --git a/lib/groups/membership.js b/lib/groups/membership.js new file mode 100644 index 0000000000..e95dd874c4 --- /dev/null +++ b/lib/groups/membership.js @@ -0,0 +1,148 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const user = require('../user'); +const cache = require('../cache'); +module.exports = function (Groups) { + Groups.getMembers = async function (groupName, start, stop) { + return await db.getSortedSetRevRange(`group:${groupName}:members`, start, stop); + }; + Groups.getMemberUsers = async function (groupNames, start, stop) { + async function get(groupName) { + const uids = await Groups.getMembers(groupName, start, stop); + return await user.getUsersFields(uids, ['uid', 'username', 'picture', 'userslug']); + } + return await Promise.all(groupNames.map(name => get(name))); + }; + Groups.getMembersOfGroups = async function (groupNames) { + return await db.getSortedSetsMembers(groupNames.map(name => `group:${name}:members`)); + }; + Groups.isMember = async function (uid, groupName) { + if (!uid || parseInt(uid, 10) <= 0 || !groupName) { + return isMemberOfEphemeralGroup(uid, groupName); + } + const cacheKey = `${uid}:${groupName}`; + let isMember = Groups.cache.get(cacheKey); + if (isMember !== undefined) { + return isMember; + } + isMember = await db.isSortedSetMember(`group:${groupName}:members`, uid); + Groups.cache.set(cacheKey, isMember); + return isMember; + }; + Groups.isMembers = async function (uids, groupName) { + if (!groupName || !uids.length) { + return uids.map(() => false); + } + if (groupName === 'guests' || groupName === 'spiders') { + return uids.map(uid => isMemberOfEphemeralGroup(uid, groupName)); + } + const cachedData = {}; + const nonCachedUids = uids.filter(uid => filterNonCached(cachedData, uid, groupName)); + if (!nonCachedUids.length) { + return uids.map(uid => cachedData[`${uid}:${groupName}`]); + } + const isMembers = await db.isSortedSetMembers(`group:${groupName}:members`, nonCachedUids); + nonCachedUids.forEach((uid, index) => { + cachedData[`${uid}:${groupName}`] = isMembers[index]; + Groups.cache.set(`${uid}:${groupName}`, isMembers[index]); + }); + return uids.map(uid => cachedData[`${uid}:${groupName}`]); + }; + Groups.isMemberOfGroups = async function (uid, groups) { + if (!uid || parseInt(uid, 10) <= 0 || !groups.length) { + return groups.map(groupName => isMemberOfEphemeralGroup(uid, groupName)); + } + const cachedData = {}; + const nonCachedGroups = groups.filter(groupName => filterNonCached(cachedData, uid, groupName)); + if (!nonCachedGroups.length) { + return groups.map(groupName => cachedData[`${uid}:${groupName}`]); + } + const nonCachedGroupsMemberSets = nonCachedGroups.map(groupName => `group:${groupName}:members`); + const isMembers = await db.isMemberOfSortedSets(nonCachedGroupsMemberSets, uid); + nonCachedGroups.forEach((groupName, index) => { + cachedData[`${uid}:${groupName}`] = isMembers[index]; + Groups.cache.set(`${uid}:${groupName}`, isMembers[index]); + }); + return groups.map(groupName => cachedData[`${uid}:${groupName}`]); + }; + function isMemberOfEphemeralGroup(uid, groupName) { + return groupName === 'guests' && parseInt(uid, 10) === 0 || groupName === 'spiders' && parseInt(uid, 10) === -1; + } + function filterNonCached(cachedData, uid, groupName) { + const isMember = Groups.cache.get(`${uid}:${groupName}`); + const isInCache = isMember !== undefined; + if (isInCache) { + cachedData[`${uid}:${groupName}`] = isMember; + } + return !isInCache; + } + Groups.isMemberOfAny = async function (uid, groups) { + if (!Array.isArray(groups) || !groups.length) { + return false; + } + const isMembers = await Groups.isMemberOfGroups(uid, groups); + return isMembers.includes(true); + }; + Groups.getMemberCount = async function (groupName) { + const count = await db.getObjectField(`group:${groupName}`, 'memberCount'); + return parseInt(count, 10); + }; + Groups.isMemberOfGroupList = async function (uid, groupListKey) { + let groupNames = await getGroupNames(groupListKey); + groupNames = Groups.removeEphemeralGroups(groupNames); + if (!groupNames.length) { + return false; + } + const isMembers = await Groups.isMemberOfGroups(uid, groupNames); + return isMembers.includes(true); + }; + Groups.isMemberOfGroupsList = async function (uid, groupListKeys) { + const members = await getGroupNames(groupListKeys); + let uniqueGroups = _.uniq(_.flatten(members)); + uniqueGroups = Groups.removeEphemeralGroups(uniqueGroups); + const isMembers = await Groups.isMemberOfGroups(uid, uniqueGroups); + const isGroupMember = _.zipObject(uniqueGroups, isMembers); + return members.map(groupNames => !!groupNames.find(name => isGroupMember[name])); + }; + Groups.isMembersOfGroupList = async function (uids, groupListKey) { + const results = uids.map(() => false); + let groupNames = await getGroupNames(groupListKey); + groupNames = Groups.removeEphemeralGroups(groupNames); + if (!groupNames.length) { + return results; + } + const isGroupMembers = await Promise.all(groupNames.map(name => Groups.isMembers(uids, name))); + isGroupMembers.forEach(isMembers => { + results.forEach((isMember, index) => { + if (!isMember && isMembers[index]) { + results[index] = true; + } + }); + }); + return results; + }; + async function getGroupNames(keys) { + const isArray = Array.isArray(keys); + keys = isArray ? keys : [keys]; + const cachedData = {}; + const nonCachedKeys = keys.filter(groupName => { + const groupMembers = cache.get(`group:${groupName}:members`); + const isInCache = groupMembers !== undefined; + if (isInCache) { + cachedData[groupName] = groupMembers; + } + return !isInCache; + }); + if (!nonCachedKeys.length) { + return isArray ? keys.map(groupName => cachedData[groupName]) : cachedData[keys[0]]; + } + const groupMembers = await db.getSortedSetsMembers(nonCachedKeys.map(name => `group:${name}:members`)); + nonCachedKeys.forEach((groupName, index) => { + cachedData[groupName] = groupMembers[index]; + cache.set(`group:${groupName}:members`, groupMembers[index]); + }); + return isArray ? keys.map(groupName => cachedData[groupName]) : cachedData[keys[0]]; + } +}; \ No newline at end of file diff --git a/lib/groups/ownership.js b/lib/groups/ownership.js new file mode 100644 index 0000000000..4b0453fa77 --- /dev/null +++ b/lib/groups/ownership.js @@ -0,0 +1,37 @@ +'use strict'; + +const db = require('../database'); +const plugins = require('../plugins'); +module.exports = function (Groups) { + Groups.ownership = {}; + Groups.ownership.isOwner = async function (uid, groupName) { + if (!(parseInt(uid, 10) > 0)) { + return false; + } + return await db.isSetMember(`group:${groupName}:owners`, uid); + }; + Groups.ownership.isOwners = async function (uids, groupName) { + if (!Array.isArray(uids)) { + return []; + } + return await db.isSetMembers(`group:${groupName}:owners`, uids); + }; + Groups.ownership.grant = async function (toUid, groupName) { + await db.setAdd(`group:${groupName}:owners`, toUid); + plugins.hooks.fire('action:group.grantOwnership', { + uid: toUid, + groupName: groupName + }); + }; + Groups.ownership.rescind = async function (toUid, groupName) { + const [numOwners, isOwner] = await Promise.all([db.setCount(`group:${groupName}:owners`), db.isSetMember(`group:${groupName}:owners`, toUid)]); + if (numOwners <= 1 && isOwner) { + throw new Error('[[error:group-needs-owner]]'); + } + await db.setRemove(`group:${groupName}:owners`, toUid); + plugins.hooks.fire('action:group.rescindOwnership', { + uid: toUid, + groupName: groupName + }); + }; +}; \ No newline at end of file diff --git a/lib/groups/posts.js b/lib/groups/posts.js new file mode 100644 index 0000000000..39f5cca686 --- /dev/null +++ b/lib/groups/posts.js @@ -0,0 +1,37 @@ +'use strict'; + +const db = require('../database'); +const privileges = require('../privileges'); +const posts = require('../posts'); +module.exports = function (Groups) { + Groups.onNewPostMade = async function (postData) { + if (!parseInt(postData.uid, 10) || postData.timestamp > Date.now()) { + return; + } + let groupNames = await Groups.getUserGroupMembership('groups:visible:createtime', [postData.uid]); + groupNames = groupNames[0]; + const groupData = await Groups.getGroupsFields(groupNames, ['memberPostCids']); + groupNames = groupNames.filter((groupName, idx) => !groupData[idx].memberPostCidsArray.length || groupData[idx].memberPostCidsArray.includes(postData.cid)); + const keys = groupNames.map(groupName => `group:${groupName}:member:pids`); + await db.sortedSetsAdd(keys, postData.timestamp, postData.pid); + await Promise.all(groupNames.map(truncateMemberPosts)); + }; + async function truncateMemberPosts(groupName) { + let lastPid = await db.getSortedSetRevRangeByScore(`group:${groupName}:member:pids`, 10, 1, Date.now(), '-inf'); + lastPid = lastPid[0]; + if (!parseInt(lastPid, 10)) { + return; + } + const score = await db.sortedSetScore(`group:${groupName}:member:pids`, lastPid); + await db.sortedSetsRemoveRangeByScore([`group:${groupName}:member:pids`], '-inf', score); + } + Groups.getLatestMemberPosts = async function (groupName, max, uid) { + const [allPids, groupData] = await Promise.all([db.getSortedSetRevRangeByScore(`group:${groupName}:member:pids`, 0, max, Date.now(), '-inf'), Groups.getGroupFields(groupName, ['memberPostCids'])]); + const cids = groupData.memberPostCidsArray; + const pids = await privileges.posts.filter('topics:read', allPids, uid); + const postData = await posts.getPostSummaryByPids(pids, uid, { + stripTags: false + }); + return postData.filter(p => p && p.topic && (!cids.length || cids.includes(p.topic.cid))); + }; +}; \ No newline at end of file diff --git a/lib/groups/search.js b/lib/groups/search.js new file mode 100644 index 0000000000..dea77fc09e --- /dev/null +++ b/lib/groups/search.js @@ -0,0 +1,76 @@ +'use strict'; + +const user = require('../user'); +const db = require('../database'); +module.exports = function (Groups) { + Groups.search = async function (query, options) { + if (!query) { + return []; + } + query = String(query).toLowerCase(); + let groupNames = Object.values(await db.getObject('groupslug:groupname')); + if (!options.hideEphemeralGroups) { + groupNames = Groups.ephemeralGroups.concat(groupNames); + } + groupNames = groupNames.filter(name => name.toLowerCase().includes(query) && name !== Groups.BANNED_USERS); + groupNames = groupNames.slice(0, 100); + let groupsData; + if (options.showMembers) { + groupsData = await Groups.getGroupsAndMembers(groupNames); + } else { + groupsData = await Groups.getGroupsData(groupNames); + } + groupsData = groupsData.filter(Boolean); + if (options.filterHidden) { + groupsData = groupsData.filter(group => !group.hidden); + } + return Groups.sort(options.sort, groupsData); + }; + Groups.sort = function (strategy, groups) { + switch (strategy) { + case 'count': + groups.sort((a, b) => a.slug > b.slug).sort((a, b) => b.memberCount - a.memberCount); + break; + case 'date': + groups.sort((a, b) => b.createtime - a.createtime); + break; + case 'alpha': + default: + groups.sort((a, b) => a.slug > b.slug ? 1 : -1); + } + return groups; + }; + Groups.searchMembers = async function (data) { + if (!data.query) { + const users = await Groups.getOwnersAndMembers(data.groupName, data.uid, 0, 19); + const matchCount = users.length; + const timing = '0.00'; + return { + users, + matchCount, + timing + }; + } + const results = await user.search({ + ...data, + paginate: false, + hardCap: -1 + }); + const uids = results.users.map(user => user && user.uid); + const isOwners = await Groups.ownership.isOwners(uids, data.groupName); + results.users.forEach((user, index) => { + if (user) { + user.isOwner = isOwners[index]; + } + }); + results.users.sort((a, b) => { + if (a.isOwner && !b.isOwner) { + return -1; + } else if (!a.isOwner && b.isOwner) { + return 1; + } + return 0; + }); + return results; + }; +}; \ No newline at end of file diff --git a/lib/groups/update.js b/lib/groups/update.js new file mode 100644 index 0000000000..aaec775473 --- /dev/null +++ b/lib/groups/update.js @@ -0,0 +1,245 @@ +'use strict'; + +const winston = require('winston'); +const categories = require('../categories'); +const plugins = require('../plugins'); +const slugify = require('../slugify'); +const db = require('../database'); +const user = require('../user'); +const batch = require('../batch'); +const meta = require('../meta'); +const cache = require('../cache'); +module.exports = function (Groups) { + Groups.update = async function (groupName, values) { + const exists = await db.exists(`group:${groupName}`); + if (!exists) { + throw new Error('[[error:no-group]]'); + } + ({ + values + } = await plugins.hooks.fire('filter:group.update', { + groupName: groupName, + values: values + })); + ['userTitleEnabled', 'private', 'hidden', 'disableJoinRequests', 'disableLeave'].forEach(prop => { + if (values.hasOwnProperty(prop) && typeof values[prop] !== 'boolean') { + values[prop] = values[prop] === 'true' || parseInt(values[prop], 10) === 1; + } + }); + const payload = { + description: values.description || '', + icon: values.icon || '', + labelColor: values.labelColor || '#000000', + textColor: values.textColor || '#ffffff' + }; + if (values.hasOwnProperty('userTitle')) { + payload.userTitle = values.userTitle || ''; + } + if (values.hasOwnProperty('userTitleEnabled')) { + payload.userTitleEnabled = values.userTitleEnabled ? '1' : '0'; + } + if (values.hasOwnProperty('hidden')) { + payload.hidden = values.hidden ? '1' : '0'; + } + if (values.hasOwnProperty('private')) { + payload.private = values.private ? '1' : '0'; + } + if (values.hasOwnProperty('disableJoinRequests')) { + payload.disableJoinRequests = values.disableJoinRequests ? '1' : '0'; + } + if (values.hasOwnProperty('disableLeave')) { + payload.disableLeave = values.disableLeave ? '1' : '0'; + } + if (values.hasOwnProperty('name')) { + await checkNameChange(groupName, values.name); + } + if (values.hasOwnProperty('private')) { + await updatePrivacy(groupName, values.private); + } + if (values.hasOwnProperty('hidden')) { + await updateVisibility(groupName, values.hidden); + } + if (values.hasOwnProperty('memberPostCids')) { + const validCids = await categories.getCidsByPrivilege('categories:cid', groupName, 'topics:read'); + const cidsArray = values.memberPostCids.split(',').map(cid => parseInt(cid.trim(), 10)).filter(Boolean); + payload.memberPostCids = cidsArray.filter(cid => validCids.includes(cid)).join(',') || ''; + } + await db.setObject(`group:${groupName}`, payload); + await Groups.renameGroup(groupName, values.name); + plugins.hooks.fire('action:group.update', { + name: groupName, + values: values + }); + }; + async function updateVisibility(groupName, hidden) { + if (hidden) { + await db.sortedSetRemoveBulk([['groups:visible:createtime', groupName], ['groups:visible:memberCount', groupName], ['groups:visible:name', `${groupName.toLowerCase()}:${groupName}`]]); + return; + } + const groupData = await db.getObjectFields(`group:${groupName}`, ['createtime', 'memberCount']); + await db.sortedSetAddBulk([['groups:visible:createtime', groupData.createtime, groupName], ['groups:visible:memberCount', groupData.memberCount, groupName], ['groups:visible:name', 0, `${groupName.toLowerCase()}:${groupName}`]]); + } + Groups.hide = async function (groupName) { + await showHide(groupName, 'hidden'); + }; + Groups.show = async function (groupName) { + await showHide(groupName, 'show'); + }; + async function showHide(groupName, hidden) { + hidden = hidden === 'hidden'; + await Promise.all([db.setObjectField(`group:${groupName}`, 'hidden', hidden ? 1 : 0), updateVisibility(groupName, hidden)]); + } + async function updatePrivacy(groupName, isPrivate) { + const groupData = await Groups.getGroupFields(groupName, ['private']); + const currentlyPrivate = groupData.private === 1; + if (!currentlyPrivate || currentlyPrivate === isPrivate) { + return; + } + const pendingUids = await db.getSetMembers(`group:${groupName}:pending`); + if (!pendingUids.length) { + return; + } + winston.verbose(`[groups.update] Group is now public, automatically adding ${pendingUids.length} new members, who were pending prior.`); + for (const uid of pendingUids) { + await Groups.join(groupName, uid); + } + await db.delete(`group:${groupName}:pending`); + } + async function checkNameChange(currentName, newName) { + if (Groups.isPrivilegeGroup(newName)) { + throw new Error('[[error:invalid-group-name]]'); + } + const currentSlug = slugify(currentName); + const newSlug = slugify(newName); + if (currentName === newName || currentSlug === newSlug) { + return; + } + Groups.validateGroupName(newName); + const [group, exists] = await Promise.all([Groups.getGroupData(currentName), Groups.existsBySlug(newSlug)]); + if (exists) { + throw new Error('[[error:group-already-exists]]'); + } + if (!group) { + throw new Error('[[error:no-group]]'); + } + if (group.system) { + throw new Error('[[error:not-allowed-to-rename-system-group]]'); + } + } + Groups.renameGroup = async function (oldName, newName) { + if (oldName === newName || !newName || String(newName).length === 0) { + return; + } + const group = await db.getObject(`group:${oldName}`); + if (!group) { + return; + } + const exists = await Groups.exists(newName); + if (exists) { + throw new Error('[[error:group-already-exists]]'); + } + await updateMemberGroupTitles(oldName, newName); + await updateNavigationItems(oldName, newName); + await updateWidgets(oldName, newName); + await updateConfig(oldName, newName); + await updateChatRooms(oldName, newName); + await db.setObject(`group:${oldName}`, { + name: newName, + slug: slugify(newName) + }); + if (!Groups.isPrivilegeGroup(oldName) && !Groups.isPrivilegeGroup(newName)) { + await db.deleteObjectField('groupslug:groupname', group.slug); + await db.setObjectField('groupslug:groupname', slugify(newName), newName); + } + const allGroups = await db.getSortedSetRange('groups:createtime', 0, -1); + const keys = allGroups.map(group => `group:${group}:members`); + await renameGroupsMember(keys, oldName, newName); + cache.del(keys); + await db.rename(`group:${oldName}`, `group:${newName}`); + await db.rename(`group:${oldName}:members`, `group:${newName}:members`); + await db.rename(`group:${oldName}:owners`, `group:${newName}:owners`); + await db.rename(`group:${oldName}:pending`, `group:${newName}:pending`); + await db.rename(`group:${oldName}:invited`, `group:${newName}:invited`); + await db.rename(`group:${oldName}:member:pids`, `group:${newName}:member:pids`); + await renameGroupsMember(['groups:createtime', 'groups:visible:createtime', 'groups:visible:memberCount'], oldName, newName); + await renameGroupsMember(['groups:visible:name'], `${oldName.toLowerCase()}:${oldName}`, `${newName.toLowerCase()}:${newName}`); + plugins.hooks.fire('action:group.rename', { + old: oldName, + new: newName + }); + Groups.cache.reset(); + }; + async function updateMemberGroupTitles(oldName, newName) { + await batch.processSortedSet(`group:${oldName}:members`, async uids => { + let usersData = await user.getUsersData(uids); + usersData = usersData.filter(userData => userData && userData.groupTitleArray.includes(oldName)); + usersData.forEach(userData => { + userData.newTitleArray = userData.groupTitleArray.map(oldTitle => oldTitle === oldName ? newName : oldTitle); + }); + await Promise.all(usersData.map(u => user.setUserField(u.uid, 'groupTitle', JSON.stringify(u.newTitleArray)))); + }, {}); + } + async function renameGroupsMember(keys, oldName, newName) { + const isMembers = await db.isMemberOfSortedSets(keys, oldName); + keys = keys.filter((key, index) => isMembers[index]); + if (!keys.length) { + return; + } + const scores = await db.sortedSetsScore(keys, oldName); + await db.sortedSetsRemove(keys, oldName); + await db.sortedSetsAdd(keys, scores, newName); + } + async function updateNavigationItems(oldName, newName) { + const navigation = require('../navigation/admin'); + const navItems = await navigation.get(); + navItems.forEach(navItem => { + if (navItem && Array.isArray(navItem.groups) && navItem.groups.includes(oldName)) { + navItem.groups.splice(navItem.groups.indexOf(oldName), 1, newName); + } + }); + navigation.unescapeFields(navItems); + await navigation.save(navItems); + } + async function updateWidgets(oldName, newName) { + const admin = require('../widgets/admin'); + const widgets = require('../widgets'); + const data = await admin.get(); + data.areas.forEach(area => { + area.widgets = area.data; + area.widgets.forEach(widget => { + if (widget && widget.data && Array.isArray(widget.data.groups) && widget.data.groups.includes(oldName)) { + widget.data.groups.splice(widget.data.groups.indexOf(oldName), 1, newName); + } + }); + }); + for (const area of data.areas) { + if (area.data.length) { + await widgets.setArea(area); + } + } + } + async function updateConfig(oldName, newName) { + const configKeys = ['groupsExemptFromPostQueue', 'groupsExemptFromNewUserRestrictions', 'groupsExemptFromMaintenanceMode']; + for (const key of configKeys) { + if (meta.config[key] && meta.config[key].includes(oldName)) { + meta.config[key].splice(meta.config[key].indexOf(oldName), 1, newName); + await meta.configs.set(key, meta.config[key]); + } + } + } + async function updateChatRooms(oldName, newName) { + const messaging = require('../messaging'); + const roomIds = await db.getSortedSetRange('chat:rooms:public', 0, -1); + const roomData = await messaging.getRoomsData(roomIds); + const bulkSet = []; + roomData.forEach(room => { + if (room && room.public && Array.isArray(room.groups) && room.groups.includes(oldName)) { + room.groups.splice(room.groups.indexOf(oldName), 1, newName); + bulkSet.push([`chat:room:${room.roomId}`, { + groups: JSON.stringify(room.groups) + }]); + } + }); + await db.setObjectBulk(bulkSet); + } +}; \ No newline at end of file diff --git a/lib/groups/user.js b/lib/groups/user.js new file mode 100644 index 0000000000..4a02b8de1d --- /dev/null +++ b/lib/groups/user.js @@ -0,0 +1,50 @@ +'use strict'; + +const db = require('../database'); +const user = require('../user'); +module.exports = function (Groups) { + Groups.getUsersFromSet = async function (set, fields = []) { + const uids = await db.getSetMembers(set); + const userData = await user.getUsersFields(uids, fields); + return userData.filter(u => u && u.uid); + }; + Groups.getUserGroups = async function (uids) { + return await Groups.getUserGroupsFromSet('groups:visible:createtime', uids); + }; + Groups.getUserGroupsFromSet = async function (set, uids) { + const memberOf = await Groups.getUserGroupMembership(set, uids); + return await Promise.all(memberOf.map(memberOf => Groups.getGroupsData(memberOf))); + }; + Groups.getUserGroupMembership = async function (set, uids) { + const groupNames = await db.getSortedSetRevRange(set, 0, -1); + return await Promise.all(uids.map(uid => findUserGroups(uid, groupNames))); + }; + async function findUserGroups(uid, groupNames) { + const isMembers = await Groups.isMemberOfGroups(uid, groupNames); + return groupNames.filter((name, i) => isMembers[i]); + } + Groups.getUserInviteGroups = async function (uid) { + let allGroups = await Groups.getNonPrivilegeGroups('groups:createtime', 0, -1); + allGroups = allGroups.filter(group => !Groups.ephemeralGroups.includes(group.name)); + const publicGroups = allGroups.filter(group => group.hidden === 0 && group.system === 0 && group.private === 0); + const adminModGroups = [{ + name: 'administrators', + displayName: 'administrators' + }, { + name: 'Global Moderators', + displayName: 'Global Moderators' + }]; + const privateGroups = allGroups.filter(group => group.hidden === 0 && group.system === 0 && group.private === 1); + const [ownership, isAdmin, isGlobalMod] = await Promise.all([Promise.all(privateGroups.map(group => Groups.ownership.isOwner(uid, group.name))), user.isAdministrator(uid), user.isGlobalModerator(uid)]); + const ownGroups = privateGroups.filter((group, index) => ownership[index]); + let inviteGroups = []; + if (isAdmin) { + inviteGroups = inviteGroups.concat(adminModGroups).concat(privateGroups); + } else if (isGlobalMod) { + inviteGroups = inviteGroups.concat(privateGroups); + } else { + inviteGroups = inviteGroups.concat(ownGroups); + } + return inviteGroups.concat(publicGroups); + }; +}; \ No newline at end of file diff --git a/lib/helpers.js b/lib/helpers.js new file mode 100644 index 0000000000..7535e19d61 --- /dev/null +++ b/lib/helpers.js @@ -0,0 +1,3 @@ +'use strict'; + +module.exports = require('../public/src/modules/helpers.common')(require('./utils'), require('benchpressjs'), require('nconf').get('relative_path')); \ No newline at end of file diff --git a/lib/image.js b/lib/image.js new file mode 100644 index 0000000000..5656179449 --- /dev/null +++ b/lib/image.js @@ -0,0 +1,169 @@ +'use strict'; + +const os = require('os'); +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const winston = require('winston'); +const file = require('./file'); +const plugins = require('./plugins'); +const meta = require('./meta'); +const image = module.exports; +function requireSharp() { + const sharp = require('sharp'); + if (os.platform() === 'win32') { + sharp.cache(false); + } + return sharp; +} +image.isFileTypeAllowed = async function (path) { + const plugins = require('./plugins'); + if (plugins.hooks.hasListeners('filter:image.isFileTypeAllowed')) { + return await plugins.hooks.fire('filter:image.isFileTypeAllowed', path); + } + const sharp = require('sharp'); + await sharp(path, { + failOnError: true + }).metadata(); +}; +image.resizeImage = async function (data) { + if (plugins.hooks.hasListeners('filter:image.resize')) { + await plugins.hooks.fire('filter:image.resize', { + path: data.path, + target: data.target, + width: data.width, + height: data.height, + quality: data.quality + }); + } else { + const sharp = requireSharp(); + const buffer = await fs.promises.readFile(data.path); + const sharpImage = sharp(buffer, { + failOnError: true, + animated: data.path.endsWith('gif') + }); + const metadata = await sharpImage.metadata(); + sharpImage.rotate(); + sharpImage.resize(data.hasOwnProperty('width') ? data.width : null, data.hasOwnProperty('height') ? data.height : null); + if (data.quality) { + switch (metadata.format) { + case 'jpeg': + { + sharpImage.jpeg({ + quality: data.quality, + mozjpeg: true + }); + break; + } + case 'png': + { + sharpImage.png({ + quality: data.quality, + compressionLevel: 9 + }); + break; + } + } + } + await sharpImage.toFile(data.target || data.path); + } +}; +image.normalise = async function (path) { + if (plugins.hooks.hasListeners('filter:image.normalise')) { + await plugins.hooks.fire('filter:image.normalise', { + path: path + }); + } else { + const sharp = requireSharp(); + await sharp(path, { + failOnError: true + }).png().toFile(`${path}.png`); + } + return `${path}.png`; +}; +image.size = async function (path) { + let imageData; + if (plugins.hooks.hasListeners('filter:image.size')) { + imageData = await plugins.hooks.fire('filter:image.size', { + path: path + }); + } else { + const sharp = requireSharp(); + imageData = await sharp(path, { + failOnError: true + }).metadata(); + } + return imageData ? { + width: imageData.width, + height: imageData.height + } : undefined; +}; +image.stripEXIF = async function (path) { + if (!meta.config.stripEXIFData || path.endsWith('.svg')) { + return; + } + try { + if (plugins.hooks.hasListeners('filter:image.stripEXIF')) { + await plugins.hooks.fire('filter:image.stripEXIF', { + path: path + }); + return; + } + const buffer = await fs.promises.readFile(path); + const sharp = requireSharp(); + await sharp(buffer, { + failOnError: true, + pages: -1 + }).rotate().toFile(path); + } catch (err) { + winston.error(err.stack); + } +}; +image.checkDimensions = async function (path) { + const meta = require('./meta'); + const result = await image.size(path); + if (result.width > meta.config.rejectImageWidth || result.height > meta.config.rejectImageHeight) { + throw new Error('[[error:invalid-image-dimensions]]'); + } + return result; +}; +image.convertImageToBase64 = async function (path) { + return await fs.promises.readFile(path, 'base64'); +}; +image.mimeFromBase64 = function (imageData) { + return imageData.slice(5, imageData.indexOf('base64') - 1); +}; +image.extensionFromBase64 = function (imageData) { + return file.typeToExtension(image.mimeFromBase64(imageData)); +}; +image.writeImageDataToTempFile = async function (imageData) { + const filename = crypto.createHash('md5').update(imageData).digest('hex'); + const type = image.mimeFromBase64(imageData); + const extension = file.typeToExtension(type); + const filepath = path.join(os.tmpdir(), filename + extension); + const buffer = Buffer.from(imageData.slice(imageData.indexOf('base64') + 7), 'base64'); + await fs.promises.writeFile(filepath, buffer, { + encoding: 'base64' + }); + return filepath; +}; +image.sizeFromBase64 = function (imageData) { + return Buffer.from(imageData.slice(imageData.indexOf('base64') + 7), 'base64').length; +}; +image.uploadImage = async function (filename, folder, imageData) { + if (plugins.hooks.hasListeners('filter:uploadImage')) { + return await plugins.hooks.fire('filter:uploadImage', { + image: imageData, + uid: imageData.uid, + folder: folder + }); + } + await image.isFileTypeAllowed(imageData.path); + const upload = await file.saveFileToLocal(filename, folder, imageData.path); + return { + url: upload.url, + path: upload.path, + name: imageData.name + }; +}; +require('./promisify')(image); \ No newline at end of file diff --git a/lib/install.js b/lib/install.js new file mode 100644 index 0000000000..99b863cd51 --- /dev/null +++ b/lib/install.js @@ -0,0 +1,501 @@ +'use strict'; + +const fs = require('fs'); +const url = require('url'); +const path = require('path'); +const prompt = require('prompt'); +const winston = require('winston'); +const nconf = require('nconf'); +const _ = require('lodash'); +const utils = require('./utils'); +const { + paths +} = require('./constants'); +const install = module.exports; +const questions = {}; +questions.main = [{ + name: 'url', + description: 'URL used to access this NodeBB', + default: nconf.get('url') || 'http://localhost:4567', + pattern: /^http(?:s)?:\/\//, + message: 'Base URL must begin with \'http://\' or \'https://\'' +}, { + name: 'secret', + description: 'Please enter a NodeBB secret', + default: nconf.get('secret') || utils.generateUUID() +}, { + name: 'submitPluginUsage', + description: 'Would you like to submit anonymous plugin usage to nbbpm?', + default: 'yes' +}, { + name: 'database', + description: 'Which database to use', + default: nconf.get('database') || 'mongo' +}]; +questions.optional = [{ + name: 'port', + default: nconf.get('port') || 4567 +}]; +function checkSetupFlagEnv() { + let setupVal = install.values; + const envConfMap = { + CONFIG: 'config', + NODEBB_CONFIG: 'config', + NODEBB_URL: 'url', + NODEBB_PORT: 'port', + NODEBB_ADMIN_USERNAME: 'admin:username', + NODEBB_ADMIN_PASSWORD: 'admin:password', + NODEBB_ADMIN_EMAIL: 'admin:email', + NODEBB_DB: 'database', + NODEBB_DB_HOST: 'host', + NODEBB_DB_PORT: 'port', + NODEBB_DB_USER: 'username', + NODEBB_DB_PASSWORD: 'password', + NODEBB_DB_NAME: 'database', + NODEBB_DB_SSL: 'ssl' + }; + const envKeys = Object.keys(process.env); + if (Object.keys(envConfMap).some(key => envKeys.includes(key))) { + winston.info('[install/checkSetupFlagEnv] checking env vars for setup info...'); + setupVal = setupVal || {}; + Object.entries(process.env).forEach(([evName, evValue]) => { + if (evName.startsWith('NODEBB_DB_')) { + setupVal[`${process.env.NODEBB_DB}:${envConfMap[evName]}`] = evValue; + } else if (evName.startsWith('NODEBB_')) { + setupVal[envConfMap[evName]] = evValue; + } + }); + setupVal['admin:password:confirm'] = setupVal['admin:password']; + } + try { + if (nconf.get('setup')) { + const setupJSON = JSON.parse(nconf.get('setup')); + setupVal = { + ...setupVal, + ...setupJSON + }; + } + } catch (err) { + winston.error('[install/checkSetupFlagEnv] invalid json in nconf.get(\'setup\'), ignoring setup values from json'); + } + if (setupVal && typeof setupVal === 'object') { + if (setupVal['admin:username'] && setupVal['admin:password'] && setupVal['admin:password:confirm'] && setupVal['admin:email']) { + install.values = setupVal; + } else { + winston.error('[install/checkSetupFlagEnv] required values are missing for automated setup:'); + if (!setupVal['admin:username']) { + winston.error(' admin:username'); + } + if (!setupVal['admin:password']) { + winston.error(' admin:password'); + } + if (!setupVal['admin:password:confirm']) { + winston.error(' admin:password:confirm'); + } + if (!setupVal['admin:email']) { + winston.error(' admin:email'); + } + process.exit(); + } + } else if (nconf.get('database')) { + install.values = install.values || {}; + install.values.database = nconf.get('database'); + } +} +function checkCIFlag() { + let ciVals; + try { + ciVals = JSON.parse(nconf.get('ci')); + } catch (e) { + ciVals = undefined; + } + if (ciVals && ciVals instanceof Object) { + if (ciVals.hasOwnProperty('host') && ciVals.hasOwnProperty('port') && ciVals.hasOwnProperty('database')) { + install.ciVals = ciVals; + } else { + winston.error('[install/checkCIFlag] required values are missing for automated CI integration:'); + if (!ciVals.hasOwnProperty('host')) { + winston.error(' host'); + } + if (!ciVals.hasOwnProperty('port')) { + winston.error(' port'); + } + if (!ciVals.hasOwnProperty('database')) { + winston.error(' database'); + } + process.exit(); + } + } +} +async function setupConfig() { + const configureDatabases = require('../install/databases'); + prompt.start(); + prompt.message = ''; + prompt.delimiter = ''; + prompt.colors = false; + let config = {}; + if (install.values) { + const redisQuestions = require('./database/redis').questions; + const mongoQuestions = require('./database/mongo').questions; + const postgresQuestions = require('./database/postgres').questions; + const allQuestions = [...questions.main, ...questions.optional, ...redisQuestions, ...mongoQuestions, ...postgresQuestions]; + allQuestions.forEach(question => { + if (install.values.hasOwnProperty(question.name)) { + config[question.name] = install.values[question.name]; + } else if (question.hasOwnProperty('default')) { + config[question.name] = question.default; + } else { + config[question.name] = undefined; + } + }); + } else { + config = await prompt.get(questions.main); + } + await configureDatabases(config); + await completeConfigSetup(config); +} +async function completeConfigSetup(config) { + if (install.ciVals) { + config.test_database = { + ...install.ciVals + }; + } + if (nconf.get('package_manager')) { + config.package_manager = nconf.get('package_manager'); + } + nconf.overrides(config); + const db = require('./database'); + await db.init(); + if (db.hasOwnProperty('createIndices')) { + await db.createIndices(); + } + if (!/^http(?:s)?:\/\//.test(config.url)) { + config.url = `http://${config.url}`; + } + const urlObj = url.parse(config.url); + if (urlObj.port && (!install.values || !install.values.hasOwnProperty('port'))) { + config.port = urlObj.port; + } + if (urlObj.path === '/') { + urlObj.path = ''; + urlObj.pathname = ''; + } + config.url = url.format(urlObj); + delete config.type; + const meta = require('./meta'); + await meta.configs.set('submitPluginUsage', config.submitPluginUsage === 'yes' ? 1 : 0); + delete config.submitPluginUsage; + await install.save(config); +} +async function setupDefaultConfigs() { + console.log('Populating database with default configs, if not already set...'); + const meta = require('./meta'); + const defaults = require(path.join(__dirname, '../', 'install/data/defaults.json')); + await meta.configs.setOnEmpty(defaults); + await meta.configs.init(); +} +async function enableDefaultTheme() { + const meta = require('./meta'); + const id = await meta.configs.get('theme:id'); + if (id) { + console.log('Previous theme detected, skipping enabling default theme'); + return; + } + const defaultTheme = nconf.get('defaultTheme') || 'nodebb-theme-harmony'; + console.log(`Enabling default theme: ${defaultTheme}`); + await meta.themes.set({ + type: 'local', + id: defaultTheme + }); +} +async function createDefaultUserGroups() { + const groups = require('./groups'); + async function createGroup(name) { + await groups.create({ + name: name, + hidden: 1, + private: 1, + system: 1, + disableLeave: 1, + disableJoinRequests: 1 + }); + } + const [verifiedExists, unverifiedExists, bannedExists] = await groups.exists(['verified-users', 'unverified-users', 'banned-users']); + if (!verifiedExists) { + await createGroup('verified-users'); + } + if (!unverifiedExists) { + await createGroup('unverified-users'); + } + if (!bannedExists) { + await createGroup('banned-users'); + } +} +async function createAdministrator() { + const Groups = require('./groups'); + const memberCount = await Groups.getMemberCount('administrators'); + if (memberCount > 0) { + console.log('Administrator found, skipping Admin setup'); + return; + } + return await createAdmin(); +} +async function createAdmin() { + const User = require('./user'); + const Groups = require('./groups'); + let password; + winston.warn('No administrators have been detected, running initial user setup\n'); + let questions = [{ + name: 'username', + description: 'Administrator username', + required: true, + type: 'string' + }, { + name: 'email', + description: 'Administrator email address', + pattern: /.+@.+/, + required: true + }]; + const passwordQuestions = [{ + name: 'password', + description: 'Password', + required: true, + hidden: true, + type: 'string' + }, { + name: 'password:confirm', + description: 'Confirm Password', + required: true, + hidden: true, + type: 'string' + }]; + async function success(results) { + if (!results) { + throw new Error('aborted'); + } + if (results['password:confirm'] !== results.password) { + winston.warn('Passwords did not match, please try again'); + return await retryPassword(results); + } + try { + User.isPasswordValid(results.password); + } catch (err) { + const [namespace, key] = err.message.slice(2, -2).split(':', 2); + if (namespace && key && err.message.startsWith('[[') && err.message.endsWith(']]')) { + const lang = require(path.join(__dirname, `../public/language/en-GB/${namespace}`)); + if (lang && lang[key]) { + err.message = lang[key]; + } + } + winston.warn(`Password error, please try again. ${err.message}`); + return await retryPassword(results); + } + const adminUid = await User.create({ + username: results.username, + password: results.password, + email: results.email + }); + await Groups.join('administrators', adminUid); + await Groups.show('administrators'); + await Groups.ownership.grant(adminUid, 'administrators'); + return password ? results : undefined; + } + async function retryPassword(originalResults) { + const results = await prompt.get(passwordQuestions); + originalResults.password = results.password; + originalResults['password:confirm'] = results['password:confirm']; + return await success(originalResults); + } + questions = questions.concat(passwordQuestions); + if (!install.values) { + const results = await prompt.get(questions); + return await success(results); + } + if (!install.values.hasOwnProperty('admin:password') && !nconf.get('admin:password')) { + console.log('Password was not provided during automated setup, generating one...'); + password = utils.generateUUID().slice(0, 8); + } + const results = { + username: install.values['admin:username'] || nconf.get('admin:username') || 'admin', + email: install.values['admin:email'] || nconf.get('admin:email') || '', + password: install.values['admin:password'] || nconf.get('admin:password') || password, + 'password:confirm': install.values['admin:password:confirm'] || nconf.get('admin:password') || password + }; + return await success(results); +} +async function createGlobalModeratorsGroup() { + const groups = require('./groups'); + const exists = await groups.exists('Global Moderators'); + if (exists) { + winston.info('Global Moderators group found, skipping creation!'); + } else { + await groups.create({ + name: 'Global Moderators', + userTitle: 'Global Moderator', + description: 'Forum wide moderators', + hidden: 0, + private: 1, + disableJoinRequests: 1 + }); + } + await groups.show('Global Moderators'); +} +async function giveGlobalPrivileges() { + const privileges = require('./privileges'); + const defaultPrivileges = ['groups:chat', 'groups:upload:post:image', 'groups:signature', 'groups:search:content', 'groups:search:users', 'groups:search:tags', 'groups:view:users', 'groups:view:tags', 'groups:view:groups', 'groups:local:login']; + await privileges.global.give(defaultPrivileges, 'registered-users'); + await privileges.global.give(defaultPrivileges.concat(['groups:ban', 'groups:upload:post:file', 'groups:view:users:info']), 'Global Moderators'); + await privileges.global.give(['groups:view:users', 'groups:view:tags', 'groups:view:groups'], 'guests'); + await privileges.global.give(['groups:view:users', 'groups:view:tags', 'groups:view:groups'], 'spiders'); +} +async function createCategories() { + const Categories = require('./categories'); + const db = require('./database'); + const cids = await db.getSortedSetRange('categories:cid', 0, -1); + if (Array.isArray(cids) && cids.length) { + console.log(`Categories OK. Found ${cids.length} categories.`); + return; + } + console.log('No categories found, populating instance with default categories'); + const default_categories = JSON.parse(await fs.promises.readFile(path.join(__dirname, '../', 'install/data/categories.json'), 'utf8')); + for (const categoryData of default_categories) { + await Categories.create(categoryData); + } +} +async function createMenuItems() { + const db = require('./database'); + const exists = await db.exists('navigation:enabled'); + if (exists) { + return; + } + const navigation = require('./navigation/admin'); + const data = require('../install/data/navigation.json'); + await navigation.save(data); +} +async function createWelcomePost() { + const db = require('./database'); + const Topics = require('./topics'); + const [content, numTopics] = await Promise.all([fs.promises.readFile(path.join(__dirname, '../', 'install/data/welcome.md'), 'utf8'), db.getObjectField('global', 'topicCount')]); + if (!parseInt(numTopics, 10)) { + console.log('Creating welcome post!'); + await Topics.post({ + uid: 1, + cid: 2, + title: 'Welcome to your NodeBB!', + content: content + }); + } +} +async function enableDefaultPlugins() { + console.log('Enabling default plugins'); + let defaultEnabled = ['nodebb-plugin-composer-default', 'nodebb-plugin-markdown', 'nodebb-plugin-mentions', 'nodebb-widget-essentials', 'nodebb-rewards-essentials', 'nodebb-plugin-emoji', 'nodebb-plugin-emoji-android']; + let customDefaults = nconf.get('defaultplugins') || nconf.get('defaultPlugins'); + winston.info(`[install/defaultPlugins] customDefaults ${String(customDefaults)}`); + if (customDefaults && customDefaults.length) { + try { + customDefaults = Array.isArray(customDefaults) ? customDefaults : JSON.parse(customDefaults); + defaultEnabled = defaultEnabled.concat(customDefaults); + } catch (e) { + winston.info('[install/enableDefaultPlugins] Invalid defaultPlugins value received. Ignoring.'); + } + } + defaultEnabled = _.uniq(defaultEnabled); + winston.info('[install/enableDefaultPlugins] activating default plugins', defaultEnabled); + const db = require('./database'); + const order = defaultEnabled.map((plugin, index) => index); + await db.sortedSetAdd('plugins:active', order, defaultEnabled); +} +async function setCopyrightWidget() { + const db = require('./database'); + const [footerJSON, footer] = await Promise.all([fs.promises.readFile(path.join(__dirname, '../', 'install/data/footer.json'), 'utf8'), db.getObjectField('widgets:global', 'footer')]); + if (!footer && footerJSON) { + await db.setObjectField('widgets:global', 'sidebar-footer', footerJSON); + } +} +async function copyFavicon() { + const file = require('./file'); + const pathToIco = path.join(nconf.get('upload_path'), 'system', 'favicon.ico'); + const defaultIco = path.join(nconf.get('base_dir'), 'public', 'favicon.ico'); + const targetExists = await file.exists(pathToIco); + const defaultExists = await file.exists(defaultIco); + if (defaultExists && !targetExists) { + try { + await fs.promises.copyFile(defaultIco, pathToIco); + } catch (err) { + winston.error(`Cannot copy favicon.ico\n${err.stack}`); + } + } +} +async function checkUpgrade() { + const upgrade = require('./upgrade'); + try { + await upgrade.check(); + } catch (err) { + if (err.message === 'schema-out-of-date') { + await upgrade.run(); + return; + } + throw err; + } +} +async function installPlugins() { + const pluginInstall = require('./plugins'); + const nbbVersion = require(paths.currentPackage).version; + await Promise.all((await pluginInstall.getActive()).map(async id => { + if (await pluginInstall.isInstalled(id)) return; + const version = await pluginInstall.suggest(id, nbbVersion); + await pluginInstall.toggleInstall(id, version.version); + })); +} +install.setup = async function () { + try { + checkSetupFlagEnv(); + checkCIFlag(); + await setupConfig(); + await setupDefaultConfigs(); + await enableDefaultTheme(); + await createCategories(); + await createDefaultUserGroups(); + const adminInfo = await createAdministrator(); + await createGlobalModeratorsGroup(); + await giveGlobalPrivileges(); + await createMenuItems(); + await createWelcomePost(); + await enableDefaultPlugins(); + await setCopyrightWidget(); + await copyFavicon(); + if (nconf.get('plugins:autoinstall')) await installPlugins(); + await checkUpgrade(); + const data = { + ...adminInfo + }; + return data; + } catch (err) { + if (err) { + winston.warn(`NodeBB Setup Aborted.\n ${err.stack}`); + process.exit(1); + } + } +}; +install.save = async function (server_conf) { + let serverConfigPath = path.join(__dirname, '../config.json'); + if (nconf.get('config')) { + serverConfigPath = path.resolve(__dirname, '../', nconf.get('config')); + } + let currentConfig = {}; + try { + currentConfig = require(serverConfigPath); + } catch (err) { + if (err.code !== 'MODULE_NOT_FOUND') { + throw err; + } + } + await fs.promises.writeFile(serverConfigPath, JSON.stringify({ + ...currentConfig, + ...server_conf + }, null, 4)); + console.log('Configuration Saved OK'); + nconf.file({ + file: serverConfigPath + }); +}; \ No newline at end of file diff --git a/lib/languages.js b/lib/languages.js new file mode 100644 index 0000000000..fe1c01bfea --- /dev/null +++ b/lib/languages.js @@ -0,0 +1,76 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const utils = require('./utils'); +const { + paths +} = require('./constants'); +const plugins = require('./plugins'); +const Languages = module.exports; +const languagesPath = path.join(__dirname, '../build/public/language'); +const files = fs.readdirSync(path.join(paths.nodeModules, '/timeago/locales')); +Languages.timeagoCodes = files.filter(f => f.startsWith('jquery.timeago')).map(f => f.split('.')[2]); +Languages.get = async function (language, namespace) { + const pathToLanguageFile = path.join(languagesPath, language, `${namespace}.json`); + if (!pathToLanguageFile.startsWith(languagesPath)) { + throw new Error('[[error:invalid-path]]'); + } + const data = await fs.promises.readFile(pathToLanguageFile, 'utf8'); + const parsed = JSON.parse(data) || {}; + const result = await plugins.hooks.fire('filter:languages.get', { + language, + namespace, + data: parsed + }); + return result.data; +}; +let codeCache = null; +Languages.listCodes = async function () { + if (codeCache && codeCache.length) { + return codeCache; + } + try { + const file = await fs.promises.readFile(path.join(languagesPath, 'metadata.json'), 'utf8'); + const parsed = JSON.parse(file); + codeCache = parsed.languages; + return parsed.languages; + } catch (err) { + if (err.code === 'ENOENT') { + return []; + } + throw err; + } +}; +let listCache = null; +Languages.list = async function () { + if (listCache && listCache.length) { + return listCache; + } + const codes = await Languages.listCodes(); + let languages = await Promise.all(codes.map(async folder => { + try { + const configPath = path.join(languagesPath, folder, 'language.json'); + const file = await fs.promises.readFile(configPath, 'utf8'); + const lang = JSON.parse(file); + return lang; + } catch (err) { + if (err.code === 'ENOENT') { + return; + } + throw err; + } + })); + languages = languages.filter(lang => lang && lang.code && lang.name && lang.dir); + listCache = languages; + return languages; +}; +Languages.userTimeagoCode = async function (userLang) { + const languageCodes = await Languages.listCodes(); + const timeagoCode = utils.userLangToTimeagoCode(userLang); + if (languageCodes.includes(userLang) && Languages.timeagoCodes.includes(timeagoCode)) { + return timeagoCode; + } + return ''; +}; +require('./promisify')(Languages); \ No newline at end of file diff --git a/lib/logger.js b/lib/logger.js new file mode 100644 index 0000000000..097ac4776d --- /dev/null +++ b/lib/logger.js @@ -0,0 +1,161 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const winston = require('winston'); +const util = require('util'); +const morgan = require('morgan'); +const file = require('./file'); +const meta = require('./meta'); +const opts = { + express: { + app: {}, + set: 0, + ofn: null + }, + streams: { + log: { + f: process.stdout + } + } +}; +const Logger = module.exports; +Logger.init = function (app) { + opts.express.app = app; + Logger.setup(); +}; +Logger.setup = function () { + Logger.setup_one('loggerPath', meta.config.loggerPath); +}; +Logger.setup_one = function (key, value) { + if (key === 'loggerPath') { + Logger.setup_one_log(value); + Logger.express_open(); + } +}; +Logger.setup_one_log = function (value) { + if (meta.config.loggerStatus > 0 || meta.config.loggerIOStatus) { + const stream = Logger.open(value); + if (stream) { + opts.streams.log.f = stream; + } else { + opts.streams.log.f = process.stdout; + } + } else { + Logger.close(opts.streams.log); + } +}; +Logger.open = function (value) { + let stream; + if (value) { + if (file.existsSync(value)) { + const stats = fs.statSync(value); + if (stats) { + if (stats.isDirectory()) { + stream = fs.createWriteStream(path.join(value, 'nodebb.log'), { + flags: 'a' + }); + } else { + stream = fs.createWriteStream(value, { + flags: 'a' + }); + } + } + } else { + stream = fs.createWriteStream(value, { + flags: 'a' + }); + } + if (stream) { + stream.on('error', err => { + winston.error(err.stack); + }); + } + } else { + stream = process.stdout; + } + return stream; +}; +Logger.close = function (stream) { + if (stream.f !== process.stdout && stream.f) { + stream.end(); + } + stream.f = null; +}; +Logger.monitorConfig = function (socket, data) { + Logger.setup_one(data.key, data.value); + Logger.io_close(socket); + Logger.io(socket); +}; +Logger.express_open = function () { + if (opts.express.set !== 1) { + opts.express.set = 1; + opts.express.app.use(Logger.expressLogger); + } + opts.express.ofn = morgan('combined', { + stream: opts.streams.log.f + }); +}; +Logger.expressLogger = function (req, res, next) { + if (meta.config.loggerStatus > 0) { + return opts.express.ofn(req, res, next); + } + return next(); +}; +Logger.prepare_io_string = function (_type, _uid, _args) { + try { + return `io: ${_uid} ${_type} ${util.inspect(Array.prototype.slice.call(_args), { + depth: 3 + })}\n`; + } catch (err) { + winston.info('Logger.prepare_io_string: Failed', err); + return 'error'; + } +}; +Logger.io_close = function (socket) { + if (!socket || !socket.io || !socket.io.sockets || !socket.io.sockets.sockets) { + return; + } + const clientsMap = socket.io.sockets.sockets; + for (const [, client] of clientsMap) { + if (client.oEmit && client.oEmit !== client.emit) { + client.emit = client.oEmit; + } + if (client.$onevent && client.$onevent !== client.onevent) { + client.onevent = client.$onevent; + } + } +}; +Logger.io = function (socket) { + if (!socket || !socket.io || !socket.io.sockets || !socket.io.sockets.sockets) { + return; + } + const clientsMap = socket.io.sockets.sockets; + for (const [, socketObj] of clientsMap) { + Logger.io_one(socketObj, socketObj.uid); + } +}; +Logger.io_one = function (socket, uid) { + function override(method, name, errorMsg) { + return (...args) => { + if (opts.streams.log.f) { + opts.streams.log.f.write(Logger.prepare_io_string(name, uid, args)); + } + try { + method.apply(socket, args); + } catch (err) { + winston.info(errorMsg, err); + } + }; + } + if (socket && meta.config.loggerIOStatus > 0) { + socket.oEmit = socket.emit; + const { + emit + } = socket; + socket.emit = override(emit, 'emit', 'Logger.io_one: emit.apply: Failed'); + socket.$onvent = socket.onevent; + const $onevent = socket.onevent; + socket.onevent = override($onevent, 'on', 'Logger.io_one: $emit.apply: Failed'); + } +}; \ No newline at end of file diff --git a/lib/messaging/create.js b/lib/messaging/create.js new file mode 100644 index 0000000000..0f6c2bc304 --- /dev/null +++ b/lib/messaging/create.js @@ -0,0 +1,122 @@ +'use strict'; + +const _ = require('lodash'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const db = require('../database'); +const user = require('../user'); +const utils = require('../utils'); +module.exports = function (Messaging) { + Messaging.sendMessage = async data => { + await Messaging.checkContent(data.content); + const inRoom = await Messaging.isUserInRoom(data.uid, data.roomId); + if (!inRoom) { + throw new Error('[[error:not-allowed]]'); + } + return await Messaging.addMessage(data); + }; + Messaging.checkContent = async content => { + if (!content) { + throw new Error('[[error:invalid-chat-message]]'); + } + const maximumChatMessageLength = meta.config.maximumChatMessageLength || 1000; + content = String(content).trim(); + let { + length + } = content; + ({ + content, + length + } = await plugins.hooks.fire('filter:messaging.checkContent', { + content, + length + })); + if (!content) { + throw new Error('[[error:invalid-chat-message]]'); + } + if (length > maximumChatMessageLength) { + throw new Error(`[[error:chat-message-too-long, ${maximumChatMessageLength}]]`); + } + }; + Messaging.addMessage = async data => { + const { + uid, + roomId + } = data; + const roomData = await Messaging.getRoomData(roomId); + if (!roomData) { + throw new Error('[[error:no-room]]'); + } + if (data.toMid) { + if (!utils.isNumber(data.toMid)) { + throw new Error('[[error:invalid-mid]]'); + } + if (!(await Messaging.canViewMessage(data.toMid, roomId, uid))) { + throw new Error('[[error:no-privileges]]'); + } + } + const mid = await db.incrObjectField('global', 'nextMid'); + const timestamp = data.timestamp || Date.now(); + let message = { + mid: mid, + content: String(data.content), + timestamp: timestamp, + fromuid: uid, + roomId: roomId + }; + if (data.toMid) { + message.toMid = data.toMid; + } + if (data.system) { + message.system = data.system; + } + if (data.ip) { + message.ip = data.ip; + } + message = await plugins.hooks.fire('filter:messaging.save', message); + await db.setObject(`message:${mid}`, message); + const isNewSet = await Messaging.isNewSet(uid, roomId, timestamp); + const tasks = [Messaging.addMessageToRoom(roomId, mid, timestamp), Messaging.markRead(uid, roomId), db.sortedSetAdd('messages:mid', timestamp, mid), db.incrObjectField('global', 'messageCount')]; + if (data.toMid) { + tasks.push(db.sortedSetAdd(`mid:${data.toMid}:replies`, timestamp, mid)); + } + if (roomData.public) { + tasks.push(db.sortedSetAdd('chat:rooms:public:lastpost', timestamp, roomId)); + } else { + let uids = await Messaging.getUidsInRoom(roomId, 0, -1); + uids = await user.blocks.filterUids(uid, uids); + tasks.push(Messaging.addRoomToUsers(roomId, uids, timestamp), Messaging.markUnread(uids.filter(uid => uid !== String(data.uid)), roomId)); + } + await Promise.all(tasks); + const messages = await Messaging.getMessagesData([mid], uid, roomId, true); + if (!messages || !messages[0]) { + return null; + } + messages[0].newSet = isNewSet; + plugins.hooks.fire('action:messaging.save', { + message: message, + data: data + }); + return messages[0]; + }; + Messaging.addSystemMessage = async (content, uid, roomId) => { + const message = await Messaging.addMessage({ + content: content, + uid: uid, + roomId: roomId, + system: 1 + }); + Messaging.notifyUsersInRoom(uid, roomId, message); + }; + Messaging.addRoomToUsers = async (roomId, uids, timestamp) => { + if (!uids.length) { + return; + } + const keys = _.uniq(uids).map(uid => `uid:${uid}:chat:rooms`); + await db.sortedSetsAdd(keys, timestamp, roomId); + }; + Messaging.addMessageToRoom = async (roomId, mid, timestamp) => { + await db.sortedSetAdd(`chat:room:${roomId}:mids`, timestamp, mid); + await db.incrObjectField(`chat:room:${roomId}`, 'messageCount'); + }; +}; \ No newline at end of file diff --git a/lib/messaging/data.js b/lib/messaging/data.js new file mode 100644 index 0000000000..dd7a831036 --- /dev/null +++ b/lib/messaging/data.js @@ -0,0 +1,161 @@ +'use strict'; + +const _ = require('lodash'); +const validator = require('validator'); +const db = require('../database'); +const user = require('../user'); +const utils = require('../utils'); +const plugins = require('../plugins'); +const intFields = ['mid', 'timestamp', 'edited', 'fromuid', 'roomId', 'deleted', 'system']; +module.exports = function (Messaging) { + Messaging.newMessageCutoff = 1000 * 60 * 3; + Messaging.getMessagesFields = async (mids, fields) => { + if (!Array.isArray(mids) || !mids.length) { + return []; + } + const keys = mids.map(mid => `message:${mid}`); + const messages = await db.getObjects(keys, fields); + return await Promise.all(messages.map(async (message, idx) => modifyMessage(message, fields, parseInt(mids[idx], 10)))); + }; + Messaging.getMessageField = async (mid, field) => { + const fields = await Messaging.getMessageFields(mid, [field]); + return fields ? fields[field] : null; + }; + Messaging.getMessageFields = async (mid, fields) => { + const messages = await Messaging.getMessagesFields([mid], fields); + return messages ? messages[0] : null; + }; + Messaging.setMessageField = async (mid, field, content) => { + await db.setObjectField(`message:${mid}`, field, content); + }; + Messaging.setMessageFields = async (mid, data) => { + await db.setObject(`message:${mid}`, data); + }; + Messaging.getMessagesData = async (mids, uid, roomId, isNew) => { + let messages = await Messaging.getMessagesFields(mids, []); + messages = messages.map((msg, idx) => { + if (msg) { + msg.messageId = parseInt(mids[idx], 10); + msg.ip = undefined; + msg.isOwner = msg.fromuid === parseInt(uid, 10); + } + return msg; + }).filter(Boolean); + messages = await user.blocks.filter(uid, 'fromuid', messages); + const users = await user.getUsersFields(messages.map(msg => msg && msg.fromuid), ['uid', 'username', 'userslug', 'picture', 'status', 'banned']); + messages.forEach((message, index) => { + message.fromUser = users[index]; + message.fromUser.banned = !!message.fromUser.banned; + message.fromUser.deleted = message.fromuid !== message.fromUser.uid && message.fromUser.uid === 0; + const self = message.fromuid === parseInt(uid, 10); + message.self = self ? 1 : 0; + message.newSet = false; + message.roomId = String(message.roomId || roomId); + }); + await parseMessages(messages, uid, roomId, isNew); + if (messages.length > 1) { + messages = messages.map((message, index) => { + if (index > 0 && message.timestamp > messages[index - 1].timestamp + Messaging.newMessageCutoff) { + message.newSet = true; + } else if (index > 0 && message.fromuid !== messages[index - 1].fromuid) { + message.newSet = true; + } else if (index > 0 && messages[index - 1].system) { + message.newSet = true; + } else if (index === 0 || message.toMid) { + message.newSet = true; + } + return message; + }); + } else if (messages.length === 1) { + const key = `chat:room:${roomId}:mids`; + const index = await db.sortedSetRank(key, messages[0].messageId); + if (index > 0) { + const mid = await db.getSortedSetRange(key, index - 1, index - 1); + const fields = await Messaging.getMessageFields(mid, ['fromuid', 'timestamp']); + if (messages[0].timestamp > fields.timestamp + Messaging.newMessageCutoff || messages[0].fromuid !== fields.fromuid || messages[0].system || messages[0].toMid) { + messages[0].newSet = true; + } + } else { + messages[0].newSet = true; + } + } + await addParentMessages(messages, uid, roomId); + const data = await plugins.hooks.fire('filter:messaging.getMessages', { + messages: messages, + uid: uid, + roomId: roomId, + isNew: isNew, + mids: mids + }); + return data && data.messages; + }; + async function addParentMessages(messages, uid, roomId) { + let parentMids = messages.map(msg => msg && msg.hasOwnProperty('toMid') ? parseInt(msg.toMid, 10) : null).filter(Boolean); + if (!parentMids.length) { + return; + } + parentMids = _.uniq(parentMids); + const canView = await Messaging.canViewMessage(parentMids, roomId, uid); + parentMids = parentMids.filter((mid, idx) => canView[idx]); + const parentMessages = await Messaging.getMessagesFields(parentMids, ['fromuid', 'content', 'timestamp', 'deleted']); + const parentUids = _.uniq(parentMessages.map(msg => msg && msg.fromuid)); + const usersMap = _.zipObject(parentUids, await user.getUsersFields(parentUids, ['uid', 'username', 'userslug', 'picture'])); + await Promise.all(parentMessages.map(async parentMsg => { + if (parentMsg.deleted && parentMsg.fromuid !== parseInt(uid, 10)) { + parentMsg.content = `

[[modules:chat.message-deleted]]

`; + return; + } + const foundMsg = messages.find(msg => parseInt(msg.mid, 10) === parseInt(parentMsg.mid, 10)); + if (foundMsg) { + parentMsg.content = foundMsg.content; + return; + } + parentMsg.content = await parseMessage(parentMsg, uid, roomId, false); + })); + const parents = {}; + parentMessages.forEach((msg, i) => { + if (usersMap[msg.fromuid]) { + msg.user = usersMap[msg.fromuid]; + parents[parentMids[i]] = msg; + } + }); + messages.forEach(msg => { + if (parents[msg.toMid]) { + msg.parent = parents[msg.toMid]; + msg.parent.mid = msg.toMid; + } + }); + } + async function parseMessages(messages, uid, roomId, isNew) { + await Promise.all(messages.map(async msg => { + if (msg.deleted && !msg.isOwner) { + msg.content = `

[[modules:chat.message-deleted]]

`; + return; + } + msg.content = await parseMessage(msg, uid, roomId, isNew); + })); + } + async function parseMessage(message, uid, roomId, isNew) { + if (message.system) { + return validator.escape(String(message.content)); + } + return await Messaging.parse(message.content, message.fromuid, uid, roomId, isNew); + } +}; +async function modifyMessage(message, fields, mid) { + if (message) { + db.parseIntFields(message, intFields, fields); + if (message.hasOwnProperty('timestamp')) { + message.timestampISO = utils.toISOString(message.timestamp); + } + if (message.hasOwnProperty('edited')) { + message.editedISO = utils.toISOString(message.edited); + } + } + const payload = await plugins.hooks.fire('filter:messaging.getFields', { + mid: mid, + message: message, + fields: fields + }); + return payload.message; +} \ No newline at end of file diff --git a/lib/messaging/delete.js b/lib/messaging/delete.js new file mode 100644 index 0000000000..a3123c4ef7 --- /dev/null +++ b/lib/messaging/delete.js @@ -0,0 +1,30 @@ +'use strict'; + +const sockets = require('../socket.io'); +const plugins = require('../plugins'); +module.exports = function (Messaging) { + Messaging.deleteMessage = async (mid, uid) => await doDeleteRestore(mid, 1, uid); + Messaging.restoreMessage = async (mid, uid) => await doDeleteRestore(mid, 0, uid); + async function doDeleteRestore(mid, state, uid) { + const field = state ? 'deleted' : 'restored'; + const msgData = await Messaging.getMessageFields(mid, ['mid', 'fromuid', 'deleted', 'roomId', 'content', 'system']); + if (msgData.deleted === state) { + throw new Error(`[[error:chat-${field}-already]]`); + } + await Messaging.setMessageField(mid, 'deleted', state); + msgData.deleted = state; + const ioRoom = sockets.in(`chat_room_${msgData.roomId}`); + if (state === 1 && ioRoom) { + ioRoom.emit('event:chats.delete', mid); + plugins.hooks.fire('action:messaging.delete', { + message: msgData + }); + } else if (state === 0 && ioRoom) { + const messages = await Messaging.getMessagesData([mid], uid, msgData.roomId, true); + ioRoom.emit('event:chats.restore', messages[0]); + plugins.hooks.fire('action:messaging.restore', { + message: msgData + }); + } + } +}; \ No newline at end of file diff --git a/lib/messaging/edit.js b/lib/messaging/edit.js new file mode 100644 index 0000000000..35002b56d7 --- /dev/null +++ b/lib/messaging/edit.js @@ -0,0 +1,83 @@ +'use strict'; + +const meta = require('../meta'); +const user = require('../user'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const sockets = require('../socket.io'); +module.exports = function (Messaging) { + Messaging.editMessage = async (uid, mid, roomId, content) => { + await Messaging.checkContent(content); + const raw = await Messaging.getMessageField(mid, 'content'); + if (raw === content) { + return; + } + const payload = await plugins.hooks.fire('filter:messaging.edit', { + content: content, + edited: Date.now() + }); + if (!String(payload.content).trim()) { + throw new Error('[[error:invalid-chat-message]]'); + } + await Messaging.setMessageFields(mid, payload); + const messages = await Messaging.getMessagesData([mid], uid, roomId, true); + if (messages[0]) { + const roomName = messages[0].deleted ? `uid_${uid}` : `chat_room_${roomId}`; + sockets.in(roomName).emit('event:chats.edit', { + messages: messages + }); + } + plugins.hooks.fire('action:messaging.edit', { + message: { + ...messages[0], + content: payload.content + } + }); + }; + const canEditDelete = async (messageId, uid, type) => { + let durationConfig = ''; + if (type === 'edit') { + durationConfig = 'chatEditDuration'; + } else if (type === 'delete') { + durationConfig = 'chatDeleteDuration'; + } + const exists = await Messaging.messageExists(messageId); + if (!exists) { + throw new Error('[[error:invalid-mid]]'); + } + const isAdminOrGlobalMod = await user.isAdminOrGlobalMod(uid); + if (meta.config.disableChat) { + throw new Error('[[error:chat-disabled]]'); + } else if (!isAdminOrGlobalMod && meta.config.disableChatMessageEditing) { + throw new Error('[[error:chat-message-editing-disabled]]'); + } + const userData = await user.getUserFields(uid, ['banned']); + if (userData.banned) { + throw new Error('[[error:user-banned]]'); + } + const canChat = await privileges.global.can(['chat', 'chat:privileged'], uid); + if (!canChat.includes(true)) { + throw new Error('[[error:no-privileges]]'); + } + const messageData = await Messaging.getMessageFields(messageId, ['fromuid', 'timestamp', 'system']); + if (isAdminOrGlobalMod && !messageData.system) { + return; + } + const chatConfigDuration = meta.config[durationConfig]; + if (chatConfigDuration && Date.now() - messageData.timestamp > chatConfigDuration * 1000) { + throw new Error(`[[error:chat-${type}-duration-expired, ${meta.config[durationConfig]}]]`); + } + if (messageData.fromuid === parseInt(uid, 10) && !messageData.system) { + return; + } + throw new Error(`[[error:cant-${type}-chat-message]]`); + }; + Messaging.canEdit = async (messageId, uid) => await canEditDelete(messageId, uid, 'edit'); + Messaging.canDelete = async (messageId, uid) => await canEditDelete(messageId, uid, 'delete'); + Messaging.canPin = async (roomId, uid) => { + const [isAdmin, isGlobalMod, inRoom, isRoomOwner] = await Promise.all([user.isAdministrator(uid), user.isGlobalModerator(uid), Messaging.isUserInRoom(uid, roomId), Messaging.isRoomOwner(uid, roomId)]); + if (!isAdmin && !isGlobalMod && (!inRoom || !isRoomOwner)) { + throw new Error('[[error:no-privileges]]'); + } + }; +}; \ No newline at end of file diff --git a/lib/messaging/index.js b/lib/messaging/index.js new file mode 100644 index 0000000000..694e21291e --- /dev/null +++ b/lib/messaging/index.js @@ -0,0 +1,334 @@ +'use strict'; + +const _ = require('lodash'); +const validator = require('validator'); +const nconf = require('nconf'); +const db = require('../database'); +const user = require('../user'); +const groups = require('../groups'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const meta = require('../meta'); +const utils = require('../utils'); +const translator = require('../translator'); +const cache = require('../cache'); +const relative_path = nconf.get('relative_path'); +const Messaging = module.exports; +require('./data')(Messaging); +require('./create')(Messaging); +require('./delete')(Messaging); +require('./edit')(Messaging); +require('./rooms')(Messaging); +require('./unread')(Messaging); +require('./notifications')(Messaging); +require('./pins')(Messaging); +Messaging.notificationSettings = Object.create(null); +Messaging.notificationSettings.NONE = 1; +Messaging.notificationSettings.ATMENTION = 2; +Messaging.notificationSettings.ALLMESSAGES = 3; +Messaging.messageExists = async mid => db.exists(`message:${mid}`); +Messaging.getMessages = async params => { + const { + callerUid, + uid, + roomId + } = params; + const isNew = params.isNew || false; + const start = params.hasOwnProperty('start') ? params.start : 0; + const stop = parseInt(start, 10) + ((params.count || 50) - 1); + const ok = await canGet('filter:messaging.canGetMessages', callerUid, uid); + if (!ok) { + return; + } + const [mids, messageCount] = await Promise.all([getMessageIds(roomId, uid, start, stop), db.getObjectField(`chat:room:${roomId}`, 'messageCount')]); + if (!mids.length) { + return []; + } + const count = parseInt(messageCount, 10) || 0; + const indices = {}; + mids.forEach((mid, index) => { + indices[mid] = count - start - index - 1; + }); + mids.reverse(); + const messageData = await Messaging.getMessagesData(mids, uid, roomId, isNew); + messageData.forEach(msg => { + msg.index = indices[msg.messageId.toString()]; + }); + return messageData; +}; +async function getMessageIds(roomId, uid, start, stop) { + const isPublic = await db.getObjectField(`chat:room:${roomId}`, 'public'); + if (parseInt(isPublic, 10) === 1) { + return await db.getSortedSetRevRange(`chat:room:${roomId}:mids`, start, stop); + } + const userjoinTimestamp = await db.sortedSetScore(`chat:room:${roomId}:uids`, uid); + return await db.getSortedSetRevRangeByScore(`chat:room:${roomId}:mids`, start, stop - start + 1, '+inf', userjoinTimestamp); +} +async function canGet(hook, callerUid, uid) { + const data = await plugins.hooks.fire(hook, { + callerUid: callerUid, + uid: uid, + canGet: parseInt(callerUid, 10) === parseInt(uid, 10) + }); + return data ? data.canGet : false; +} +Messaging.parse = async (message, fromuid, uid, roomId, isNew) => { + const parsed = await plugins.hooks.fire('filter:parse.raw', String(message || '')); + let messageData = { + message: message, + parsed: parsed, + fromuid: fromuid, + uid: uid, + roomId: roomId, + isNew: isNew, + parsedMessage: parsed + }; + messageData = await plugins.hooks.fire('filter:messaging.parse', messageData); + return messageData ? messageData.parsedMessage : ''; +}; +Messaging.isNewSet = async (uid, roomId, timestamp) => { + const setKey = `chat:room:${roomId}:mids`; + const messages = await db.getSortedSetRevRangeWithScores(setKey, 0, 0); + if (messages && messages.length) { + return parseInt(timestamp, 10) > parseInt(messages[0].score, 10) + Messaging.newMessageCutoff; + } + return true; +}; +Messaging.getPublicRoomIdsFromSet = async function (set) { + const cacheKey = `${set}:all`; + let allRoomIds = cache.get(cacheKey); + if (allRoomIds === undefined) { + allRoomIds = await db.getSortedSetRange(set, 0, -1); + cache.set(cacheKey, allRoomIds); + } + return allRoomIds.slice(); +}; +Messaging.getPublicRooms = async (callerUid, uid) => { + const ok = await canGet('filter:messaging.canGetPublicChats', callerUid, uid); + if (!ok) { + return null; + } + const allRoomIds = await Messaging.getPublicRoomIdsFromSet('chat:rooms:public:order'); + const allRoomData = await Messaging.getRoomsData(allRoomIds); + const isAdmin = await privileges.users.isAdministrator(callerUid); + const checks = await Promise.all(allRoomData.map(room => room && (!Array.isArray(room.groups) || !room.groups.length || isAdmin || groups.isMemberOfAny(uid, room && room.groups)))); + const roomData = allRoomData.filter((room, idx) => room && checks[idx]); + const roomIds = roomData.map(r => r.roomId); + const userReadTimestamps = await db.getObjectFields(`uid:${uid}:chat:rooms:read`, roomIds); + const maxUnread = 50; + const unreadCounts = await Promise.all(roomIds.map(async roomId => { + const cutoff = userReadTimestamps[roomId] || '-inf'; + const unreadMids = await db.getSortedSetRangeByScore(`chat:room:${roomId}:mids`, 0, maxUnread + 1, cutoff, '+inf'); + return unreadMids.length; + })); + roomData.forEach((r, idx) => { + const count = unreadCounts[idx]; + r.unreadCountText = count > maxUnread ? `${maxUnread}+` : String(count); + r.unreadCount = count; + r.unread = count > 0; + r.icon = Messaging.getRoomIcon(r); + }); + return roomData; +}; +Messaging.getRecentChats = async (callerUid, uid, start, stop) => { + const ok = await canGet('filter:messaging.canGetRecentChats', callerUid, uid); + if (!ok) { + throw new Error('[[error:no-privileges]]'); + } + const roomIds = await db.getSortedSetRevRange(`uid:${uid}:chat:rooms`, start, stop); + async function getUsers(roomIds) { + const arrayOfUids = await Promise.all(roomIds.map(roomId => Messaging.getUidsInRoom(roomId, 0, 9))); + const uniqUids = _.uniq(_.flatten(arrayOfUids)).filter(_uid => _uid && parseInt(_uid, 10) !== parseInt(uid, 10)); + const uidToUser = _.zipObject(uniqUids, await user.getUsersFields(uniqUids, ['uid', 'username', 'userslug', 'picture', 'status', 'lastonline'])); + return arrayOfUids.map(uids => uids.map(uid => uidToUser[uid])); + } + const results = await utils.promiseParallel({ + roomData: Messaging.getRoomsData(roomIds), + unread: db.isSortedSetMembers(`uid:${uid}:chat:rooms:unread`, roomIds), + users: getUsers(roomIds), + teasers: Messaging.getTeasers(uid, roomIds), + settings: user.getSettings(uid) + }); + await Promise.all(results.roomData.map(async (room, index) => { + if (room) { + room.users = results.users[index]; + room.groupChat = room.users.length > 2; + room.unread = results.unread[index]; + room.teaser = results.teasers[index]; + room.users.forEach(userData => { + if (userData && parseInt(userData.uid, 10)) { + userData.status = user.getStatus(userData); + } + }); + room.users = room.users.filter(user => user && parseInt(user.uid, 10)); + room.lastUser = room.users[0]; + room.usernames = Messaging.generateUsernames(room, uid); + room.chatWithMessage = await Messaging.generateChatWithMessage(room, uid, results.settings.userLang); + } + })); + results.roomData = results.roomData.filter(Boolean); + const ref = { + rooms: results.roomData, + nextStart: stop + 1 + }; + return await plugins.hooks.fire('filter:messaging.getRecentChats', { + rooms: ref.rooms, + nextStart: ref.nextStart, + uid: uid, + callerUid: callerUid + }); +}; +Messaging.generateUsernames = function (room, excludeUid) { + const users = room.users.filter(u => u && parseInt(u.uid, 10) !== excludeUid); + const usernames = users.map(u => u.username); + if (users.length > 3) { + return translator.compile('modules:chat.usernames-and-x-others', usernames.slice(0, 2).join(', '), room.userCount - 2); + } + return usernames.join(', '); +}; +Messaging.generateChatWithMessage = async function (room, callerUid, userLang) { + const users = room.users.filter(u => u && parseInt(u.uid, 10) !== callerUid); + const usernames = users.map(u => `${u.username}`); + let compiled = ''; + if (!users.length) { + return '[[modules:chat.no-users-in-room]]'; + } + if (users.length > 3) { + compiled = translator.compile('modules:chat.chat-with-usernames-and-x-others', usernames.slice(0, 2).join(', '), room.userCount - 2); + } else { + compiled = translator.compile('modules:chat.chat-with-usernames', usernames.join(', ')); + } + return utils.decodeHTMLEntities(await translator.translate(compiled, userLang)); +}; +Messaging.getTeaser = async (uid, roomId) => { + const teasers = await Messaging.getTeasers(uid, [roomId]); + return teasers[0]; +}; +Messaging.getTeasers = async (uid, roomIds) => { + const mids = await Promise.all(roomIds.map(roomId => Messaging.getLatestUndeletedMessage(uid, roomId))); + const [teasers, blockedUids] = await Promise.all([Messaging.getMessagesFields(mids, ['fromuid', 'content', 'timestamp']), user.blocks.list(uid)]); + const uids = _.uniq(teasers.map(t => t && t.fromuid).filter(uid => uid && !blockedUids.includes(uid))); + const userMap = _.zipObject(uids, await user.getUsersFields(uids, ['uid', 'username', 'userslug', 'picture', 'status', 'lastonline'])); + return await Promise.all(roomIds.map(async (roomId, idx) => { + const teaser = teasers[idx]; + if (!teaser || !teaser.fromuid) { + return null; + } + if (userMap[teaser.fromuid]) { + teaser.user = userMap[teaser.fromuid]; + } + teaser.content = validator.escape(String(utils.stripHTMLTags(utils.decodeHTMLEntities(teaser.content)))); + teaser.roomId = roomId; + const payload = await plugins.hooks.fire('filter:messaging.getTeaser', { + teaser: teaser + }); + return payload.teaser; + })); +}; +Messaging.getLatestUndeletedMessage = async (uid, roomId) => { + let done = false; + let latestMid = null; + let index = 0; + let mids; + while (!done) { + mids = await getMessageIds(roomId, uid, index, index); + if (mids.length) { + const states = await Messaging.getMessageFields(mids[0], ['deleted', 'system']); + done = !states.deleted && !states.system; + if (done) { + latestMid = mids[0]; + } + index += 1; + } else { + done = true; + } + } + return latestMid; +}; +Messaging.canMessageUser = async (uid, toUid) => { + if (meta.config.disableChat || uid <= 0) { + throw new Error('[[error:chat-disabled]]'); + } + if (parseInt(uid, 10) === parseInt(toUid, 10)) { + throw new Error('[[error:cant-chat-with-yourself]]'); + } + const [exists, isTargetPrivileged, canChat, canChatWithPrivileged] = await Promise.all([user.exists(toUid), user.isPrivileged(toUid), privileges.global.can('chat', uid), privileges.global.can('chat:privileged', uid), checkReputation(uid)]); + if (!exists) { + throw new Error('[[error:no-user]]'); + } + if (!canChat && !(canChatWithPrivileged && isTargetPrivileged)) { + throw new Error('[[error:no-privileges]]'); + } + const [settings, isAdmin, isModerator, isFollowing, isBlocked] = await Promise.all([user.getSettings(toUid), user.isAdministrator(uid), user.isModeratorOfAnyCategory(uid), user.isFollowing(toUid, uid), user.blocks.is(uid, toUid)]); + if (isBlocked || settings.restrictChat && !isAdmin && !isModerator && !isFollowing) { + throw new Error('[[error:chat-restricted]]'); + } + await plugins.hooks.fire('static:messaging.canMessageUser', { + uid: uid, + toUid: toUid + }); +}; +Messaging.canMessageRoom = async (uid, roomId) => { + if (meta.config.disableChat || uid <= 0) { + throw new Error('[[error:chat-disabled]]'); + } + const [roomData, inRoom, canChat] = await Promise.all([Messaging.getRoomData(roomId), Messaging.isUserInRoom(uid, roomId), privileges.global.can(['chat', 'chat:privileged'], uid), checkReputation(uid), user.checkMuted(uid)]); + if (!roomData) { + throw new Error('[[error:no-room]]'); + } + if (!inRoom) { + throw new Error('[[error:not-in-room]]'); + } + if (!canChat.includes(true)) { + throw new Error('[[error:no-privileges]]'); + } + await plugins.hooks.fire('static:messaging.canMessageRoom', { + uid: uid, + roomId: roomId + }); +}; +async function checkReputation(uid) { + if (meta.config['reputation:disabled']) { + return; + } + const [reputation, isPrivileged] = await Promise.all([user.getUserField(uid, 'reputation'), user.isPrivileged(uid)]); + if (!isPrivileged && meta.config['min:rep:chat'] > reputation) { + throw new Error(`[[error:not-enough-reputation-to-chat, ${meta.config['min:rep:chat']}]]`); + } +} +Messaging.hasPrivateChat = async (uid, withUid) => { + if (parseInt(uid, 10) === parseInt(withUid, 10) || parseInt(uid, 10) <= 0 || parseInt(withUid, 10) <= 0) { + return 0; + } + const results = await utils.promiseParallel({ + myRooms: db.getSortedSetRevRange(`uid:${uid}:chat:rooms`, 0, -1), + theirRooms: db.getSortedSetRevRange(`uid:${withUid}:chat:rooms`, 0, -1) + }); + const roomIds = results.myRooms.filter(roomId => roomId && results.theirRooms.includes(roomId)); + if (!roomIds.length) { + return 0; + } + let index = 0; + let roomId = 0; + while (index < roomIds.length && !roomId) { + const count = await Messaging.getUserCountInRoom(roomIds[index]); + if (count === 2) { + roomId = roomIds[index]; + } else { + index += 1; + } + } + return roomId; +}; +Messaging.canViewMessage = async (mids, roomId, uid) => { + let single = false; + if (!Array.isArray(mids) && isFinite(mids)) { + mids = [mids]; + single = true; + } + const isPublic = parseInt(await db.getObjectField(`chat:room:${roomId}`, 'public'), 10) === 1; + const [midTimestamps, userTimestamp] = await Promise.all([db.sortedSetScores(`chat:room:${roomId}:mids`, mids), db.sortedSetScore(`chat:room:${roomId}:uids`, uid)]); + const canView = midTimestamps.map(midTimestamp => !!(midTimestamp && userTimestamp && (isPublic || userTimestamp <= midTimestamp))); + return single ? canView.pop() : canView; +}; +require('../promisify')(Messaging); \ No newline at end of file diff --git a/lib/messaging/notifications.js b/lib/messaging/notifications.js new file mode 100644 index 0000000000..2c187f8a9d --- /dev/null +++ b/lib/messaging/notifications.js @@ -0,0 +1,112 @@ +'use strict'; + +const winston = require('winston'); +const batch = require('../batch'); +const db = require('../database'); +const notifications = require('../notifications'); +const user = require('../user'); +const io = require('../socket.io'); +const plugins = require('../plugins'); +module.exports = function (Messaging) { + Messaging.setUserNotificationSetting = async (uid, roomId, value) => { + if (parseInt(value, 10) === -1) { + return await db.deleteObjectField(`chat:room:${roomId}:notification:settings`, uid); + } + await db.setObjectField(`chat:room:${roomId}:notification:settings`, uid, parseInt(value, 10)); + }; + Messaging.getUidsNotificationSetting = async (uids, roomId) => { + const [settings, roomData] = await Promise.all([db.getObjectFields(`chat:room:${roomId}:notification:settings`, uids), Messaging.getRoomData(roomId, ['notificationSetting'])]); + return uids.map(uid => parseInt(settings[uid] || roomData.notificationSetting, 10)); + }; + Messaging.markRoomNotificationsRead = async (uid, roomId) => { + const chatNids = await db.getSortedSetScan({ + key: `uid:${uid}:notifications:unread`, + match: `chat_${roomId}_*` + }); + if (chatNids.length) { + await notifications.markReadMultiple(chatNids, uid); + await user.notifications.pushCount(uid); + } + }; + Messaging.notifyUsersInRoom = async (fromUid, roomId, messageObj) => { + const isPublic = parseInt(await db.getObjectField(`chat:room:${roomId}`, 'public'), 10) === 1; + let data = { + roomId: roomId, + fromUid: fromUid, + message: messageObj, + public: isPublic + }; + data = await plugins.hooks.fire('filter:messaging.notify', data); + if (!data) { + return; + } + io.in(`chat_room_${roomId}`).emit('event:chats.receive', data); + const unreadData = { + roomId, + fromUid, + public: isPublic + }; + if (isPublic && !messageObj.system) { + io.in(`chat_room_public_${roomId}`).emit('event:chats.public.unread', unreadData); + } + if (messageObj.system) { + return; + } + if (!isPublic) { + const uids = await Messaging.getAllUidsInRoomFromSet(`chat:room:${roomId}:uids:online`); + Messaging.pushUnreadCount(uids, unreadData); + } + try { + await sendNotification(fromUid, roomId, messageObj); + } catch (err) { + winston.error(`[messaging/notifications] Unabled to send notification\n${err.stack}`); + } + }; + async function sendNotification(fromUid, roomId, messageObj) { + fromUid = parseInt(fromUid, 10); + const [settings, roomData, realtimeUids] = await Promise.all([db.getObject(`chat:room:${roomId}:notification:settings`), Messaging.getRoomData(roomId), io.getUidsInRoom(`chat_room_${roomId}`)]); + const roomDefault = roomData.notificationSetting; + const uidsToNotify = []; + const { + ALLMESSAGES + } = Messaging.notificationSettings; + await batch.processSortedSet(`chat:room:${roomId}:uids:online`, async uids => { + uids = uids.filter(uid => parseInt(settings && settings[uid] || roomDefault, 10) === ALLMESSAGES && fromUid !== parseInt(uid, 10) && !realtimeUids.includes(parseInt(uid, 10))); + const hasRead = await Messaging.hasRead(uids, roomId); + uidsToNotify.push(...uids.filter((uid, index) => !hasRead[index])); + }, { + reverse: true, + batch: 500, + interval: 100 + }); + if (uidsToNotify.length) { + const { + displayname + } = messageObj.fromUser; + const isGroupChat = await Messaging.isGroupChat(roomId); + const roomName = roomData.roomName || `[[modules:chat.room-id, ${roomId}]]`; + const notifData = { + type: isGroupChat ? 'new-group-chat' : 'new-chat', + subject: roomData.roomName ? `[[email:notif.chat.new-message-from-user-in-room, ${displayname}, ${roomName}]]` : `[[email:notif.chat.new-message-from-user, ${displayname}]]`, + bodyShort: isGroupChat || roomData.roomName ? `[[notifications:new-message-in, ${roomName}]]` : `[[notifications:new-message-from, ${displayname}]]`, + bodyLong: messageObj.content, + nid: `chat_${roomId}_${fromUid}_${Date.now()}`, + mergeId: `new-chat|${roomId}`, + from: fromUid, + roomId, + roomName, + path: `/chats/${messageObj.roomId}` + }; + if (roomData.public) { + const icon = Messaging.getRoomIcon(roomData); + notifData.type = 'new-public-chat'; + notifData.roomIcon = icon; + notifData.subject = `[[email:notif.chat.new-message-from-user-in-room, ${displayname}, ${roomName}]]`; + notifData.bodyShort = `[[notifications:user-posted-in-public-room, ${displayname}, ${icon}, ${roomName}]]`; + notifData.mergeId = `notifications:user-posted-in-public-room|${roomId}`; + } + const notification = await notifications.create(notifData); + await notifications.push(notification, uidsToNotify); + } + } +}; \ No newline at end of file diff --git a/lib/messaging/pins.js b/lib/messaging/pins.js new file mode 100644 index 0000000000..c68f9f79ca --- /dev/null +++ b/lib/messaging/pins.js @@ -0,0 +1,36 @@ +'use strict'; + +const db = require('../database'); +module.exports = function (Messaging) { + Messaging.pinMessage = async (mid, roomId) => { + const isMessageInRoom = await db.isSortedSetMember(`chat:room:${roomId}:mids`, mid); + if (isMessageInRoom) { + await db.sortedSetAdd(`chat:room:${roomId}:mids:pinned`, Date.now(), mid); + await Messaging.setMessageFields(mid, { + pinned: 1 + }); + } + }; + Messaging.unpinMessage = async (mid, roomId) => { + const isMessageInRoom = await db.isSortedSetMember(`chat:room:${roomId}:mids`, mid); + if (isMessageInRoom) { + await db.sortedSetRemove(`chat:room:${roomId}:mids:pinned`, mid); + await Messaging.setMessageFields(mid, { + pinned: 0 + }); + } + }; + Messaging.getPinnedMessages = async (roomId, uid, start, stop) => { + const mids = await db.getSortedSetRevRange(`chat:room:${roomId}:mids:pinned`, start, stop); + if (!mids.length) { + return []; + } + const messageData = await Messaging.getMessagesData(mids, uid, roomId, true); + messageData.forEach((msg, i) => { + if (msg) { + msg.index = start + i; + } + }); + return messageData; + }; +}; \ No newline at end of file diff --git a/lib/messaging/rooms.js b/lib/messaging/rooms.js new file mode 100644 index 0000000000..8ea698fc99 --- /dev/null +++ b/lib/messaging/rooms.js @@ -0,0 +1,411 @@ +'use strict'; + +const _ = require('lodash'); +const validator = require('validator'); +const winston = require('winston'); +const db = require('../database'); +const user = require('../user'); +const groups = require('../groups'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const meta = require('../meta'); +const io = require('../socket.io'); +const cache = require('../cache'); +const cacheCreate = require('../cacheCreate'); +const roomUidCache = cacheCreate({ + name: 'chat:room:uids', + max: 500, + ttl: 0 +}); +const intFields = ['roomId', 'timestamp', 'userCount', 'messageCount']; +module.exports = function (Messaging) { + Messaging.getRoomData = async (roomId, fields = []) => { + const roomData = await Messaging.getRoomsData([roomId], fields); + return roomData[0]; + }; + Messaging.getRoomsData = async (roomIds, fields = []) => { + if (fields.includes('notificationSetting') && !fields.includes('public')) { + fields.push('public'); + } + const roomData = await db.getObjects(roomIds.map(roomId => `chat:room:${roomId}`), fields); + modifyRoomData(roomData, fields); + return roomData; + }; + function modifyRoomData(rooms, fields) { + rooms.forEach(data => { + if (data) { + db.parseIntFields(data, intFields, fields); + data.roomName = validator.escape(String(data.roomName || '')); + data.public = parseInt(data.public, 10) === 1; + data.groupChat = data.userCount > 2; + if (!fields.length || fields.includes('notificationSetting')) { + data.notificationSetting = data.notificationSetting || (data.public ? Messaging.notificationSettings.ATMENTION : Messaging.notificationSettings.ALLMESSAGES); + } + if (data.hasOwnProperty('groups') || !fields.length || fields.includes('groups')) { + try { + data.groups = JSON.parse(data.groups || '[]'); + } catch (err) { + winston.error(err.stack); + data.groups = []; + } + } + } + }); + } + Messaging.newRoom = async (uid, data) => { + if (Array.isArray(data)) { + data = { + uids: data + }; + } + if (data.hasOwnProperty('roomName')) { + checkRoomName(data.roomName); + } + const now = Date.now(); + const roomId = await db.incrObjectField('global', 'nextChatRoomId'); + const room = { + roomId: roomId, + timestamp: now, + notificationSetting: data.notificationSetting, + messageCount: 0 + }; + if (data.hasOwnProperty('roomName') && data.roomName) { + room.roomName = String(data.roomName).trim(); + } + if (Array.isArray(data.groups) && data.groups.length) { + room.groups = JSON.stringify(data.groups); + } + const isPublic = data.type === 'public'; + if (isPublic) { + room.public = 1; + } + await Promise.all([db.setObject(`chat:room:${roomId}`, room), db.sortedSetAdd('chat:rooms', now, roomId), db.sortedSetAdd(`chat:room:${roomId}:owners`, now, uid), db.sortedSetsAdd([`chat:room:${roomId}:uids`, `chat:room:${roomId}:uids:online`], now, uid)]); + await Promise.all([Messaging.addUsersToRoom(uid, data.uids, roomId), isPublic ? db.sortedSetAddBulk([['chat:rooms:public', now, roomId], ['chat:rooms:public:order', roomId, roomId]]) : Messaging.addRoomToUsers(roomId, [uid].concat(data.uids), now)]); + cache.del(['chat:rooms:public:all', 'chat:rooms:public:order:all']); + if (!isPublic) { + await Messaging.addSystemMessage('user-join', uid, roomId); + } + return roomId; + }; + Messaging.deleteRooms = async roomIds => { + if (!roomIds) { + throw new Error('[[error:invalid-data]]'); + } + if (!Array.isArray(roomIds)) { + roomIds = [roomIds]; + } + await Promise.all(roomIds.map(async roomId => { + const uids = await db.getSortedSetMembers(`chat:room:${roomId}:uids`); + const keys = uids.map(uid => `uid:${uid}:chat:rooms`).concat(uids.map(uid => `uid:${uid}:chat:rooms:unread`)); + await db.sortedSetsRemove(keys, roomId); + })); + await Promise.all([db.deleteAll([...roomIds.map(id => `chat:room:${id}`), ...roomIds.map(id => `chat:room:${id}:uids`), ...roomIds.map(id => `chat:room:${id}:owners`), ...roomIds.map(id => `chat:room:${id}:uids:online`), ...roomIds.map(id => `chat:room:${id}:notification:settings`)]), db.sortedSetRemove(['chat:rooms', 'chat:rooms:public', 'chat:rooms:public:order', 'chat:rooms:public:lastpost'], roomIds)]); + cache.del(['chat:rooms:public:all', 'chat:rooms:public:order:all']); + }; + Messaging.isUserInRoom = async (uid, roomIds) => { + let single = false; + if (!Array.isArray(roomIds)) { + roomIds = [roomIds]; + single = true; + } + const inRooms = await db.isMemberOfSortedSets(roomIds.map(id => `chat:room:${id}:uids`), uid); + const data = await Promise.all(roomIds.map(async (roomId, idx) => { + const data = await plugins.hooks.fire('filter:messaging.isUserInRoom', { + uid: uid, + roomId: roomId, + inRoom: inRooms[idx] + }); + return data.inRoom; + })); + return single ? data.pop() : data; + }; + Messaging.isUsersInRoom = async (uids, roomId) => { + let single = false; + if (!Array.isArray(uids)) { + uids = [uids]; + single = true; + } + const inRooms = await db.isSortedSetMembers(`chat:room:${roomId}:uids`, uids); + const data = await plugins.hooks.fire('filter:messaging.isUsersInRoom', { + uids: uids, + roomId: roomId, + inRooms: inRooms + }); + return single ? data.inRooms.pop() : data.inRooms; + }; + Messaging.roomExists = async roomId => db.exists(`chat:room:${roomId}`); + Messaging.getUserCountInRoom = async roomId => db.sortedSetCard(`chat:room:${roomId}:uids`); + Messaging.isRoomOwner = async (uids, roomId) => { + const isArray = Array.isArray(uids); + if (!isArray) { + uids = [uids]; + } + const isOwners = await db.isSortedSetMembers(`chat:room:${roomId}:owners`, uids); + const result = await Promise.all(isOwners.map(async (isOwner, index) => { + const payload = await plugins.hooks.fire('filter:messaging.isRoomOwner', { + uid: uids[index], + roomId, + isOwner + }); + return payload.isOwner; + })); + return isArray ? result : result[0]; + }; + Messaging.toggleOwner = async (uid, roomId, state = null) => { + if (!(parseInt(uid, 10) > 0) || !roomId) { + throw new Error('[[error:invalid-data]]'); + } + const isOwner = await Messaging.isRoomOwner(uid, roomId); + if (state !== null) { + if (state === isOwner) { + return false; + } + } else { + state = !isOwner; + } + if (state) { + await db.sortedSetAdd(`chat:room:${roomId}:owners`, Date.now(), uid); + } else { + await db.sortedSetRemove(`chat:room:${roomId}:owners`, uid); + } + }; + Messaging.isRoomPublic = async function (roomId) { + return parseInt(await db.getObjectField(`chat:room:${roomId}`, 'public'), 10) === 1; + }; + Messaging.addUsersToRoom = async function (uid, uids, roomId) { + uids = _.uniq(uids); + const inRoom = await Messaging.isUserInRoom(uid, roomId); + const payload = await plugins.hooks.fire('filter:messaging.addUsersToRoom', { + uid, + uids, + roomId, + inRoom + }); + if (!payload.inRoom) { + throw new Error('[[error:cant-add-users-to-chat-room]]'); + } + await addUidsToRoom(payload.uids, roomId); + }; + async function addUidsToRoom(uids, roomId) { + const now = Date.now(); + const timestamps = uids.map(() => now); + await Promise.all([db.sortedSetAdd(`chat:room:${roomId}:uids`, timestamps, uids), db.sortedSetAdd(`chat:room:${roomId}:uids:online`, timestamps, uids)]); + await updateUserCount([roomId]); + await Promise.all(uids.map(uid => Messaging.addSystemMessage('user-join', uid, roomId))); + } + Messaging.removeUsersFromRoom = async (uid, uids, roomId) => { + const [isOwner, userCount] = await Promise.all([Messaging.isRoomOwner(uid, roomId), Messaging.getUserCountInRoom(roomId)]); + const payload = await plugins.hooks.fire('filter:messaging.removeUsersFromRoom', { + uid, + uids, + roomId, + isOwner, + userCount + }); + if (!payload.isOwner) { + throw new Error('[[error:cant-remove-users-from-chat-room]]'); + } + await Messaging.leaveRoom(payload.uids, payload.roomId); + }; + Messaging.isGroupChat = async function (roomId) { + return (await Messaging.getRoomData(roomId)).groupChat; + }; + async function updateUserCount(roomIds) { + const userCounts = await db.sortedSetsCard(roomIds.map(roomId => `chat:room:${roomId}:uids`)); + const countMap = _.zipObject(roomIds, userCounts); + const groupChats = roomIds.filter((roomId, index) => userCounts[index] > 2); + const privateChats = roomIds.filter((roomId, index) => userCounts[index] <= 2); + await db.setObjectBulk([...groupChats.map(id => [`chat:room:${id}`, { + groupChat: 1, + userCount: countMap[id] + }]), ...privateChats.map(id => [`chat:room:${id}`, { + groupChat: 0, + userCount: countMap[id] + }])]); + roomUidCache.del(roomIds.map(id => `chat:room:${id}:users`)); + } + Messaging.leaveRoom = async (uids, roomId) => { + const isInRoom = await Promise.all(uids.map(uid => Messaging.isUserInRoom(uid, roomId))); + uids = uids.filter((uid, index) => isInRoom[index]); + const keys = uids.map(uid => `uid:${uid}:chat:rooms`).concat(uids.map(uid => `uid:${uid}:chat:rooms:unread`)); + await Promise.all([db.sortedSetRemove([`chat:room:${roomId}:uids`, `chat:room:${roomId}:owners`, `chat:room:${roomId}:uids:online`], uids), db.sortedSetsRemove(keys, roomId)]); + await Promise.all(uids.map(uid => Messaging.addSystemMessage('user-leave', uid, roomId))); + await updateOwner(roomId); + await updateUserCount([roomId]); + }; + Messaging.leaveRooms = async (uid, roomIds) => { + const isInRoom = await Promise.all(roomIds.map(roomId => Messaging.isUserInRoom(uid, roomId))); + roomIds = roomIds.filter((roomId, index) => isInRoom[index]); + const roomKeys = [...roomIds.map(roomId => `chat:room:${roomId}:uids`), ...roomIds.map(roomId => `chat:room:${roomId}:owners`), ...roomIds.map(roomId => `chat:room:${roomId}:uids:online`)]; + await Promise.all([db.sortedSetsRemove(roomKeys, uid), db.sortedSetRemove([`uid:${uid}:chat:rooms`, `uid:${uid}:chat:rooms:unread`], roomIds)]); + await Promise.all(roomIds.map(roomId => updateOwner(roomId)).concat(roomIds.map(roomId => Messaging.addSystemMessage('user-leave', uid, roomId)))); + await updateUserCount(roomIds); + }; + async function updateOwner(roomId) { + let nextOwner = await db.getSortedSetRange(`chat:room:${roomId}:owners`, 0, 0); + if (!nextOwner.length) { + nextOwner = await db.getSortedSetRange(`chat:room:${roomId}:uids`, 0, 0); + const newOwner = nextOwner[0] || 0; + if (parseInt(newOwner, 10) > 0) { + await db.sortedSetAdd(`chat:room:${roomId}:owners`, Date.now(), newOwner); + } + } + } + Messaging.getAllUidsInRoomFromSet = async function (set) { + const cacheKey = `${set}:all`; + let uids = roomUidCache.get(cacheKey); + if (uids !== undefined) { + return uids; + } + uids = await Messaging.getUidsInRoomFromSet(set, 0, -1); + roomUidCache.set(cacheKey, uids); + return uids; + }; + Messaging.getUidsInRoomFromSet = async (set, start, stop, reverse = false) => db[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](set, start, stop); + Messaging.getUidsInRoom = async (roomId, start, stop, reverse = false) => db[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](`chat:room:${roomId}:uids`, start, stop); + Messaging.getUsersInRoom = async (roomId, start, stop, reverse = false) => { + const users = await Messaging.getUsersInRoomFromSet(`chat:room:${roomId}:uids`, roomId, start, stop, reverse); + return users; + }; + Messaging.getUsersInRoomFromSet = async (set, roomId, start, stop, reverse = false) => { + const uids = await Messaging.getUidsInRoomFromSet(set, start, stop, reverse); + const [users, isOwners] = await Promise.all([user.getUsersFields(uids, ['uid', 'username', 'picture', 'status']), Messaging.isRoomOwner(uids, roomId)]); + return users.map((user, index) => { + user.index = start + index; + user.isOwner = isOwners[index]; + return user; + }); + }; + Messaging.renameRoom = async function (uid, roomId, newName) { + newName = String(newName).trim(); + checkRoomName(newName); + const payload = await plugins.hooks.fire('filter:chat.renameRoom', { + uid: uid, + roomId: roomId, + newName: newName + }); + const isOwner = await Messaging.isRoomOwner(payload.uid, payload.roomId); + if (!isOwner) { + throw new Error('[[error:no-privileges]]'); + } + await db.setObjectField(`chat:room:${payload.roomId}`, 'roomName', payload.newName); + await Messaging.addSystemMessage(`room-rename, ${payload.newName.replace(',', ',')}`, payload.uid, payload.roomId); + plugins.hooks.fire('action:chat.renameRoom', { + roomId: payload.roomId, + newName: payload.newName + }); + }; + function checkRoomName(roomName) { + if (!roomName && roomName !== '') { + throw new Error('[[error:invalid-room-name]]'); + } + if (roomName.length > meta.config.maximumChatRoomNameLength) { + throw new Error(`[[error:chat-room-name-too-long, ${meta.config.maximumChatRoomNameLength}]]`); + } + } + Messaging.canReply = async (roomId, uid) => { + const inRoom = await db.isSortedSetMember(`chat:room:${roomId}:uids`, uid); + const data = await plugins.hooks.fire('filter:messaging.canReply', { + uid: uid, + roomId: roomId, + inRoom: inRoom, + canReply: inRoom + }); + return data.canReply; + }; + Messaging.loadRoom = async (uid, data) => { + const { + roomId + } = data; + const [room, inRoom, canChat, isAdmin, isGlobalMod] = await Promise.all([Messaging.getRoomData(roomId), Messaging.isUserInRoom(uid, roomId), privileges.global.can(['chat', 'chat:privileged'], uid), user.isAdministrator(uid), user.isGlobalModerator(uid)]); + if (!room || !room.public && !inRoom || room.public && Array.isArray(room.groups) && room.groups.length && !isAdmin && !(await groups.isMemberOfAny(uid, room.groups))) { + return null; + } + if (!canChat.includes(true)) { + throw new Error('[[error:no-privileges]]'); + } + if (room.public && !inRoom) { + await addUidsToRoom([uid], roomId); + room.userCount += 1; + } else if (inRoom) { + await db.sortedSetAdd(`chat:room:${roomId}:uids:online`, Date.now(), uid); + } + async function getNotificationOptions() { + const userSetting = await db.getObjectField(`chat:room:${roomId}:notification:settings`, uid); + const roomDefault = room.notificationSetting; + const currentSetting = userSetting || roomDefault; + const labels = { + [Messaging.notificationSettings.NONE]: { + label: '[[modules:chat.notification-setting-none]]', + icon: 'fa-ban' + }, + [Messaging.notificationSettings.ATMENTION]: { + label: '[[modules:chat.notification-setting-at-mention-only]]', + icon: 'fa-at' + }, + [Messaging.notificationSettings.ALLMESSAGES]: { + label: '[[modules:chat.notification-setting-all-messages]]', + icon: 'fa-comment-o' + } + }; + const options = [{ + label: '[[modules:chat.notification-setting-room-default]]', + subLabel: labels[roomDefault].label || '', + icon: labels[roomDefault].icon, + value: -1, + selected: userSetting === null + }]; + Object.keys(labels).forEach(key => { + options.push({ + label: labels[key].label, + icon: labels[key].icon, + value: key, + selected: parseInt(userSetting, 10) === parseInt(key, 10) + }); + }); + return { + options, + selectedIcon: labels[currentSetting].icon + }; + } + const [canReply, users, messages, settings, isOwner, onlineUids, notifOptions] = await Promise.all([Messaging.canReply(roomId, uid), Messaging.getUsersInRoomFromSet(`chat:room:${roomId}:uids:online`, roomId, 0, 39, true), Messaging.getMessages({ + callerUid: uid, + start: data.start || 0, + uid: data.uid || uid, + roomId: roomId, + isNew: false + }), user.getSettings(uid), Messaging.isRoomOwner(uid, roomId), io.getUidsInRoom(`chat_room_${roomId}`), getNotificationOptions(), Messaging.markRoomNotificationsRead(uid, roomId)]); + users.forEach(user => { + if (user) { + user.online = parseInt(user.uid, 10) === parseInt(uid, 10) || onlineUids.includes(String(user.uid)); + } + }); + room.messages = messages; + room.isOwner = isOwner; + room.users = users; + room.canReply = canReply; + room.groupChat = users.length > 2; + room.icon = Messaging.getRoomIcon(room); + room.usernames = Messaging.generateUsernames(room, uid); + room.chatWithMessage = await Messaging.generateChatWithMessage(room, uid, settings.userLang); + room.maximumUsersInChatRoom = meta.config.maximumUsersInChatRoom; + room.maximumChatMessageLength = meta.config.maximumChatMessageLength; + room.showUserInput = !room.maximumUsersInChatRoom || room.maximumUsersInChatRoom > 2; + room.isAdminOrGlobalMod = isAdmin || isGlobalMod; + room.isAdmin = isAdmin; + room.notificationOptions = notifOptions.options; + room.notificationOptionsIcon = notifOptions.selectedIcon; + room.composerActions = []; + const payload = await plugins.hooks.fire('filter:messaging.loadRoom', { + uid, + data, + room + }); + return payload.room; + }; + const globalUserGroups = ['registered-users', 'verified-users', 'unverified-users', 'banned-users']; + Messaging.getRoomIcon = function (roomData) { + const hasGroups = Array.isArray(roomData.groups) && roomData.groups.length; + return !hasGroups || roomData.groups.some(group => globalUserGroups.includes(group)) ? 'fa-hashtag' : 'fa-lock'; + }; +}; \ No newline at end of file diff --git a/lib/messaging/unread.js b/lib/messaging/unread.js new file mode 100644 index 0000000000..0d01df113d --- /dev/null +++ b/lib/messaging/unread.js @@ -0,0 +1,54 @@ +'use strict'; + +const db = require('../database'); +const io = require('../socket.io'); +module.exports = function (Messaging) { + Messaging.getUnreadCount = async uid => { + if (!(parseInt(uid, 10) > 0)) { + return 0; + } + return await db.sortedSetCard(`uid:${uid}:chat:rooms:unread`); + }; + Messaging.pushUnreadCount = async (uids, data = null) => { + if (!Array.isArray(uids)) { + uids = [uids]; + } + uids = uids.filter(uid => parseInt(uid, 10) > 0); + if (!uids.length) { + return; + } + uids.forEach(uid => { + io.in(`uid_${uid}`).emit('event:unread.updateChatCount', data); + }); + }; + Messaging.markRead = async (uid, roomId) => { + await Promise.all([db.sortedSetRemove(`uid:${uid}:chat:rooms:unread`, roomId), db.setObjectField(`uid:${uid}:chat:rooms:read`, roomId, Date.now())]); + }; + Messaging.hasRead = async (uids, roomId) => { + if (!uids.length) { + return []; + } + const roomData = await Messaging.getRoomData(roomId); + if (!roomData) { + return uids.map(() => false); + } + if (roomData.public) { + const [userTimestamps, mids] = await Promise.all([db.getObjectsFields(uids.map(uid => `uid:${uid}:chat:rooms:read`), [roomId]), db.getSortedSetRevRangeWithScores(`chat:room:${roomId}:mids`, 0, 0)]); + const lastMsgTimestamp = mids[0] ? mids[0].score : 0; + return uids.map((uid, index) => !userTimestamps[index] || !userTimestamps[index][roomId] || parseInt(userTimestamps[index][roomId], 10) > lastMsgTimestamp); + } + const isMembers = await db.isMemberOfSortedSets(uids.map(uid => `uid:${uid}:chat:rooms:unread`), roomId); + return uids.map((uid, index) => !isMembers[index]); + }; + Messaging.markAllRead = async uid => { + await db.delete(`uid:${uid}:chat:rooms:unread`); + }; + Messaging.markUnread = async (uids, roomId) => { + const exists = await Messaging.roomExists(roomId); + if (!exists) { + return; + } + const keys = uids.map(uid => `uid:${uid}:chat:rooms:unread`); + await db.sortedSetsAdd(keys, Date.now(), roomId); + }; +}; \ No newline at end of file diff --git a/lib/meta/aliases.js b/lib/meta/aliases.js new file mode 100644 index 0000000000..13d5ed1e83 --- /dev/null +++ b/lib/meta/aliases.js @@ -0,0 +1,29 @@ +'use strict'; + +const _ = require('lodash'); +const chalk = require('chalk'); +const aliases = { + 'plugin static dirs': ['staticdirs'], + 'requirejs modules': ['rjs', 'modules'], + 'client js bundle': ['clientjs', 'clientscript', 'clientscripts'], + 'admin js bundle': ['adminjs', 'adminscript', 'adminscripts'], + javascript: ['js'], + 'client side styles': ['clientcss', 'clientscss', 'clientstyles', 'clientstyle'], + 'admin control panel styles': ['admincss', 'adminscss', 'adminstyles', 'adminstyle', 'acpcss', 'acpscss', 'acpstyles', 'acpstyle'], + styles: ['css', 'scss', 'style'], + templates: ['tpl'], + languages: ['lang', 'i18n'] +}; +exports.aliases = aliases; +function buildTargets() { + let length = 0; + const output = Object.keys(aliases).map(name => { + const arr = aliases[name]; + if (name.length > length) { + length = name.length; + } + return [name, arr.join(', ')]; + }).map(tuple => ` ${chalk.magenta(_.padEnd(`"${tuple[0]}"`, length + 2))} | ${tuple[1]}`).join('\n'); + process.stdout.write('\n\n Build targets:\n' + `${chalk.green(`\n ${_.padEnd('Target', length + 2)} | Aliases`)}` + `${chalk.blue('\n ------------------------------------------------------\n')}` + `${output}\n\n`); +} +exports.buildTargets = buildTargets; \ No newline at end of file diff --git a/lib/meta/blacklist.js b/lib/meta/blacklist.js new file mode 100644 index 0000000000..b25fb67370 --- /dev/null +++ b/lib/meta/blacklist.js @@ -0,0 +1,147 @@ +'use strict'; + +const ipaddr = require('ipaddr.js'); +const winston = require('winston'); +const _ = require('lodash'); +const validator = require('validator'); +const db = require('../database'); +const pubsub = require('../pubsub'); +const plugins = require('../plugins'); +const analytics = require('../analytics'); +const Blacklist = module.exports; +Blacklist._rules = {}; +Blacklist.load = async function () { + let rules = await Blacklist.get(); + rules = Blacklist.validate(rules); + winston.verbose(`[meta/blacklist] Loading ${rules.valid.length} blacklist rule(s)${rules.duplicateCount > 0 ? `, ignored ${rules.duplicateCount} duplicate(s)` : ''}`); + if (rules.invalid.length) { + winston.warn(`[meta/blacklist] ${rules.invalid.length} invalid blacklist rule(s) were ignored.`); + } + Blacklist._rules = { + ipv4: rules.ipv4, + ipv6: rules.ipv6, + cidr: rules.cidr, + cidr6: rules.cidr6 + }; +}; +pubsub.on('blacklist:reload', Blacklist.load); +Blacklist.save = async function (rules) { + await db.setObject('ip-blacklist-rules', { + rules: rules + }); + await Blacklist.load(); + pubsub.publish('blacklist:reload'); +}; +Blacklist.addRule = async function (rule) { + const { + valid + } = Blacklist.validate(rule); + if (!valid.length) { + throw new Error('[[error:invalid-rule]]'); + } + let rules = await Blacklist.get(); + rules = `${rules}\n${valid[0]}`; + await Blacklist.save(rules); +}; +Blacklist.get = async function () { + const data = await db.getObject('ip-blacklist-rules'); + return data && data.rules; +}; +Blacklist.test = async function (clientIp) { + if (!clientIp) { + return; + } + clientIp = clientIp.split(':').length === 2 ? clientIp.split(':')[0] : clientIp; + if (!validator.isIP(clientIp)) { + throw new Error('[[error:invalid-ip]]'); + } + const rules = Blacklist._rules; + function checkCidrRange(clientIP) { + if (!rules.cidr.length) { + return false; + } + let addr; + try { + addr = ipaddr.parse(clientIP); + } catch (err) { + winston.error(`[meta/blacklist] Error parsing client IP : ${clientIp}`); + throw err; + } + return rules.cidr.some(subnet => { + const cidr = ipaddr.parseCIDR(subnet); + if (addr.kind() !== cidr[0].kind()) { + return false; + } + return addr.match(cidr); + }); + } + if (rules.ipv4.includes(clientIp) || rules.ipv6.includes(clientIp) || checkCidrRange(clientIp)) { + const err = new Error('[[error:blacklisted-ip]]'); + err.code = 'blacklisted-ip'; + analytics.increment('blacklist'); + throw err; + } + try { + await plugins.hooks.fire('filter:blacklist.test', { + ip: clientIp + }); + } catch (err) { + analytics.increment('blacklist'); + throw err; + } +}; +Blacklist.validate = function (rules) { + rules = (rules || '').split('\n'); + const ipv4 = []; + const ipv6 = []; + const cidr = []; + const invalid = []; + let duplicateCount = 0; + const inlineCommentMatch = /#.*$/; + const whitelist = ['127.0.0.1', '::1', '::ffff:0:127.0.0.1']; + rules = rules.map(rule => { + rule = rule.replace(inlineCommentMatch, '').trim(); + return rule.length && !rule.startsWith('#') ? rule : null; + }).filter(Boolean); + const uniqRules = _.uniq(rules); + duplicateCount += rules.length - uniqRules.length; + rules = uniqRules; + rules = rules.filter(rule => { + let addr; + let isRange = false; + try { + addr = ipaddr.parse(rule); + } catch (e) {} + try { + addr = ipaddr.parseCIDR(rule); + isRange = true; + } catch (e) {} + if (!addr || whitelist.includes(rule)) { + invalid.push(validator.escape(rule)); + return false; + } + if (!isRange) { + if (addr.kind() === 'ipv4' && ipaddr.IPv4.isValid(rule)) { + ipv4.push(rule); + return true; + } + if (addr.kind() === 'ipv6' && ipaddr.IPv6.isValid(rule)) { + ipv6.push(rule); + return true; + } + } else { + cidr.push(rule); + return true; + } + return false; + }); + return { + numRules: rules.length + invalid.length, + ipv4: ipv4, + ipv6: ipv6, + cidr: cidr, + valid: rules, + invalid: invalid, + duplicateCount: duplicateCount + }; +}; \ No newline at end of file diff --git a/lib/meta/build.js b/lib/meta/build.js new file mode 100644 index 0000000000..53ee574b38 --- /dev/null +++ b/lib/meta/build.js @@ -0,0 +1,203 @@ +'use strict'; + +const os = require('os'); +const winston = require('winston'); +const nconf = require('nconf'); +const _ = require('lodash'); +const path = require('path'); +const { + mkdirp +} = require('mkdirp'); +const chalk = require('chalk'); +const cacheBuster = require('./cacheBuster'); +const { + aliases +} = require('./aliases'); +let meta; +const targetHandlers = { + 'plugin static dirs': async function () { + await meta.js.linkStatics(); + }, + 'requirejs modules': async function (parallel) { + await meta.js.buildModules(parallel); + }, + 'client js bundle': async function (parallel) { + await meta.js.buildBundle('client', parallel); + }, + 'admin js bundle': async function (parallel) { + await meta.js.buildBundle('admin', parallel); + }, + javascript: ['plugin static dirs', 'requirejs modules', 'client js bundle', 'admin js bundle'], + 'client side styles': async function (parallel) { + await meta.css.buildBundle('client', parallel); + }, + 'admin control panel styles': async function (parallel) { + await meta.css.buildBundle('admin', parallel); + }, + styles: ['client side styles', 'admin control panel styles'], + templates: async function () { + await meta.templates.compile(); + }, + languages: async function () { + await meta.languages.build(); + } +}; +const aliasMap = Object.keys(aliases).reduce((prev, key) => { + const arr = aliases[key]; + arr.forEach(alias => { + prev[alias] = key; + }); + prev[key] = key; + return prev; +}, {}); +async function beforeBuild(targets) { + const db = require('../database'); + process.stdout.write(`${chalk.green(' started')}\n`); + try { + await db.init(); + meta = require('./index'); + await meta.themes.setupPaths(); + const plugins = require('../plugins'); + await plugins.prepareForBuild(targets); + await mkdirp(path.join(__dirname, '../../build/public')); + } catch (err) { + winston.error(`[build] Encountered error preparing for build`); + throw err; + } +} +const allTargets = Object.keys(targetHandlers).filter(name => typeof targetHandlers[name] === 'function'); +async function buildTargets(targets, parallel, options) { + const length = Math.max(...targets.map(name => name.length)); + const jsTargets = targets.filter(target => targetHandlers.javascript.includes(target)); + const otherTargets = targets.filter(target => !targetHandlers.javascript.includes(target)); + async function buildJSTargets() { + await Promise.all(jsTargets.map(target => step(target, parallel, `${_.padStart(target, length)} `))); + if (options.webpack || options.watch) { + await exports.webpack(options); + } + } + if (parallel) { + await Promise.all([buildJSTargets(), ...otherTargets.map(target => step(target, parallel, `${_.padStart(target, length)} `))]); + } else { + for (const target of targets) { + await step(target, parallel, `${_.padStart(target, length)} `); + } + if (options.webpack || options.watch) { + await exports.webpack(options); + } + } +} +async function step(target, parallel, targetStr) { + const startTime = Date.now(); + winston.info(`[build] ${targetStr} build started`); + try { + await targetHandlers[target](parallel); + const time = (Date.now() - startTime) / 1000; + winston.info(`[build] ${targetStr} build completed in ${time}sec`); + } catch (err) { + winston.error(`[build] ${targetStr} build failed`); + throw err; + } +} +exports.build = async function (targets, options) { + if (!options) { + options = {}; + } + if (targets === true) { + targets = allTargets; + } else if (!Array.isArray(targets)) { + targets = targets.split(','); + } + let series = nconf.get('series') || options.series; + if (series === undefined) { + winston.verbose('[build] Querying CPU core count for build strategy'); + const cpus = os.cpus(); + series = cpus.length < 4; + winston.verbose(`[build] System returned ${cpus.length} cores, opting for ${series ? 'series' : 'parallel'} build strategy`); + } + targets = targets.map(target => { + target = target.toLowerCase().replace(/-/g, ''); + if (!aliasMap[target]) { + winston.warn(`[build] Unknown target: ${target}`); + if (target.includes(',')) { + winston.warn('[build] Are you specifying multiple targets? Separate them with spaces:'); + winston.warn('[build] e.g. `./nodebb build adminjs tpl`'); + } + return false; + } + return aliasMap[target]; + }).filter(Boolean); + targets = _.uniq(_.flatMap(targets, target => Array.isArray(targetHandlers[target]) ? targetHandlers[target] : target)); + winston.verbose(`[build] building the following targets: ${targets.join(', ')}`); + if (!targets) { + winston.info('[build] No valid targets supplied. Aborting.'); + return; + } + try { + await beforeBuild(targets); + const threads = parseInt(nconf.get('threads'), 10); + if (threads) { + require('./minifier').maxThreads = threads - 1; + } + if (!series) { + winston.info('[build] Building in parallel mode'); + } else { + winston.info('[build] Building in series mode'); + } + const startTime = Date.now(); + await buildTargets(targets, !series, options); + const totalTime = (Date.now() - startTime) / 1000; + await cacheBuster.write(); + winston.info(`[build] Asset compilation successful. Completed in ${totalTime}sec.`); + } catch (err) { + winston.error(`[build] Encountered error during build step`); + throw err; + } +}; +function getWebpackConfig() { + return require(process.env.NODE_ENV !== 'development' ? '../../webpack.prod' : '../../webpack.dev'); +} +exports.webpack = async function (options) { + winston.info(`[build] ${options.watch ? 'Watching' : 'Bundling'} with Webpack.`); + const webpack = require('webpack'); + const fs = require('fs'); + const util = require('util'); + const plugins = require('../plugins/data'); + const activePlugins = (await plugins.getActive()).map(p => p.id); + if (!activePlugins.includes('nodebb-plugin-composer-default')) { + activePlugins.push('nodebb-plugin-composer-default'); + } + await fs.promises.writeFile(path.resolve(__dirname, '../../build/active_plugins.json'), JSON.stringify(activePlugins)); + const webpackCfg = getWebpackConfig(); + const compiler = webpack(webpackCfg); + const webpackRun = util.promisify(compiler.run).bind(compiler); + const webpackWatch = util.promisify(compiler.watch).bind(compiler); + try { + let stats; + if (options.watch) { + stats = await webpackWatch(webpackCfg.watchOptions); + compiler.hooks.assetEmitted.tap('nbbWatchPlugin', file => { + console.log(`webpack:assetEmitted > ${webpackCfg.output.publicPath}${file}`); + }); + } else { + stats = await webpackRun(); + } + if (stats.hasErrors() || stats.hasWarnings()) { + console.log(stats.toString('minimal')); + } else { + const statsJson = stats.toJson(); + winston.info(`[build] ${options.watch ? 'Watching' : 'Bundling'} took ${statsJson.time} ms`); + } + } catch (err) { + console.error(err.stack || err); + if (err.details) { + console.error(err.details); + } + } +}; +exports.buildAll = async function () { + await exports.build(allTargets, { + webpack: true + }); +}; +require('../promisify')(exports); \ No newline at end of file diff --git a/lib/meta/cacheBuster.js b/lib/meta/cacheBuster.js new file mode 100644 index 0000000000..56adefb07a --- /dev/null +++ b/lib/meta/cacheBuster.js @@ -0,0 +1,35 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { + mkdirp +} = require('mkdirp'); +const winston = require('winston'); +const filePath = path.join(__dirname, '../../build/cache-buster'); +let cached; +function generate() { + return (Math.random() * 1e18).toString(32).slice(0, 11); +} +exports.write = async function write() { + await mkdirp(path.dirname(filePath)); + await fs.promises.writeFile(filePath, generate()); +}; +exports.read = async function read() { + if (cached) { + return cached; + } + try { + const buster = await fs.promises.readFile(filePath, 'utf8'); + if (!buster || buster.length !== 11) { + winston.warn(`[cache-buster] cache buster string invalid: expected /[a-z0-9]{11}/, got \`${buster}\``); + return generate(); + } + cached = buster; + return cached; + } catch (err) { + winston.warn('[cache-buster] could not read cache buster', err); + return generate(); + } +}; +require('../promisify')(exports); \ No newline at end of file diff --git a/lib/meta/configs.js b/lib/meta/configs.js new file mode 100644 index 0000000000..f748b07e46 --- /dev/null +++ b/lib/meta/configs.js @@ -0,0 +1,210 @@ +'use strict'; + +const nconf = require('nconf'); +const path = require('path'); +const winston = require('winston'); +const db = require('../database'); +const pubsub = require('../pubsub'); +const Meta = require('./index'); +const cacheBuster = require('./cacheBuster'); +const defaults = require('../../install/data/defaults.json'); +const Configs = module.exports; +Meta.config = {}; +function deserialize(config) { + const deserialized = {}; + Object.keys(config).forEach(key => { + const defaultType = typeof defaults[key]; + const type = typeof config[key]; + const number = parseFloat(config[key]); + if (defaultType === 'string' && type === 'number') { + deserialized[key] = String(config[key]); + } else if (defaultType === 'number' && type === 'string') { + if (!isNaN(number) && isFinite(config[key])) { + deserialized[key] = number; + } else { + deserialized[key] = defaults[key]; + } + } else if (config[key] === 'true') { + deserialized[key] = true; + } else if (config[key] === 'false') { + deserialized[key] = false; + } else if (config[key] === null) { + deserialized[key] = defaults[key]; + } else if (defaultType === 'undefined' && !isNaN(number) && isFinite(config[key])) { + deserialized[key] = number; + } else if (Array.isArray(defaults[key]) && !Array.isArray(config[key])) { + try { + deserialized[key] = JSON.parse(config[key] || '[]'); + } catch (err) { + winston.error(err.stack); + deserialized[key] = defaults[key]; + } + } else { + deserialized[key] = config[key]; + } + }); + return deserialized; +} +function serialize(config) { + const serialized = {}; + Object.keys(config).forEach(key => { + const defaultType = typeof defaults[key]; + const type = typeof config[key]; + const number = parseFloat(config[key]); + if (defaultType === 'string' && type === 'number') { + serialized[key] = String(config[key]); + } else if (defaultType === 'number' && type === 'string') { + if (!isNaN(number) && isFinite(config[key])) { + serialized[key] = number; + } else { + serialized[key] = defaults[key]; + } + } else if (config[key] === null) { + serialized[key] = defaults[key]; + } else if (defaultType === 'undefined' && !isNaN(number) && isFinite(config[key])) { + serialized[key] = number; + } else if (Array.isArray(defaults[key]) && Array.isArray(config[key])) { + serialized[key] = JSON.stringify(config[key]); + } else { + serialized[key] = config[key]; + } + }); + return serialized; +} +Configs.deserialize = deserialize; +Configs.serialize = serialize; +Configs.init = async function () { + const config = await Configs.list(); + const buster = await cacheBuster.read(); + config['cache-buster'] = `v=${buster || Date.now()}`; + Meta.config = config; +}; +Configs.list = async function () { + return await Configs.getFields([]); +}; +Configs.get = async function (field) { + const values = await Configs.getFields([field]); + return values.hasOwnProperty(field) && values[field] !== undefined ? values[field] : null; +}; +Configs.getFields = async function (fields) { + let values; + if (fields.length) { + values = await db.getObjectFields('config', fields); + } else { + values = await db.getObject('config'); + } + values = { + ...defaults, + ...(values ? deserialize(values) : {}) + }; + if (!fields.length) { + values.version = nconf.get('version'); + values.registry = nconf.get('registry'); + } + return values; +}; +Configs.set = async function (field, value) { + if (!field) { + throw new Error('[[error:invalid-data]]'); + } + await Configs.setMultiple({ + [field]: value + }); +}; +Configs.setMultiple = async function (data) { + await processConfig(data); + data = serialize(data); + await db.setObject('config', data); + updateConfig(deserialize(data)); +}; +Configs.setOnEmpty = async function (values) { + const data = await db.getObject('config'); + values = serialize(values); + const config = { + ...values, + ...(data ? serialize(data) : {}) + }; + await db.setObject('config', config); +}; +Configs.remove = async function (field) { + await db.deleteObjectField('config', field); +}; +Configs.cookie = { + get: () => { + const cookie = {}; + if (nconf.get('cookieDomain') || Meta.config.cookieDomain) { + cookie.domain = nconf.get('cookieDomain') || Meta.config.cookieDomain; + } + if (nconf.get('secure')) { + cookie.secure = true; + } + const relativePath = nconf.get('relative_path'); + if (relativePath !== '') { + cookie.path = relativePath; + } + cookie.sameSite = 'Lax'; + return cookie; + } +}; +async function processConfig(data) { + ensureInteger(data, 'maximumUsernameLength', 1); + ensureInteger(data, 'minimumUsernameLength', 1); + ensureInteger(data, 'minimumPasswordLength', 1); + ensureInteger(data, 'maximumAboutMeLength', 0); + if (data.minimumUsernameLength > data.maximumUsernameLength) { + throw new Error('[[error:invalid-data]]'); + } + require('../social').postSharing = null; + await Promise.all([saveRenderedCss(data), getLogoSize(data)]); +} +function ensureInteger(data, field, min) { + if (data.hasOwnProperty(field)) { + data[field] = parseInt(data[field], 10); + if (!(data[field] >= min)) { + throw new Error('[[error:invalid-data]]'); + } + } +} +async function saveRenderedCss(data) { + if (!data.customCSS) { + return; + } + const sass = require('../utils').getSass(); + const scssOutput = await sass.compileStringAsync(data.customCSS, {}); + data.renderedCustomCSS = scssOutput.css.toString(); +} +async function getLogoSize(data) { + const image = require('../image'); + if (!data['brand:logo']) { + return; + } + let size; + try { + size = await image.size(path.join(nconf.get('upload_path'), 'system', 'site-logo-x50.png')); + } catch (err) { + if (err.code === 'ENOENT') { + winston.warn('[logo] The email-safe logo doesn\'t seem to have been created, please re-upload your site logo.'); + size = { + height: 0, + width: 0 + }; + } else { + throw err; + } + } + data['brand:emailLogo'] = nconf.get('url') + path.join(nconf.get('upload_url'), 'system', 'site-logo-x50.png'); + data['brand:emailLogo:height'] = size.height; + data['brand:emailLogo:width'] = size.width; +} +function updateConfig(config) { + updateLocalConfig(config); + pubsub.publish('config:update', config); +} +function updateLocalConfig(config) { + Object.assign(Meta.config, config); +} +pubsub.on('config:update', config => { + if (typeof config === 'object' && Meta.config) { + updateLocalConfig(config); + } +}); \ No newline at end of file diff --git a/lib/meta/css.js b/lib/meta/css.js new file mode 100644 index 0000000000..d834688b62 --- /dev/null +++ b/lib/meta/css.js @@ -0,0 +1,217 @@ +'use strict'; + +const _ = require('lodash'); +const winston = require('winston'); +const nconf = require('nconf'); +const fs = require('fs'); +const path = require('path'); +const { + mkdirp +} = require('mkdirp'); +const plugins = require('../plugins'); +const db = require('../database'); +const file = require('../file'); +const minifier = require('./minifier'); +const utils = require('../utils'); +const CSS = module.exports; +CSS.supportedSkins = ['cerulean', 'cosmo', 'cyborg', 'darkly', 'flatly', 'journal', 'litera', 'lumen', 'lux', 'materia', 'minty', 'morph', 'pulse', 'quartz', 'sandstone', 'simplex', 'sketchy', 'slate', 'solar', 'spacelab', 'superhero', 'united', 'vapor', 'yeti', 'zephyr']; +const buildImports = { + client: function (source, themeData) { + return [boostrapImport(themeData), '@import "@adactive/bootstrap-tagsinput/src/bootstrap-tagsinput";', source, '@import "jquery-ui";', '@import "cropperjs/dist/cropper";'].join('\n'); + }, + admin: function (source) { + return ['@import "admin/overrides";', '@import "bootstrap/scss/bootstrap";', '@import "mixins";', '@import "fontawesome/loader";', getFontawesomeStyle(), '@import "@adactive/bootstrap-tagsinput/src/bootstrap-tagsinput";', '@import "generics";', '@import "responsive-utilities";', '@import "admin/admin";', source, '@import "jquery-ui";'].join('\n'); + } +}; +function boostrapImport(themeData) { + const { + bootswatchSkin, + bsVariables, + isCustomSkin + } = themeData; + function bsvariables() { + if (bootswatchSkin) { + if (isCustomSkin) { + return themeData._variables || ''; + } + return `@import "bootswatch/dist/${bootswatchSkin}/variables";`; + } + return bsVariables; + } + return [bsvariables(), '@import "bootstrap/scss/mixins/banner";', '@include bsBanner("");', '@import "bootstrap/scss/functions";', '@import "./scss/overrides";', '@import "overrides.scss";', '@import "bootstrap/scss/variables";', '@import "bootstrap/scss/variables-dark";', '@import "bootstrap/scss/maps";', '@import "bootstrap/scss/mixins";', '@import "bootstrap/scss/utilities";', '@import "bootstrap/scss/root";', '@import "bootstrap/scss/reboot";', '@import "bootstrap/scss/type";', '@import "bootstrap/scss/images";', '@import "bootstrap/scss/containers";', '@import "bootstrap/scss/grid";', '@import "bootstrap/scss/tables";', '@import "bootstrap/scss/forms";', '@import "bootstrap/scss/buttons";', '@import "bootstrap/scss/transitions";', '@import "bootstrap/scss/dropdown";', '@import "bootstrap/scss/button-group";', '@import "bootstrap/scss/nav";', '@import "bootstrap/scss/navbar";', '@import "bootstrap/scss/card";', '@import "bootstrap/scss/accordion";', '@import "bootstrap/scss/breadcrumb";', '@import "bootstrap/scss/pagination";', '@import "bootstrap/scss/badge";', '@import "bootstrap/scss/alert";', '@import "bootstrap/scss/progress";', '@import "bootstrap/scss/list-group";', '@import "bootstrap/scss/close";', '@import "bootstrap/scss/toasts";', '@import "bootstrap/scss/modal";', '@import "bootstrap/scss/tooltip";', '@import "bootstrap/scss/popover";', '@import "bootstrap/scss/carousel";', '@import "bootstrap/scss/spinners";', '@import "bootstrap/scss/offcanvas";', '@import "bootstrap/scss/placeholders";', '@import "bootstrap/scss/helpers";', '@import "responsive-utilities";', '@import "bootstrap/scss/utilities/api";', '@import "fontawesome/loader";', getFontawesomeStyle(), '@import "mixins";', '@import "generics";', '@import "client";', '@import "./theme";', bootswatchSkin && !isCustomSkin ? `@import "bootswatch/dist/${bootswatchSkin}/bootswatch";` : ''].join('\n'); +} +function getFontawesomeStyle() { + const styles = utils.getFontawesomeStyles(); + return styles.map(style => `@import "fontawesome/style-${style}";`).join('\n'); +} +async function copyFontAwesomeFiles() { + await mkdirp(path.join(__dirname, '../../build/public/fontawesome/webfonts')); + const fonts = await fs.promises.opendir(path.join(utils.getFontawesomePath(), '/webfonts')); + const copyOperations = []; + for await (const file of fonts) { + if (file.isFile() && file.name.match(/\.(woff2|ttf|eot)?$/)) { + copyOperations.push(fs.promises.copyFile(path.join(fonts.path, file.name), path.join(__dirname, '../../build/public/fontawesome/webfonts/', file.name))); + } + } + await Promise.all(copyOperations); +} +async function filterMissingFiles(filepaths) { + const exists = await Promise.all(filepaths.map(async filepath => { + const exists = await file.exists(path.join(__dirname, '../../node_modules', filepath)); + if (!exists) { + winston.warn(`[meta/css] File not found! ${filepath}`); + } + return exists; + })); + return filepaths.filter((filePath, i) => exists[i]); +} +async function getImports(files, extension) { + const pluginDirectories = []; + let source = ''; + function pathToImport(file) { + if (!file) { + return ''; + } + const parsed = path.parse(file); + const newFile = path.join(parsed.dir, parsed.name); + return `\n@import "${newFile.replace(/\\/g, '/')}";`; + } + files.forEach(styleFile => { + if (styleFile.endsWith(extension)) { + source += pathToImport(styleFile); + } else { + pluginDirectories.push(styleFile); + } + }); + await Promise.all(pluginDirectories.map(async directory => { + const styleFiles = await file.walk(directory); + styleFiles.forEach(styleFile => { + source += pathToImport(styleFile); + }); + })); + return source; +} +async function getBundleMetadata(target) { + const paths = [path.join(__dirname, '../../node_modules'), path.join(__dirname, '../../public/scss'), path.join(__dirname, '../../public/fontawesome/scss'), path.join(utils.getFontawesomePath(), 'scss')]; + let skin; + let isCustomSkin = false; + if (target.startsWith('client-')) { + skin = target.split('-').slice(1).join('-'); + const isBootswatchSkin = CSS.supportedSkins.includes(skin); + isCustomSkin = !isBootswatchSkin && (await CSS.isCustomSkin(skin)); + target = 'client'; + if (!isBootswatchSkin && !isCustomSkin) { + skin = ''; + } + } + let themeData = null; + if (target === 'client') { + themeData = await db.getObjectFields('config', ['theme:type', 'theme:id', 'useBSVariables', 'bsVariables']); + const themeId = themeData['theme:id'] || 'nodebb-theme-harmony'; + const baseThemePath = path.join(nconf.get('themes_path'), themeData['theme:type'] && themeData['theme:type'] === 'local' ? themeId : 'nodebb-theme-harmony'); + paths.unshift(baseThemePath); + paths.unshift(`${baseThemePath}/node_modules`); + themeData.bsVariables = parseInt(themeData.useBSVariables, 10) === 1 ? themeData.bsVariables || '' : ''; + themeData.bootswatchSkin = skin; + themeData.isCustomSkin = isCustomSkin; + const customSkin = isCustomSkin ? await CSS.getCustomSkin(skin) : null; + themeData._variables = customSkin && customSkin._variables; + } + const [scssImports, cssImports, acpScssImports] = await Promise.all([filterGetImports(plugins.scssFiles, '.scss'), filterGetImports(plugins.cssFiles, '.css'), target === 'client' ? '' : filterGetImports(plugins.acpScssFiles, '.scss')]); + async function filterGetImports(files, extension) { + const filteredFiles = await filterMissingFiles(files); + return await getImports(filteredFiles, extension); + } + let imports = `${cssImports}\n${scssImports}\n${acpScssImports}`; + imports = buildImports[target](imports, themeData); + return { + paths: paths, + imports: imports + }; +} +CSS.getSkinSwitcherOptions = async function (uid) { + const user = require('../user'); + const meta = require('./index'); + const [userSettings, customSkins] = await Promise.all([user.getSettings(uid), CSS.getCustomSkins()]); + const foundCustom = customSkins.find(skin => skin.value === meta.config.bootswatchSkin); + const defaultSkin = foundCustom ? foundCustom.name : _.capitalize(meta.config.bootswatchSkin) || '[[user:no-skin]]'; + const defaultSkins = [{ + name: `[[user:default, ${defaultSkin}]]`, + value: '', + selected: userSettings.bootswatchSkin === '' + }, { + name: '[[user:no-skin]]', + value: 'noskin', + selected: userSettings.bootswatchSkin === 'noskin' + }]; + const lightSkins = ['cerulean', 'cosmo', 'flatly', 'journal', 'litera', 'lumen', 'lux', 'materia', 'minty', 'morph', 'pulse', 'sandstone', 'simplex', 'sketchy', 'spacelab', 'united', 'yeti', 'zephyr']; + const darkSkins = ['cyborg', 'darkly', 'quartz', 'slate', 'solar', 'superhero', 'vapor']; + function parseSkins(skins) { + skins = skins.map(skin => ({ + name: _.capitalize(skin), + value: skin + })); + skins.forEach(skin => { + skin.selected = skin.value === userSettings.bootswatchSkin; + }); + return skins; + } + return await plugins.hooks.fire('filter:meta.css.getSkinSwitcherOptions', { + default: defaultSkins, + custom: customSkins.map(s => ({ + ...s, + selected: s.value === userSettings.bootswatchSkin + })), + light: parseSkins(lightSkins), + dark: parseSkins(darkSkins) + }); +}; +CSS.getCustomSkins = async function (opts = {}) { + const meta = require('./index'); + const slugify = require('../slugify'); + const { + loadVariables + } = opts; + const customSkins = await meta.settings.get('custom-skins'); + const returnSkins = []; + if (customSkins && Array.isArray(customSkins['custom-skin-list'])) { + customSkins['custom-skin-list'].forEach(customSkin => { + if (customSkin) { + returnSkins.push({ + name: customSkin['custom-skin-name'], + value: slugify(customSkin['custom-skin-name']), + _variables: loadVariables ? customSkin._variables : undefined + }); + } + }); + } + return returnSkins; +}; +CSS.isSkinValid = async function (skin) { + return CSS.supportedSkins.includes(skin) || (await CSS.isCustomSkin(skin)); +}; +CSS.isCustomSkin = async function (skin) { + const skins = await CSS.getCustomSkins(); + return !!skins.find(s => s.value === skin); +}; +CSS.getCustomSkin = async function (skin) { + const skins = await CSS.getCustomSkins({ + loadVariables: true + }); + return skins.find(s => s.value === skin); +}; +CSS.buildBundle = async function (target, fork) { + if (target === 'client') { + let files = await fs.promises.readdir(path.join(__dirname, '../../build/public')); + files = files.filter(f => f.match(/^client.*\.css$/)); + await Promise.all(files.map(f => fs.promises.unlink(path.join(__dirname, '../../build/public', f)))); + } + const data = await getBundleMetadata(target); + const minify = process.env.NODE_ENV !== 'development'; + const { + ltr, + rtl + } = await minifier.css.bundle(data.imports, data.paths, minify, fork); + await Promise.all([fs.promises.writeFile(path.join(__dirname, '../../build/public', `${target}.css`), ltr.code), fs.promises.writeFile(path.join(__dirname, '../../build/public', `${target}-rtl.css`), rtl.code), copyFontAwesomeFiles()]); + return [ltr.code, rtl.code]; +}; \ No newline at end of file diff --git a/lib/meta/debugFork.js b/lib/meta/debugFork.js new file mode 100644 index 0000000000..8dc056a6fa --- /dev/null +++ b/lib/meta/debugFork.js @@ -0,0 +1,28 @@ +'use strict'; + +const { + fork +} = require('child_process'); +let debugArg = process.execArgv.find(arg => /^--(debug|inspect)/.test(arg)); +const debugging = !!debugArg; +debugArg = debugArg ? debugArg.replace('-brk', '').split('=') : ['--debug', 5859]; +let lastAddress = parseInt(debugArg[1], 10); +function debugFork(modulePath, args, options) { + let execArgv = []; + if (global.v8debug || debugging) { + lastAddress += 1; + execArgv = [`${debugArg[0]}=${lastAddress}`, '--nolazy']; + } + if (!Array.isArray(args)) { + options = args; + args = []; + } + options = options || {}; + options = { + ...options, + execArgv: execArgv + }; + return fork(modulePath, args, options); +} +debugFork.debugging = debugging; +module.exports = debugFork; \ No newline at end of file diff --git a/lib/meta/dependencies.js b/lib/meta/dependencies.js new file mode 100644 index 0000000000..2113461c36 --- /dev/null +++ b/lib/meta/dependencies.js @@ -0,0 +1,62 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const semver = require('semver'); +const winston = require('winston'); +const chalk = require('chalk'); +const pkg = require('../../package.json'); +const { + paths, + pluginNamePattern +} = require('../constants'); +const Dependencies = module.exports; +let depsMissing = false; +let depsOutdated = false; +Dependencies.check = async function () { + const modules = Object.keys(pkg.dependencies); + winston.verbose('Checking dependencies for outdated modules'); + await Promise.all(modules.map(module => Dependencies.checkModule(module))); + if (depsMissing) { + throw new Error('dependencies-missing'); + } else if (depsOutdated && global.env !== 'development') { + throw new Error('dependencies-out-of-date'); + } +}; +Dependencies.checkModule = async function (moduleName) { + try { + let pkgData = await fs.promises.readFile(path.join(paths.nodeModules, moduleName, 'package.json'), 'utf8'); + pkgData = Dependencies.parseModuleData(moduleName, pkgData); + const satisfies = Dependencies.doesSatisfy(pkgData, pkg.dependencies[moduleName]); + return satisfies; + } catch (err) { + if (err.code === 'ENOENT' && pluginNamePattern.test(moduleName)) { + winston.warn(`[meta/dependencies] Bundled plugin ${moduleName} not found, skipping dependency check.`); + return true; + } + throw err; + } +}; +Dependencies.parseModuleData = function (moduleName, pkgData) { + try { + pkgData = JSON.parse(pkgData); + } catch (e) { + winston.warn(`[${chalk.red('missing')}] ${chalk.bold(moduleName)} is a required dependency but could not be found\n`); + depsMissing = true; + return null; + } + return pkgData; +}; +Dependencies.doesSatisfy = function (moduleData, packageJSONVersion) { + if (!moduleData) { + return false; + } + const versionOk = !semver.validRange(packageJSONVersion) || semver.satisfies(moduleData.version, packageJSONVersion); + const githubRepo = moduleData._resolved && moduleData._resolved.includes('//github.com'); + const satisfies = versionOk || githubRepo; + if (!satisfies) { + winston.warn(`[${chalk.yellow('outdated')}] ${chalk.bold(moduleData.name)} installed v${moduleData.version}, package.json requires ${packageJSONVersion}\n`); + depsOutdated = true; + } + return satisfies; +}; \ No newline at end of file diff --git a/lib/meta/errors.js b/lib/meta/errors.js new file mode 100644 index 0000000000..9cd97e526a --- /dev/null +++ b/lib/meta/errors.js @@ -0,0 +1,48 @@ +'use strict'; + +const winston = require('winston'); +const validator = require('validator'); +const cronJob = require('cron').CronJob; +const db = require('../database'); +const analytics = require('../analytics'); +const Errors = module.exports; +let counters = {}; +new cronJob('0 * * * * *', () => { + Errors.writeData(); +}, null, true); +Errors.writeData = async function () { + try { + const _counters = { + ...counters + }; + counters = {}; + const keys = Object.keys(_counters); + if (!keys.length) { + return; + } + for (const key of keys) { + await db.sortedSetIncrBy('errors:404', _counters[key], key); + } + } catch (err) { + winston.error(err.stack); + } +}; +Errors.log404 = function (route) { + if (!route) { + return; + } + route = route.slice(0, 512).replace(/\/$/, ''); + analytics.increment('errors:404'); + counters[route] = counters[route] || 0; + counters[route] += 1; +}; +Errors.get = async function (escape) { + const data = await db.getSortedSetRevRangeWithScores('errors:404', 0, 199); + data.forEach(nfObject => { + nfObject.value = escape ? validator.escape(String(nfObject.value || '')) : nfObject.value; + }); + return data; +}; +Errors.clear = async function () { + await db.delete('errors:404'); +}; \ No newline at end of file diff --git a/lib/meta/index.js b/lib/meta/index.js new file mode 100644 index 0000000000..1ae6c7d940 --- /dev/null +++ b/lib/meta/index.js @@ -0,0 +1,61 @@ +'use strict'; + +const winston = require('winston'); +const os = require('os'); +const nconf = require('nconf'); +const pubsub = require('../pubsub'); +const slugify = require('../slugify'); +const Meta = module.exports; +Meta.reloadRequired = false; +Meta.configs = require('./configs'); +Meta.themes = require('./themes'); +Meta.js = require('./js'); +Meta.css = require('./css'); +Meta.settings = require('./settings'); +Meta.logs = require('./logs'); +Meta.errors = require('./errors'); +Meta.tags = require('./tags'); +Meta.dependencies = require('./dependencies'); +Meta.templates = require('./templates'); +Meta.blacklist = require('./blacklist'); +Meta.languages = require('./languages'); +const user = require('../user'); +const groups = require('../groups'); +Meta.userOrGroupExists = async function (slug) { + const isArray = Array.isArray(slug); + if (isArray && slug.some(slug => !slug) || !isArray && !slug) { + throw new Error('[[error:invalid-data]]'); + } + slug = isArray ? slug.map(s => slugify(s, false)) : slugify(slug); + const [userExists, groupExists] = await Promise.all([user.existsBySlug(slug), groups.existsBySlug(slug)]); + return isArray ? slug.map((s, i) => userExists[i] || groupExists[i]) : userExists || groupExists; +}; +if (nconf.get('isPrimary')) { + pubsub.on('meta:restart', data => { + if (data.hostname !== os.hostname()) { + restart(); + } + }); +} +Meta.restart = function () { + pubsub.publish('meta:restart', { + hostname: os.hostname() + }); + restart(); +}; +function restart() { + if (process.send) { + process.send({ + action: 'restart' + }); + } else { + winston.error('[meta.restart] Could not restart, are you sure NodeBB was started with `./nodebb start`?'); + } +} +Meta.getSessionTTLSeconds = function () { + const ttlDays = 60 * 60 * 24 * Meta.config.loginDays; + const ttlSeconds = Meta.config.loginSeconds; + const ttl = ttlSeconds || ttlDays || 1209600; + return ttl; +}; +require('../promisify')(Meta); \ No newline at end of file diff --git a/lib/meta/js.js b/lib/meta/js.js new file mode 100644 index 0000000000..64f1d0ba50 --- /dev/null +++ b/lib/meta/js.js @@ -0,0 +1,100 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const { + mkdirp +} = require('mkdirp'); +const file = require('../file'); +const plugins = require('../plugins'); +const minifier = require('./minifier'); +const JS = module.exports; +JS.scripts = { + base: ['node_modules/@adactive/bootstrap-tagsinput/src/bootstrap-tagsinput.js', 'node_modules/jquery-serializeobject/jquery.serializeObject.js', 'node_modules/jquery-deserialize/src/jquery.deserialize.js', 'public/vendor/bootbox/wrapper.js'], + modules: {} +}; +const basePath = path.resolve(__dirname, '../..'); +async function linkModules() { + const { + modules + } = JS.scripts; + await Promise.all([mkdirp(path.join(__dirname, '../../build/public/src/admin/plugins')), mkdirp(path.join(__dirname, '../../build/public/src/client/plugins'))]); + await Promise.all(Object.keys(modules).map(async relPath => { + const srcPath = path.join(__dirname, '../../', modules[relPath]); + const destPath = path.join(__dirname, '../../build/public/src/modules', relPath); + const destDir = path.dirname(destPath); + const [stats] = await Promise.all([fs.promises.stat(srcPath), mkdirp(destDir)]); + if (stats.isDirectory()) { + await file.linkDirs(srcPath, destPath, true); + } else { + const relPath = path.relative(destDir, srcPath).split(path.sep).join(path.posix.sep); + await fs.promises.writeFile(destPath, `module.exports = require('${relPath}');`); + } + })); +} +const moduleDirs = ['modules', 'admin', 'client']; +async function clearModules() { + const builtPaths = moduleDirs.map(p => path.join(__dirname, '../../build/public/src', p)); + await Promise.all(builtPaths.map(builtPath => fs.promises.rm(builtPath, { + recursive: true, + force: true + }))); +} +JS.buildModules = async function () { + await clearModules(); + const fse = require('fs-extra'); + await fse.copy(path.join(__dirname, `../../public/src`), path.join(__dirname, `../../build/public/src`)); + await linkModules(); +}; +JS.linkStatics = async function () { + await fs.promises.rm(path.join(__dirname, '../../build/public/plugins'), { + recursive: true, + force: true + }); + plugins.staticDirs['core/inter'] = path.join(basePath, 'node_modules//@fontsource/inter/files'); + plugins.staticDirs['core/poppins'] = path.join(basePath, 'node_modules//@fontsource/poppins/files'); + await Promise.all(Object.keys(plugins.staticDirs).map(async mappedPath => { + const sourceDir = plugins.staticDirs[mappedPath]; + const destDir = path.join(__dirname, '../../build/public/plugins', mappedPath); + await mkdirp(path.dirname(destDir)); + await file.linkDirs(sourceDir, destDir, true); + })); +}; +async function getBundleScriptList(target) { + const pluginDirectories = []; + if (target === 'admin') { + target = 'acp'; + } + let pluginScripts = plugins[`${target}Scripts`].filter(path => { + if (path.endsWith('.js')) { + return true; + } + pluginDirectories.push(path); + return false; + }); + await Promise.all(pluginDirectories.map(async directory => { + const scripts = await file.walk(directory); + pluginScripts = pluginScripts.concat(scripts); + })); + pluginScripts = JS.scripts.base.concat(pluginScripts).map(script => { + const srcPath = path.resolve(basePath, script).replace(/\\/g, '/'); + return { + srcPath: srcPath, + filename: path.relative(basePath, srcPath).replace(/\\/g, '/') + }; + }); + return pluginScripts; +} +JS.buildBundle = async function (target, fork) { + const filename = `scripts-${target}.js`; + const files = await getBundleScriptList(target); + const filePath = path.join(__dirname, '../../build/public', filename); + await minifier.js.bundle({ + files: files, + filename: filename, + destPath: filePath + }, fork); +}; +JS.killMinifier = function () { + minifier.killAll(); +}; \ No newline at end of file diff --git a/lib/meta/languages.js b/lib/meta/languages.js new file mode 100644 index 0000000000..c685abbd9f --- /dev/null +++ b/lib/meta/languages.js @@ -0,0 +1,106 @@ +'use strict'; + +const _ = require('lodash'); +const nconf = require('nconf'); +const path = require('path'); +const fs = require('fs'); +const { + mkdirp +} = require('mkdirp'); +const file = require('../file'); +const Plugins = require('../plugins'); +const { + paths +} = require('../constants'); +const buildLanguagesPath = path.join(paths.baseDir, 'build/public/language'); +const coreLanguagesPath = path.join(paths.baseDir, 'public/language'); +async function getTranslationMetadata() { + const paths = await file.walk(coreLanguagesPath); + let languages = []; + let namespaces = []; + paths.forEach(p => { + if (!p.endsWith('.json')) { + return; + } + const rel = path.relative(coreLanguagesPath, p).split(/[/\\]/); + const language = rel.shift().replace('_', '-').replace('@', '-x-'); + const namespace = rel.join('/').replace(/\.json$/, ''); + if (!language || !namespace) { + return; + } + languages.push(language); + namespaces.push(namespace); + }); + languages = _.union(languages, Plugins.languageData.languages).sort().filter(Boolean); + namespaces = _.union(namespaces, Plugins.languageData.namespaces).sort().filter(Boolean); + const configLangs = nconf.get('languages'); + if (process.env.NODE_ENV === 'development' && Array.isArray(configLangs) && configLangs.length) { + languages = configLangs; + } + await mkdirp(buildLanguagesPath); + const result = { + languages: languages, + namespaces: namespaces + }; + await fs.promises.writeFile(path.join(buildLanguagesPath, 'metadata.json'), JSON.stringify(result)); + return result; +} +async function writeLanguageFile(language, namespace, translations) { + const dev = process.env.NODE_ENV === 'development'; + const filePath = path.join(buildLanguagesPath, language, `${namespace}.json`); + await mkdirp(path.dirname(filePath)); + await fs.promises.writeFile(filePath, JSON.stringify(translations, null, dev ? 2 : 0)); +} +async function buildTranslations(ref) { + const { + namespaces + } = ref; + const { + languages + } = ref; + const plugins = _.values(Plugins.pluginsData).filter(plugin => typeof plugin.languages === 'string'); + const promises = []; + namespaces.forEach(namespace => { + languages.forEach(language => { + promises.push(buildNamespaceLanguage(language, namespace, plugins)); + }); + }); + await Promise.all(promises); +} +async function buildNamespaceLanguage(lang, namespace, plugins) { + const translations = {}; + await assignFileToTranslations(translations, path.join(coreLanguagesPath, lang, `${namespace}.json`)); + await Promise.all(plugins.map(pluginData => addPlugin(translations, pluginData, lang, namespace))); + if (Object.keys(translations).length) { + await writeLanguageFile(lang, namespace, translations); + } +} +async function addPlugin(translations, pluginData, lang, namespace) { + if (pluginData.languageData && !pluginData.languageData.namespaces.includes(namespace)) { + return; + } + const pathToPluginLanguageFolder = path.join(paths.nodeModules, pluginData.id, pluginData.languages); + const defaultLang = pluginData.defaultLang || 'en-GB'; + const langs = _.uniq([defaultLang.replace('-', '_').replace('-x-', '@'), defaultLang.replace('_', '-').replace('@', '-x-'), lang.replace('-', '_').replace('-x-', '@'), lang]); + for (const language of langs) { + await assignFileToTranslations(translations, path.join(pathToPluginLanguageFolder, language, `${namespace}.json`)); + } +} +async function assignFileToTranslations(translations, path) { + try { + const fileData = await fs.promises.readFile(path, 'utf8'); + Object.assign(translations, JSON.parse(fileData)); + } catch (err) { + if (err.code !== 'ENOENT') { + throw err; + } + } +} +exports.build = async function buildLanguages() { + await fs.promises.rm(buildLanguagesPath, { + recursive: true, + force: true + }); + const data = await getTranslationMetadata(); + await buildTranslations(data); +}; \ No newline at end of file diff --git a/lib/meta/logs.js b/lib/meta/logs.js new file mode 100644 index 0000000000..2a807a88a9 --- /dev/null +++ b/lib/meta/logs.js @@ -0,0 +1,12 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs'); +const Logs = module.exports; +Logs.path = path.resolve(__dirname, '../../logs/output.log'); +Logs.get = async function () { + return await fs.promises.readFile(Logs.path, 'utf-8'); +}; +Logs.clear = async function () { + await fs.promises.truncate(Logs.path, 0); +}; \ No newline at end of file diff --git a/lib/meta/minifier.js b/lib/meta/minifier.js new file mode 100644 index 0000000000..289eb59f24 --- /dev/null +++ b/lib/meta/minifier.js @@ -0,0 +1,184 @@ +'use strict'; + +const fs = require('fs'); +const os = require('os'); +const async = require('async'); +const winston = require('winston'); +const postcss = require('postcss'); +const autoprefixer = require('autoprefixer'); +const clean = require('postcss-clean'); +const rtlcss = require('rtlcss'); +const sass = require('../utils').getSass(); +const fork = require('./debugFork'); +require('../file'); +const Minifier = module.exports; +const pool = []; +const free = []; +let maxThreads = 0; +Object.defineProperty(Minifier, 'maxThreads', { + get: function () { + return maxThreads; + }, + set: function (val) { + maxThreads = val; + if (!process.env.minifier_child) { + winston.verbose(`[minifier] utilizing a maximum of ${maxThreads} additional threads`); + } + }, + configurable: true, + enumerable: true +}); +Minifier.maxThreads = Math.max(1, os.cpus().length - 1); +Minifier.killAll = function () { + pool.forEach(child => { + child.kill('SIGTERM'); + }); + pool.length = 0; + free.length = 0; +}; +function getChild() { + if (free.length) { + return free.shift(); + } + const proc = fork(__filename, [], { + cwd: __dirname, + env: { + minifier_child: true + } + }); + pool.push(proc); + return proc; +} +function freeChild(proc) { + proc.removeAllListeners(); + free.push(proc); +} +function removeChild(proc) { + const i = pool.indexOf(proc); + if (i !== -1) { + pool.splice(i, 1); + } +} +function forkAction(action) { + return new Promise((resolve, reject) => { + const proc = getChild(); + proc.on('message', message => { + freeChild(proc); + if (message.type === 'error') { + return reject(new Error(message.message)); + } + if (message.type === 'end') { + resolve(message.result); + } + }); + proc.on('error', err => { + proc.kill(); + removeChild(proc); + reject(err); + }); + proc.send({ + type: 'action', + action: action + }); + }); +} +const actions = {}; +if (process.env.minifier_child) { + process.on('message', async message => { + if (message.type === 'action') { + const { + action + } = message; + if (typeof actions[action.act] !== 'function') { + process.send({ + type: 'error', + message: 'Unknown action' + }); + return; + } + try { + const result = await actions[action.act](action); + process.send({ + type: 'end', + result: result + }); + } catch (err) { + process.send({ + type: 'error', + message: err.stack || err.message || 'unknown error' + }); + } + } + }); +} +async function executeAction(action, fork) { + if (fork && pool.length - free.length < Minifier.maxThreads) { + return await forkAction(action); + } + if (typeof actions[action.act] !== 'function') { + throw new Error('Unknown action'); + } + return await actions[action.act](action); +} +actions.concat = async function concat(data) { + if (data.files && data.files.length) { + const files = await async.mapLimit(data.files, 1000, async ref => await fs.promises.readFile(ref.srcPath, 'utf8')); + const output = files.join('\n;'); + await fs.promises.writeFile(data.destPath, output); + } +}; +Minifier.js = {}; +Minifier.js.bundle = async function (data, fork) { + return await executeAction({ + act: 'concat', + files: data.files, + filename: data.filename, + destPath: data.destPath + }, fork); +}; +actions.buildCSS = async function buildCSS(data) { + let css = ''; + try { + const scssOutput = await sass.compileStringAsync(data.source, { + loadPaths: data.paths + }); + css = scssOutput.css.toString(); + } catch (err) { + console.error(err.stack); + } + async function processScss(direction) { + if (direction === 'rtl') { + css = await postcss([rtlcss()]).process(css, { + from: undefined + }); + } + const postcssArgs = [autoprefixer]; + if (data.minify) { + postcssArgs.push(clean({ + processImportFrom: ['local'] + })); + } + return await postcss(postcssArgs).process(css, { + from: undefined + }); + } + const [ltrresult, rtlresult] = await Promise.all([processScss('ltr'), processScss('rtl')]); + return { + ltr: { + code: ltrresult.css + }, + rtl: { + code: rtlresult.css + } + }; +}; +Minifier.css = {}; +Minifier.css.bundle = async function (source, paths, minify, fork) { + return await executeAction({ + act: 'buildCSS', + source: source, + paths: paths, + minify: minify + }, fork); +}; +require('../promisify')(exports); \ No newline at end of file diff --git a/lib/meta/settings.js b/lib/meta/settings.js new file mode 100644 index 0000000000..8557be5cef --- /dev/null +++ b/lib/meta/settings.js @@ -0,0 +1,111 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const plugins = require('../plugins'); +const Meta = require('./index'); +const pubsub = require('../pubsub'); +const cache = require('../cache'); +const Settings = module.exports; +Settings.get = async function (hash) { + const cached = cache.get(`settings:${hash}`); + if (cached) { + return _.cloneDeep(cached); + } + const [data, sortedLists] = await Promise.all([db.getObject(`settings:${hash}`), db.getSetMembers(`settings:${hash}:sorted-lists`)]); + const values = data || {}; + await Promise.all(sortedLists.map(async list => { + const members = await db.getSortedSetRange(`settings:${hash}:sorted-list:${list}`, 0, -1); + const keys = members.map(order => `settings:${hash}:sorted-list:${list}:${order}`); + values[list] = []; + const objects = await db.getObjects(keys); + objects.forEach(obj => { + values[list].push(obj); + }); + })); + const result = await plugins.hooks.fire('filter:settings.get', { + plugin: hash, + values: values + }); + cache.set(`settings:${hash}`, result.values); + return _.cloneDeep(result.values); +}; +Settings.getOne = async function (hash, field) { + const data = await Settings.get(hash); + return data[field] !== undefined ? data[field] : null; +}; +Settings.set = async function (hash, values, quiet) { + quiet = quiet || false; + ({ + plugin: hash, + settings: values, + quiet + } = await plugins.hooks.fire('filter:settings.set', { + plugin: hash, + settings: values, + quiet + })); + const sortedListData = {}; + for (const [key, value] of Object.entries(values)) { + if (Array.isArray(value) && typeof value[0] !== 'string') { + sortedListData[key] = value; + delete values[key]; + } + } + const sortedLists = Object.keys(sortedListData); + if (sortedLists.length) { + await db.setRemove(`settings:${hash}:sorted-lists`, sortedLists.filter(list => !sortedListData[list].length)); + await db.setAdd(`settings:${hash}:sorted-lists`, sortedLists); + await Promise.all(sortedLists.map(async list => { + const numItems = await db.sortedSetCard(`settings:${hash}:sorted-list:${list}`); + const deleteKeys = [`settings:${hash}:sorted-list:${list}`]; + for (let x = 0; x < numItems; x++) { + deleteKeys.push(`settings:${hash}:sorted-list:${list}:${x}`); + } + await db.deleteAll(deleteKeys); + })); + const sortedSetData = []; + const objectData = []; + sortedLists.forEach(list => { + const arr = sortedListData[list]; + arr.forEach((data, order) => { + sortedSetData.push([`settings:${hash}:sorted-list:${list}`, order, order]); + objectData.push([`settings:${hash}:sorted-list:${list}:${order}`, data]); + }); + }); + await Promise.all([db.sortedSetAddBulk(sortedSetData), db.setObjectBulk(objectData)]); + } + if (Object.keys(values).length) { + await db.setObject(`settings:${hash}`, values); + } + cache.del(`settings:${hash}`); + plugins.hooks.fire('action:settings.set', { + plugin: hash, + settings: { + ...values, + ...sortedListData + }, + quiet + }); + pubsub.publish(`action:settings.set.${hash}`, values); + if (!Meta.reloadRequired && !quiet) { + Meta.reloadRequired = true; + } +}; +Settings.setOne = async function (hash, field, value) { + const data = {}; + data[field] = value; + await Settings.set(hash, data); +}; +Settings.setOnEmpty = async function (hash, values) { + const settings = (await Settings.get(hash)) || {}; + const empty = {}; + Object.keys(values).forEach(key => { + if (!settings.hasOwnProperty(key)) { + empty[key] = values[key]; + } + }); + if (Object.keys(empty).length) { + await Settings.set(hash, empty); + } +}; \ No newline at end of file diff --git a/lib/meta/tags.js b/lib/meta/tags.js new file mode 100644 index 0000000000..932ce7a479 --- /dev/null +++ b/lib/meta/tags.js @@ -0,0 +1,259 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const plugins = require('../plugins'); +const Meta = require('./index'); +const utils = require('../utils'); +const Tags = module.exports; +const url = nconf.get('url'); +const relative_path = nconf.get('relative_path'); +const upload_url = nconf.get('upload_url'); +Tags.parse = async (req, data, meta, link) => { + const isAPI = req.res && req.res.locals && req.res.locals.isAPI; + const defaultTags = isAPI ? [] : [{ + name: 'viewport', + content: 'width=device-width, initial-scale=1.0' + }, { + name: 'content-type', + content: 'text/html; charset=UTF-8', + noEscape: true + }, { + name: 'apple-mobile-web-app-capable', + content: 'yes' + }, { + name: 'mobile-web-app-capable', + content: 'yes' + }, { + property: 'og:site_name', + content: Meta.config.title || 'NodeBB' + }, { + name: 'msapplication-badge', + content: `frequency=30; polling-uri=${url}/sitemap.xml`, + noEscape: true + }, { + name: 'theme-color', + content: Meta.config.themeColor || '#ffffff' + }]; + if (Meta.config.keywords && !isAPI) { + defaultTags.push({ + name: 'keywords', + content: Meta.config.keywords + }); + } + if (Meta.config['brand:logo'] && !isAPI) { + defaultTags.push({ + name: 'msapplication-square150x150logo', + content: Meta.config['brand:logo'], + noEscape: true + }); + } + const faviconPath = `${relative_path}/assets/uploads/system/favicon.ico`; + const cacheBuster = `${Meta.config['cache-buster'] ? `?${Meta.config['cache-buster']}` : ''}`; + const defaultLinks = isAPI ? [] : [{ + rel: 'icon', + type: 'image/x-icon', + href: `${faviconPath}${cacheBuster}` + }, { + rel: 'manifest', + href: `${relative_path}/manifest.webmanifest`, + crossorigin: `use-credentials` + }]; + if (plugins.hooks.hasListeners('filter:search.query') && !isAPI) { + defaultLinks.push({ + rel: 'search', + type: 'application/opensearchdescription+xml', + title: utils.escapeHTML(String(Meta.config.title || Meta.config.browserTitle || 'NodeBB')), + href: `${relative_path}/osd.xml` + }); + } + if (!isAPI) { + addTouchIcons(defaultLinks); + } + const results = await utils.promiseParallel({ + tags: plugins.hooks.fire('filter:meta.getMetaTags', { + req: req, + data: data, + tags: defaultTags + }), + links: plugins.hooks.fire('filter:meta.getLinkTags', { + req: req, + data: data, + links: defaultLinks + }) + }); + meta = results.tags.tags.concat(meta || []).map(tag => { + if (!tag || typeof tag.content !== 'string') { + winston.warn('Invalid meta tag. ', tag); + return tag; + } + if (!tag.noEscape) { + const attributes = Object.keys(tag); + attributes.forEach(attr => { + tag[attr] = utils.escapeHTML(String(tag[attr])); + }); + } + return tag; + }); + await addSiteOGImage(meta); + addIfNotExists(meta, 'property', 'og:title', Meta.config.title || 'NodeBB'); + const ogUrl = url + (req.originalUrl !== '/' ? stripRelativePath(req.originalUrl) : ''); + addIfNotExists(meta, 'property', 'og:url', ogUrl); + addIfNotExists(meta, 'name', 'description', Meta.config.description); + addIfNotExists(meta, 'property', 'og:description', Meta.config.description); + link = results.links.links.concat(link || []); + if (isAPI) { + const whitelist = ['canonical', 'alternate', 'up']; + link = link.filter(link => whitelist.some(val => val === link.rel)); + } + link = link.map(tag => { + if (!tag.noEscape) { + const attributes = Object.keys(tag); + attributes.forEach(attr => { + tag[attr] = utils.escapeHTML(String(tag[attr])); + }); + } + return tag; + }); + return { + meta, + link + }; +}; +function addTouchIcons(defaultLinks) { + if (Meta.config['brand:touchIcon']) { + defaultLinks.push({ + rel: 'apple-touch-icon', + href: `${relative_path + upload_url}/system/touchicon-orig.png` + }, { + rel: 'icon', + sizes: '36x36', + href: `${relative_path + upload_url}/system/touchicon-36.png` + }, { + rel: 'icon', + sizes: '48x48', + href: `${relative_path + upload_url}/system/touchicon-48.png` + }, { + rel: 'icon', + sizes: '72x72', + href: `${relative_path + upload_url}/system/touchicon-72.png` + }, { + rel: 'icon', + sizes: '96x96', + href: `${relative_path + upload_url}/system/touchicon-96.png` + }, { + rel: 'icon', + sizes: '144x144', + href: `${relative_path + upload_url}/system/touchicon-144.png` + }, { + rel: 'icon', + sizes: '192x192', + href: `${relative_path + upload_url}/system/touchicon-192.png` + }); + } else { + defaultLinks.push({ + rel: 'apple-touch-icon', + href: `${relative_path}/assets/images/touch/512.png` + }, { + rel: 'icon', + sizes: '36x36', + href: `${relative_path}/assets/images/touch/36.png` + }, { + rel: 'icon', + sizes: '48x48', + href: `${relative_path}/assets/images/touch/48.png` + }, { + rel: 'icon', + sizes: '72x72', + href: `${relative_path}/assets/images/touch/72.png` + }, { + rel: 'icon', + sizes: '96x96', + href: `${relative_path}/assets/images/touch/96.png` + }, { + rel: 'icon', + sizes: '144x144', + href: `${relative_path}/assets/images/touch/144.png` + }, { + rel: 'icon', + sizes: '192x192', + href: `${relative_path}/assets/images/touch/192.png` + }, { + rel: 'icon', + sizes: '512x512', + href: `${relative_path}/assets/images/touch/512.png` + }); + } +} +function addIfNotExists(meta, keyName, tagName, value) { + const exists = meta.some(tag => tag[keyName] === tagName); + if (!exists && value) { + meta.push({ + content: utils.escapeHTML(String(value)), + [keyName]: tagName + }); + } +} +function stripRelativePath(url) { + if (url.startsWith(relative_path)) { + return url.slice(relative_path.length); + } + return url; +} +async function addSiteOGImage(meta) { + const key = Meta.config['og:image'] ? 'og:image' : 'brand:logo'; + let ogImage = stripRelativePath(Meta.config[key] || ''); + if (ogImage && !ogImage.startsWith('http')) { + ogImage = url + ogImage; + } + const { + images + } = await plugins.hooks.fire('filter:meta.addSiteOGImage', { + images: [{ + url: ogImage || `${url}/assets/images/logo@3x.png`, + width: ogImage ? Meta.config[`${key}:width`] : 963, + height: ogImage ? Meta.config[`${key}:height`] : 225 + }] + }); + const properties = ['url', 'secure_url', 'type', 'width', 'height', 'alt']; + images.forEach(image => { + for (const property of properties) { + if (image.hasOwnProperty(property)) { + switch (property) { + case 'url': + { + meta.push({ + property: 'og:image', + content: image.url, + noEscape: true + }, { + property: 'og:image:url', + content: image.url, + noEscape: true + }); + break; + } + case 'secure_url': + { + meta.push({ + property: `og:${property}`, + content: image[property], + noEscape: true + }); + break; + } + case 'type': + case 'alt': + case 'width': + case 'height': + { + meta.push({ + property: `og:image:${property}`, + content: String(image[property]) + }); + } + } + } + } + }); +} \ No newline at end of file diff --git a/lib/meta/templates.js b/lib/meta/templates.js new file mode 100644 index 0000000000..b63498197f --- /dev/null +++ b/lib/meta/templates.js @@ -0,0 +1,119 @@ +'use strict'; + +const { + mkdirp +} = require('mkdirp'); +const winston = require('winston'); +const path = require('path'); +const fs = require('fs'); +const nconf = require('nconf'); +const _ = require('lodash'); +const Benchpress = require('benchpressjs'); +const plugins = require('../plugins'); +const file = require('../file'); +const { + themeNamePattern, + paths +} = require('../constants'); +const viewsPath = nconf.get('views_dir'); +const Templates = module.exports; +async function processImports(paths, templatePath, source) { + const regex = //; + const matches = source.match(regex); + if (!matches) { + return source; + } + const partial = matches[1]; + if (paths[partial] && templatePath !== partial) { + const partialSource = await fs.promises.readFile(paths[partial], 'utf8'); + source = source.replace(regex, partialSource); + return await processImports(paths, templatePath, source); + } + winston.warn(`[meta/templates] Partial not loaded: ${matches[1]}`); + source = source.replace(regex, ''); + return await processImports(paths, templatePath, source); +} +Templates.processImports = processImports; +async function getTemplateDirs(activePlugins) { + const pluginTemplates = activePlugins.map(id => { + if (themeNamePattern.test(id)) { + return nconf.get('theme_templates_path'); + } + if (!plugins.pluginsData[id]) { + return ''; + } + return path.join(paths.nodeModules, id, plugins.pluginsData[id].templates || 'templates'); + }).filter(Boolean); + let themeConfig = require(nconf.get('theme_config')); + let theme = themeConfig.baseTheme; + let themePath; + let themeTemplates = []; + while (theme) { + themePath = path.join(nconf.get('themes_path'), theme); + themeConfig = require(path.join(themePath, 'theme.json')); + themeTemplates.push(path.join(themePath, themeConfig.templates || 'templates')); + theme = themeConfig.baseTheme; + } + themeTemplates = _.uniq(themeTemplates.reverse()); + const coreTemplatesPath = nconf.get('core_templates_path'); + let templateDirs = _.uniq([coreTemplatesPath].concat(themeTemplates, pluginTemplates)); + templateDirs = await Promise.all(templateDirs.map(async path => (await file.exists(path)) ? path : false)); + return templateDirs.filter(Boolean); +} +async function getTemplateFiles(dirs) { + const buckets = await Promise.all(dirs.map(async dir => { + let files = await file.walk(dir); + files = files.filter(path => path.endsWith('.tpl')).map(file => ({ + name: path.relative(dir, file).replace(/\\/g, '/'), + path: file + })); + return files; + })); + const dict = {}; + buckets.forEach(files => { + files.forEach(file => { + dict[file.name] = file.path; + }); + }); + return dict; +} +async function compileTemplate(filename, source) { + let paths = await file.walk(viewsPath); + paths = _.fromPairs(paths.map(p => { + const relative = path.relative(viewsPath, p).replace(/\\/g, '/'); + return [relative, p]; + })); + source = await processImports(paths, filename, source); + const compiled = await Benchpress.precompile(source, { + filename + }); + return await fs.promises.writeFile(path.join(viewsPath, filename.replace(/\.tpl$/, '.js')), compiled); +} +Templates.compileTemplate = compileTemplate; +async function compile() { + await fs.promises.rm(viewsPath, { + recursive: true, + force: true + }); + await mkdirp(viewsPath); + let files = await plugins.getActive(); + files = await getTemplateDirs(files); + files = await getTemplateFiles(files); + const minify = process.env.NODE_ENV !== 'development'; + await Promise.all(Object.keys(files).map(async name => { + const filePath = files[name]; + let imported = await fs.promises.readFile(filePath, 'utf8'); + imported = await processImports(files, name, imported); + await mkdirp(path.join(viewsPath, path.dirname(name))); + if (minify) { + imported = imported.split('\n').map(line => line.trim()).filter(Boolean).join('\n'); + } + await fs.promises.writeFile(path.join(viewsPath, name), imported); + const compiled = await Benchpress.precompile(imported, { + filename: name + }); + await fs.promises.writeFile(path.join(viewsPath, name.replace(/\.tpl$/, '.js')), compiled); + })); + winston.verbose('[meta/templates] Successfully compiled templates.'); +} +Templates.compile = compile; \ No newline at end of file diff --git a/lib/meta/themes.js b/lib/meta/themes.js new file mode 100644 index 0000000000..ca8a3c8c5d --- /dev/null +++ b/lib/meta/themes.js @@ -0,0 +1,151 @@ +'use strict'; + +const path = require('path'); +const nconf = require('nconf'); +const winston = require('winston'); +const _ = require('lodash'); +const fs = require('fs'); +const file = require('../file'); +const db = require('../database'); +const Meta = require('./index'); +const events = require('../events'); +const utils = require('../utils'); +const { + themeNamePattern +} = require('../constants'); +const Themes = module.exports; +Themes.get = async () => { + const themePath = nconf.get('themes_path'); + if (typeof themePath !== 'string') { + return []; + } + let themes = await getThemes(themePath); + themes = _.flatten(themes).filter(Boolean); + themes = await Promise.all(themes.map(async theme => { + const config = path.join(themePath, theme, 'theme.json'); + const pack = path.join(themePath, theme, 'package.json'); + try { + const [configFile, packageFile] = await Promise.all([fs.promises.readFile(config, 'utf8'), fs.promises.readFile(pack, 'utf8')]); + const configObj = JSON.parse(configFile); + const packageObj = JSON.parse(packageFile); + configObj.id = packageObj.name; + configObj.type = 'local'; + if (configObj.screenshot) { + configObj.screenshot_url = `${nconf.get('relative_path')}/css/previews/${encodeURIComponent(configObj.id)}`; + } else { + configObj.screenshot_url = `${nconf.get('relative_path')}/assets/images/themes/default.png`; + } + return configObj; + } catch (err) { + if (err.code === 'ENOENT') { + return false; + } + winston.error(`[themes] Unable to parse theme.json ${theme}`); + return false; + } + })); + return themes.filter(Boolean); +}; +async function getThemes(themePath) { + let dirs = await fs.promises.readdir(themePath); + dirs = dirs.filter(dir => themeNamePattern.test(dir) || dir.startsWith('@')); + return await Promise.all(dirs.map(async dir => { + try { + const dirpath = path.join(themePath, dir); + const stat = await fs.promises.stat(dirpath); + if (!stat.isDirectory()) { + return false; + } + if (!dir.startsWith('@')) { + return dir; + } + const themes = await getThemes(path.join(themePath, dir)); + return themes.map(theme => path.join(dir, theme)); + } catch (err) { + if (err.code === 'ENOENT') { + return false; + } + throw err; + } + })); +} +Themes.set = async data => { + switch (data.type) { + case 'local': + { + const current = await Meta.configs.get('theme:id'); + const score = await db.sortedSetScore('plugins:active', current); + await db.sortedSetRemove('plugins:active', current); + await db.sortedSetAdd('plugins:active', score || 0, data.id); + if (current !== data.id) { + const pathToThemeJson = path.join(nconf.get('themes_path'), data.id, 'theme.json'); + if (!pathToThemeJson.startsWith(nconf.get('themes_path'))) { + throw new Error('[[error:invalid-theme-id]]'); + } + let config = await fs.promises.readFile(pathToThemeJson, 'utf8'); + config = JSON.parse(config); + const activePluginsConfig = nconf.get('plugins:active'); + if (!activePluginsConfig) { + const score = await db.sortedSetScore('plugins:active', current); + await db.sortedSetRemove('plugins:active', current); + await db.sortedSetAdd('plugins:active', score || 0, data.id); + } else if (!activePluginsConfig.includes(data.id)) { + winston.error(`When defining active plugins in configuration, changing themes requires adding the theme '${data.id}' to the list of active plugins before updating it in the ACP`); + throw new Error('[[error:theme-not-set-in-configuration]]'); + } + Themes.setPath(config); + await Meta.configs.setMultiple({ + 'theme:type': data.type, + 'theme:id': data.id, + 'theme:staticDir': config.staticDir ? config.staticDir : '', + 'theme:templates': config.templates ? config.templates : '', + 'theme:src': '', + bootswatchSkin: '' + }); + await events.log({ + type: 'theme-set', + uid: parseInt(data.uid, 10) || 0, + ip: data.ip || '127.0.0.1', + text: data.id + }); + Meta.reloadRequired = true; + } + break; + } + case 'bootswatch': + await Meta.configs.setMultiple({ + 'theme:src': data.src, + bootswatchSkin: data.id.toLowerCase() + }); + break; + } +}; +Themes.setupPaths = async () => { + const data = await utils.promiseParallel({ + themesData: Themes.get(), + currentThemeId: Meta.configs.get('theme:id') + }); + const themeId = data.currentThemeId || 'nodebb-theme-harmony'; + if (process.env.NODE_ENV === 'development') { + winston.info(`[themes] Using theme ${themeId}`); + } + const themeObj = data.themesData.find(themeObj => themeObj.id === themeId); + if (!themeObj) { + throw new Error('theme-not-found'); + } + Themes.setPath(themeObj); +}; +Themes.setPath = function (themeObj) { + let themePath; + const fallback = path.join(nconf.get('themes_path'), themeObj.id, 'templates'); + if (themeObj.templates) { + themePath = path.join(nconf.get('themes_path'), themeObj.id, themeObj.templates); + } else if (file.existsSync(fallback)) { + themePath = fallback; + } else { + winston.error('[themes] Unable to resolve this theme\'s templates. Expected to be at "templates/" or defined in the "templates" property of "theme.json"'); + throw new Error('theme-missing-templates'); + } + nconf.set('theme_templates_path', themePath); + nconf.set('theme_config', path.join(nconf.get('themes_path'), themeObj.id, 'theme.json')); +}; \ No newline at end of file diff --git a/lib/middleware/admin.js b/lib/middleware/admin.js new file mode 100644 index 0000000000..52c3db55c2 --- /dev/null +++ b/lib/middleware/admin.js @@ -0,0 +1,71 @@ +'use strict'; + +const nconf = require('nconf'); +const user = require('../user'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const helpers = require('./helpers'); +const controllers = { + admin: require('../controllers/admin'), + helpers: require('../controllers/helpers') +}; +const middleware = module.exports; +middleware.buildHeader = helpers.try(async (req, res, next) => { + res.locals.renderAdminHeader = true; + if (req.method === 'GET') { + await require('./index').applyCSRFasync(req, res); + } + res.locals.config = await controllers.admin.loadConfig(req); + next(); +}); +middleware.checkPrivileges = helpers.try(async (req, res, next) => { + if (req.uid <= 0) { + return controllers.helpers.notAllowed(req, res); + } + const path = req.path.replace(/^(\/api)?(\/v3)?\/admin\/?/g, ''); + if (path) { + const privilege = privileges.admin.resolve(path); + if (!(await privileges.admin.can(privilege, req.uid))) { + return controllers.helpers.notAllowed(req, res); + } + } else { + const privilegeSet = await privileges.admin.get(req.uid); + if (!Object.values(privilegeSet).some(Boolean)) { + return controllers.helpers.notAllowed(req, res); + } + } + const hasPassword = await user.hasPassword(req.uid); + if (!hasPassword) { + return next(); + } + const loginTime = req.session.meta ? req.session.meta.datetime : 0; + const adminReloginDuration = meta.config.adminReloginDuration * 60000; + const disabled = meta.config.adminReloginDuration === 0; + if (disabled || loginTime && parseInt(loginTime, 10) > Date.now() - adminReloginDuration) { + const timeLeft = parseInt(loginTime, 10) - (Date.now() - adminReloginDuration); + if (req.session.meta && timeLeft < Math.min(60000, adminReloginDuration)) { + req.session.meta.datetime += Math.min(60000, adminReloginDuration); + } + return next(); + } + let returnTo = req.path; + if (nconf.get('relative_path')) { + returnTo = req.path.replace(new RegExp(`^${nconf.get('relative_path')}`), ''); + } + returnTo = returnTo.replace(/^\/api/, ''); + req.session.returnTo = returnTo; + req.session.forceLogin = 1; + await plugins.hooks.fire('response:auth.relogin', { + req, + res + }); + if (res.headersSent) { + return; + } + if (res.locals.isAPI) { + controllers.helpers.formatApiResponse(401, res); + } else { + res.redirect(`${nconf.get('relative_path')}/login?local=1`); + } +}); \ No newline at end of file diff --git a/lib/middleware/assert.js b/lib/middleware/assert.js new file mode 100644 index 0000000000..6a5871de5e --- /dev/null +++ b/lib/middleware/assert.js @@ -0,0 +1,102 @@ +'use strict'; + +const path = require('path'); +const nconf = require('nconf'); +const file = require('../file'); +const user = require('../user'); +const groups = require('../groups'); +const categories = require('../categories'); +const topics = require('../topics'); +const posts = require('../posts'); +const messaging = require('../messaging'); +const flags = require('../flags'); +const slugify = require('../slugify'); +const helpers = require('./helpers'); +const controllerHelpers = require('../controllers/helpers'); +const Assert = module.exports; +Assert.user = helpers.try(async (req, res, next) => { + if (!(await user.exists(req.params.uid))) { + return controllerHelpers.formatApiResponse(404, res, new Error('[[error:no-user]]')); + } + next(); +}); +Assert.group = helpers.try(async (req, res, next) => { + const name = await groups.getGroupNameByGroupSlug(req.params.slug); + if (!name || !(await groups.exists(name))) { + return controllerHelpers.formatApiResponse(404, res, new Error('[[error:no-group]]')); + } + next(); +}); +Assert.category = helpers.try(async (req, res, next) => { + if (!(await categories.exists(req.params.cid))) { + return controllerHelpers.formatApiResponse(404, res, new Error('[[error:no-category]]')); + } + next(); +}); +Assert.topic = helpers.try(async (req, res, next) => { + if (!(await topics.exists(req.params.tid))) { + return controllerHelpers.formatApiResponse(404, res, new Error('[[error:no-topic]]')); + } + next(); +}); +Assert.post = helpers.try(async (req, res, next) => { + if (!(await posts.exists(req.params.pid))) { + return controllerHelpers.formatApiResponse(404, res, new Error('[[error:no-post]]')); + } + next(); +}); +Assert.flag = helpers.try(async (req, res, next) => { + const canView = await flags.canView(req.params.flagId, req.uid); + if (!canView) { + return controllerHelpers.formatApiResponse(404, res, new Error('[[error:no-flag]]')); + } + next(); +}); +Assert.path = helpers.try(async (req, res, next) => { + if (req.body.path.startsWith('file:///')) { + req.body.path = new URL(req.body.path).pathname; + } + if (req.body.path.startsWith(nconf.get('upload_url'))) { + req.body.path = req.body.path.slice(nconf.get('upload_url').length); + } + const pathToFile = path.join(nconf.get('upload_path'), req.body.path); + res.locals.cleanedPath = pathToFile; + if (!pathToFile.startsWith(nconf.get('upload_path'))) { + return controllerHelpers.formatApiResponse(403, res, new Error('[[error:invalid-path]]')); + } + if (!(await file.exists(pathToFile))) { + return controllerHelpers.formatApiResponse(404, res, new Error('[[error:invalid-path]]')); + } + next(); +}); +Assert.folderName = helpers.try(async (req, res, next) => { + const folderName = slugify(path.basename(req.body.folderName.trim())); + const folderPath = path.join(res.locals.cleanedPath, folderName); + if (!folderName) { + return controllerHelpers.formatApiResponse(403, res, new Error('[[error:invalid-path]]')); + } + if (await file.exists(folderPath)) { + return controllerHelpers.formatApiResponse(403, res, new Error('[[error:folder-exists]]')); + } + res.locals.folderPath = folderPath; + next(); +}); +Assert.room = helpers.try(async (req, res, next) => { + if (!isFinite(req.params.roomId)) { + return controllerHelpers.formatApiResponse(400, res, new Error('[[error:invalid-data]]')); + } + const [exists, inRoom] = await Promise.all([messaging.roomExists(req.params.roomId), messaging.isUserInRoom(req.uid, req.params.roomId)]); + if (!exists) { + return controllerHelpers.formatApiResponse(404, res, new Error('[[error:chat-room-does-not-exist]]')); + } + if (!inRoom) { + return controllerHelpers.formatApiResponse(403, res, new Error('[[error:no-privileges]]')); + } + next(); +}); +Assert.message = helpers.try(async (req, res, next) => { + if (!isFinite(req.params.mid) || !(await messaging.messageExists(req.params.mid)) || !(await messaging.canViewMessage(req.params.mid, req.params.roomId, req.uid))) { + return controllerHelpers.formatApiResponse(400, res, new Error('[[error:invalid-mid]]')); + } + next(); +}); \ No newline at end of file diff --git a/lib/middleware/csrf.js b/lib/middleware/csrf.js new file mode 100644 index 0000000000..b5ab1b425f --- /dev/null +++ b/lib/middleware/csrf.js @@ -0,0 +1,28 @@ +'use strict'; + +const { + csrfSync +} = require('csrf-sync'); +const { + generateToken, + csrfSynchronisedProtection, + isRequestValid +} = csrfSync({ + getTokenFromRequest: req => { + if (req.headers['x-csrf-token']) { + return req.headers['x-csrf-token']; + } else if (req.body && req.body.csrf_token) { + return req.body.csrf_token; + } else if (req.body && req.body._csrf) { + return req.body._csrf; + } else if (req.query && req.query._csrf) { + return req.query._csrf; + } + }, + size: 64 +}); +module.exports = { + generateToken, + csrfSynchronisedProtection, + isRequestValid +}; \ No newline at end of file diff --git a/lib/middleware/expose.js b/lib/middleware/expose.js new file mode 100644 index 0000000000..63e0626aac --- /dev/null +++ b/lib/middleware/expose.js @@ -0,0 +1,34 @@ +'use strict'; + +const user = require('../user'); +const privileges = require('../privileges'); +const utils = require('../utils'); +module.exports = function (middleware) { + middleware.exposeAdmin = async (req, res, next) => { + res.locals.isAdmin = false; + if (!req.user) { + return next(); + } + res.locals.isAdmin = await user.isAdministrator(req.user.uid); + next(); + }; + middleware.exposePrivileges = async (req, res, next) => { + const hash = await utils.promiseParallel({ + isAdmin: user.isAdministrator(req.user.uid), + isGmod: user.isGlobalModerator(req.user.uid), + isPrivileged: user.isPrivileged(req.user.uid) + }); + if (req.params.uid) { + hash.isSelf = parseInt(req.params.uid, 10) === req.user.uid; + } + res.locals.privileges = hash; + next(); + }; + middleware.exposePrivilegeSet = async (req, res, next) => { + res.locals.privileges = { + ...(await privileges.global.get(req.user.uid)), + ...(await privileges.admin.get(req.user.uid)) + }; + next(); + }; +}; \ No newline at end of file diff --git a/lib/middleware/header.js b/lib/middleware/header.js new file mode 100644 index 0000000000..413d1cf3d6 --- /dev/null +++ b/lib/middleware/header.js @@ -0,0 +1,27 @@ +'use strict'; + +const plugins = require('../plugins'); +const helpers = require('./helpers'); +const controllers = { + api: require('../controllers/api') +}; +const middleware = module.exports; +middleware.buildHeader = helpers.try(async (req, res, next) => { + await doBuildHeader(req, res); + next(); +}); +middleware.buildHeaderAsync = async (req, res) => { + await doBuildHeader(req, res); +}; +async function doBuildHeader(req, res) { + res.locals.renderHeader = true; + res.locals.isAPI = false; + if (req.method === 'GET') { + await require('./index').applyCSRFasync(req, res); + } + await plugins.hooks.fire('filter:middleware.buildHeader', { + req: req, + locals: res.locals + }); + res.locals.config = await controllers.api.loadConfig(req); +} \ No newline at end of file diff --git a/lib/middleware/headers.js b/lib/middleware/headers.js new file mode 100644 index 0000000000..bbc8f3bb51 --- /dev/null +++ b/lib/middleware/headers.js @@ -0,0 +1,100 @@ +'use strict'; + +const os = require('os'); +const winston = require('winston'); +const _ = require('lodash'); +const meta = require('../meta'); +const languages = require('../languages'); +const helpers = require('./helpers'); +const plugins = require('../plugins'); +module.exports = function (middleware) { + middleware.addHeaders = helpers.try((req, res, next) => { + const headers = { + 'X-Powered-By': encodeURI(meta.config['powered-by'] || 'NodeBB'), + 'Access-Control-Allow-Methods': encodeURI(meta.config['access-control-allow-methods'] || ''), + 'Access-Control-Allow-Headers': encodeURI(meta.config['access-control-allow-headers'] || '') + }; + if (meta.config['csp-frame-ancestors']) { + headers['Content-Security-Policy'] = `frame-ancestors ${meta.config['csp-frame-ancestors']}`; + if (meta.config['csp-frame-ancestors'] === '\'none\'') { + headers['X-Frame-Options'] = 'DENY'; + } + } else { + headers['Content-Security-Policy'] = 'frame-ancestors \'self\''; + headers['X-Frame-Options'] = 'SAMEORIGIN'; + } + if (meta.config['access-control-allow-origin']) { + let origins = meta.config['access-control-allow-origin'].split(','); + origins = origins.map(origin => origin && origin.trim()); + if (origins.includes(req.get('origin'))) { + headers['Access-Control-Allow-Origin'] = encodeURI(req.get('origin')); + headers.Vary = headers.Vary ? `${headers.Vary}, Origin` : 'Origin'; + } + } + if (meta.config['access-control-allow-origin-regex']) { + let originsRegex = meta.config['access-control-allow-origin-regex'].split(','); + originsRegex = originsRegex.map(origin => { + try { + origin = new RegExp(origin.trim()); + } catch (err) { + winston.error(`[middleware.addHeaders] Invalid RegExp For access-control-allow-origin ${origin}`); + origin = null; + } + return origin; + }); + originsRegex.forEach(regex => { + if (regex && regex.test(req.get('origin'))) { + headers['Access-Control-Allow-Origin'] = encodeURI(req.get('origin')); + headers.Vary = headers.Vary ? `${headers.Vary}, Origin` : 'Origin'; + } + }); + } + if (meta.config['permissions-policy']) { + headers['Permissions-Policy'] = meta.config['permissions-policy']; + } + if (meta.config['access-control-allow-credentials']) { + headers['Access-Control-Allow-Credentials'] = meta.config['access-control-allow-credentials']; + } + if (process.env.NODE_ENV === 'development') { + headers['X-Upstream-Hostname'] = os.hostname().replace(/[^0-9A-Za-z-.]/g, ''); + } + for (const [key, value] of Object.entries(headers)) { + if (value) { + res.setHeader(key, value); + } + } + next(); + }); + middleware.autoLocale = helpers.try(async (req, res, next) => { + await plugins.hooks.fire('filter:middleware.autoLocale', { + req: req, + res: res + }); + if (req.query.lang) { + const langs = await listCodes(); + if (!langs.includes(req.query.lang)) { + req.query.lang = meta.config.defaultLang; + } + return next(); + } + if (meta.config.autoDetectLang && req.uid === 0) { + const langs = await listCodes(); + const lang = req.acceptsLanguages(langs); + if (!lang) { + return next(); + } + req.query.lang = lang; + } + next(); + }); + async function listCodes() { + const defaultLang = meta.config.defaultLang || 'en-GB'; + try { + const codes = await languages.listCodes(); + return _.uniq([defaultLang, ...codes]); + } catch (err) { + winston.error(`[middleware/autoLocale] Could not retrieve languages codes list! ${err.stack}`); + return [defaultLang]; + } + } +}; \ No newline at end of file diff --git a/lib/middleware/helpers.js b/lib/middleware/helpers.js new file mode 100644 index 0000000000..295815406d --- /dev/null +++ b/lib/middleware/helpers.js @@ -0,0 +1,71 @@ +'use strict'; + +const winston = require('winston'); +const validator = require('validator'); +const slugify = require('../slugify'); +const meta = require('../meta'); +const helpers = module.exports; +helpers.try = function (middleware) { + if (middleware && middleware.constructor && middleware.constructor.name === 'AsyncFunction') { + return async function (req, res, next) { + try { + await middleware(req, res, next); + } catch (err) { + next(err); + } + }; + } + return function (req, res, next) { + try { + middleware(req, res, next); + } catch (err) { + next(err); + } + }; +}; +helpers.buildBodyClass = function (req, res, templateData = {}) { + const clean = req.path.replace(/^\/api/, '').replace(/^\/|\/$/g, ''); + const parts = clean.split('/').slice(0, 3); + parts.forEach((p, index) => { + try { + p = slugify(decodeURIComponent(p)); + } catch (err) { + winston.error(`Error decoding URI: ${p}`); + winston.error(err.stack); + p = ''; + } + p = validator.escape(String(p)); + parts[index] = index ? `${parts[0]}-${p}` : `page-${p || 'home'}`; + }); + const { + template + } = templateData; + if (template) { + parts.push(`template-${template.name.split('/').join('-')}`); + } + if (template && template.topic) { + parts.push(`page-topic-category-${templateData.category.cid}`); + parts.push(`page-topic-category-${slugify(templateData.category.name)}`); + } + if (template && template.chats && templateData.roomId) { + parts.push(`page-user-chats-${templateData.roomId}`); + } + if (Array.isArray(templateData.breadcrumbs)) { + templateData.breadcrumbs.forEach(crumb => { + if (crumb && crumb.hasOwnProperty('cid')) { + parts.push(`parent-category-${crumb.cid}`); + } + }); + } + if (templateData && templateData.bodyClasses) { + parts.push(...templateData.bodyClasses); + } + parts.push(`page-status-${res.statusCode}`); + parts.push(`theme-${(meta.config['theme:id'] || '').split('-')[2]}`); + if (req.loggedIn) { + parts.push('user-loggedin'); + } else { + parts.push('user-guest'); + } + return parts.join(' '); +}; \ No newline at end of file diff --git a/lib/middleware/index.js b/lib/middleware/index.js new file mode 100644 index 0000000000..28c37bfadf --- /dev/null +++ b/lib/middleware/index.js @@ -0,0 +1,249 @@ +'use strict'; + +const async = require('async'); +const path = require('path'); +const validator = require('validator'); +const nconf = require('nconf'); +const toobusy = require('toobusy-js'); +const util = require('util'); +const multipart = require('connect-multiparty'); +const { + csrfSynchronisedProtection +} = require('./csrf'); +const plugins = require('../plugins'); +const meta = require('../meta'); +const user = require('../user'); +const groups = require('../groups'); +const analytics = require('../analytics'); +const privileges = require('../privileges'); +const cacheCreate = require('../cache/lru'); +const helpers = require('./helpers'); +const api = require('../api'); +const controllers = { + api: require('../controllers/api'), + helpers: require('../controllers/helpers') +}; +const delayCache = cacheCreate({ + ttl: 1000 * 60, + max: 200 +}); +const multipartMiddleware = multipart(); +const middleware = module.exports; +const relative_path = nconf.get('relative_path'); +middleware.regexes = { + timestampedUpload: /^\d+-.+$/ +}; +const csrfMiddleware = csrfSynchronisedProtection; +middleware.applyCSRF = function (req, res, next) { + if (req.uid >= 0) { + csrfMiddleware(req, res, next); + } else { + next(); + } +}; +middleware.applyCSRFasync = util.promisify(middleware.applyCSRF); +middleware.ensureLoggedIn = (req, res, next) => { + if (!req.loggedIn) { + return controllers.helpers.notAllowed(req, res); + } + setImmediate(next); +}; +Object.assign(middleware, { + admin: require('./admin'), + ...require('./header') +}); +require('./render')(middleware); +require('./maintenance')(middleware); +require('./user')(middleware); +middleware.uploads = require('./uploads'); +require('./headers')(middleware); +require('./expose')(middleware); +middleware.assert = require('./assert'); +middleware.stripLeadingSlashes = function stripLeadingSlashes(req, res, next) { + const target = req.originalUrl.replace(relative_path, ''); + if (target.startsWith('//')) { + return res.redirect(relative_path + target.replace(/^\/+/, '/')); + } + next(); +}; +middleware.pageView = helpers.try(async (req, res, next) => { + if (req.loggedIn) { + await Promise.all([user.updateOnlineUsers(req.uid), user.updateLastOnlineTime(req.uid)]); + } + next(); + await analytics.pageView({ + ip: req.ip, + uid: req.uid + }); + plugins.hooks.fire('action:middleware.pageView', { + req: req + }); +}); +middleware.pluginHooks = helpers.try(async (req, res, next) => { + await async.each(plugins.loadedHooks['filter:router.page'] || [], (hookObj, next) => { + hookObj.method(req, res, next); + }); + await plugins.hooks.fire('response:router.page', { + req: req, + res: res + }); + if (!res.headersSent) { + next(); + } +}); +middleware.validateFiles = function validateFiles(req, res, next) { + if (!req.files.files) { + return next(new Error(['[[error:invalid-files]]'])); + } + if (Array.isArray(req.files.files) && req.files.files.length) { + return next(); + } + if (typeof req.files.files === 'object') { + req.files.files = [req.files.files]; + return next(); + } + return next(new Error(['[[error:invalid-files]]'])); +}; +middleware.prepareAPI = function prepareAPI(req, res, next) { + res.locals.isAPI = true; + next(); +}; +middleware.logApiUsage = async function logApiUsage(req, res, next) { + if (req.headers.hasOwnProperty('authorization')) { + const [, token] = req.headers.authorization.split(' '); + await api.utils.tokens.log(token); + } + next(); +}; +middleware.routeTouchIcon = function routeTouchIcon(req, res) { + if (meta.config['brand:touchIcon'] && validator.isURL(meta.config['brand:touchIcon'])) { + return res.redirect(meta.config['brand:touchIcon']); + } + let iconPath = ''; + if (meta.config['brand:touchIcon']) { + iconPath = path.join(nconf.get('upload_path'), meta.config['brand:touchIcon'].replace(/assets\/uploads/, '')); + } else { + iconPath = path.join(nconf.get('base_dir'), 'public/images/touch/512.png'); + } + return res.sendFile(iconPath, { + maxAge: req.app.enabled('cache') ? 5184000000 : 0 + }); +}; +middleware.privateTagListing = helpers.try(async (req, res, next) => { + const canView = await privileges.global.can('view:tags', req.uid); + if (!canView) { + return controllers.helpers.notAllowed(req, res); + } + next(); +}); +middleware.exposeGroupName = helpers.try(async (req, res, next) => { + await expose('groupName', groups.getGroupNameByGroupSlug, 'slug', req, res, next); +}); +middleware.exposeUid = helpers.try(async (req, res, next) => { + await expose('uid', user.getUidByUserslug, 'userslug', req, res, next); +}); +async function expose(exposedField, method, field, req, res, next) { + if (!req.params.hasOwnProperty(field)) { + return next(); + } + const value = await method(String(req.params[field]).toLowerCase()); + if (!value) { + next('route'); + return; + } + res.locals[exposedField] = value; + next(); +} +middleware.privateUploads = function privateUploads(req, res, next) { + if (req.loggedIn || !meta.config.privateUploads) { + return next(); + } + if (req.path.startsWith(`${nconf.get('relative_path')}/assets/uploads/files`)) { + const extensions = (meta.config.privateUploadsExtensions || '').split(',').filter(Boolean); + let ext = path.extname(req.path); + ext = ext ? ext.replace(/^\./, '') : ext; + if (!extensions.length || extensions.includes(ext)) { + return res.status(403).json('not-allowed'); + } + } + next(); +}; +middleware.busyCheck = function busyCheck(req, res, next) { + if (global.env === 'production' && meta.config.eventLoopCheckEnabled && toobusy()) { + analytics.increment('errors:503'); + res.status(503).type('text/html').sendFile(path.join(__dirname, '../../public/503.html')); + } else { + setImmediate(next); + } +}; +middleware.applyBlacklist = async function applyBlacklist(req, res, next) { + try { + await meta.blacklist.test(req.ip); + next(); + } catch (err) { + next(err); + } +}; +middleware.delayLoading = function delayLoading(req, res, next) { + let timesSeen = delayCache.get(req.ip) || 0; + if (timesSeen > 10) { + return res.sendStatus(429); + } + delayCache.set(req.ip, timesSeen += 1); + setTimeout(next, 1000); +}; +middleware.buildSkinAsset = helpers.try(async (req, res, next) => { + const targetSkin = path.basename(req.originalUrl).split('.css')[0].replace(/-rtl$/, ''); + if (!targetSkin) { + return next(); + } + const skins = (await meta.css.getCustomSkins()).map(skin => skin.value); + const found = skins.concat(meta.css.supportedSkins).find(skin => `client-${skin}` === targetSkin); + if (!found) { + return next(); + } + await plugins.prepareForBuild(['client side styles']); + const [ltr, rtl] = await meta.css.buildBundle(targetSkin, true); + require('../meta/minifier').killAll(); + res.status(200).type('text/css').send(req.originalUrl.includes('-rtl') ? rtl : ltr); +}); +middleware.addUploadHeaders = function addUploadHeaders(req, res, next) { + let basename = path.basename(req.path); + const extname = path.extname(req.path); + if (req.path.startsWith('/uploads/files/') && middleware.regexes.timestampedUpload.test(basename)) { + basename = basename.slice(14); + res.header('Content-Disposition', `${extname.startsWith('.htm') ? 'attachment' : 'inline'}; filename="${basename}"`); + } + next(); +}; +middleware.validateAuth = helpers.try(async (req, res, next) => { + try { + await plugins.hooks.fire('static:auth.validate', { + user: res.locals.user, + strategy: res.locals.strategy + }); + next(); + } catch (err) { + const regenerateSession = util.promisify(cb => req.session.regenerate(cb)); + await regenerateSession(); + req.uid = 0; + req.loggedIn = false; + next(err); + } +}); +middleware.checkRequired = function (fields, req, res, next) { + const missing = fields.filter(field => !req.body.hasOwnProperty(field) && !req.query.hasOwnProperty(field)); + if (!missing.length) { + return next(); + } + controllers.helpers.formatApiResponse(400, res, new Error(`[[error:required-parameters-missing, ${missing.join(' ')}]]`)); +}; +middleware.handleMultipart = (req, res, next) => { + const { + 'content-type': contentType + } = req.headers; + if (contentType && !contentType.startsWith('multipart/form-data')) { + return next(); + } + multipartMiddleware(req, res, next); +}; \ No newline at end of file diff --git a/lib/middleware/maintenance.js b/lib/middleware/maintenance.js new file mode 100644 index 0000000000..bb98209c11 --- /dev/null +++ b/lib/middleware/maintenance.js @@ -0,0 +1,39 @@ +'use strict'; + +const util = require('util'); +const nconf = require('nconf'); +const meta = require('../meta'); +const user = require('../user'); +const groups = require('../groups'); +const helpers = require('./helpers'); +const controllerHelpers = require('../controllers/helpers'); +module.exports = function (middleware) { + middleware.maintenanceMode = helpers.try(async (req, res, next) => { + if (!meta.config.maintenanceMode) { + return next(); + } + const hooksAsync = util.promisify(middleware.pluginHooks); + await hooksAsync(req, res); + const url = req.url.replace(nconf.get('relative_path'), ''); + if (url.startsWith('/login') || url.startsWith('/api/login')) { + return next(); + } + const [isAdmin, isMemberOfExempt] = await Promise.all([user.isAdministrator(req.uid), groups.isMemberOfAny(req.uid, meta.config.groupsExemptFromMaintenanceMode)]); + if (isAdmin || isMemberOfExempt) { + return next(); + } + if (req.originalUrl.startsWith(`${nconf.get('relative_path')}/api/v3/`)) { + return controllerHelpers.formatApiResponse(meta.config.maintenanceModeStatus, res); + } + res.status(meta.config.maintenanceModeStatus); + const data = { + site_title: meta.config.title || 'NodeBB', + message: meta.config.maintenanceModeMessage + }; + if (res.locals.isAPI) { + return res.json(data); + } + await middleware.buildHeaderAsync(req, res); + res.render('503', data); + }); +}; \ No newline at end of file diff --git a/lib/middleware/ratelimit.js b/lib/middleware/ratelimit.js new file mode 100644 index 0000000000..5a32b0128e --- /dev/null +++ b/lib/middleware/ratelimit.js @@ -0,0 +1,24 @@ +'use strict'; + +const winston = require('winston'); +const ratelimit = module.exports; +const allowedCalls = 100; +const timeframe = 10000; +ratelimit.isFlooding = function (socket) { + socket.callsPerSecond = socket.callsPerSecond || 0; + socket.elapsedTime = socket.elapsedTime || 0; + socket.lastCallTime = socket.lastCallTime || Date.now(); + socket.callsPerSecond += 1; + const now = Date.now(); + socket.elapsedTime += now - socket.lastCallTime; + if (socket.callsPerSecond > allowedCalls && socket.elapsedTime < timeframe) { + winston.warn(`Flooding detected! Calls : ${socket.callsPerSecond}, Duration : ${socket.elapsedTime}`); + return true; + } + if (socket.elapsedTime >= timeframe) { + socket.elapsedTime = 0; + socket.callsPerSecond = 0; + } + socket.lastCallTime = now; + return false; +}; \ No newline at end of file diff --git a/lib/middleware/render.js b/lib/middleware/render.js new file mode 100644 index 0000000000..c5a2f33ff4 --- /dev/null +++ b/lib/middleware/render.js @@ -0,0 +1,483 @@ +'use strict'; + +const _ = require('lodash'); +const nconf = require('nconf'); +const validator = require('validator'); +const jsesc = require('jsesc'); +const winston = require('winston'); +const semver = require('semver'); +const db = require('../database'); +const navigation = require('../navigation'); +const translator = require('../translator'); +const privileges = require('../privileges'); +const languages = require('../languages'); +const plugins = require('../plugins'); +const user = require('../user'); +const topics = require('../topics'); +const messaging = require('../messaging'); +const flags = require('../flags'); +const meta = require('../meta'); +const widgets = require('../widgets'); +const utils = require('../utils'); +const helpers = require('./helpers'); +const versions = require('../admin/versions'); +const controllersHelpers = require('../controllers/helpers'); +const relative_path = nconf.get('relative_path'); +module.exports = function (middleware) { + middleware.processRender = function processRender(req, res, next) { + const { + render + } = res; + res.render = async function renderOverride(template, options, fn) { + const self = this; + const { + req + } = this; + async function renderMethod(template, options, fn) { + options = options || {}; + if (typeof options === 'function') { + fn = options; + options = {}; + } + options.loggedIn = req.uid > 0; + options.loggedInUser = await getLoggedInUser(req); + options.relative_path = relative_path; + options.template = { + name: template, + [template]: true + }; + options.url = req.baseUrl + req.path.replace(/^\/api/, ''); + options.bodyClass = helpers.buildBodyClass(req, res, options); + if (req.loggedIn) { + res.set('cache-control', 'private'); + } + const buildResult = await plugins.hooks.fire(`filter:${template}.build`, { + req: req, + res: res, + templateData: options + }); + if (res.headersSent) { + return; + } + const templateToRender = buildResult.templateData.templateToRender || template; + const renderResult = await plugins.hooks.fire('filter:middleware.render', { + req: req, + res: res, + templateData: buildResult.templateData + }); + if (res.headersSent) { + return; + } + options = renderResult.templateData; + options._header = { + tags: await meta.tags.parse(req, renderResult, res.locals.metaTags, res.locals.linkTags) + }; + options.widgets = await widgets.render(req.uid, { + template: `${template}.tpl`, + url: options.url, + templateData: options, + req: req, + res: res + }); + res.locals.template = template; + options._locals = undefined; + if (res.locals.isAPI) { + if (req.route && req.route.path === '/api/') { + options.title = '[[pages:home]]'; + } + req.app.set('json spaces', global.env === 'development' || req.query.pretty ? 4 : 0); + return res.json(options); + } + const optionsString = JSON.stringify(options).replace(/<\//g, '<\\/'); + const headerFooterData = await loadHeaderFooterData(req, res, options); + const results = await utils.promiseParallel({ + header: renderHeaderFooter('renderHeader', req, res, options, headerFooterData), + content: renderContent(render, templateToRender, req, res, options), + footer: renderHeaderFooter('renderFooter', req, res, options, headerFooterData) + }); + const str = `${results.header + (res.locals.postHeader || '') + results.content}${res.locals.preFooter || ''}${results.footer}`; + if (typeof fn !== 'function') { + self.send(str); + } else { + fn(null, str); + } + } + try { + await renderMethod(template, options, fn); + } catch (err) { + next(err); + } + }; + next(); + }; + async function getLoggedInUser(req) { + if (req.user) { + return await user.getUserData(req.uid); + } + return { + uid: req.uid === -1 ? -1 : 0, + username: '[[global:guest]]', + picture: user.getDefaultAvatar(), + 'icon:text': '?', + 'icon:bgColor': '#aaa' + }; + } + async function loadHeaderFooterData(req, res, options) { + if (res.locals.renderHeader) { + return await loadClientHeaderFooterData(req, res, options); + } else if (res.locals.renderAdminHeader) { + return await loadAdminHeaderFooterData(req, res, options); + } + return null; + } + async function loadClientHeaderFooterData(req, res, options) { + const registrationType = meta.config.registrationType || 'normal'; + res.locals.config = res.locals.config || {}; + const templateValues = { + title: meta.config.title || '', + 'title:url': meta.config['title:url'] || '', + description: meta.config.description || '', + 'cache-buster': meta.config['cache-buster'] || '', + 'brand:logo': meta.config['brand:logo'] || '', + 'brand:logo:url': meta.config['brand:logo:url'] || '', + 'brand:logo:alt': meta.config['brand:logo:alt'] || '', + 'brand:logo:display': meta.config['brand:logo'] ? '' : 'hide', + allowRegistration: registrationType === 'normal', + searchEnabled: plugins.hooks.hasListeners('filter:search.query'), + postQueueEnabled: !!meta.config.postQueue, + registrationQueueEnabled: meta.config.registrationApprovalType !== 'normal' || meta.config.registrationType === 'invite-only' || meta.config.registrationType === 'admin-invite-only', + config: res.locals.config, + relative_path, + bodyClass: options.bodyClass, + widgets: options.widgets + }; + templateValues.configJSON = jsesc(JSON.stringify(res.locals.config), { + isScriptContext: true + }); + const title = translator.unescape(utils.stripHTMLTags(options.title || '')); + const results = await utils.promiseParallel({ + isAdmin: user.isAdministrator(req.uid), + isGlobalMod: user.isGlobalModerator(req.uid), + isModerator: user.isModeratorOfAnyCategory(req.uid), + privileges: privileges.global.get(req.uid), + blocks: user.blocks.list(req.uid), + user: user.getUserData(req.uid), + isEmailConfirmSent: req.uid <= 0 ? false : await user.email.isValidationPending(req.uid), + languageDirection: translator.translate('[[language:dir]]', res.locals.config.userLang), + timeagoCode: languages.userTimeagoCode(res.locals.config.userLang), + browserTitle: translator.translate(controllersHelpers.buildTitle(title)), + navigation: navigation.get(req.uid), + roomIds: req.uid > 0 ? db.getSortedSetRevRange(`uid:${req.uid}:chat:rooms`, 0, 0) : [] + }); + const unreadData = { + '': {}, + new: {}, + watched: {}, + unreplied: {} + }; + results.user.unreadData = unreadData; + results.user.isAdmin = results.isAdmin; + results.user.isGlobalMod = results.isGlobalMod; + results.user.isMod = !!results.isModerator; + results.user.privileges = results.privileges; + results.user.blocks = results.blocks; + results.user.timeagoCode = results.timeagoCode; + results.user[results.user.status] = true; + results.user.lastRoomId = results.roomIds.length ? results.roomIds[0] : null; + results.user.email = String(results.user.email); + results.user['email:confirmed'] = results.user['email:confirmed'] === 1; + results.user.isEmailConfirmSent = !!results.isEmailConfirmSent; + templateValues.bootswatchSkin = res.locals.config.bootswatchSkin || ''; + templateValues.browserTitle = results.browserTitle; + ({ + navigation: templateValues.navigation, + unreadCount: templateValues.unreadCount + } = await appendUnreadCounts({ + uid: req.uid, + query: req.query, + navigation: results.navigation, + unreadData + })); + templateValues.isAdmin = results.user.isAdmin; + templateValues.isGlobalMod = results.user.isGlobalMod; + templateValues.showModMenu = results.user.isAdmin || results.user.isGlobalMod || results.user.isMod; + templateValues.canChat = (results.privileges.chat || results.privileges['chat:privileged']) && meta.config.disableChat !== 1; + templateValues.user = results.user; + templateValues.userJSON = jsesc(JSON.stringify(results.user), { + isScriptContext: true + }); + templateValues.useCustomCSS = meta.config.useCustomCSS && meta.config.customCSS; + templateValues.customCSS = templateValues.useCustomCSS ? meta.config.renderedCustomCSS || '' : ''; + templateValues.useCustomHTML = meta.config.useCustomHTML; + templateValues.customHTML = templateValues.useCustomHTML ? meta.config.customHTML : ''; + templateValues.maintenanceHeader = meta.config.maintenanceMode && !results.isAdmin; + templateValues.defaultLang = meta.config.defaultLang || 'en-GB'; + templateValues.userLang = res.locals.config.userLang; + templateValues.languageDirection = results.languageDirection; + if (req.query.noScriptMessage) { + templateValues.noScriptMessage = validator.escape(String(req.query.noScriptMessage)); + } + templateValues.template = { + name: res.locals.template + }; + templateValues.template[res.locals.template] = true; + if (options.hasOwnProperty('_header')) { + templateValues.metaTags = options._header.tags.meta; + templateValues.linkTags = options._header.tags.link; + } + if (req.route && req.route.path === '/') { + modifyTitle(templateValues); + } + return templateValues; + } + async function loadAdminHeaderFooterData(req, res, options) { + const custom_header = { + plugins: [], + authentication: [] + }; + res.locals.config = res.locals.config || {}; + const results = await utils.promiseParallel({ + userData: user.getUserFields(req.uid, ['username', 'userslug', 'email', 'picture', 'email:confirmed']), + scripts: getAdminScripts(), + custom_header: plugins.hooks.fire('filter:admin.header.build', custom_header), + configs: meta.configs.list(), + latestVersion: getLatestVersion(), + privileges: privileges.admin.get(req.uid), + tags: meta.tags.parse(req, {}, [], []), + languageDirection: translator.translate('[[language:dir]]', res.locals.config.acpLang) + }); + const { + userData + } = results; + userData.uid = req.uid; + userData['email:confirmed'] = userData['email:confirmed'] === 1; + userData.privileges = results.privileges; + let acpPath = req.path.slice(1).split('/'); + acpPath.forEach((path, i) => { + acpPath[i] = path.charAt(0).toUpperCase() + path.slice(1); + }); + acpPath = acpPath.join(' > '); + const version = nconf.get('version'); + res.locals.config.userLang = res.locals.config.acpLang || res.locals.config.userLang; + res.locals.config.isRTL = results.languageDirection === 'rtl'; + const templateValues = { + config: res.locals.config, + configJSON: jsesc(JSON.stringify(res.locals.config), { + isScriptContext: true + }), + relative_path: res.locals.config.relative_path, + adminConfigJSON: encodeURIComponent(JSON.stringify(results.configs)), + metaTags: results.tags.meta, + linkTags: results.tags.link, + user: userData, + userJSON: jsesc(JSON.stringify(userData), { + isScriptContext: true + }), + plugins: results.custom_header.plugins, + authentication: results.custom_header.authentication, + scripts: results.scripts, + 'cache-buster': meta.config['cache-buster'] || '', + env: !!process.env.NODE_ENV, + title: `${acpPath || 'Dashboard'} | NodeBB Admin Control Panel`, + bodyClass: options.bodyClass, + version: version, + latestVersion: results.latestVersion, + upgradeAvailable: results.latestVersion && semver.gt(results.latestVersion, version), + showManageMenu: results.privileges.superadmin || ['categories', 'privileges', 'users', 'admins-mods', 'groups', 'tags', 'settings'].some(priv => results.privileges[`admin:${priv}`]), + defaultLang: meta.config.defaultLang || 'en-GB', + acpLang: res.locals.config.acpLang, + languageDirection: results.languageDirection + }; + templateValues.template = { + name: res.locals.template + }; + templateValues.template[res.locals.template] = true; + return templateValues; + } + function renderContent(render, tpl, req, res, options) { + return new Promise((resolve, reject) => { + render.call(res, tpl, options, async (err, str) => { + if (err) reject(err);else resolve(await translate(str, getLang(req, res))); + }); + }); + } + async function renderHeader(req, res, options, headerFooterData) { + const hookReturn = await plugins.hooks.fire('filter:middleware.renderHeader', { + req: req, + res: res, + templateValues: headerFooterData, + templateData: headerFooterData, + data: options + }); + return await req.app.renderAsync('header', hookReturn.templateData); + } + async function renderFooter(req, res, options, headerFooterData) { + const hookReturn = await plugins.hooks.fire('filter:middleware.renderFooter', { + req, + res, + templateValues: headerFooterData, + templateData: headerFooterData, + data: options + }); + const scripts = await plugins.hooks.fire('filter:scripts.get', []); + hookReturn.templateData.scripts = scripts.map(script => ({ + src: script + })); + hookReturn.templateData.useCustomJS = meta.config.useCustomJS; + hookReturn.templateData.customJS = hookReturn.templateData.useCustomJS ? meta.config.customJS : ''; + hookReturn.templateData.isSpider = req.uid === -1; + return await req.app.renderAsync('footer', hookReturn.templateData); + } + async function renderAdminHeader(req, res, options, headerFooterData) { + const hookReturn = await plugins.hooks.fire('filter:middleware.renderAdminHeader', { + req, + res, + templateValues: headerFooterData, + templateData: headerFooterData, + data: options + }); + return await req.app.renderAsync('admin/header', hookReturn.templateData); + } + async function renderAdminFooter(req, res, options, headerFooterData) { + const hookReturn = await plugins.hooks.fire('filter:middleware.renderAdminFooter', { + req, + res, + templateValues: headerFooterData, + templateData: headerFooterData, + data: options + }); + return await req.app.renderAsync('admin/footer', hookReturn.templateData); + } + async function renderHeaderFooter(method, req, res, options, headerFooterData) { + let str = ''; + if (res.locals.renderHeader) { + if (method === 'renderHeader') { + str = await renderHeader(req, res, options, headerFooterData); + } else if (method === 'renderFooter') { + str = await renderFooter(req, res, options, headerFooterData); + } + } else if (res.locals.renderAdminHeader) { + if (method === 'renderHeader') { + str = await renderAdminHeader(req, res, options, headerFooterData); + } else if (method === 'renderFooter') { + str = await renderAdminFooter(req, res, options, headerFooterData); + } + } + return await translate(str, getLang(req, res)); + } + function getLang(req, res) { + let language = res.locals.config && res.locals.config.userLang || 'en-GB'; + if (res.locals.renderAdminHeader) { + language = res.locals.config && res.locals.config.acpLang || 'en-GB'; + } + return req.query.lang ? validator.escape(String(req.query.lang)) : language; + } + async function translate(str, language) { + const translated = await translator.translate(str, language); + return translator.unescape(translated); + } + async function appendUnreadCounts({ + uid, + navigation, + unreadData, + query + }) { + const originalRoutes = navigation.map(nav => nav.originalRoute); + const calls = { + unreadData: topics.getUnreadData({ + uid: uid, + query: query + }), + unreadChatCount: messaging.getUnreadCount(uid), + unreadNotificationCount: user.notifications.getUnreadCount(uid), + unreadFlagCount: async function () { + if (originalRoutes.includes('/flags') && (await user.isPrivileged(uid))) { + return flags.getCount({ + uid, + query, + filters: { + quick: 'unresolved', + cid: (await user.isAdminOrGlobalMod(uid)) ? [] : await user.getModeratedCids(uid) + } + }); + } + return 0; + }() + }; + const results = await utils.promiseParallel(calls); + const unreadCounts = results.unreadData.counts; + const unreadCount = { + topic: unreadCounts[''] || 0, + newTopic: unreadCounts.new || 0, + watchedTopic: unreadCounts.watched || 0, + unrepliedTopic: unreadCounts.unreplied || 0, + mobileUnread: 0, + unreadUrl: '/unread', + chat: results.unreadChatCount || 0, + notification: results.unreadNotificationCount || 0, + flags: results.unreadFlagCount || 0 + }; + Object.keys(unreadCount).forEach(key => { + if (unreadCount[key] > 99) { + unreadCount[key] = '99+'; + } + }); + const { + tidsByFilter + } = results.unreadData; + navigation = navigation.map(item => { + function modifyNavItem(item, route, filter, content) { + if (item && item.originalRoute === route) { + unreadData[filter] = _.zipObject(tidsByFilter[filter], tidsByFilter[filter].map(() => true)); + item.content = content; + unreadCount.mobileUnread = content; + unreadCount.unreadUrl = route; + if (unreadCounts[filter] > 0) { + item.iconClass += ' unread-count'; + } + } + } + modifyNavItem(item, '/unread', '', unreadCount.topic); + modifyNavItem(item, '/unread?filter=new', 'new', unreadCount.newTopic); + modifyNavItem(item, '/unread?filter=watched', 'watched', unreadCount.watchedTopic); + modifyNavItem(item, '/unread?filter=unreplied', 'unreplied', unreadCount.unrepliedTopic); + ['flags'].forEach(prop => { + if (item && item.originalRoute === `/${prop}` && unreadCount[prop] > 0) { + item.iconClass += ' unread-count'; + item.content = unreadCount.flags; + } + }); + return item; + }); + return { + navigation, + unreadCount + }; + } + function modifyTitle(obj) { + const title = controllersHelpers.buildTitle(meta.config.homePageTitle || '[[pages:home]]'); + obj.browserTitle = title; + if (obj.metaTags) { + obj.metaTags.forEach((tag, i) => { + if (tag.property === 'og:title') { + obj.metaTags[i].content = title; + } + }); + } + return title; + } + async function getAdminScripts() { + const scripts = await plugins.hooks.fire('filter:admin.scripts.get', []); + return scripts.map(script => ({ + src: script + })); + } + async function getLatestVersion() { + try { + return await versions.getLatestVersion(); + } catch (err) { + winston.error(`[acp] Failed to fetch latest version${err.stack}`); + } + return null; + } +}; \ No newline at end of file diff --git a/lib/middleware/uploads.js b/lib/middleware/uploads.js new file mode 100644 index 0000000000..16d1e19809 --- /dev/null +++ b/lib/middleware/uploads.js @@ -0,0 +1,31 @@ +'use strict'; + +const cacheCreate = require('../cache/ttl'); +const meta = require('../meta'); +const helpers = require('./helpers'); +const user = require('../user'); +let cache; +exports.clearCache = function () { + if (cache) { + cache.clear(); + } +}; +exports.ratelimit = helpers.try(async (req, res, next) => { + const { + uid + } = req; + if (!meta.config.uploadRateLimitThreshold || uid && (await user.isAdminOrGlobalMod(uid))) { + return next(); + } + if (!cache) { + cache = cacheCreate({ + ttl: meta.config.uploadRateLimitCooldown * 1000 + }); + } + const count = (cache.get(`${req.ip}:uploaded_file_count`) || 0) + req.files.files.length; + if (count > meta.config.uploadRateLimitThreshold) { + return next(new Error(['[[error:upload-ratelimit-reached]]'])); + } + cache.set(`${req.ip}:uploaded_file_count`, count); + next(); +}); \ No newline at end of file diff --git a/lib/middleware/user.js b/lib/middleware/user.js new file mode 100644 index 0000000000..1f1d9ad6fa --- /dev/null +++ b/lib/middleware/user.js @@ -0,0 +1,270 @@ +'use strict'; + +const winston = require('winston'); +const passport = require('passport'); +const nconf = require('nconf'); +const path = require('path'); +const util = require('util'); +const meta = require('../meta'); +const user = require('../user'); +const groups = require('../groups'); +const topics = require('../topics'); +const privileges = require('../privileges'); +const privilegeHelpers = require('../privileges/helpers'); +const plugins = require('../plugins'); +const helpers = require('./helpers'); +const auth = require('../routes/authentication'); +const writeRouter = require('../routes/write'); +const accountHelpers = require('../controllers/accounts/helpers'); +const controllers = { + helpers: require('../controllers/helpers'), + authentication: require('../controllers/authentication') +}; +const passportAuthenticateAsync = function (req, res) { + return new Promise((resolve, reject) => { + passport.authenticate('core.api', (err, user) => { + if (err) { + reject(err); + } else { + resolve(user); + res.on('finish', writeRouter.cleanup.bind(null, req)); + } + })(req, res); + }); +}; +module.exports = function (middleware) { + async function authenticate(req, res) { + async function finishLogin(req, user) { + const loginAsync = util.promisify(req.login).bind(req); + await loginAsync(user, { + keepSessionInfo: true + }); + await controllers.authentication.onSuccessfulLogin(req, user.uid, false); + req.uid = parseInt(user.uid, 10); + req.loggedIn = req.uid > 0; + return true; + } + if (res.locals.isAPI && (req.loggedIn || !req.headers.hasOwnProperty('authorization'))) { + await middleware.applyCSRFasync(req, res); + } + if (req.loggedIn) { + return true; + } else if (req.headers.hasOwnProperty('authorization')) { + const user = await passportAuthenticateAsync(req, res); + if (!user) { + return true; + } + if (user.hasOwnProperty('uid')) { + return await finishLogin(req, user); + } else if (user.hasOwnProperty('master') && user.master === true) { + if (req.body.hasOwnProperty('_uid') || req.query.hasOwnProperty('_uid')) { + user.uid = req.body._uid || req.query._uid; + delete user.master; + return await finishLogin(req, user); + } + throw new Error('[[error:api.master-token-no-uid]]'); + } else { + winston.warn('[api/authenticate] Unable to find user after verifying token'); + return true; + } + } + await plugins.hooks.fire('response:middleware.authenticate', { + req: req, + res: res, + next: function () {} + }); + if (!res.headersSent) { + auth.setAuthVars(req); + } + return !res.headersSent; + } + middleware.authenticateRequest = helpers.try(async (req, res, next) => { + const { + skip + } = await plugins.hooks.fire('filter:middleware.authenticate', { + skip: { + post: ['/api/v3/utilities/login'] + } + }); + const mountedPath = path.join(req.baseUrl, req.path).replace(nconf.get('relative_path'), ''); + const method = req.method.toLowerCase(); + if (skip[method] && skip[method].includes(mountedPath)) { + return next(); + } + if (!(await authenticate(req, res))) { + return; + } + next(); + }); + middleware.ensureSelfOrGlobalPrivilege = helpers.try(async (req, res, next) => { + await ensureSelfOrMethod(user.isAdminOrGlobalMod, req, res, next); + }); + middleware.ensureSelfOrPrivileged = helpers.try(async (req, res, next) => { + await ensureSelfOrMethod(user.isPrivileged, req, res, next); + }); + async function ensureSelfOrMethod(method, req, res, next) { + if (!req.loggedIn) { + return controllers.helpers.notAllowed(req, res); + } + if (req.uid === parseInt(res.locals.uid, 10)) { + return next(); + } + const allowed = await method(req.uid); + if (!allowed) { + return controllers.helpers.notAllowed(req, res); + } + return next(); + } + middleware.canViewUsers = helpers.try(async (req, res, next) => { + if (parseInt(res.locals.uid, 10) === req.uid) { + return next(); + } + const canView = await privileges.global.can('view:users', req.uid); + if (canView) { + return next(); + } + controllers.helpers.notAllowed(req, res); + }); + middleware.canViewGroups = helpers.try(async (req, res, next) => { + const canView = await privileges.global.can('view:groups', req.uid); + if (canView) { + return next(); + } + controllers.helpers.notAllowed(req, res); + }); + middleware.canChat = helpers.try(async (req, res, next) => { + const canChat = await privileges.global.can(['chat', 'chat:privileged'], req.uid); + if (canChat.includes(true)) { + return next(); + } + controllers.helpers.notAllowed(req, res); + }); + middleware.checkAccountPermissions = helpers.try(async (req, res, next) => { + if (!req.loggedIn) { + return controllers.helpers.notAllowed(req, res); + } + if (!['uid', 'userslug'].some(param => req.params.hasOwnProperty(param))) { + return controllers.helpers.notAllowed(req, res); + } + const uid = req.params.uid || (await user.getUidByUserslug(req.params.userslug)); + let allowed = await privileges.users.canEdit(req.uid, uid); + if (allowed) { + return next(); + } + if (/user\/.+\/info$/.test(req.path)) { + allowed = await privileges.global.can('view:users:info', req.uid); + } + if (allowed) { + return next(); + } + controllers.helpers.notAllowed(req, res); + }); + middleware.redirectToAccountIfLoggedIn = helpers.try(async (req, res, next) => { + if (req.session.forceLogin || req.uid <= 0) { + return next(); + } + const userslug = await user.getUserField(req.uid, 'userslug'); + controllers.helpers.redirect(res, `/user/${userslug}`); + }); + middleware.redirectUidToUserslug = helpers.try(async (req, res, next) => { + const uid = parseInt(req.params.uid, 10); + if (uid <= 0) { + return next(); + } + const [canView, userslug] = await Promise.all([privileges.global.can('view:users', req.uid), user.getUserField(uid, 'userslug')]); + if (!userslug || !canView && req.uid !== uid) { + return next(); + } + const path = req.url.replace(/^\/api/, '').replace(`/uid/${uid}`, () => `/user/${userslug}`); + controllers.helpers.redirect(res, path, true); + }); + middleware.redirectMeToUserslug = helpers.try(async (req, res) => { + const userslug = await user.getUserField(req.uid, 'userslug'); + if (!userslug) { + return controllers.helpers.notAllowed(req, res); + } + const path = req.url.replace(/^(\/api)?\/me/, () => `/user/${userslug}`); + controllers.helpers.redirect(res, path); + }); + middleware.redirectToHomeIfBanned = helpers.try(async (req, res, next) => { + if (req.loggedIn) { + const canLoginIfBanned = await user.bans.canLoginIfBanned(req.uid); + if (!canLoginIfBanned) { + req.logout(() => { + res.redirect('/'); + }); + return; + } + } + next(); + }); + middleware.requireUser = function (req, res, next) { + if (req.loggedIn) { + return next(); + } + res.status(403).render('403', { + title: '[[global:403.title]]' + }); + }; + middleware.buildAccountData = async (req, res, next) => { + const lowercaseSlug = req.params.userslug.toLowerCase(); + if (req.params.userslug !== lowercaseSlug) { + if (res.locals.isAPI) { + req.params.userslug = lowercaseSlug; + } else { + const newPath = req.path.replace(new RegExp(`/${req.params.userslug}`), () => `/${lowercaseSlug}`); + return res.redirect(`${nconf.get('relative_path')}${newPath}`); + } + } + res.locals.userData = await accountHelpers.getUserDataByUserSlug(req.params.userslug, req.uid, req.query); + if (!res.locals.userData) { + return next('route'); + } + next(); + }; + middleware.registrationComplete = async function registrationComplete(req, res, next) { + const path = req.path.startsWith('/api/') ? req.path.replace('/api', '') : req.path; + if (meta.config.requireEmailAddress && (await requiresEmailConfirmation(req))) { + req.session.registration = { + ...req.session.registration, + uid: req.uid, + updateEmail: true + }; + } + if (!req.session.hasOwnProperty('registration')) { + return setImmediate(next); + } + const { + allowed + } = await plugins.hooks.fire('filter:middleware.registrationComplete', { + allowed: ['/register/complete', '/confirm/'] + }); + if (allowed.includes(path) || allowed.some(p => path.startsWith(p))) { + return setImmediate(next); + } + req.session.registration.uid = req.session.registration.uid || req.uid; + controllers.helpers.redirect(res, '/register/complete'); + }; + async function requiresEmailConfirmation(req) { + if (req.uid <= 0) { + return false; + } + const [confirmed, isAdmin] = await Promise.all([groups.isMember(req.uid, 'verified-users'), user.isAdministrator(req.uid)]); + if (confirmed || isAdmin) { + return false; + } + let cid; + let privilege; + if (req.params.hasOwnProperty('category_id')) { + cid = req.params.category_id; + privilege = 'read'; + } else if (req.params.hasOwnProperty('topic_id')) { + cid = await topics.getTopicField(req.params.topic_id, 'cid'); + privilege = 'topics:read'; + } else { + return false; + } + const [registeredAllowed, verifiedAllowed] = await Promise.all([privilegeHelpers.isAllowedTo([privilege], 'registered-users', cid), privilegeHelpers.isAllowedTo([privilege], 'verified-users', cid)]); + return !registeredAllowed.pop() && verifiedAllowed.pop(); + } +}; \ No newline at end of file diff --git a/lib/navigation/admin.js b/lib/navigation/admin.js new file mode 100644 index 0000000000..aa58680fbe --- /dev/null +++ b/lib/navigation/admin.js @@ -0,0 +1,94 @@ +'use strict'; + +const validator = require('validator'); +const winston = require('winston'); +const plugins = require('../plugins'); +const db = require('../database'); +const pubsub = require('../pubsub'); +const admin = module.exports; +let cache = null; +pubsub.on('admin:navigation:save', () => { + cache = null; +}); +admin.save = async function (data) { + const order = Object.keys(data); + const bulkSet = []; + data.forEach((item, index) => { + item.order = order[index]; + if (item.hasOwnProperty('groups')) { + item.groups = JSON.stringify(item.groups); + } + bulkSet.push([`navigation:enabled:${item.order}`, item]); + }); + cache = null; + pubsub.publish('admin:navigation:save'); + const ids = await db.getSortedSetRange('navigation:enabled', 0, -1); + await db.deleteAll(ids.map(id => `navigation:enabled:${id}`)); + await db.setObjectBulk(bulkSet); + await db.delete('navigation:enabled'); + await db.sortedSetAdd('navigation:enabled', order, order); +}; +admin.getAdmin = async function () { + const [enabled, available] = await Promise.all([admin.get(), getAvailable()]); + return { + enabled: enabled, + available: available + }; +}; +const fieldsToEscape = ['iconClass', 'class', 'route', 'id', 'text', 'textClass', 'title']; +admin.escapeFields = navItems => toggleEscape(navItems, true); +admin.unescapeFields = navItems => toggleEscape(navItems, false); +function toggleEscape(navItems, flag) { + navItems.forEach(item => { + if (item) { + fieldsToEscape.forEach(field => { + if (item.hasOwnProperty(field)) { + item[field] = validator[flag ? 'escape' : 'unescape'](String(item[field])); + } + }); + } + }); +} +admin.get = async function () { + if (cache) { + return cache.map(item => ({ + ...item + })); + } + const ids = await db.getSortedSetRange('navigation:enabled', 0, -1); + const data = await db.getObjects(ids.map(id => `navigation:enabled:${id}`)); + cache = data.filter(Boolean).map(item => { + if (item.hasOwnProperty('groups')) { + try { + item.groups = JSON.parse(item.groups); + } catch (err) { + winston.error(err.stack); + item.groups = []; + } + } + item.groups = item.groups || []; + if (item.groups && !Array.isArray(item.groups)) { + item.groups = [item.groups]; + } + return item; + }); + admin.escapeFields(cache); + return cache.map(item => ({ + ...item + })); +}; +async function getAvailable() { + const core = require('../../install/data/navigation.json').map(item => { + item.core = true; + item.id = item.id || ''; + return item; + }); + const navItems = await plugins.hooks.fire('filter:navigation.available', core); + navItems.forEach(item => { + if (item && !item.hasOwnProperty('enabled')) { + item.enabled = true; + } + }); + return navItems; +} +require('../promisify')(admin); \ No newline at end of file diff --git a/lib/navigation/index.js b/lib/navigation/index.js new file mode 100644 index 0000000000..1a307a04ea --- /dev/null +++ b/lib/navigation/index.js @@ -0,0 +1,26 @@ +'use strict'; + +const nconf = require('nconf'); +const validator = require('validator'); +const admin = require('./admin'); +const groups = require('../groups'); +const navigation = module.exports; +const relative_path = nconf.get('relative_path'); +navigation.get = async function (uid) { + let data = await admin.get(); + data = data.filter(item => item && item.enabled).map(item => { + item.originalRoute = validator.unescape(item.route); + if (!item.route.startsWith('http')) { + item.route = relative_path + item.route; + } + return item; + }); + const pass = await Promise.all(data.map(async navItem => { + if (!navItem.groups.length) { + return true; + } + return await groups.isMemberOfAny(uid, navItem.groups); + })); + return data.filter((navItem, i) => pass[i]); +}; +require('../promisify')(navigation); \ No newline at end of file diff --git a/lib/notifications.js b/lib/notifications.js new file mode 100644 index 0000000000..f3dad79bd5 --- /dev/null +++ b/lib/notifications.js @@ -0,0 +1,422 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const cron = require('cron').CronJob; +const nconf = require('nconf'); +const _ = require('lodash'); +const db = require('./database'); +const User = require('./user'); +const posts = require('./posts'); +const groups = require('./groups'); +const meta = require('./meta'); +const batch = require('./batch'); +const plugins = require('./plugins'); +const utils = require('./utils'); +const emailer = require('./emailer'); +const ttlCache = require('./cache/ttl'); +const Notifications = module.exports; +const notificationCache = ttlCache({ + ttl: (meta.config.notificationSendDelay || 60) * 1000, + noDisposeOnSet: true, + dispose: sendEmail +}); +Notifications.baseTypes = ['notificationType_upvote', 'notificationType_new-topic', 'notificationType_new-topic-with-tag', 'notificationType_new-topic-in-category', 'notificationType_new-reply', 'notificationType_post-edit', 'notificationType_follow', 'notificationType_new-chat', 'notificationType_new-group-chat', 'notificationType_new-public-chat', 'notificationType_group-invite', 'notificationType_group-leave', 'notificationType_group-request-membership', 'notificationType_new-reward']; +Notifications.privilegedTypes = ['notificationType_new-register', 'notificationType_post-queue', 'notificationType_new-post-flag', 'notificationType_new-user-flag']; +const notificationPruneCutoff = 2592000000; +const intFields = ['datetime', 'from', 'importance', 'tid', 'pid', 'roomId']; +Notifications.getAllNotificationTypes = async function () { + const results = await plugins.hooks.fire('filter:user.notificationTypes', { + types: Notifications.baseTypes.slice(), + privilegedTypes: Notifications.privilegedTypes.slice() + }); + return results.types.concat(results.privilegedTypes); +}; +Notifications.startJobs = function () { + winston.verbose('[notifications.init] Registering jobs.'); + new cron('*/30 * * * *', Notifications.prune, null, true); +}; +Notifications.get = async function (nid) { + const notifications = await Notifications.getMultiple([nid]); + return Array.isArray(notifications) && notifications.length ? notifications[0] : null; +}; +Notifications.getMultiple = async function (nids) { + if (!Array.isArray(nids) || !nids.length) { + return []; + } + const keys = nids.map(nid => `notifications:${nid}`); + const notifications = await db.getObjects(keys); + const userKeys = notifications.map(n => n && n.from); + const usersData = await User.getUsersFields(userKeys, ['username', 'userslug', 'picture']); + notifications.forEach((notification, index) => { + if (notification) { + intFields.forEach(field => { + if (notification.hasOwnProperty(field)) { + notification[field] = parseInt(notification[field], 10) || 0; + } + }); + if (notification.path && !notification.path.startsWith('http')) { + notification.path = nconf.get('relative_path') + notification.path; + } + notification.datetimeISO = utils.toISOString(notification.datetime); + if (notification.bodyLong) { + notification.bodyLong = utils.stripHTMLTags(notification.bodyLong, ['img', 'p', 'a']); + } + notification.user = usersData[index]; + if (notification.user && notification.from) { + notification.image = notification.user.picture || null; + if (notification.user.username === '[[global:guest]]') { + notification.bodyShort = notification.bodyShort.replace(/([\s\S]*?),[\s\S]*?,([\s\S]*?)/, '$1, [[global:guest]], $2'); + } + } else if (notification.image === 'brand:logo' || !notification.image) { + notification.image = meta.config['brand:logo'] || `${nconf.get('relative_path')}/logo.png`; + } + } + }); + return notifications; +}; +Notifications.filterExists = async function (nids) { + const exists = await db.isSortedSetMembers('notifications', nids); + return nids.filter((nid, idx) => exists[idx]); +}; +Notifications.findRelated = async function (mergeIds, set) { + mergeIds = mergeIds.filter(Boolean); + if (!mergeIds.length) { + return []; + } + const nids = await db.getSortedSetRevRange(set, 0, -1); + const keys = nids.map(nid => `notifications:${nid}`); + const notificationData = await db.getObjectsFields(keys, ['mergeId']); + const notificationMergeIds = notificationData.map(notifObj => String(notifObj.mergeId)); + const mergeSet = new Set(mergeIds.map(id => String(id))); + return nids.filter((nid, idx) => mergeSet.has(notificationMergeIds[idx])); +}; +Notifications.create = async function (data) { + if (!data.nid) { + throw new Error('[[error:no-notification-id]]'); + } + data.importance = data.importance || 5; + const oldNotif = await db.getObject(`notifications:${data.nid}`); + if (oldNotif && parseInt(oldNotif.pid, 10) === parseInt(data.pid, 10) && parseInt(oldNotif.importance, 10) > parseInt(data.importance, 10)) { + return null; + } + const now = Date.now(); + data.datetime = now; + const result = await plugins.hooks.fire('filter:notifications.create', { + data: data + }); + if (!result.data) { + return null; + } + await Promise.all([db.sortedSetAdd('notifications', now, data.nid), db.setObject(`notifications:${data.nid}`, data)]); + return data; +}; +Notifications.push = async function (notification, uids) { + if (!notification || !notification.nid) { + return; + } + uids = Array.isArray(uids) ? _.uniq(uids) : [uids]; + if (!uids.length) { + return; + } + setTimeout(() => { + batch.processArray(uids, async uids => { + await pushToUids(uids, notification); + }, { + interval: 1000, + batch: 500 + }, err => { + if (err) { + winston.error(err.stack); + } + }); + }, 500); +}; +async function pushToUids(uids, notification) { + async function sendNotification(uids) { + if (!uids.length) { + return; + } + const cutoff = Date.now() - notificationPruneCutoff; + const unreadKeys = uids.map(uid => `uid:${uid}:notifications:unread`); + const readKeys = uids.map(uid => `uid:${uid}:notifications:read`); + await Promise.all([db.sortedSetsAdd(unreadKeys, notification.datetime, notification.nid), db.sortedSetsRemove(readKeys, notification.nid)]); + await db.sortedSetsRemoveRangeByScore(unreadKeys.concat(readKeys), '-inf', cutoff); + const websockets = require('./socket.io'); + if (websockets.server) { + await Promise.all(uids.map(async uid => { + await plugins.hooks.fire('filter:sockets.sendNewNoticationToUid', { + uid, + notification + }); + websockets.in(`uid_${uid}`).emit('event:new_notification', notification); + })); + } + } + async function getUidsBySettings(uids) { + const uidsToNotify = []; + const uidsToEmail = []; + const usersSettings = await User.getMultipleUserSettings(uids); + usersSettings.forEach(userSettings => { + const setting = userSettings[`notificationType_${notification.type}`] || 'notification'; + if (setting === 'notification' || setting === 'notificationemail') { + uidsToNotify.push(userSettings.uid); + } + if (setting === 'email' || setting === 'notificationemail') { + uidsToEmail.push(userSettings.uid); + } + }); + return { + uidsToNotify: uidsToNotify, + uidsToEmail: uidsToEmail + }; + } + uids = await User.blocks.filterUids(notification.from, uids); + const data = await plugins.hooks.fire('filter:notification.push', { + notification, + uids + }); + if (!data || !data.notification || !data.uids || !data.uids.length) { + return; + } + notification = data.notification; + let results = { + uidsToNotify: data.uids, + uidsToEmail: [] + }; + if (notification.type) { + results = await getUidsBySettings(data.uids); + } + await sendNotification(results.uidsToNotify); + const delayNotificationTypes = ['new-chat', 'new-group-chat', 'new-public-chat']; + if (delayNotificationTypes.includes(notification.type)) { + const cacheKey = `${notification.mergeId}|${results.uidsToEmail.join(',')}`; + if (notificationCache.has(cacheKey)) { + const payload = notificationCache.get(cacheKey); + notification.bodyLong = [payload.notification.bodyLong, notification.bodyLong].join('\n'); + } + notificationCache.set(cacheKey, { + uids: results.uidsToEmail, + notification + }); + } else { + await sendEmail({ + uids: results.uidsToEmail, + notification + }); + } + plugins.hooks.fire('action:notification.pushed', { + notification, + uids: results.uidsToNotify, + uidsNotified: results.uidsToNotify, + uidsEmailed: results.uidsToEmail + }); +} +async function sendEmail({ + uids, + notification +}, mergeId, reason) { + if (reason && reason !== 'stale') { + return; + } + if (['new-reply', 'new-chat'].includes(notification.type)) { + notification['cta-type'] = notification.type; + } + let body = notification.bodyLong || ''; + if (meta.config.removeEmailNotificationImages) { + body = body.replace(/]*>/, ''); + } + body = posts.relativeToAbsolute(body, posts.urlRegex); + body = posts.relativeToAbsolute(body, posts.imgRegex); + let errorLogged = false; + await async.eachLimit(uids, 3, async uid => { + await emailer.send('notification', uid, { + path: notification.path, + notification_url: notification.path.startsWith('http') ? notification.path : nconf.get('url') + notification.path, + subject: utils.stripHTMLTags(notification.subject || '[[notifications:new-notification]]'), + intro: utils.stripHTMLTags(notification.bodyShort), + body: body, + notification: notification, + showUnsubscribe: true + }).catch(err => { + if (!errorLogged) { + winston.error(`[emailer.send] ${err.stack}`); + errorLogged = true; + } + }); + }); +} +Notifications.pushGroup = async function (notification, groupName) { + if (!notification) { + return; + } + const members = await groups.getMembers(groupName, 0, -1); + await Notifications.push(notification, members); +}; +Notifications.pushGroups = async function (notification, groupNames) { + if (!notification) { + return; + } + let groupMembers = await groups.getMembersOfGroups(groupNames); + groupMembers = _.uniq(_.flatten(groupMembers)); + await Notifications.push(notification, groupMembers); +}; +Notifications.rescind = async function (nids) { + nids = Array.isArray(nids) ? nids : [nids]; + await Promise.all([db.sortedSetRemove('notifications', nids), db.deleteAll(nids.map(nid => `notifications:${nid}`))]); +}; +Notifications.markRead = async function (nid, uid) { + if (parseInt(uid, 10) <= 0 || !nid) { + return; + } + await Notifications.markReadMultiple([nid], uid); +}; +Notifications.markUnread = async function (nid, uid) { + if (!(parseInt(uid, 10) > 0) || !nid) { + return; + } + const notification = await db.getObject(`notifications:${nid}`); + if (!notification) { + throw new Error('[[error:no-notification]]'); + } + notification.datetime = notification.datetime || Date.now(); + await Promise.all([db.sortedSetRemove(`uid:${uid}:notifications:read`, nid), db.sortedSetAdd(`uid:${uid}:notifications:unread`, notification.datetime, nid)]); +}; +Notifications.markReadMultiple = async function (nids, uid) { + nids = nids.filter(Boolean); + if (!Array.isArray(nids) || !nids.length || !(parseInt(uid, 10) > 0)) { + return; + } + let notificationKeys = nids.map(nid => `notifications:${nid}`); + let mergeIds = await db.getObjectsFields(notificationKeys, ['mergeId']); + mergeIds = _.uniq(mergeIds.map(set => set.mergeId)); + const relatedNids = await Notifications.findRelated(mergeIds, `uid:${uid}:notifications:unread`); + notificationKeys = _.union(nids, relatedNids).map(nid => `notifications:${nid}`); + let notificationData = await db.getObjectsFields(notificationKeys, ['nid', 'datetime']); + notificationData = notificationData.filter(n => n && n.nid); + nids = notificationData.map(n => n.nid); + const datetimes = notificationData.map(n => n && n.datetime || Date.now()); + await Promise.all([db.sortedSetRemove(`uid:${uid}:notifications:unread`, nids), db.sortedSetAdd(`uid:${uid}:notifications:read`, datetimes, nids)]); +}; +Notifications.markAllRead = async function (uid) { + const nids = await db.getSortedSetRevRange(`uid:${uid}:notifications:unread`, 0, 99); + await Notifications.markReadMultiple(nids, uid); +}; +Notifications.prune = async function () { + const cutoffTime = Date.now() - notificationPruneCutoff; + const nids = await db.getSortedSetRangeByScore('notifications', 0, 500, '-inf', cutoffTime); + if (!nids.length) { + return; + } + try { + await Promise.all([db.sortedSetRemove('notifications', nids), db.deleteAll(nids.map(nid => `notifications:${nid}`))]); + await batch.processSortedSet('users:joindate', async uids => { + const unread = uids.map(uid => `uid:${uid}:notifications:unread`); + const read = uids.map(uid => `uid:${uid}:notifications:read`); + await db.sortedSetsRemoveRangeByScore(unread.concat(read), '-inf', cutoffTime); + }, { + batch: 500, + interval: 100 + }); + } catch (err) { + if (err) { + winston.error(`Encountered error pruning notifications\n${err.stack}`); + } + } +}; +Notifications.merge = async function (notifications) { + const mergeIds = ['notifications:upvoted-your-post-in', 'notifications:user-started-following-you', 'notifications:user-posted-to', 'notifications:user-flagged-post-in', 'notifications:user-flagged-user', 'new-chat', 'notifications:user-posted-in-public-room', 'new-register', 'post-queue']; + notifications = mergeIds.reduce((notifications, mergeId) => { + const isolated = notifications.filter(n => n && n.hasOwnProperty('mergeId') && n.mergeId.split('|')[0] === mergeId); + if (isolated.length <= 1) { + return notifications; + } + const differentiators = isolated.reduce((cur, next) => { + const differentiator = next.mergeId.split('|')[1] || 0; + if (!cur.includes(differentiator)) { + cur.push(differentiator); + } + return cur; + }, []); + differentiators.forEach(differentiator => { + function typeFromLength(items) { + if (items.length === 2) { + return 'dual'; + } else if (items.length === 3) { + return 'triple'; + } + return 'multiple'; + } + let set; + if (differentiator === 0 && differentiators.length === 1) { + set = isolated; + } else { + set = isolated.filter(n => n.mergeId === `${mergeId}|${differentiator}`); + } + const modifyIndex = notifications.indexOf(set[0]); + if (modifyIndex === -1 || set.length === 1) { + return notifications; + } + const notifObj = notifications[modifyIndex]; + switch (mergeId) { + case 'new-chat': + { + const { + roomId, + roomName, + type, + user + } = set[0]; + const isGroupChat = type === 'new-group-chat'; + notifObj.bodyShort = isGroupChat || roomName !== `[[modules:chat.room-id, ${roomId}]]` ? `[[notifications:new-messages-in, ${set.length}, ${roomName}]]` : `[[notifications:new-messages-from, ${set.length}, ${user.displayname}]]`; + break; + } + case 'notifications:user-posted-in-public-room': + { + const usernames = _.uniq(set.map(notifObj => notifObj && notifObj.user && notifObj.user.displayname)); + if (usernames.length === 2 || usernames.length === 3) { + notifObj.bodyShort = `[[${mergeId}-${typeFromLength(usernames)}, ${usernames.join(', ')}, ${notifObj.roomIcon}, ${notifObj.roomName}]]`; + } else if (usernames.length > 3) { + notifObj.bodyShort = `[[${mergeId}-${typeFromLength(usernames)}, ${usernames.slice(0, 2).join(', ')}, ${usernames.length - 2}, ${notifObj.roomIcon}, ${notifObj.roomName}]]`; + } + notifObj.path = set[set.length - 1].path; + break; + } + case 'notifications:upvoted-your-post-in': + case 'notifications:user-started-following-you': + case 'notifications:user-posted-to': + case 'notifications:user-flagged-post-in': + case 'notifications:user-flagged-user': + { + const usernames = _.uniq(set.map(notifObj => notifObj && notifObj.user && notifObj.user.username)); + const numUsers = usernames.length; + const title = utils.decodeHTMLEntities(notifications[modifyIndex].topicTitle || ''); + let titleEscaped = title.replace(/%/g, '%').replace(/,/g, ','); + titleEscaped = titleEscaped ? `, ${titleEscaped}` : ''; + if (numUsers === 2 || numUsers === 3) { + notifications[modifyIndex].bodyShort = `[[${mergeId}-${typeFromLength(usernames)}, ${usernames.join(', ')}${titleEscaped}]]`; + } else if (numUsers > 2) { + notifications[modifyIndex].bodyShort = `[[${mergeId}-${typeFromLength(usernames)}, ${usernames.slice(0, 2).join(', ')}, ${numUsers - 2}${titleEscaped}]]`; + } + notifications[modifyIndex].path = set[set.length - 1].path; + } + break; + case 'new-register': + notifications[modifyIndex].bodyShort = `[[notifications:${mergeId}-multiple, ${set.length}]]`; + break; + } + notifications = notifications.filter((notifObj, idx) => { + if (!notifObj || !notifObj.mergeId) { + return true; + } + return !(notifObj.mergeId === mergeId + (differentiator ? `|${differentiator}` : '') && idx !== modifyIndex); + }); + }); + return notifications; + }, notifications); + const data = await plugins.hooks.fire('filter:notifications.merge', { + notifications: notifications + }); + return data && data.notifications; +}; +require('./promisify')(Notifications); \ No newline at end of file diff --git a/lib/pagination.js b/lib/pagination.js new file mode 100644 index 0000000000..3fbcac701c --- /dev/null +++ b/lib/pagination.js @@ -0,0 +1,119 @@ +'use strict'; + +const qs = require('querystring'); +const _ = require('lodash'); +const pagination = module.exports; +pagination.create = function (currentPage, pageCount, queryObj) { + if (pageCount <= 1) { + return { + prev: { + page: 1, + active: currentPage > 1 + }, + next: { + page: 1, + active: currentPage < pageCount + }, + first: { + page: 1, + active: currentPage === 1 + }, + last: { + page: 1, + active: currentPage === pageCount + }, + rel: [], + pages: [], + currentPage: 1, + pageCount: 1 + }; + } + pageCount = parseInt(pageCount, 10); + let pagesToShow = [1, 2, pageCount - 1, pageCount]; + currentPage = parseInt(currentPage, 10) || 1; + const previous = Math.max(1, currentPage - 1); + const next = Math.min(pageCount, currentPage + 1); + let startPage = Math.max(1, currentPage - 2); + if (startPage > pageCount - 5) { + startPage -= 2 - (pageCount - currentPage); + } + let i; + for (i = 0; i < 5; i += 1) { + pagesToShow.push(startPage + i); + } + pagesToShow = _.uniq(pagesToShow).filter(page => page > 0 && page <= pageCount).sort((a, b) => a - b); + queryObj = { + ...(queryObj || {}) + }; + delete queryObj._; + const pages = pagesToShow.map(page => { + queryObj.page = page; + return { + page: page, + active: page === currentPage, + qs: qs.stringify(queryObj) + }; + }); + for (i = pages.length - 1; i > 0; i -= 1) { + if (pages[i].page - 2 === pages[i - 1].page) { + pages.splice(i, 0, { + page: pages[i].page - 1, + active: false, + qs: qs.stringify(queryObj) + }); + } else if (pages[i].page - 1 !== pages[i - 1].page) { + pages.splice(i, 0, { + separator: true + }); + } + } + const data = { + rel: [], + pages: pages, + currentPage: currentPage, + pageCount: pageCount + }; + queryObj.page = previous; + data.prev = { + page: previous, + active: currentPage > 1, + qs: qs.stringify(queryObj) + }; + queryObj.page = next; + data.next = { + page: next, + active: currentPage < pageCount, + qs: qs.stringify(queryObj) + }; + queryObj.page = 1; + data.first = { + page: 1, + active: currentPage === 1, + qs: qs.stringify(queryObj) + }; + queryObj.page = pageCount; + data.last = { + page: pageCount, + active: currentPage === pageCount, + qs: qs.stringify(queryObj) + }; + if (currentPage < pageCount) { + data.rel.push({ + rel: 'next', + href: `?${qs.stringify({ + ...queryObj, + page: next + })}` + }); + } + if (currentPage > 1) { + data.rel.push({ + rel: 'prev', + href: `?${qs.stringify({ + ...queryObj, + page: previous + })}` + }); + } + return data; +}; \ No newline at end of file diff --git a/lib/password.js b/lib/password.js new file mode 100644 index 0000000000..a909117304 --- /dev/null +++ b/lib/password.js @@ -0,0 +1,28 @@ +'use strict'; + +const path = require('path'); +const crypto = require('crypto'); +const workerpool = require('workerpool'); +const pool = workerpool.pool(path.join(__dirname, '/password_worker.js'), { + minWorkers: 1 +}); +exports.hash = async function (rounds, password) { + password = crypto.createHash('sha512').update(password).digest('hex'); + return await pool.exec('hash', [password, rounds]); +}; +exports.compare = async function (password, hash, shaWrapped) { + const fakeHash = await getFakeHash(); + if (shaWrapped) { + password = crypto.createHash('sha512').update(password).digest('hex'); + } + return await pool.exec('compare', [password, hash || fakeHash]); +}; +let fakeHashCache; +async function getFakeHash() { + if (fakeHashCache) { + return fakeHashCache; + } + fakeHashCache = await exports.hash(12, Math.random().toString()); + return fakeHashCache; +} +require('./promisify')(exports); \ No newline at end of file diff --git a/lib/password_worker.js b/lib/password_worker.js new file mode 100644 index 0000000000..543a946b88 --- /dev/null +++ b/lib/password_worker.js @@ -0,0 +1,15 @@ +'use strict'; + +const workerpool = require('workerpool'); +const bcrypt = require('bcryptjs'); +async function hash(password, rounds) { + const salt = await bcrypt.genSalt(parseInt(rounds, 10)); + return await bcrypt.hash(password, salt); +} +async function compare(password, hash) { + return await bcrypt.compare(String(password || ''), String(hash || '')); +} +workerpool.worker({ + hash: hash, + compare: compare +}); \ No newline at end of file diff --git a/lib/plugins/data.js b/lib/plugins/data.js new file mode 100644 index 0000000000..b0cd04b8a7 --- /dev/null +++ b/lib/plugins/data.js @@ -0,0 +1,203 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const winston = require('winston'); +const _ = require('lodash'); +const nconf = require('nconf'); +const db = require('../database'); +const file = require('../file'); +const { + paths +} = require('../constants'); +const Data = module.exports; +const basePath = path.join(__dirname, '../../'); +async function getActiveIds() { + if (nconf.get('plugins:active')) { + return nconf.get('plugins:active'); + } + return await db.getSortedSetRange('plugins:active', 0, -1); +} +Data.getPluginPaths = async function () { + const plugins = await getActiveIds(); + const pluginPaths = plugins.filter(plugin => plugin && typeof plugin === 'string').map(plugin => path.join(paths.nodeModules, plugin)); + const exists = await Promise.all(pluginPaths.map(file.exists)); + exists.forEach((exists, i) => { + if (!exists) { + winston.warn(`[plugins] "${plugins[i]}" is active but not installed.`); + } + }); + return pluginPaths.filter((p, i) => exists[i]); +}; +Data.loadPluginInfo = async function (pluginPath) { + const [packageJson, pluginJson] = await Promise.all([fs.promises.readFile(path.join(pluginPath, 'package.json'), 'utf8'), fs.promises.readFile(path.join(pluginPath, 'plugin.json'), 'utf8')]); + let pluginData; + let packageData; + try { + pluginData = JSON.parse(pluginJson); + packageData = JSON.parse(packageJson); + pluginData.license = parseLicense(packageData); + pluginData.id = packageData.name; + pluginData.name = packageData.name; + pluginData.description = packageData.description; + pluginData.version = packageData.version; + pluginData.repository = packageData.repository; + pluginData.nbbpm = packageData.nbbpm; + pluginData.path = pluginPath; + } catch (err) { + const pluginDir = path.basename(pluginPath); + winston.error(`[plugins/${pluginDir}] Error in plugin.json or package.json!${err.stack}`); + throw new Error('[[error:parse-error]]'); + } + return pluginData; +}; +function parseLicense(packageData) { + try { + const licenseData = require(`spdx-license-list/licenses/${packageData.license}`); + return { + name: licenseData.name, + text: licenseData.licenseText + }; + } catch (e) { + return null; + } +} +Data.getActive = async function () { + const pluginPaths = await Data.getPluginPaths(); + return await Promise.all(pluginPaths.map(p => Data.loadPluginInfo(p))); +}; +Data.getStaticDirectories = async function (pluginData) { + const validMappedPath = /^[\w\-_]+$/; + if (!pluginData.staticDirs) { + return; + } + const dirs = Object.keys(pluginData.staticDirs); + if (!dirs.length) { + return; + } + const staticDirs = {}; + async function processDir(route) { + if (!validMappedPath.test(route)) { + winston.warn(`[plugins/${pluginData.id}] Invalid mapped path specified: ${route}. Path must adhere to: ${validMappedPath.toString()}`); + return; + } + const dirPath = await resolveModulePath(pluginData.path, pluginData.staticDirs[route]); + if (!dirPath) { + winston.warn(`[plugins/${pluginData.id}] Invalid mapped path specified: ${route} => ${pluginData.staticDirs[route]}`); + return; + } + try { + const stats = await fs.promises.stat(dirPath); + if (!stats.isDirectory()) { + winston.warn(`[plugins/${pluginData.id}] Mapped path '${route} => ${dirPath}' is not a directory.`); + return; + } + staticDirs[`${pluginData.id}/${route}`] = dirPath; + } catch (err) { + if (err.code === 'ENOENT') { + winston.warn(`[plugins/${pluginData.id}] Mapped path '${route} => ${dirPath}' not found.`); + return; + } + throw err; + } + } + await Promise.all(dirs.map(route => processDir(route))); + winston.verbose(`[plugins] found ${Object.keys(staticDirs).length} static directories for ${pluginData.id}`); + return staticDirs; +}; +Data.getFiles = async function (pluginData, type) { + if (!Array.isArray(pluginData[type]) || !pluginData[type].length) { + return; + } + winston.verbose(`[plugins] Found ${pluginData[type].length} ${type} file(s) for plugin ${pluginData.id}`); + return pluginData[type].map(file => path.join(pluginData.id, file)); +}; +async function resolveModulePath(basePath, modulePath) { + const isNodeModule = /node_modules/; + const currentPath = path.join(basePath, modulePath); + const exists = await file.exists(currentPath); + if (exists) { + return currentPath; + } + if (!isNodeModule.test(modulePath)) { + winston.warn(`[plugins] File not found: ${currentPath} (Ignoring)`); + return; + } + const dirPath = path.dirname(basePath); + if (dirPath === basePath) { + winston.warn(`[plugins] File not found: ${currentPath} (Ignoring)`); + return; + } + return await resolveModulePath(dirPath, modulePath); +} +Data.getScripts = async function getScripts(pluginData, target) { + target = target === 'client' ? 'scripts' : 'acpScripts'; + const input = pluginData[target]; + if (!Array.isArray(input) || !input.length) { + return; + } + const scripts = []; + for (const filePath of input) { + const modulePath = await resolveModulePath(pluginData.path, filePath); + if (modulePath) { + scripts.push(modulePath); + } + } + if (scripts.length) { + winston.verbose(`[plugins] Found ${scripts.length} js file(s) for plugin ${pluginData.id}`); + } + return scripts; +}; +Data.getModules = async function getModules(pluginData) { + if (!pluginData.modules || !pluginData.hasOwnProperty('modules')) { + return; + } + let pluginModules = pluginData.modules; + if (Array.isArray(pluginModules)) { + const strip = parseInt(pluginData.modulesStrip, 10) || 0; + pluginModules = pluginModules.reduce((prev, modulePath) => { + let key; + if (strip) { + key = modulePath.replace(new RegExp(`.?(/[^/]+){${strip}}/`), ''); + } else { + key = path.basename(modulePath); + } + prev[key] = modulePath; + return prev; + }, {}); + } + const modules = {}; + async function processModule(key) { + const modulePath = await resolveModulePath(pluginData.path, pluginModules[key]); + if (modulePath) { + modules[key] = path.relative(basePath, modulePath); + } + } + await Promise.all(Object.keys(pluginModules).map(key => processModule(key))); + const len = Object.keys(modules).length; + winston.verbose(`[plugins] Found ${len} AMD-style module(s) for plugin ${pluginData.id}`); + return modules; +}; +Data.getLanguageData = async function getLanguageData(pluginData) { + if (typeof pluginData.languages !== 'string') { + return; + } + const pathToFolder = path.join(paths.nodeModules, pluginData.id, pluginData.languages); + const filepaths = await file.walk(pathToFolder); + const namespaces = []; + const languages = []; + filepaths.forEach(p => { + const rel = path.relative(pathToFolder, p).split(/[/\\]/); + const language = rel.shift().replace('_', '-').replace('@', '-x-'); + const namespace = rel.join('/').replace(/\.json$/, ''); + if (!language || !namespace) { + return; + } + languages.push(language); + namespaces.push(namespace); + }); + return { + languages: _.uniq(languages), + namespaces: _.uniq(namespaces) + }; +}; \ No newline at end of file diff --git a/lib/plugins/hooks.js b/lib/plugins/hooks.js new file mode 100644 index 0000000000..ed3051f364 --- /dev/null +++ b/lib/plugins/hooks.js @@ -0,0 +1,284 @@ +'use strict'; + +const winston = require('winston'); +const plugins = require('.'); +const utils = require('../utils'); +const als = require('../als'); +const Hooks = module.exports; +Hooks._deprecated = new Map([['filter:email.send', { + new: 'static:email.send', + since: 'v1.17.0', + until: 'v2.0.0' +}], ['filter:router.page', { + new: 'response:router.page', + since: 'v1.15.3', + until: 'v2.1.0' +}], ['filter:post.purge', { + new: 'filter:posts.purge', + since: 'v1.19.6', + until: 'v2.1.0' +}], ['action:post.purge', { + new: 'action:posts.purge', + since: 'v1.19.6', + until: 'v2.1.0' +}], ['filter:user.verify.code', { + new: 'filter:user.verify', + since: 'v2.2.0', + until: 'v3.0.0' +}], ['filter:flags.getFilters', { + new: 'filter:flags.init', + since: 'v2.7.0', + until: 'v3.0.0' +}], ['filter:privileges.global.list', { + new: 'static:privileges.global.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.global.groups.list', { + new: 'static:privileges.global.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.global.list_human', { + new: 'static:privileges.global.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.global.groups.list_human', { + new: 'static:privileges.global.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.list', { + new: 'static:privileges.categories.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.groups.list', { + new: 'static:privileges.categories.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.list_human', { + new: 'static:privileges.categories.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.groups.list_human', { + new: 'static:privileges.categories.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.admin.list', { + new: 'static:privileges.admin.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.admin.groups.list', { + new: 'static:privileges.admin.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.admin.list_human', { + new: 'static:privileges.admin.init', + since: 'v3.5.0', + until: 'v4.0.0' +}], ['filter:privileges.admin.groups.list_human', { + new: 'static:privileges.admin.init', + since: 'v3.5.0', + until: 'v4.0.0' +}]]); +Hooks.internals = { + _register: function (data) { + plugins.loadedHooks[data.hook] = plugins.loadedHooks[data.hook] || []; + plugins.loadedHooks[data.hook].push(data); + } +}; +const hookTypeToMethod = { + filter: fireFilterHook, + action: fireActionHook, + static: fireStaticHook, + response: fireResponseHook +}; +Hooks.register = function (id, data) { + if (!data.hook || !data.method) { + winston.warn(`[plugins/${id}] registerHook called with invalid data.hook/method`, data); + return; + } + if (Hooks._deprecated.has(data.hook)) { + const deprecation = Hooks._deprecated.get(data.hook); + if (!deprecation.hasOwnProperty('affected')) { + deprecation.affected = new Set(); + } + deprecation.affected.add(id); + Hooks._deprecated.set(data.hook, deprecation); + } + data.id = id; + if (!data.priority) { + data.priority = 10; + } + if (Array.isArray(data.method) && data.method.every(method => typeof method === 'function' || typeof method === 'string')) { + data.method.forEach(method => { + const singularData = { + ...data, + method: method + }; + Hooks.register(id, singularData); + }); + } else if (typeof data.method === 'string' && data.method.length > 0) { + const method = data.method.split('.').reduce((memo, prop) => { + if (memo && memo[prop]) { + return memo[prop]; + } + return null; + }, plugins.libraries[data.id]); + data.method = method; + Hooks.internals._register(data); + } else if (typeof data.method === 'function') { + Hooks.internals._register(data); + } else { + winston.warn(`[plugins/${id}] Hook method mismatch: ${data.hook} => ${data.method}`); + } +}; +Hooks.unregister = function (id, hook, method) { + const hooks = plugins.loadedHooks[hook] || []; + plugins.loadedHooks[hook] = hooks.filter(hookData => hookData && hookData.id !== id && hookData.method !== method); +}; +Hooks.fire = async function (hook, params) { + const hookList = plugins.loadedHooks[hook]; + const hookType = hook.split(':')[0]; + if (global.env === 'development' && hook !== 'action:plugins.firehook' && hook !== 'filter:plugins.firehook') { + winston.debug(`[plugins/fireHook] ${hook}`); + } + if (!hookTypeToMethod[hookType]) { + winston.warn(`[plugins] Unknown hookType: ${hookType}, hook : ${hook}`); + return; + } + let deleteCaller = false; + if (params && typeof params === 'object' && !Array.isArray(params) && !params.hasOwnProperty('caller')) { + params.caller = als.getStore(); + deleteCaller = true; + } + const result = await hookTypeToMethod[hookType](hook, hookList, params); + if (hook !== 'action:plugins.firehook' && hook !== 'filter:plugins.firehook') { + const payload = await Hooks.fire('filter:plugins.firehook', { + hook: hook, + params: result || params + }); + Hooks.fire('action:plugins.firehook', payload); + } + if (result !== undefined) { + if (deleteCaller && result && result.hasOwnProperty('caller')) { + delete result.caller; + } + return result; + } +}; +Hooks.hasListeners = function (hook) { + return !!(plugins.loadedHooks[hook] && plugins.loadedHooks[hook].length > 0); +}; +function hookHandlerPromise(hook, hookObj, params) { + return new Promise((resolve, reject) => { + let resolved = false; + function _resolve(result) { + if (resolved) { + winston.warn(`[plugins] ${hook} already resolved in plugin ${hookObj.id}`); + return; + } + resolved = true; + resolve(result); + } + const returned = hookObj.method(params, (err, result) => { + if (err) reject(err);else _resolve(result); + }); + if (utils.isPromise(returned)) { + returned.then(payload => _resolve(payload), err => reject(err)); + return; + } + if (hook.startsWith('filter:') && returned !== undefined) { + _resolve(returned); + } else if (hook.startsWith('static:') && hookObj.method.length <= 1) { + _resolve(); + } + }); +} +async function fireFilterHook(hook, hookList, params) { + if (!Array.isArray(hookList) || !hookList.length) { + return params; + } + async function fireMethod(hookObj, params) { + if (typeof hookObj.method !== 'function') { + if (global.env === 'development') { + winston.warn(`[plugins] Expected method for hook '${hook}' in plugin '${hookObj.id}' not found, skipping.`); + } + return params; + } + if (hookObj.method.constructor && hookObj.method.constructor.name === 'AsyncFunction') { + return await hookObj.method(params); + } + return hookHandlerPromise(hook, hookObj, params); + } + for (const hookObj of hookList) { + params = await fireMethod(hookObj, params); + } + return params; +} +async function fireActionHook(hook, hookList, params) { + if (!Array.isArray(hookList) || !hookList.length) { + return; + } + for (const hookObj of hookList) { + if (typeof hookObj.method !== 'function') { + if (global.env === 'development') { + winston.warn(`[plugins] Expected method for hook '${hook}' in plugin '${hookObj.id}' not found, skipping.`); + } + } else { + await hookObj.method(params); + } + } +} +const timeout = (prom, time, error) => { + let timer; + return Promise.race([prom, new Promise((resolve, reject) => { + timer = setTimeout(reject, time, new Error(error)); + })]).finally(() => clearTimeout(timer)); +}; +async function fireStaticHook(hook, hookList, params) { + if (!Array.isArray(hookList) || !hookList.length) { + return; + } + const noErrorHooks = ['static:app.load', 'static:assets.prepare', 'static:app.preload']; + async function fireMethod(hookObj, params) { + if (typeof hookObj.method !== 'function') { + if (global.env === 'development') { + winston.warn(`[plugins] Expected method for hook '${hook}' in plugin '${hookObj.id}' not found, skipping.`); + } + return params; + } + if (hookObj.method.constructor && hookObj.method.constructor.name === 'AsyncFunction') { + return timeout(hookObj.method(params), 10000, 'timeout'); + } + return hookHandlerPromise(hook, hookObj, params); + } + for (const hookObj of hookList) { + try { + await fireMethod(hookObj, params); + } catch (err) { + if (err && err.message === 'timeout') { + winston.warn(`[plugins] Callback timed out, hook '${hook}' in plugin '${hookObj.id}'`); + } else { + if (!noErrorHooks.includes(hook)) { + throw err; + } + winston.error(`[plugins] Error executing '${hook}' in plugin '${hookObj.id}'\n${err.stack}`); + } + } + } +} +async function fireResponseHook(hook, hookList, params) { + if (!Array.isArray(hookList) || !hookList.length) { + return; + } + for (const hookObj of hookList) { + if (typeof hookObj.method !== 'function') { + if (global.env === 'development') { + winston.warn(`[plugins] Expected method for hook '${hook}' in plugin '${hookObj.id}' not found, skipping.`); + } + } else { + if (params.res.headersSent) { + return; + } + await hookObj.method(params); + } + } +} \ No newline at end of file diff --git a/lib/plugins/index.js b/lib/plugins/index.js new file mode 100644 index 0000000000..3bc30d0fd7 --- /dev/null +++ b/lib/plugins/index.js @@ -0,0 +1,289 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const winston = require('winston'); +const semver = require('semver'); +const nconf = require('nconf'); +const chalk = require('chalk'); +const request = require('../request'); +const user = require('../user'); +const posts = require('../posts'); +const { + pluginNamePattern, + themeNamePattern, + paths +} = require('../constants'); +let app; +let middleware; +const Plugins = module.exports; +require('./install')(Plugins); +require('./load')(Plugins); +require('./usage')(Plugins); +Plugins.data = require('./data'); +Plugins.hooks = require('./hooks'); +Plugins.getPluginPaths = Plugins.data.getPluginPaths; +Plugins.loadPluginInfo = Plugins.data.loadPluginInfo; +Plugins.pluginsData = {}; +Plugins.libraries = {}; +Plugins.loadedHooks = {}; +Plugins.staticDirs = {}; +Plugins.cssFiles = []; +Plugins.scssFiles = []; +Plugins.acpScssFiles = []; +Plugins.clientScripts = []; +Plugins.acpScripts = []; +Plugins.libraryPaths = []; +Plugins.versionWarning = []; +Plugins.languageData = {}; +Plugins.loadedPlugins = []; +Plugins.initialized = false; +Plugins.requireLibrary = function (pluginData) { + let libraryPath; + try { + libraryPath = pluginData.path; + Plugins.libraries[pluginData.id] = require(libraryPath); + } catch (e) { + if (pluginData.library) { + winston.warn(` [plugins/${pluginData.id}] The plugin.json field "library" is deprecated. Please use the package.json field "main" instead.`); + winston.verbose(`[plugins/${pluginData.id}] See https://github.com/NodeBB/NodeBB/issues/8686`); + libraryPath = path.join(pluginData.path, pluginData.library); + Plugins.libraries[pluginData.id] = require(libraryPath); + } else { + throw e; + } + } + Plugins.libraryPaths.push(libraryPath); +}; +Plugins.init = async function (nbbApp, nbbMiddleware) { + if (Plugins.initialized) { + return; + } + if (nbbApp) { + app = nbbApp; + middleware = nbbMiddleware; + } + if (global.env === 'development') { + winston.verbose('[plugins] Initializing plugins system'); + } + await Plugins.reload(); + if (global.env === 'development') { + winston.info('[plugins] Plugins OK'); + } + Plugins.initialized = true; +}; +Plugins.reload = async function () { + Plugins.libraries = {}; + Plugins.loadedHooks = {}; + Plugins.staticDirs = {}; + Plugins.versionWarning = []; + Plugins.cssFiles.length = 0; + Plugins.scssFiles.length = 0; + Plugins.acpScssFiles.length = 0; + Plugins.clientScripts.length = 0; + Plugins.acpScripts.length = 0; + Plugins.libraryPaths.length = 0; + Plugins.loadedPlugins.length = 0; + await user.addInterstitials(); + const paths = await Plugins.getPluginPaths(); + for (const path of paths) { + await Plugins.loadPlugin(path); + } + if (Plugins.versionWarning.length && nconf.get('isPrimary')) { + console.log(''); + winston.warn('[plugins/load] The following plugins may not be compatible with your version of NodeBB. This may cause unintended behaviour or crashing. In the event of an unresponsive NodeBB caused by this plugin, run `./nodebb reset -p PLUGINNAME` to disable it.'); + for (let x = 0, numPlugins = Plugins.versionWarning.length; x < numPlugins; x += 1) { + console.log(`${chalk.yellow(' * ') + Plugins.versionWarning[x]}`); + } + console.log(''); + } + posts.registerHooks(); + Plugins.hooks._deprecated.forEach((deprecation, hook) => { + if (!deprecation.affected || !deprecation.affected.size) { + return; + } + const replacement = deprecation.hasOwnProperty('new') ? `Please use ${chalk.yellow(deprecation.new)} instead.` : 'There is no alternative.'; + winston.warn(`[plugins/load] ${chalk.white.bgRed.bold('DEPRECATION')} The hook ${chalk.yellow(hook)} has been deprecated as of ${deprecation.since}, and slated for removal in ${deprecation.until}. ${replacement} The following plugins are still listening for this hook:`); + deprecation.affected.forEach(id => console.log(` ${chalk.yellow('*')} ${id}`)); + }); + Object.keys(Plugins.loadedHooks).forEach(hook => { + Plugins.loadedHooks[hook].sort((a, b) => a.priority - b.priority); + }); + await posts.configureSanitize(); +}; +Plugins.reloadRoutes = async function (params) { + const controllers = require('../controllers'); + await Plugins.hooks.fire('static:app.load', { + app: app, + router: params.router, + middleware: middleware, + controllers: controllers + }); + winston.verbose('[plugins] All plugins reloaded and rerouted'); +}; +Plugins.get = async function (id) { + const url = `${nconf.get('registry') || 'https://packages.nodebb.org'}/api/v1/plugins/${id}`; + const { + response, + body + } = await request.get(url); + if (!response.ok) { + throw new Error(`[[error:unable-to-load-plugin, ${id}]]`); + } + let normalised = await Plugins.normalise([body ? body.payload : {}]); + normalised = normalised.filter(plugin => plugin.id === id); + return normalised.length ? normalised[0] : undefined; +}; +Plugins.list = async function (matching) { + if (matching === undefined) { + matching = true; + } + const { + version + } = require(paths.currentPackage); + const url = `${nconf.get('registry') || 'https://packages.nodebb.org'}/api/v1/plugins${matching !== false ? `?version=${version}` : ''}`; + try { + const { + response, + body + } = await request.get(url); + if (!response.ok) { + throw new Error(`[[error:unable-to-load-plugins-from-nbbpm]]`); + } + return await Plugins.normalise(body); + } catch (err) { + winston.error(`Error loading ${url}`, err); + return await Plugins.normalise([]); + } +}; +Plugins.listTrending = async () => { + const url = `${nconf.get('registry') || 'https://packages.nodebb.org'}/api/v1/analytics/top/week`; + const { + response, + body + } = await request.get(url); + if (!response.ok) { + throw new Error(`[[error:unable-to-load-trending-plugins]]`); + } + return body; +}; +Plugins.normalise = async function (apiReturn) { + const pluginMap = {}; + const { + dependencies + } = require(paths.currentPackage); + apiReturn = Array.isArray(apiReturn) ? apiReturn : []; + apiReturn.forEach(packageData => { + packageData.id = packageData.name; + packageData.installed = false; + packageData.active = false; + packageData.url = packageData.url || (packageData.repository ? packageData.repository.url : ''); + pluginMap[packageData.name] = packageData; + }); + let installedPlugins = await Plugins.showInstalled(); + installedPlugins = installedPlugins.filter(plugin => plugin && !plugin.system); + installedPlugins.forEach(plugin => { + if (plugin.error) { + pluginMap[plugin.id] = pluginMap[plugin.id] || {}; + pluginMap[plugin.id].installed = true; + pluginMap[plugin.id].error = true; + return; + } + pluginMap[plugin.id] = pluginMap[plugin.id] || {}; + pluginMap[plugin.id].id = pluginMap[plugin.id].id || plugin.id; + pluginMap[plugin.id].name = plugin.name || pluginMap[plugin.id].name; + pluginMap[plugin.id].description = plugin.description; + pluginMap[plugin.id].url = pluginMap[plugin.id].url || plugin.url; + pluginMap[plugin.id].installed = true; + pluginMap[plugin.id].isTheme = themeNamePattern.test(plugin.id); + pluginMap[plugin.id].error = plugin.error || false; + pluginMap[plugin.id].active = plugin.active; + pluginMap[plugin.id].version = plugin.version; + pluginMap[plugin.id].settingsRoute = plugin.settingsRoute; + pluginMap[plugin.id].license = plugin.license; + if (dependencies.hasOwnProperty(plugin.id) && semver.valid(dependencies[plugin.id])) { + pluginMap[plugin.id].latest = dependencies[plugin.id]; + } else { + pluginMap[plugin.id].latest = pluginMap[plugin.id].latest || plugin.version; + } + pluginMap[plugin.id].outdated = semver.gt(pluginMap[plugin.id].latest, pluginMap[plugin.id].version); + }); + if (nconf.get('plugins:active')) { + nconf.get('plugins:active').forEach(id => { + pluginMap[id] = pluginMap[id] || {}; + pluginMap[id].active = true; + }); + } + const pluginArray = Object.values(pluginMap); + pluginArray.sort((a, b) => { + if (a.name > b.name) { + return 1; + } else if (a.name < b.name) { + return -1; + } + return 0; + }); + return pluginArray; +}; +Plugins.nodeModulesPath = paths.nodeModules; +Plugins.showInstalled = async function () { + const dirs = await fs.promises.readdir(Plugins.nodeModulesPath); + let pluginPaths = await findNodeBBModules(dirs); + pluginPaths = pluginPaths.map(dir => path.join(Plugins.nodeModulesPath, dir)); + async function load(file) { + try { + const pluginData = await Plugins.loadPluginInfo(file); + const isActive = await Plugins.isActive(pluginData.name); + delete pluginData.hooks; + delete pluginData.library; + pluginData.active = isActive; + pluginData.installed = true; + pluginData.error = false; + return pluginData; + } catch (err) { + winston.error(err.stack); + } + } + const plugins = await Promise.all(pluginPaths.map(file => load(file))); + return plugins.filter(Boolean); +}; +async function findNodeBBModules(dirs) { + const pluginPaths = []; + await Promise.all(dirs.map(async dirname => { + const dirPath = path.join(Plugins.nodeModulesPath, dirname); + const isDir = await isDirectory(dirPath); + if (!isDir) { + return; + } + if (pluginNamePattern.test(dirname)) { + pluginPaths.push(dirname); + return; + } + if (dirname[0] === '@') { + const subdirs = await fs.promises.readdir(dirPath); + await Promise.all(subdirs.map(async subdir => { + if (!pluginNamePattern.test(subdir)) { + return; + } + const subdirPath = path.join(dirPath, subdir); + const isDir = await isDirectory(subdirPath); + if (isDir) { + pluginPaths.push(`${dirname}/${subdir}`); + } + })); + } + })); + return pluginPaths; +} +async function isDirectory(dirPath) { + try { + const stats = await fs.promises.stat(dirPath); + return stats.isDirectory(); + } catch (err) { + if (err.code !== 'ENOENT') { + throw err; + } + return false; + } +} +require('../promisify')(Plugins); \ No newline at end of file diff --git a/lib/plugins/install.js b/lib/plugins/install.js new file mode 100644 index 0000000000..e1ca4d6551 --- /dev/null +++ b/lib/plugins/install.js @@ -0,0 +1,175 @@ +'use strict'; + +const winston = require('winston'); +const path = require('path'); +const fs = require('fs').promises; +const nconf = require('nconf'); +const os = require('os'); +const cproc = require('child_process'); +const util = require('util'); +const request = require('../request'); +const db = require('../database'); +const meta = require('../meta'); +const pubsub = require('../pubsub'); +const { + paths, + pluginNamePattern +} = require('../constants'); +const pkgInstall = require('../cli/package-install'); +const packageManager = pkgInstall.getPackageManager(); +let packageManagerExecutable = packageManager; +const packageManagerCommands = { + yarn: { + install: 'add', + uninstall: 'remove' + }, + npm: { + install: 'install', + uninstall: 'uninstall' + }, + cnpm: { + install: 'install', + uninstall: 'uninstall' + }, + pnpm: { + install: 'install', + uninstall: 'uninstall' + } +}; +if (process.platform === 'win32') { + packageManagerExecutable += '.cmd'; +} +module.exports = function (Plugins) { + if (nconf.get('isPrimary')) { + pubsub.on('plugins:toggleInstall', data => { + if (data.hostname !== os.hostname()) { + toggleInstall(data.id, data.version); + } + }); + pubsub.on('plugins:upgrade', data => { + if (data.hostname !== os.hostname()) { + upgrade(data.id, data.version); + } + }); + } + Plugins.toggleActive = async function (id) { + if (nconf.get('plugins:active')) { + winston.error('Cannot activate plugins while plugin state is set in the configuration (config.json, environmental variables or terminal arguments), please modify the configuration instead'); + throw new Error('[[error:plugins-set-in-configuration]]'); + } + if (!pluginNamePattern.test(id)) { + throw new Error('[[error:invalid-plugin-id]]'); + } + const isActive = await Plugins.isActive(id); + if (isActive) { + await db.sortedSetRemove('plugins:active', id); + } else { + const count = await db.sortedSetCard('plugins:active'); + await db.sortedSetAdd('plugins:active', count, id); + } + meta.reloadRequired = true; + const hook = isActive ? 'deactivate' : 'activate'; + Plugins.hooks.fire(`action:plugin.${hook}`, { + id: id + }); + return { + id: id, + active: !isActive + }; + }; + Plugins.checkWhitelist = async function (id, version) { + const { + response, + body + } = await request.get(`https://packages.nodebb.org/api/v1/plugins/${encodeURIComponent(id)}`); + if (!response.ok) { + throw new Error(`[[error:cant-connect-to-nbbpm]]`); + } + if (body && body.code === 'ok' && (version === 'latest' || body.payload.valid.includes(version))) { + return; + } + throw new Error('[[error:plugin-not-whitelisted]]'); + }; + Plugins.suggest = async function (pluginId, nbbVersion) { + const { + response, + body + } = await request.get(`https://packages.nodebb.org/api/v1/suggest?package=${encodeURIComponent(pluginId)}&version=${encodeURIComponent(nbbVersion)}`); + if (!response.ok) { + throw new Error(`[[error:cant-connect-to-nbbpm]]`); + } + return body; + }; + Plugins.toggleInstall = async function (id, version) { + pubsub.publish('plugins:toggleInstall', { + hostname: os.hostname(), + id: id, + version: version + }); + return await toggleInstall(id, version); + }; + const runPackageManagerCommandAsync = util.promisify(runPackageManagerCommand); + async function toggleInstall(id, version) { + const [installed, active] = await Promise.all([Plugins.isInstalled(id), Plugins.isActive(id)]); + const type = installed ? 'uninstall' : 'install'; + if (active && !nconf.get('plugins:active')) { + await Plugins.toggleActive(id); + } + await runPackageManagerCommandAsync(type, id, version || 'latest'); + const pluginData = await Plugins.get(id); + Plugins.hooks.fire(`action:plugin.${type}`, { + id: id, + version: version + }); + return pluginData; + } + function runPackageManagerCommand(command, pkgName, version, callback) { + cproc.execFile(packageManagerExecutable, [packageManagerCommands[packageManager][command], pkgName + (command === 'install' && version ? `@${version}` : ''), '--save'], (err, stdout) => { + if (err) { + return callback(err); + } + winston.verbose(`[plugins/${command}] ${stdout}`); + callback(); + }); + } + Plugins.upgrade = async function (id, version) { + pubsub.publish('plugins:upgrade', { + hostname: os.hostname(), + id: id, + version: version + }); + return await upgrade(id, version); + }; + async function upgrade(id, version) { + await runPackageManagerCommandAsync('install', id, version || 'latest'); + const isActive = await Plugins.isActive(id); + meta.reloadRequired = isActive; + return isActive; + } + Plugins.isInstalled = async function (id) { + const pluginDir = path.join(paths.nodeModules, id); + try { + const stats = await fs.stat(pluginDir); + return stats.isDirectory(); + } catch (err) { + return false; + } + }; + Plugins.isActive = async function (id) { + if (nconf.get('plugins:active')) { + return nconf.get('plugins:active').includes(id); + } + return await db.isSortedSetMember('plugins:active', id); + }; + Plugins.getActive = async function () { + if (nconf.get('plugins:active')) { + return nconf.get('plugins:active'); + } + return await db.getSortedSetRange('plugins:active', 0, -1); + }; + Plugins.autocomplete = async fragment => { + const pluginDir = paths.nodeModules; + const plugins = (await fs.readdir(pluginDir)).filter(filename => filename.startsWith(fragment)); + return plugins.length === 1 ? plugins.pop() : fragment; + }; +}; \ No newline at end of file diff --git a/lib/plugins/load.js b/lib/plugins/load.js new file mode 100644 index 0000000000..932ef0c46d --- /dev/null +++ b/lib/plugins/load.js @@ -0,0 +1,153 @@ +'use strict'; + +const semver = require('semver'); +const async = require('async'); +const winston = require('winston'); +const nconf = require('nconf'); +const _ = require('lodash'); +const meta = require('../meta'); +const { + themeNamePattern +} = require('../constants'); +module.exports = function (Plugins) { + async function registerPluginAssets(pluginData, fields) { + function add(dest, arr) { + dest.push(...(arr || [])); + } + const handlers = { + staticDirs: function (next) { + Plugins.data.getStaticDirectories(pluginData, next); + }, + cssFiles: function (next) { + Plugins.data.getFiles(pluginData, 'css', next); + }, + scssFiles: function (next) { + Plugins.data.getFiles(pluginData, 'scss', next); + }, + acpScssFiles: function (next) { + Plugins.data.getFiles(pluginData, 'acpScss', next); + }, + clientScripts: function (next) { + Plugins.data.getScripts(pluginData, 'client', next); + }, + acpScripts: function (next) { + Plugins.data.getScripts(pluginData, 'acp', next); + }, + modules: function (next) { + Plugins.data.getModules(pluginData, next); + }, + languageData: function (next) { + Plugins.data.getLanguageData(pluginData, next); + } + }; + let methods = {}; + if (Array.isArray(fields)) { + fields.forEach(field => { + methods[field] = handlers[field]; + }); + } else { + methods = handlers; + } + const results = await async.parallel(methods); + Object.assign(Plugins.staticDirs, results.staticDirs || {}); + add(Plugins.cssFiles, results.cssFiles); + add(Plugins.scssFiles, results.scssFiles); + add(Plugins.acpScssFiles, results.acpScssFiles); + add(Plugins.clientScripts, results.clientScripts); + add(Plugins.acpScripts, results.acpScripts); + Object.assign(meta.js.scripts.modules, results.modules || {}); + if (results.languageData) { + Plugins.languageData.languages = _.union(Plugins.languageData.languages, results.languageData.languages); + Plugins.languageData.namespaces = _.union(Plugins.languageData.namespaces, results.languageData.namespaces); + pluginData.languageData = results.languageData; + } + Plugins.pluginsData[pluginData.id] = pluginData; + } + Plugins.prepareForBuild = async function (targets) { + const map = { + 'plugin static dirs': ['staticDirs'], + 'requirejs modules': ['modules'], + 'client js bundle': ['clientScripts'], + 'admin js bundle': ['acpScripts'], + 'client side styles': ['cssFiles', 'scssFiles'], + 'admin control panel styles': ['cssFiles', 'scssFiles', 'acpScssFiles'], + languages: ['languageData'] + }; + const fields = _.uniq(_.flatMap(targets, target => map[target] || [])); + fields.forEach(field => { + switch (field) { + case 'clientScripts': + case 'acpScripts': + case 'cssFiles': + case 'scssFiles': + case 'acpScssFiles': + Plugins[field].length = 0; + break; + case 'languageData': + Plugins.languageData.languages = []; + Plugins.languageData.namespaces = []; + break; + } + }); + winston.verbose(`[plugins] loading the following fields from plugin data: ${fields.join(', ')}`); + const plugins = await Plugins.data.getActive(); + await Promise.all(plugins.map(p => registerPluginAssets(p, fields))); + }; + Plugins.loadPlugin = async function (pluginPath) { + let pluginData; + try { + pluginData = await Plugins.data.loadPluginInfo(pluginPath); + } catch (err) { + if (err.message === '[[error:parse-error]]') { + return; + } + if (!themeNamePattern.test(pluginPath)) { + throw err; + } + return; + } + checkVersion(pluginData); + try { + registerHooks(pluginData); + await registerPluginAssets(pluginData); + } catch (err) { + winston.error(err.stack); + winston.verbose(`[plugins] Could not load plugin : ${pluginData.id}`); + return; + } + if (!pluginData.private) { + Plugins.loadedPlugins.push({ + id: pluginData.id, + version: pluginData.version + }); + } + winston.verbose(`[plugins] Loaded plugin: ${pluginData.id}`); + }; + function checkVersion(pluginData) { + function add() { + if (!Plugins.versionWarning.includes(pluginData.id)) { + Plugins.versionWarning.push(pluginData.id); + } + } + if (pluginData.nbbpm && pluginData.nbbpm.compatibility && semver.validRange(pluginData.nbbpm.compatibility)) { + if (!semver.satisfies(nconf.get('version'), pluginData.nbbpm.compatibility)) { + add(); + } + } else { + add(); + } + } + function registerHooks(pluginData) { + try { + if (!Plugins.libraries[pluginData.id]) { + Plugins.requireLibrary(pluginData); + } + if (Array.isArray(pluginData.hooks)) { + pluginData.hooks.forEach(hook => Plugins.hooks.register(pluginData.id, hook)); + } + } catch (err) { + winston.warn(`[plugins] Unable to load library for: ${pluginData.id}`); + throw err; + } + } +}; \ No newline at end of file diff --git a/lib/plugins/usage.js b/lib/plugins/usage.js new file mode 100644 index 0000000000..b7ce1b14cf --- /dev/null +++ b/lib/plugins/usage.js @@ -0,0 +1,42 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const crypto = require('crypto'); +const cronJob = require('cron').CronJob; +const request = require('../request'); +const pkg = require('../../package.json'); +const meta = require('../meta'); +module.exports = function (Plugins) { + Plugins.startJobs = function () { + new cronJob('0 0 0 * * *', async () => { + await Plugins.submitUsageData(); + }, null, true); + }; + Plugins.submitUsageData = async function () { + if (!meta.config.submitPluginUsage || !Plugins.loadedPlugins.length || global.env !== 'production') { + return; + } + const hash = crypto.createHash('sha256'); + hash.update(nconf.get('url')); + const url = `${nconf.get('registry') || 'https://packages.nodebb.org'}/api/v1/plugin/usage`; + try { + const { + response, + body + } = await request.post(url, { + body: { + id: hash.digest('hex'), + version: pkg.version, + plugins: Plugins.loadedPlugins + }, + timeout: 5000 + }); + if (!response.ok) { + winston.error(`[plugins.submitUsageData] received ${response.status} ${body}`); + } + } catch (err) { + winston.error(err.stack); + } + }; +}; \ No newline at end of file diff --git a/lib/posts/bookmarks.js b/lib/posts/bookmarks.js new file mode 100644 index 0000000000..b84560cb76 --- /dev/null +++ b/lib/posts/bookmarks.js @@ -0,0 +1,53 @@ +'use strict'; + +const db = require('../database'); +const plugins = require('../plugins'); +module.exports = function (Posts) { + Posts.bookmark = async function (pid, uid) { + return await toggleBookmark('bookmark', pid, uid); + }; + Posts.unbookmark = async function (pid, uid) { + return await toggleBookmark('unbookmark', pid, uid); + }; + async function toggleBookmark(type, pid, uid) { + if (parseInt(uid, 10) <= 0) { + throw new Error('[[error:not-logged-in]]'); + } + const isBookmarking = type === 'bookmark'; + const [postData, hasBookmarked] = await Promise.all([Posts.getPostFields(pid, ['pid', 'uid']), Posts.hasBookmarked(pid, uid)]); + if (isBookmarking && hasBookmarked) { + throw new Error('[[error:already-bookmarked]]'); + } + if (!isBookmarking && !hasBookmarked) { + throw new Error('[[error:already-unbookmarked]]'); + } + if (isBookmarking) { + await db.sortedSetAdd(`uid:${uid}:bookmarks`, Date.now(), pid); + } else { + await db.sortedSetRemove(`uid:${uid}:bookmarks`, pid); + } + await db[isBookmarking ? 'setAdd' : 'setRemove'](`pid:${pid}:users_bookmarked`, uid); + postData.bookmarks = await db.setCount(`pid:${pid}:users_bookmarked`); + await Posts.setPostField(pid, 'bookmarks', postData.bookmarks); + plugins.hooks.fire(`action:post.${type}`, { + pid: pid, + uid: uid, + owner: postData.uid, + current: hasBookmarked ? 'bookmarked' : 'unbookmarked' + }); + return { + post: postData, + isBookmarked: isBookmarking + }; + } + Posts.hasBookmarked = async function (pid, uid) { + if (parseInt(uid, 10) <= 0) { + return Array.isArray(pid) ? pid.map(() => false) : false; + } + if (Array.isArray(pid)) { + const sets = pid.map(pid => `pid:${pid}:users_bookmarked`); + return await db.isMemberOfSets(sets, uid); + } + return await db.isSetMember(`pid:${pid}:users_bookmarked`, uid); + }; +}; \ No newline at end of file diff --git a/lib/posts/cache.js b/lib/posts/cache.js new file mode 100644 index 0000000000..a9ad419fc7 --- /dev/null +++ b/lib/posts/cache.js @@ -0,0 +1,29 @@ +'use strict'; + +let cache = null; +exports.getOrCreate = function () { + if (!cache) { + const cacheCreate = require('../cache/lru'); + const meta = require('../meta'); + cache = cacheCreate({ + name: 'post', + maxSize: meta.config.postCacheSize, + sizeCalculation: function (n) { + return n.length || 1; + }, + ttl: 0, + enabled: global.env === 'production' + }); + } + return cache; +}; +exports.del = function (pid) { + if (cache) { + cache.del(pid); + } +}; +exports.reset = function () { + if (cache) { + cache.reset(); + } +}; \ No newline at end of file diff --git a/lib/posts/category.js b/lib/posts/category.js new file mode 100644 index 0000000000..f32ee12fb4 --- /dev/null +++ b/lib/posts/category.js @@ -0,0 +1,33 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const topics = require('../topics'); +module.exports = function (Posts) { + Posts.getCidByPid = async function (pid) { + const tid = await Posts.getPostField(pid, 'tid'); + return await topics.getTopicField(tid, 'cid'); + }; + Posts.getCidsByPids = async function (pids) { + const postData = await Posts.getPostsFields(pids, ['tid']); + const tids = _.uniq(postData.map(post => post && post.tid).filter(Boolean)); + const topicData = await topics.getTopicsFields(tids, ['cid']); + const tidToTopic = _.zipObject(tids, topicData); + const cids = postData.map(post => tidToTopic[post.tid] && tidToTopic[post.tid].cid); + return cids; + }; + Posts.filterPidsByCid = async function (pids, cid) { + if (!cid) { + return pids; + } + if (!Array.isArray(cid) || cid.length === 1) { + return await filterPidsBySingleCid(pids, cid); + } + const pidsArr = await Promise.all(cid.map(c => Posts.filterPidsByCid(pids, c))); + return _.union(...pidsArr); + }; + async function filterPidsBySingleCid(pids, cid) { + const isMembers = await db.isSortedSetMembers(`cid:${parseInt(cid, 10)}:pids`, pids); + return pids.filter((pid, index) => pid && isMembers[index]); + } +}; \ No newline at end of file diff --git a/lib/posts/create.js b/lib/posts/create.js new file mode 100644 index 0000000000..7fa5bf25c0 --- /dev/null +++ b/lib/posts/create.js @@ -0,0 +1,78 @@ +'use strict'; + +const _ = require('lodash'); +const meta = require('../meta'); +const db = require('../database'); +const plugins = require('../plugins'); +const user = require('../user'); +const topics = require('../topics'); +const categories = require('../categories'); +const groups = require('../groups'); +const privileges = require('../privileges'); +module.exports = function (Posts) { + Posts.create = async function (data) { + const { + uid + } = data; + const { + tid + } = data; + const content = data.content.toString(); + const timestamp = data.timestamp || Date.now(); + const isMain = data.isMain || false; + if (!uid && parseInt(uid, 10) !== 0) { + throw new Error('[[error:invalid-uid]]'); + } + if (data.toPid) { + await checkToPid(data.toPid, uid); + } + const pid = await db.incrObjectField('global', 'nextPid'); + let postData = { + pid: pid, + uid: uid, + tid: tid, + content: content, + timestamp: timestamp + }; + if (data.toPid) { + postData.toPid = data.toPid; + } + if (data.ip && meta.config.trackIpPerPost) { + postData.ip = data.ip; + } + if (data.handle && !parseInt(uid, 10)) { + postData.handle = data.handle; + } + let result = await plugins.hooks.fire('filter:post.create', { + post: postData, + data: data + }); + postData = result.post; + await db.setObject(`post:${postData.pid}`, postData); + const topicData = await topics.getTopicFields(tid, ['cid', 'pinned']); + postData.cid = topicData.cid; + await Promise.all([db.sortedSetAdd('posts:pid', timestamp, postData.pid), db.incrObjectField('global', 'postCount'), user.onNewPostMade(postData), topics.onNewPostMade(postData), categories.onNewPostMade(topicData.cid, topicData.pinned, postData), groups.onNewPostMade(postData), addReplyTo(postData, timestamp), Posts.uploads.sync(postData.pid)]); + result = await plugins.hooks.fire('filter:post.get', { + post: postData, + uid: data.uid + }); + result.post.isMain = isMain; + plugins.hooks.fire('action:post.save', { + post: _.clone(result.post) + }); + return result.post; + }; + async function addReplyTo(postData, timestamp) { + if (!postData.toPid) { + return; + } + await Promise.all([db.sortedSetAdd(`pid:${postData.toPid}:replies`, timestamp, postData.pid), db.incrObjectField(`post:${postData.toPid}`, 'replies')]); + } + async function checkToPid(toPid, uid) { + const [toPost, canViewToPid] = await Promise.all([Posts.getPostFields(toPid, ['pid', 'deleted']), privileges.posts.can('posts:view_deleted', toPid, uid)]); + const toPidExists = !!toPost.pid; + if (!toPidExists || toPost.deleted && !canViewToPid) { + throw new Error('[[error:invalid-pid]]'); + } + } +}; \ No newline at end of file diff --git a/lib/posts/data.js b/lib/posts/data.js new file mode 100644 index 0000000000..809766a01a --- /dev/null +++ b/lib/posts/data.js @@ -0,0 +1,65 @@ +'use strict'; + +const db = require('../database'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const intFields = ['uid', 'pid', 'tid', 'deleted', 'timestamp', 'upvotes', 'downvotes', 'deleterUid', 'edited', 'replies', 'bookmarks']; +module.exports = function (Posts) { + Posts.getPostsFields = async function (pids, fields) { + if (!Array.isArray(pids) || !pids.length) { + return []; + } + const keys = pids.map(pid => `post:${pid}`); + const postData = await db.getObjects(keys, fields); + const result = await plugins.hooks.fire('filter:post.getFields', { + pids: pids, + posts: postData, + fields: fields + }); + result.posts.forEach(post => modifyPost(post, fields)); + return result.posts; + }; + Posts.getPostData = async function (pid) { + const posts = await Posts.getPostsFields([pid], []); + return posts && posts.length ? posts[0] : null; + }; + Posts.getPostsData = async function (pids) { + return await Posts.getPostsFields(pids, []); + }; + Posts.getPostField = async function (pid, field) { + const post = await Posts.getPostFields(pid, [field]); + return post ? post[field] : null; + }; + Posts.getPostFields = async function (pid, fields) { + const posts = await Posts.getPostsFields([pid], fields); + return posts ? posts[0] : null; + }; + Posts.setPostField = async function (pid, field, value) { + await Posts.setPostFields(pid, { + [field]: value + }); + }; + Posts.setPostFields = async function (pid, data) { + await db.setObject(`post:${pid}`, data); + plugins.hooks.fire('action:post.setFields', { + data: { + ...data, + pid + } + }); + }; +}; +function modifyPost(post, fields) { + if (post) { + db.parseIntFields(post, intFields, fields); + if (post.hasOwnProperty('upvotes') && post.hasOwnProperty('downvotes')) { + post.votes = post.upvotes - post.downvotes; + } + if (post.hasOwnProperty('timestamp')) { + post.timestampISO = utils.toISOString(post.timestamp); + } + if (post.hasOwnProperty('edited')) { + post.editedISO = post.edited !== 0 ? utils.toISOString(post.edited) : ''; + } + } +} \ No newline at end of file diff --git a/lib/posts/delete.js b/lib/posts/delete.js new file mode 100644 index 0000000000..b4815ee055 --- /dev/null +++ b/lib/posts/delete.js @@ -0,0 +1,194 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const topics = require('../topics'); +const categories = require('../categories'); +const user = require('../user'); +const notifications = require('../notifications'); +const plugins = require('../plugins'); +const flags = require('../flags'); +module.exports = function (Posts) { + Posts.delete = async function (pid, uid) { + return await deleteOrRestore('delete', pid, uid); + }; + Posts.restore = async function (pid, uid) { + return await deleteOrRestore('restore', pid, uid); + }; + async function deleteOrRestore(type, pid, uid) { + const isDeleting = type === 'delete'; + await plugins.hooks.fire(`filter:post.${type}`, { + pid: pid, + uid: uid + }); + await Posts.setPostFields(pid, { + deleted: isDeleting ? 1 : 0, + deleterUid: isDeleting ? uid : 0 + }); + const postData = await Posts.getPostFields(pid, ['pid', 'tid', 'uid', 'content', 'timestamp']); + const topicData = await topics.getTopicFields(postData.tid, ['tid', 'cid', 'pinned']); + postData.cid = topicData.cid; + await Promise.all([topics.updateLastPostTimeFromLastPid(postData.tid), topics.updateTeaser(postData.tid), isDeleting ? db.sortedSetRemove(`cid:${topicData.cid}:pids`, pid) : db.sortedSetAdd(`cid:${topicData.cid}:pids`, postData.timestamp, pid)]); + await categories.updateRecentTidForCid(postData.cid); + plugins.hooks.fire(`action:post.${type}`, { + post: _.clone(postData), + uid: uid + }); + if (type === 'delete') { + await flags.resolveFlag('post', pid, uid); + } + return postData; + } + Posts.purge = async function (pids, uid) { + pids = Array.isArray(pids) ? pids : [pids]; + let postData = await Posts.getPostsData(pids); + pids = pids.filter((pid, index) => !!postData[index]); + postData = postData.filter(Boolean); + if (!postData.length) { + return; + } + const uniqTids = _.uniq(postData.map(p => p.tid)); + const topicData = await topics.getTopicsFields(uniqTids, ['tid', 'cid', 'pinned', 'postcount']); + const tidToTopic = _.zipObject(uniqTids, topicData); + postData.forEach(p => { + p.topic = tidToTopic[p.tid]; + p.cid = tidToTopic[p.tid] && tidToTopic[p.tid].cid; + }); + await Promise.all(postData.map(p => plugins.hooks.fire('filter:post.purge', { + post: p, + pid: p.pid, + uid: uid + }))); + await plugins.hooks.fire('filter:posts.purge', { + posts: postData, + pids: postData.map(p => p.pid), + uid: uid + }); + await Promise.all([deleteFromTopicUserNotification(postData), deleteFromCategoryRecentPosts(postData), deleteFromUsersBookmarks(pids), deleteFromUsersVotes(pids), deleteFromReplies(postData), deleteFromGroups(pids), deleteDiffs(pids), deleteFromUploads(pids), db.sortedSetsRemove(['posts:pid', 'posts:votes', 'posts:flagged'], pids)]); + await resolveFlags(postData, uid); + Promise.all(postData.map(p => plugins.hooks.fire('action:post.purge', { + post: p, + uid: uid + }))); + plugins.hooks.fire('action:posts.purge', { + posts: postData, + uid: uid + }); + await db.deleteAll(postData.map(p => `post:${p.pid}`)); + }; + async function deleteFromTopicUserNotification(postData) { + const bulkRemove = []; + postData.forEach(p => { + bulkRemove.push([`tid:${p.tid}:posts`, p.pid]); + bulkRemove.push([`tid:${p.tid}:posts:votes`, p.pid]); + bulkRemove.push([`uid:${p.uid}:posts`, p.pid]); + bulkRemove.push([`cid:${p.cid}:uid:${p.uid}:pids`, p.pid]); + bulkRemove.push([`cid:${p.cid}:uid:${p.uid}:pids:votes`, p.pid]); + }); + await db.sortedSetRemoveBulk(bulkRemove); + const incrObjectBulk = [['global', { + postCount: -postData.length + }]]; + const postsByCategory = _.groupBy(postData, p => parseInt(p.cid, 10)); + for (const [cid, posts] of Object.entries(postsByCategory)) { + incrObjectBulk.push([`category:${cid}`, { + post_count: -posts.length + }]); + } + const postsByTopic = _.groupBy(postData, p => parseInt(p.tid, 10)); + const topicPostCountTasks = []; + const topicTasks = []; + const zsetIncrBulk = []; + const tids = []; + for (const [tid, posts] of Object.entries(postsByTopic)) { + tids.push(tid); + incrObjectBulk.push([`topic:${tid}`, { + postcount: -posts.length + }]); + if (posts.length && posts[0]) { + const topicData = posts[0].topic; + const newPostCount = topicData.postcount - posts.length; + topicPostCountTasks.push(['topics:posts', newPostCount, tid]); + if (!topicData.pinned) { + zsetIncrBulk.push([`cid:${topicData.cid}:tids:posts`, -posts.length, tid]); + } + } + topicTasks.push(topics.updateTeaser(tid)); + topicTasks.push(topics.updateLastPostTimeFromLastPid(tid)); + const postsByUid = _.groupBy(posts, p => parseInt(p.uid, 10)); + for (const [uid, uidPosts] of Object.entries(postsByUid)) { + zsetIncrBulk.push([`tid:${tid}:posters`, -uidPosts.length, uid]); + } + topicTasks.push(db.sortedSetIncrByBulk(zsetIncrBulk)); + } + await Promise.all([db.incrObjectFieldByBulk(incrObjectBulk), db.sortedSetAddBulk(topicPostCountTasks), ...topicTasks, user.updatePostCount(_.uniq(postData.map(p => p.uid))), notifications.rescind(...postData.map(p => `new_post:tid:${p.tid}:pid:${p.pid}:uid:${p.uid}`))]); + const tidPosterZsets = tids.map(tid => `tid:${tid}:posters`); + await db.sortedSetsRemoveRangeByScore(tidPosterZsets, '-inf', 0); + const posterCounts = await db.sortedSetsCard(tidPosterZsets); + await db.setObjectBulk(tids.map((tid, idx) => [`topic:${tid}`, { + postercount: posterCounts[idx] || 0 + }])); + } + async function deleteFromCategoryRecentPosts(postData) { + const uniqCids = _.uniq(postData.map(p => p.cid)); + const sets = uniqCids.map(cid => `cid:${cid}:pids`); + await db.sortedSetRemove(sets, postData.map(p => p.pid)); + await Promise.all(uniqCids.map(categories.updateRecentTidForCid)); + } + async function deleteFromUsersBookmarks(pids) { + const arrayOfUids = await db.getSetsMembers(pids.map(pid => `pid:${pid}:users_bookmarked`)); + const bulkRemove = []; + pids.forEach((pid, index) => { + arrayOfUids[index].forEach(uid => { + bulkRemove.push([`uid:${uid}:bookmarks`, pid]); + }); + }); + await db.sortedSetRemoveBulk(bulkRemove); + await db.deleteAll(pids.map(pid => `pid:${pid}:users_bookmarked`)); + } + async function deleteFromUsersVotes(pids) { + const [upvoters, downvoters] = await Promise.all([db.getSetsMembers(pids.map(pid => `pid:${pid}:upvote`)), db.getSetsMembers(pids.map(pid => `pid:${pid}:downvote`))]); + const bulkRemove = []; + pids.forEach((pid, index) => { + upvoters[index].forEach(upvoterUid => { + bulkRemove.push([`uid:${upvoterUid}:upvote`, pid]); + }); + downvoters[index].forEach(downvoterUid => { + bulkRemove.push([`uid:${downvoterUid}:downvote`, pid]); + }); + }); + await Promise.all([db.sortedSetRemoveBulk(bulkRemove), db.deleteAll([...pids.map(pid => `pid:${pid}:upvote`), ...pids.map(pid => `pid:${pid}:downvote`)])]); + } + async function deleteFromReplies(postData) { + const arrayOfReplyPids = await db.getSortedSetsMembers(postData.map(p => `pid:${p.pid}:replies`)); + const allReplyPids = _.flatten(arrayOfReplyPids); + const promises = [db.deleteObjectFields(allReplyPids.map(pid => `post:${pid}`), ['toPid']), db.deleteAll(postData.map(p => `pid:${p.pid}:replies`))]; + const postsWithParents = postData.filter(p => parseInt(p.toPid, 10)); + const bulkRemove = postsWithParents.map(p => [`pid:${p.toPid}:replies`, p.pid]); + promises.push(db.sortedSetRemoveBulk(bulkRemove)); + await Promise.all(promises); + const parentPids = _.uniq(postsWithParents.map(p => p.toPid)); + const counts = await db.sortedSetsCard(parentPids.map(pid => `pid:${pid}:replies`)); + await db.setObjectBulk(parentPids.map((pid, index) => [`post:${pid}`, { + replies: counts[index] + }])); + } + async function deleteFromGroups(pids) { + const groupNames = await db.getSortedSetMembers('groups:visible:createtime'); + const keys = groupNames.map(groupName => `group:${groupName}:member:pids`); + await db.sortedSetRemove(keys, pids); + } + async function deleteDiffs(pids) { + const timestamps = await Promise.all(pids.map(pid => Posts.diffs.list(pid))); + await db.deleteAll([...pids.map(pid => `post:${pid}:diffs`), ..._.flattenDeep(pids.map((pid, index) => timestamps[index].map(t => `diff:${pid}.${t}`)))]); + } + async function deleteFromUploads(pids) { + await Promise.all(pids.map(Posts.uploads.dissociateAll)); + } + async function resolveFlags(postData, uid) { + const flaggedPosts = postData.filter(p => p && parseInt(p.flagId, 10)); + await Promise.all(flaggedPosts.map(p => flags.update(p.flagId, uid, { + state: 'resolved' + }))); + } +}; \ No newline at end of file diff --git a/lib/posts/diffs.js b/lib/posts/diffs.js new file mode 100644 index 0000000000..0a3287050e --- /dev/null +++ b/lib/posts/diffs.js @@ -0,0 +1,143 @@ +'use strict'; + +const validator = require('validator'); +const diff = require('diff'); +const db = require('../database'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const translator = require('../translator'); +const topics = require('../topics'); +module.exports = function (Posts) { + const Diffs = {}; + Posts.diffs = Diffs; + Diffs.exists = async function (pid) { + if (meta.config.enablePostHistory !== 1) { + return false; + } + const numDiffs = await db.listLength(`post:${pid}:diffs`); + return !!numDiffs; + }; + Diffs.get = async function (pid, since) { + const timestamps = await Diffs.list(pid); + if (!since) { + since = 0; + } + const keys = timestamps.filter(t => (parseInt(t, 10) || 0) > since).map(t => `diff:${pid}.${t}`); + return await db.getObjects(keys); + }; + Diffs.list = async function (pid) { + return await db.getListRange(`post:${pid}:diffs`, 0, -1); + }; + Diffs.save = async function (data) { + const { + pid, + uid, + oldContent, + newContent, + edited, + topic + } = data; + const editTimestamp = edited || Date.now(); + const diffData = { + uid: uid, + pid: pid + }; + if (oldContent !== newContent) { + diffData.patch = diff.createPatch('', newContent, oldContent); + } + if (topic.renamed) { + diffData.title = topic.oldTitle; + } + if (topic.tagsupdated && Array.isArray(topic.oldTags)) { + diffData.tags = topic.oldTags.map(tag => tag && tag.value).filter(Boolean).join(','); + } + await Promise.all([db.listPrepend(`post:${pid}:diffs`, editTimestamp), db.setObject(`diff:${pid}.${editTimestamp}`, diffData)]); + }; + Diffs.load = async function (pid, since, uid) { + since = getValidatedTimestamp(since); + const post = await postDiffLoad(pid, since, uid); + post.content = String(post.content || ''); + const result = await plugins.hooks.fire('filter:parse.post', { + postData: post + }); + result.postData.content = translator.escape(result.postData.content); + return result.postData; + }; + Diffs.restore = async function (pid, since, uid, req) { + since = getValidatedTimestamp(since); + const post = await postDiffLoad(pid, since, uid); + return await Posts.edit({ + uid: uid, + pid: pid, + content: post.content, + req: req, + timestamp: since, + title: post.topic.title, + tags: post.topic.tags.map(tag => tag.value) + }); + }; + Diffs.delete = async function (pid, timestamp, uid) { + getValidatedTimestamp(timestamp); + const [post, diffs, timestamps] = await Promise.all([Posts.getPostSummaryByPids([pid], uid, { + parse: false + }), Diffs.get(pid), Diffs.list(pid)]); + const timestampIndex = timestamps.indexOf(timestamp); + const lastTimestampIndex = timestamps.length - 1; + if (timestamp === String(post[0].timestamp)) { + return Promise.all([db.delete(`diff:${pid}.${timestamps[lastTimestampIndex]}`), db.listRemoveAll(`post:${pid}:diffs`, timestamps[lastTimestampIndex])]); + } + if (timestampIndex === 0 || timestampIndex === -1) { + throw new Error('[[error:invalid-data]]'); + } + const postContent = validator.unescape(post[0].content); + const versionContents = {}; + for (let i = 0, content = postContent; i < timestamps.length; ++i) { + versionContents[timestamps[i]] = applyPatch(content, diffs[i]); + content = versionContents[timestamps[i]]; + } + for (let i = lastTimestampIndex; i >= timestampIndex; --i) { + const newContentIndex = i === timestampIndex ? i - 2 : i - 1; + const timestampToUpdate = newContentIndex + 1; + const newContent = newContentIndex < 0 ? postContent : versionContents[timestamps[newContentIndex]]; + const patch = diff.createPatch('', newContent, versionContents[timestamps[i]]); + await db.setObject(`diff:${pid}.${timestamps[timestampToUpdate]}`, { + patch + }); + } + return Promise.all([db.delete(`diff:${pid}.${timestamp}`), db.listRemoveAll(`post:${pid}:diffs`, timestamp)]); + }; + async function postDiffLoad(pid, since, uid) { + const [post, diffs] = await Promise.all([Posts.getPostSummaryByPids([pid], uid, { + parse: false + }), Posts.diffs.get(pid, since)]); + post[0].content = diffs.reduce(applyPatch, validator.unescape(post[0].content)); + const titleDiffs = diffs.filter(d => d.hasOwnProperty('title') && d.title); + if (titleDiffs.length && post[0].topic) { + post[0].topic.title = validator.unescape(String(titleDiffs[titleDiffs.length - 1].title)); + } + const tagDiffs = diffs.filter(d => d.hasOwnProperty('tags') && d.tags); + if (tagDiffs.length && post[0].topic) { + const tags = tagDiffs[tagDiffs.length - 1].tags.split(',').map(tag => ({ + value: tag + })); + post[0].topic.tags = topics.getTagData(tags); + } + return post[0]; + } + function getValidatedTimestamp(timestamp) { + timestamp = parseInt(timestamp, 10); + if (isNaN(timestamp)) { + throw new Error('[[error:invalid-data]]'); + } + return timestamp; + } + function applyPatch(content, aDiff) { + if (aDiff && aDiff.patch) { + const result = diff.applyPatch(content, aDiff.patch, { + fuzzFactor: 1 + }); + return typeof result === 'string' ? result : content; + } + return content; + } +}; \ No newline at end of file diff --git a/lib/posts/edit.js b/lib/posts/edit.js new file mode 100644 index 0000000000..bc3bf5b1e6 --- /dev/null +++ b/lib/posts/edit.js @@ -0,0 +1,185 @@ +'use strict'; + +const validator = require('validator'); +const _ = require('lodash'); +const db = require('../database'); +const meta = require('../meta'); +const topics = require('../topics'); +const user = require('../user'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const pubsub = require('../pubsub'); +const utils = require('../utils'); +const slugify = require('../slugify'); +const translator = require('../translator'); +module.exports = function (Posts) { + pubsub.on('post:edit', pid => { + require('./cache').del(pid); + }); + Posts.edit = async function (data) { + const canEdit = await privileges.posts.canEdit(data.pid, data.uid); + if (!canEdit.flag) { + throw new Error(canEdit.message); + } + const postData = await Posts.getPostData(data.pid); + if (!postData) { + throw new Error('[[error:no-post]]'); + } + const topicData = await topics.getTopicFields(postData.tid, ['cid', 'mainPid', 'title', 'timestamp', 'scheduled', 'slug', 'tags']); + await scheduledTopicCheck(data, topicData); + const oldContent = postData.content; + const editPostData = getEditPostData(data, topicData, postData); + if (data.handle) { + editPostData.handle = data.handle; + } + const result = await plugins.hooks.fire('filter:post.edit', { + req: data.req, + post: editPostData, + data: data, + uid: data.uid + }); + const [editor, topic] = await Promise.all([user.getUserFields(data.uid, ['username', 'userslug']), editMainPost(data, postData, topicData)]); + await Posts.setPostFields(data.pid, result.post); + const contentChanged = data.content !== oldContent || topic.renamed || topic.tagsupdated; + if (meta.config.enablePostHistory === 1 && contentChanged) { + await Posts.diffs.save({ + pid: data.pid, + uid: data.uid, + oldContent: oldContent, + newContent: data.content, + edited: editPostData.edited, + topic + }); + } + await Posts.uploads.sync(data.pid); + postData.deleted = !!postData.deleted; + const returnPostData = { + ...postData, + ...result.post + }; + returnPostData.cid = topic.cid; + returnPostData.topic = topic; + returnPostData.editedISO = utils.toISOString(editPostData.edited); + returnPostData.changed = contentChanged; + returnPostData.oldContent = oldContent; + returnPostData.newContent = data.content; + await topics.notifyFollowers(returnPostData, data.uid, { + type: 'post-edit', + bodyShort: translator.compile('notifications:user-edited-post', editor.username, topic.title), + nid: `edit_post:${data.pid}:uid:${data.uid}` + }); + await topics.syncBacklinks(returnPostData); + plugins.hooks.fire('action:post.edit', { + post: _.clone(returnPostData), + data: data, + uid: data.uid + }); + require('./cache').del(String(postData.pid)); + pubsub.publish('post:edit', String(postData.pid)); + await Posts.parsePost(returnPostData); + return { + topic: topic, + editor: editor, + post: returnPostData + }; + }; + async function editMainPost(data, postData, topicData) { + const { + tid + } = postData; + const title = data.title ? data.title.trim() : ''; + const isMain = parseInt(data.pid, 10) === parseInt(topicData.mainPid, 10); + if (!isMain) { + return { + tid: tid, + cid: topicData.cid, + title: topicData.title, + isMainPost: false, + renamed: false, + tagsupdated: false + }; + } + const newTopicData = { + tid: tid, + cid: topicData.cid, + uid: postData.uid, + mainPid: data.pid, + timestamp: rescheduling(data, topicData) ? data.timestamp : topicData.timestamp + }; + if (title) { + newTopicData.title = title; + newTopicData.slug = `${tid}/${slugify(title) || 'topic'}`; + } + const tagsupdated = Array.isArray(data.tags) && !_.isEqual(data.tags, topicData.tags.map(tag => tag.value)); + if (tagsupdated) { + const canTag = await privileges.categories.can('topics:tag', topicData.cid, data.uid); + if (!canTag) { + throw new Error('[[error:no-privileges]]'); + } + await topics.validateTags(data.tags, topicData.cid, data.uid, tid); + } + const results = await plugins.hooks.fire('filter:topic.edit', { + req: data.req, + topic: newTopicData, + data: data + }); + await db.setObject(`topic:${tid}`, results.topic); + if (tagsupdated) { + await topics.updateTopicTags(tid, data.tags); + } + const tags = await topics.getTopicTagsObjects(tid); + if (rescheduling(data, topicData)) { + await topics.scheduled.reschedule(newTopicData); + } + newTopicData.tags = data.tags; + newTopicData.oldTitle = topicData.title; + const renamed = title && translator.escape(validator.escape(String(title))) !== topicData.title; + plugins.hooks.fire('action:topic.edit', { + topic: newTopicData, + uid: data.uid + }); + return { + tid: tid, + cid: newTopicData.cid, + uid: postData.uid, + title: validator.escape(String(title)), + oldTitle: topicData.title, + slug: newTopicData.slug || topicData.slug, + isMainPost: true, + renamed: renamed, + tagsupdated: tagsupdated, + tags: tags, + oldTags: topicData.tags, + rescheduled: rescheduling(data, topicData) + }; + } + async function scheduledTopicCheck(data, topicData) { + if (!topicData.scheduled) { + return; + } + const canSchedule = await privileges.categories.can('topics:schedule', topicData.cid, data.uid); + if (!canSchedule) { + throw new Error('[[error:no-privileges]]'); + } + const isMain = parseInt(data.pid, 10) === parseInt(topicData.mainPid, 10); + if (isMain && isNaN(data.timestamp)) { + throw new Error('[[error:invalid-data]]'); + } + } + function getEditPostData(data, topicData, postData) { + const editPostData = { + content: data.content, + editor: data.uid + }; + editPostData.edited = topicData.scheduled ? (postData.edited || postData.timestamp) + 1 : Date.now(); + if (rescheduling(data, topicData)) { + editPostData.edited = data.timestamp; + editPostData.timestamp = data.timestamp; + } + return editPostData; + } + function rescheduling(data, topicData) { + const isMain = parseInt(data.pid, 10) === parseInt(topicData.mainPid, 10); + return isMain && topicData.scheduled && topicData.timestamp !== data.timestamp; + } +}; \ No newline at end of file diff --git a/lib/posts/index.js b/lib/posts/index.js new file mode 100644 index 0000000000..8c70698612 --- /dev/null +++ b/lib/posts/index.js @@ -0,0 +1,96 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const utils = require('../utils'); +const user = require('../user'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const Posts = module.exports; +require('./data')(Posts); +require('./create')(Posts); +require('./delete')(Posts); +require('./edit')(Posts); +require('./parse')(Posts); +require('./user')(Posts); +require('./topics')(Posts); +require('./category')(Posts); +require('./summary')(Posts); +require('./recent')(Posts); +require('./tools')(Posts); +require('./votes')(Posts); +require('./bookmarks')(Posts); +require('./queue')(Posts); +require('./diffs')(Posts); +require('./uploads')(Posts); +Posts.exists = async function (pids) { + return await db.exists(Array.isArray(pids) ? pids.map(pid => `post:${pid}`) : `post:${pids}`); +}; +Posts.getPidsFromSet = async function (set, start, stop, reverse) { + if (isNaN(start) || isNaN(stop)) { + return []; + } + return await db[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](set, start, stop); +}; +Posts.getPostsByPids = async function (pids, uid) { + if (!Array.isArray(pids) || !pids.length) { + return []; + } + let posts = await Posts.getPostsData(pids); + posts = await Promise.all(posts.map(Posts.parsePost)); + const data = await plugins.hooks.fire('filter:post.getPosts', { + posts: posts, + uid: uid + }); + if (!data || !Array.isArray(data.posts)) { + return []; + } + return data.posts.filter(Boolean); +}; +Posts.getPostSummariesFromSet = async function (set, uid, start, stop) { + let pids = await db.getSortedSetRevRange(set, start, stop); + pids = await privileges.posts.filter('topics:read', pids, uid); + const posts = await Posts.getPostSummaryByPids(pids, uid, { + stripTags: false + }); + return { + posts: posts, + nextStart: stop + 1 + }; +}; +Posts.getPidIndex = async function (pid, tid, topicPostSort) { + const set = topicPostSort === 'most_votes' ? `tid:${tid}:posts:votes` : `tid:${tid}:posts`; + const reverse = topicPostSort === 'newest_to_oldest' || topicPostSort === 'most_votes'; + const index = await db[reverse ? 'sortedSetRevRank' : 'sortedSetRank'](set, pid); + if (!utils.isNumber(index)) { + return 0; + } + return utils.isNumber(index) ? parseInt(index, 10) + 1 : 0; +}; +Posts.getPostIndices = async function (posts, uid) { + if (!Array.isArray(posts) || !posts.length) { + return []; + } + const settings = await user.getSettings(uid); + const byVotes = settings.topicPostSort === 'most_votes'; + let sets = posts.map(p => byVotes ? `tid:${p.tid}:posts:votes` : `tid:${p.tid}:posts`); + const reverse = settings.topicPostSort === 'newest_to_oldest' || settings.topicPostSort === 'most_votes'; + const uniqueSets = _.uniq(sets); + let method = reverse ? 'sortedSetsRevRanks' : 'sortedSetsRanks'; + if (uniqueSets.length === 1) { + method = reverse ? 'sortedSetRevRanks' : 'sortedSetRanks'; + sets = uniqueSets[0]; + } + const pids = posts.map(post => post.pid); + const indices = await db[method](sets, pids); + return indices.map(index => utils.isNumber(index) ? parseInt(index, 10) + 1 : 0); +}; +Posts.modifyPostByPrivilege = function (post, privileges) { + if (post && post.deleted && !(post.selfPost || privileges['posts:view_deleted'])) { + post.content = '[[topic:post-is-deleted]]'; + if (post.user) { + post.user.signature = ''; + } + } +}; +require('../promisify')(Posts); \ No newline at end of file diff --git a/lib/posts/parse.js b/lib/posts/parse.js new file mode 100644 index 0000000000..a0eeea7834 --- /dev/null +++ b/lib/posts/parse.js @@ -0,0 +1,142 @@ +'use strict'; + +const nconf = require('nconf'); +const url = require('url'); +const winston = require('winston'); +const sanitize = require('sanitize-html'); +const _ = require('lodash'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const translator = require('../translator'); +const utils = require('../utils'); +const postCache = require('./cache'); +let sanitizeConfig = { + allowedTags: sanitize.defaults.allowedTags.concat(['ins', 'del', 'img', 'button', 'video', 'audio', 'source', 'iframe', 'embed']), + allowedAttributes: { + ...sanitize.defaults.allowedAttributes, + a: ['href', 'name', 'hreflang', 'media', 'rel', 'target', 'type'], + img: ['alt', 'height', 'ismap', 'src', 'usemap', 'width', 'srcset'], + iframe: ['height', 'name', 'src', 'width'], + video: ['autoplay', 'playsinline', 'controls', 'height', 'loop', 'muted', 'poster', 'preload', 'src', 'width'], + audio: ['autoplay', 'controls', 'loop', 'muted', 'preload', 'src'], + source: ['type', 'src', 'srcset', 'sizes', 'media', 'height', 'width'], + embed: ['height', 'src', 'type', 'width'] + }, + globalAttributes: ['accesskey', 'class', 'contenteditable', 'dir', 'draggable', 'dropzone', 'hidden', 'id', 'lang', 'spellcheck', 'style', 'tabindex', 'title', 'translate', 'aria-expanded', 'data-*'], + allowedClasses: { + ...sanitize.defaults.allowedClasses + } +}; +module.exports = function (Posts) { + Posts.urlRegex = { + regex: /href="([^"]+)"/g, + length: 6 + }; + Posts.imgRegex = { + regex: /src="([^"]+)"/g, + length: 5 + }; + Posts.parsePost = async function (postData) { + if (!postData) { + return postData; + } + postData.content = String(postData.content || ''); + const cache = postCache.getOrCreate(); + const pid = String(postData.pid); + const cachedContent = cache.get(pid); + if (postData.pid && cachedContent !== undefined) { + postData.content = cachedContent; + return postData; + } + const data = await plugins.hooks.fire('filter:parse.post', { + postData: postData + }); + data.postData.content = translator.escape(data.postData.content); + if (data.postData.pid) { + cache.set(pid, data.postData.content); + } + return data.postData; + }; + Posts.parseSignature = async function (userData, uid) { + userData.signature = sanitizeSignature(userData.signature || ''); + return await plugins.hooks.fire('filter:parse.signature', { + userData: userData, + uid: uid + }); + }; + Posts.relativeToAbsolute = function (content, regex) { + if (!content) { + return content; + } + let parsed; + let current = regex.regex.exec(content); + let absolute; + while (current !== null) { + if (current[1]) { + try { + parsed = url.parse(current[1]); + if (!parsed.protocol) { + if (current[1].startsWith('/')) { + absolute = nconf.get('base_url') + current[1]; + } else { + absolute = `//${current[1]}`; + } + content = content.slice(0, current.index + regex.length) + absolute + content.slice(current.index + regex.length + current[1].length); + } + } catch (err) { + winston.verbose(err.messsage); + } + } + current = regex.regex.exec(content); + } + return content; + }; + Posts.sanitize = function (content) { + return sanitize(content, { + allowedTags: sanitizeConfig.allowedTags, + allowedAttributes: sanitizeConfig.allowedAttributes, + allowedClasses: sanitizeConfig.allowedClasses + }); + }; + Posts.configureSanitize = async () => { + sanitizeConfig.allowedTags.forEach(tag => { + sanitizeConfig.allowedAttributes[tag] = _.union(sanitizeConfig.allowedAttributes[tag], sanitizeConfig.globalAttributes); + }); + sanitizeConfig = await plugins.hooks.fire('filter:sanitize.config', sanitizeConfig); + }; + Posts.registerHooks = () => { + plugins.hooks.register('core', { + hook: 'filter:parse.post', + method: async data => { + data.postData.content = Posts.sanitize(data.postData.content); + return data; + } + }); + plugins.hooks.register('core', { + hook: 'filter:parse.raw', + method: async content => Posts.sanitize(content) + }); + plugins.hooks.register('core', { + hook: 'filter:parse.aboutme', + method: async content => Posts.sanitize(content) + }); + plugins.hooks.register('core', { + hook: 'filter:parse.signature', + method: async data => { + data.userData.signature = Posts.sanitize(data.userData.signature); + return data; + } + }); + }; + function sanitizeSignature(signature) { + signature = translator.escape(signature); + const tagsToStrip = []; + if (meta.config['signatures:disableLinks']) { + tagsToStrip.push('a'); + } + if (meta.config['signatures:disableImages']) { + tagsToStrip.push('img'); + } + return utils.stripHTMLTags(signature, tagsToStrip); + } +}; \ No newline at end of file diff --git a/lib/posts/queue.js b/lib/posts/queue.js new file mode 100644 index 0000000000..ca93408157 --- /dev/null +++ b/lib/posts/queue.js @@ -0,0 +1,366 @@ +'use strict'; + +const _ = require('lodash'); +const validator = require('validator'); +const nconf = require('nconf'); +const db = require('../database'); +const user = require('../user'); +const meta = require('../meta'); +const groups = require('../groups'); +const topics = require('../topics'); +const categories = require('../categories'); +const notifications = require('../notifications'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const cache = require('../cache'); +const socketHelpers = require('../socket.io/helpers'); +module.exports = function (Posts) { + Posts.getQueuedPosts = async (filter = {}, options = {}) => { + options = { + metadata: true, + ...options + }; + let postData = _.cloneDeep(cache.get('post-queue')); + if (!postData) { + const ids = await db.getSortedSetRange('post:queue', 0, -1); + const keys = ids.map(id => `post:queue:${id}`); + postData = await db.getObjects(keys); + postData.forEach(data => { + if (data) { + data.data = JSON.parse(data.data); + data.data.timestampISO = utils.toISOString(data.data.timestamp); + } + }); + const uids = postData.map(data => data && data.uid); + const userData = await user.getUsersFields(uids, ['username', 'userslug', 'picture', 'joindate', 'postcount', 'reputation']); + postData.forEach((postData, index) => { + if (postData) { + postData.user = userData[index]; + if (postData.user.uid === 0 && postData.data.handle) { + postData.user.username = validator.escape(String(postData.data.handle)); + postData.user.displayname = postData.user.username; + postData.user.fullname = postData.user.username; + } + postData.data.rawContent = validator.escape(String(postData.data.content)); + postData.data.title = validator.escape(String(postData.data.title || '')); + } + }); + cache.set('post-queue', _.cloneDeep(postData)); + } + if (filter.id) { + postData = postData.filter(p => p.id === filter.id); + } + if (options.metadata) { + await Promise.all(postData.map(addMetaData)); + } + if (utils.isNumber(filter.tid)) { + const tid = parseInt(filter.tid, 10); + postData = postData.filter(item => item.data.tid && parseInt(item.data.tid, 10) === tid); + } else if (Array.isArray(filter.tid)) { + const tids = filter.tid.map(tid => parseInt(tid, 10)); + postData = postData.filter(item => item.data.tid && tids.includes(parseInt(item.data.tid, 10))); + } + return postData; + }; + async function addMetaData(postData) { + if (!postData) { + return; + } + postData.topic = { + cid: 0 + }; + if (postData.data.cid) { + postData.topic = { + cid: parseInt(postData.data.cid, 10) + }; + } else if (postData.data.tid) { + postData.topic = await topics.getTopicFields(postData.data.tid, ['title', 'cid', 'lastposttime']); + } + postData.category = await categories.getCategoryData(postData.topic.cid); + const result = await plugins.hooks.fire('filter:parse.post', { + postData: postData.data + }); + postData.data.content = result.postData.content; + } + Posts.canUserPostContentWithLinks = async function (uid, content) { + if (!content) { + return true; + } + const [reputation, isPrivileged] = await Promise.all([user.getUserField(uid, 'reputation'), user.isPrivileged(uid)]); + if (!isPrivileged && reputation < meta.config['min:rep:post-links']) { + const parsed = await plugins.hooks.fire('filter:parse.raw', String(content)); + const matches = parsed.matchAll(/]*href="([^"]+)"[^>]*>/g); + let external = 0; + for (const [, href] of matches) { + const internal = utils.isInternalURI(new URL(href, nconf.get('url')), new URL(nconf.get('url')), nconf.get('relative_path')); + if (!internal) { + external += 1; + } + } + return external === 0; + } + return true; + }; + Posts.shouldQueue = async function (uid, data) { + let shouldQueue = meta.config.postQueue; + if (shouldQueue) { + const [userData, isPrivileged, isMemberOfExempt, categoryQueueEnabled] = await Promise.all([user.getUserFields(uid, ['uid', 'reputation', 'postcount']), user.isPrivileged(uid), groups.isMemberOfAny(uid, meta.config.groupsExemptFromPostQueue), isCategoryQueueEnabled(data)]); + shouldQueue = categoryQueueEnabled && !isPrivileged && !isMemberOfExempt && (!userData.uid || userData.reputation < meta.config.postQueueReputationThreshold || userData.postcount <= 0 || !(await Posts.canUserPostContentWithLinks(uid, data.content))); + } + const result = await plugins.hooks.fire('filter:post.shouldQueue', { + shouldQueue: !!shouldQueue, + uid: uid, + data: data + }); + return result.shouldQueue; + }; + async function isCategoryQueueEnabled(data) { + const type = getType(data); + const cid = await getCid(type, data); + if (!cid) { + return true; + } + return await categories.getCategoryField(cid, 'postQueue'); + } + function getType(data) { + if (data.hasOwnProperty('tid')) { + return 'reply'; + } else if (data.hasOwnProperty('cid')) { + return 'topic'; + } + throw new Error('[[error:invalid-type]]'); + } + async function removeQueueNotification(id) { + await notifications.rescind(`post-queue-${id}`); + const data = await getParsedObject(id); + if (!data) { + return; + } + const cid = await getCid(data.type, data); + const uids = await getNotificationUids(cid); + uids.forEach(uid => user.notifications.pushCount(uid)); + } + async function getNotificationUids(cid) { + const results = await Promise.all([groups.getMembersOfGroups(['administrators', 'Global Moderators']), categories.getModeratorUids([cid])]); + return _.uniq(_.flattenDeep(results)); + } + Posts.addToQueue = async function (data) { + const type = getType(data); + const now = Date.now(); + const id = `${type}-${now}`; + await canPost(type, data); + let payload = { + id: id, + uid: data.uid, + type: type, + data: data + }; + payload = await plugins.hooks.fire('filter:post-queue.save', payload); + payload.data = JSON.stringify(data); + await db.sortedSetAdd('post:queue', now, id); + await db.setObject(`post:queue:${id}`, payload); + await user.setUserField(data.uid, 'lastqueuetime', now); + cache.del('post-queue'); + const cid = await getCid(type, data); + const uids = await getNotificationUids(cid); + const bodyLong = await parseBodyLong(cid, type, data); + const notifObj = await notifications.create({ + type: 'post-queue', + nid: `post-queue-${id}`, + mergeId: 'post-queue', + bodyShort: '[[notifications:post-awaiting-review]]', + bodyLong: bodyLong, + path: `/post-queue/${id}` + }); + await notifications.push(notifObj, uids); + return { + id: id, + type: type, + queued: true, + message: '[[success:post-queued]]' + }; + }; + async function parseBodyLong(cid, type, data) { + const url = nconf.get('url'); + const [content, category, userData] = await Promise.all([plugins.hooks.fire('filter:parse.raw', data.content), categories.getCategoryFields(cid, ['name', 'slug']), user.getUserFields(data.uid, ['uid', 'username'])]); + category.url = `${url}/category/${category.slug}`; + if (userData.uid > 0) { + userData.url = `${url}/uid/${userData.uid}`; + } + const topic = { + cid: cid, + title: data.title, + tid: data.tid + }; + if (type === 'reply') { + topic.title = await topics.getTopicField(data.tid, 'title'); + topic.url = `${url}/topic/${data.tid}`; + } + const { + app + } = require('../webserver'); + return await app.renderAsync('emails/partials/post-queue-body', { + content: content, + category: category, + user: userData, + topic: topic + }); + } + async function getCid(type, data) { + if (type === 'topic') { + return data.cid; + } else if (type === 'reply') { + return await topics.getTopicField(data.tid, 'cid'); + } + return null; + } + async function canPost(type, data) { + const cid = await getCid(type, data); + const typeToPrivilege = { + topic: 'topics:create', + reply: 'topics:reply' + }; + topics.checkContent(data.content); + if (type === 'topic') { + topics.checkTitle(data.title); + if (data.tags) { + await topics.validateTags(data.tags, cid, data.uid); + } + } + const [canPost] = await Promise.all([privileges.categories.can(typeToPrivilege[type], cid, data.uid), user.isReadyToQueue(data.uid, cid)]); + if (!canPost) { + throw new Error('[[error:no-privileges]]'); + } + } + Posts.removeFromQueue = async function (id) { + const data = await getParsedObject(id); + if (!data) { + return null; + } + const result = await plugins.hooks.fire('filter:post-queue:removeFromQueue', { + data: data + }); + await removeFromQueue(id); + plugins.hooks.fire('action:post-queue:removeFromQueue', { + data: result.data + }); + return result.data; + }; + async function removeFromQueue(id) { + await removeQueueNotification(id); + await db.sortedSetRemove('post:queue', id); + await db.delete(`post:queue:${id}`); + cache.del('post-queue'); + } + Posts.submitFromQueue = async function (id) { + let data = await getParsedObject(id); + if (!data) { + return null; + } + const result = await plugins.hooks.fire('filter:post-queue:submitFromQueue', { + data: data + }); + data = result.data; + if (data.type === 'topic') { + const result = await createTopic(data.data); + data.pid = result.postData.pid; + } else if (data.type === 'reply') { + const result = await createReply(data.data); + data.pid = result.pid; + } + await removeFromQueue(id); + plugins.hooks.fire('action:post-queue:submitFromQueue', { + data: data + }); + return data; + }; + Posts.getFromQueue = async function (id) { + return await getParsedObject(id); + }; + async function getParsedObject(id) { + const data = await db.getObject(`post:queue:${id}`); + if (!data) { + return null; + } + data.data = JSON.parse(data.data); + data.data.fromQueue = true; + return data; + } + async function createTopic(data) { + const result = await topics.post(data); + socketHelpers.notifyNew(data.uid, 'newTopic', { + posts: [result.postData], + topic: result.topicData + }); + return result; + } + async function createReply(data) { + const postData = await topics.reply(data); + const result = { + posts: [postData], + 'reputation:disabled': !!meta.config['reputation:disabled'], + 'downvote:disabled': !!meta.config['downvote:disabled'] + }; + socketHelpers.notifyNew(data.uid, 'newPost', result); + return postData; + } + Posts.editQueuedContent = async function (uid, editData) { + const [canEditQueue, data] = await Promise.all([Posts.canEditQueue(uid, editData, 'edit'), getParsedObject(editData.id)]); + if (!data) { + throw new Error('[[error:no-post]]'); + } + if (!canEditQueue) { + throw new Error('[[error:no-privileges]]'); + } + if (editData.content !== undefined) { + data.data.content = editData.content; + } + if (editData.title !== undefined) { + data.data.title = editData.title; + } + if (editData.cid !== undefined) { + data.data.cid = editData.cid; + } + await db.setObjectField(`post:queue:${editData.id}`, 'data', JSON.stringify(data.data)); + cache.del('post-queue'); + }; + Posts.canEditQueue = async function (uid, editData, action) { + const [isAdminOrGlobalMod, data] = await Promise.all([user.isAdminOrGlobalMod(uid), getParsedObject(editData.id)]); + if (!data) { + return false; + } + const selfPost = parseInt(uid, 10) === parseInt(data.uid, 10); + if (isAdminOrGlobalMod || (action === 'reject' || action === 'edit') && selfPost) { + return true; + } + let cid; + if (data.type === 'topic') { + cid = data.data.cid; + } else if (data.type === 'reply') { + cid = await topics.getTopicField(data.data.tid, 'cid'); + } + const isModerator = await user.isModerator(uid, cid); + let isModeratorOfTargetCid = true; + if (editData.cid) { + isModeratorOfTargetCid = await user.isModerator(uid, editData.cid); + } + return isModerator && isModeratorOfTargetCid; + }; + Posts.updateQueuedPostsTopic = async function (newTid, tids) { + const postData = await Posts.getQueuedPosts({ + tid: tids + }, { + metadata: false + }); + if (postData.length) { + postData.forEach(post => { + post.data.tid = newTid; + }); + await db.setObjectBulk(postData.map(p => [`post:queue:${p.id}`, { + data: JSON.stringify(p.data) + }])); + cache.del('post-queue'); + } + }; +}; \ No newline at end of file diff --git a/lib/posts/recent.js b/lib/posts/recent.js new file mode 100644 index 0000000000..cb9f620e85 --- /dev/null +++ b/lib/posts/recent.js @@ -0,0 +1,29 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const privileges = require('../privileges'); +module.exports = function (Posts) { + const terms = { + day: 86400000, + week: 604800000, + month: 2592000000 + }; + Posts.getRecentPosts = async function (uid, start, stop, term) { + let min = 0; + if (terms[term]) { + min = Date.now() - terms[term]; + } + const count = parseInt(stop, 10) === -1 ? stop : stop - start + 1; + let pids = await db.getSortedSetRevRangeByScore('posts:pid', start, count, '+inf', min); + pids = await privileges.posts.filter('topics:read', pids, uid); + return await Posts.getPostSummaryByPids(pids, uid, { + stripTags: true + }); + }; + Posts.getRecentPosterUids = async function (start, stop) { + const pids = await db.getSortedSetRevRange('posts:pid', start, stop); + const postData = await Posts.getPostsFields(pids, ['uid']); + return _.uniq(postData.map(p => p && p.uid).filter(uid => parseInt(uid, 10))); + }; +}; \ No newline at end of file diff --git a/lib/posts/summary.js b/lib/posts/summary.js new file mode 100644 index 0000000000..8c5d247814 --- /dev/null +++ b/lib/posts/summary.js @@ -0,0 +1,84 @@ +'use strict'; + +const validator = require('validator'); +const _ = require('lodash'); +const topics = require('../topics'); +const user = require('../user'); +const plugins = require('../plugins'); +const categories = require('../categories'); +const utils = require('../utils'); +module.exports = function (Posts) { + Posts.getPostSummaryByPids = async function (pids, uid, options) { + if (!Array.isArray(pids) || !pids.length) { + return []; + } + options.stripTags = options.hasOwnProperty('stripTags') ? options.stripTags : false; + options.parse = options.hasOwnProperty('parse') ? options.parse : true; + options.extraFields = options.hasOwnProperty('extraFields') ? options.extraFields : []; + const fields = ['pid', 'tid', 'content', 'uid', 'timestamp', 'deleted', 'upvotes', 'downvotes', 'replies', 'handle'].concat(options.extraFields); + let posts = await Posts.getPostsFields(pids, fields); + posts = posts.filter(Boolean); + posts = await user.blocks.filter(uid, posts); + const uids = _.uniq(posts.map(p => p && p.uid)); + const tids = _.uniq(posts.map(p => p && p.tid)); + const [users, topicsAndCategories] = await Promise.all([user.getUsersFields(uids, ['uid', 'username', 'userslug', 'picture', 'status']), getTopicAndCategories(tids)]); + const uidToUser = toObject('uid', users); + const tidToTopic = toObject('tid', topicsAndCategories.topics); + const cidToCategory = toObject('cid', topicsAndCategories.categories); + posts.forEach(post => { + if (!uidToUser.hasOwnProperty(post.uid)) { + post.uid = 0; + } + post.user = uidToUser[post.uid]; + Posts.overrideGuestHandle(post, post.handle); + post.handle = undefined; + post.topic = tidToTopic[post.tid]; + post.category = post.topic && cidToCategory[post.topic.cid]; + post.isMainPost = post.topic && post.pid === post.topic.mainPid; + post.deleted = post.deleted === 1; + post.timestampISO = utils.toISOString(post.timestamp); + }); + posts = posts.filter(post => tidToTopic[post.tid]); + posts = await parsePosts(posts, options); + const result = await plugins.hooks.fire('filter:post.getPostSummaryByPids', { + posts: posts, + uid: uid + }); + return result.posts; + }; + async function parsePosts(posts, options) { + return await Promise.all(posts.map(async post => { + if (!post.content || !options.parse) { + post.content = post.content ? validator.escape(String(post.content)) : post.content; + return post; + } + post = await Posts.parsePost(post); + if (options.stripTags) { + post.content = stripTags(post.content); + } + return post; + })); + } + async function getTopicAndCategories(tids) { + const topicsData = await topics.getTopicsFields(tids, ['uid', 'tid', 'title', 'cid', 'tags', 'slug', 'deleted', 'scheduled', 'postcount', 'mainPid', 'teaserPid']); + const cids = _.uniq(topicsData.map(topic => topic && topic.cid)); + const categoriesData = await categories.getCategoriesFields(cids, ['cid', 'name', 'icon', 'slug', 'parentCid', 'bgColor', 'color', 'backgroundImage', 'imageClass']); + return { + topics: topicsData, + categories: categoriesData + }; + } + function toObject(key, data) { + const obj = {}; + for (let i = 0; i < data.length; i += 1) { + obj[data[i][key]] = data[i]; + } + return obj; + } + function stripTags(content) { + if (content) { + return utils.stripHTMLTags(content, utils.stripTags); + } + return content; + } +}; \ No newline at end of file diff --git a/lib/posts/tools.js b/lib/posts/tools.js new file mode 100644 index 0000000000..014fa814c5 --- /dev/null +++ b/lib/posts/tools.js @@ -0,0 +1,35 @@ +'use strict'; + +const privileges = require('../privileges'); +module.exports = function (Posts) { + Posts.tools = {}; + Posts.tools.delete = async function (uid, pid) { + return await togglePostDelete(uid, pid, true); + }; + Posts.tools.restore = async function (uid, pid) { + return await togglePostDelete(uid, pid, false); + }; + async function togglePostDelete(uid, pid, isDelete) { + const [postData, canDelete] = await Promise.all([Posts.getPostData(pid), privileges.posts.canDelete(pid, uid)]); + if (!postData) { + throw new Error('[[error:no-post]]'); + } + if (postData.deleted && isDelete) { + throw new Error('[[error:post-already-deleted]]'); + } else if (!postData.deleted && !isDelete) { + throw new Error('[[error:post-already-restored]]'); + } + if (!canDelete.flag) { + throw new Error(canDelete.message); + } + let post; + if (isDelete) { + require('./cache').del(pid); + post = await Posts.delete(pid, uid); + } else { + post = await Posts.restore(pid, uid); + post = await Posts.parsePost(post); + } + return post; + } +}; \ No newline at end of file diff --git a/lib/posts/topics.js b/lib/posts/topics.js new file mode 100644 index 0000000000..ce1d0e2b17 --- /dev/null +++ b/lib/posts/topics.js @@ -0,0 +1,43 @@ +'use strict'; + +const topics = require('../topics'); +const user = require('../user'); +const utils = require('../utils'); +module.exports = function (Posts) { + Posts.getPostsFromSet = async function (set, start, stop, uid, reverse) { + const pids = await Posts.getPidsFromSet(set, start, stop, reverse); + const posts = await Posts.getPostsByPids(pids, uid); + return await user.blocks.filter(uid, posts); + }; + Posts.isMain = async function (pids) { + const isArray = Array.isArray(pids); + pids = isArray ? pids : [pids]; + const postData = await Posts.getPostsFields(pids, ['tid']); + const topicData = await topics.getTopicsFields(postData.map(t => t.tid), ['mainPid']); + const result = pids.map((pid, i) => parseInt(pid, 10) === parseInt(topicData[i].mainPid, 10)); + return isArray ? result : result[0]; + }; + Posts.getTopicFields = async function (pid, fields) { + const tid = await Posts.getPostField(pid, 'tid'); + return await topics.getTopicFields(tid, fields); + }; + Posts.generatePostPath = async function (pid, uid) { + const paths = await Posts.generatePostPaths([pid], uid); + return Array.isArray(paths) && paths.length ? paths[0] : null; + }; + Posts.generatePostPaths = async function (pids, uid) { + const postData = await Posts.getPostsFields(pids, ['pid', 'tid']); + const tids = postData.map(post => post && post.tid); + const [indices, topicData] = await Promise.all([Posts.getPostIndices(postData, uid), topics.getTopicsFields(tids, ['slug'])]); + const paths = pids.map((pid, index) => { + const slug = topicData[index] ? topicData[index].slug : null; + const postIndex = utils.isNumber(indices[index]) ? parseInt(indices[index], 10) + 1 : null; + if (slug && postIndex) { + const index = postIndex === 1 ? '' : `/${postIndex}`; + return `/topic/${slug}${index}`; + } + return null; + }); + return paths; + }; +}; \ No newline at end of file diff --git a/lib/posts/uploads.js b/lib/posts/uploads.js new file mode 100644 index 0000000000..cb4a80738d --- /dev/null +++ b/lib/posts/uploads.js @@ -0,0 +1,173 @@ +'use strict'; + +const nconf = require('nconf'); +const fs = require('fs').promises; +const crypto = require('crypto'); +const path = require('path'); +const winston = require('winston'); +const mime = require('mime'); +const validator = require('validator'); +const cronJob = require('cron').CronJob; +const chalk = require('chalk'); +const db = require('../database'); +const image = require('../image'); +const user = require('../user'); +const topics = require('../topics'); +const file = require('../file'); +const meta = require('../meta'); +module.exports = function (Posts) { + Posts.uploads = {}; + const md5 = filename => crypto.createHash('md5').update(filename).digest('hex'); + const pathPrefix = path.join(nconf.get('upload_path')); + const searchRegex = /\/assets\/uploads\/(files\/[^\s")]+\.?[\w]*)/g; + const _getFullPath = relativePath => path.join(pathPrefix, relativePath); + const _filterValidPaths = async filePaths => (await Promise.all(filePaths.map(async filePath => { + const fullPath = _getFullPath(filePath); + return fullPath.startsWith(pathPrefix) && (await file.exists(fullPath)) ? filePath : false; + }))).filter(Boolean); + const runJobs = nconf.get('runJobs'); + if (runJobs) { + new cronJob('0 2 * * 0', async () => { + const orphans = await Posts.uploads.cleanOrphans(); + if (orphans.length) { + winston.info(`[posts/uploads] Deleting ${orphans.length} orphaned uploads...`); + orphans.forEach(relPath => { + process.stdout.write(`${chalk.red(' - ')} ${relPath}`); + }); + } + }, null, true); + } + Posts.uploads.sync = async function (pid) { + const [content, currentUploads, isMainPost] = await Promise.all([Posts.getPostField(pid, 'content'), Posts.uploads.list(pid), Posts.isMain(pid)]); + let match = searchRegex.exec(content); + const uploads = []; + while (match) { + uploads.push(match[1].replace('-resized', '')); + match = searchRegex.exec(content); + } + if (isMainPost) { + const tid = await Posts.getPostField(pid, 'tid'); + let thumbs = await topics.thumbs.get(tid); + thumbs = thumbs.map(thumb => thumb.path).filter(path => !validator.isURL(path, { + require_protocol: true + })); + thumbs = thumbs.map(t => t.slice(1)); + uploads.push(...thumbs); + } + const add = uploads.filter(path => !currentUploads.includes(path)); + const remove = currentUploads.filter(path => !uploads.includes(path)); + await Promise.all([Posts.uploads.associate(pid, add), Posts.uploads.dissociate(pid, remove)]); + }; + Posts.uploads.list = async function (pid) { + return await db.getSortedSetMembers(`post:${pid}:uploads`); + }; + Posts.uploads.listWithSizes = async function (pid) { + const paths = await Posts.uploads.list(pid); + const sizes = (await db.getObjects(paths.map(path => `upload:${md5(path)}`))) || []; + return sizes.map((sizeObj, idx) => ({ + ...sizeObj, + name: paths[idx] + })); + }; + Posts.uploads.getOrphans = async () => { + let files = await fs.readdir(_getFullPath('/files')); + files = files.filter(filename => filename !== '.gitignore'); + const tsPrefix = /^\d{13}-/; + files = files.filter(filename => tsPrefix.test(filename)); + files = await Promise.all(files.map(async filename => (await Posts.uploads.isOrphan(`files/${filename}`)) ? `files/${filename}` : null)); + files = files.filter(Boolean); + return files; + }; + Posts.uploads.cleanOrphans = async () => { + const now = Date.now(); + const expiration = now - 1000 * 60 * 60 * 24 * meta.config.orphanExpiryDays; + const days = meta.config.orphanExpiryDays; + if (!days) { + return []; + } + let orphans = await Posts.uploads.getOrphans(); + orphans = await Promise.all(orphans.map(async relPath => { + const { + mtimeMs + } = await fs.stat(_getFullPath(relPath)); + return mtimeMs < expiration ? relPath : null; + })); + orphans = orphans.filter(Boolean); + await Promise.all(orphans.map(async relPath => { + await file.delete(_getFullPath(relPath)); + })); + return orphans; + }; + Posts.uploads.isOrphan = async function (filePath) { + const length = await db.sortedSetCard(`upload:${md5(filePath)}:pids`); + return length === 0; + }; + Posts.uploads.getUsage = async function (filePaths) { + if (!Array.isArray(filePaths)) { + filePaths = [filePaths]; + } + if (process.platform === 'win32') { + filePaths.forEach(file => { + file.path = file.path.split(path.sep).join(path.posix.sep); + }); + } + const keys = filePaths.map(fileObj => `upload:${md5(fileObj.path.replace('-resized', ''))}:pids`); + return await Promise.all(keys.map(k => db.getSortedSetRange(k, 0, -1))); + }; + Posts.uploads.associate = async function (pid, filePaths) { + filePaths = !Array.isArray(filePaths) ? [filePaths] : filePaths; + if (!filePaths.length) { + return; + } + filePaths = await _filterValidPaths(filePaths); + const now = Date.now(); + const scores = filePaths.map((p, i) => now + i); + const bulkAdd = filePaths.map(path => [`upload:${md5(path)}:pids`, now, pid]); + await Promise.all([db.sortedSetAdd(`post:${pid}:uploads`, scores, filePaths), db.sortedSetAddBulk(bulkAdd), Posts.uploads.saveSize(filePaths)]); + }; + Posts.uploads.dissociate = async function (pid, filePaths) { + filePaths = !Array.isArray(filePaths) ? [filePaths] : filePaths; + if (!filePaths.length) { + return; + } + const bulkRemove = filePaths.map(path => [`upload:${md5(path)}:pids`, pid]); + const promises = [db.sortedSetRemove(`post:${pid}:uploads`, filePaths), db.sortedSetRemoveBulk(bulkRemove)]; + await Promise.all(promises); + if (!meta.config.preserveOrphanedUploads) { + const deletePaths = (await Promise.all(filePaths.map(async filePath => (await Posts.uploads.isOrphan(filePath)) ? filePath : false))).filter(Boolean); + const uploaderUids = (await db.getObjectsFields(deletePaths.map(path => `upload:${md5(path)}`, ['uid']))).map(o => o ? o.uid || null : null); + await Promise.all(uploaderUids.map((uid, idx) => uid && isFinite(uid) ? user.deleteUpload(uid, uid, deletePaths[idx]) : null).filter(Boolean)); + await Posts.uploads.deleteFromDisk(deletePaths); + } + }; + Posts.uploads.dissociateAll = async pid => { + const current = await Posts.uploads.list(pid); + await Posts.uploads.dissociate(pid, current); + }; + Posts.uploads.deleteFromDisk = async filePaths => { + if (typeof filePaths === 'string') { + filePaths = [filePaths]; + } else if (!Array.isArray(filePaths)) { + throw new Error(`[[error:wrong-parameter-type, filePaths, ${typeof filePaths}, array]]`); + } + filePaths = (await _filterValidPaths(filePaths)).map(_getFullPath); + await Promise.all(filePaths.map(file.delete)); + }; + Posts.uploads.saveSize = async filePaths => { + filePaths = filePaths.filter(fileName => { + const type = mime.getType(fileName); + return type && type.match(/image./); + }); + await Promise.all(filePaths.map(async fileName => { + try { + const size = await image.size(_getFullPath(fileName)); + await db.setObject(`upload:${md5(fileName)}`, { + width: size.width, + height: size.height + }); + } catch (err) { + winston.error(`[posts/uploads] Error while saving post upload sizes (${fileName}): ${err.message}`); + } + })); + }; +}; \ No newline at end of file diff --git a/lib/posts/user.js b/lib/posts/user.js new file mode 100644 index 0000000000..1a1d002916 --- /dev/null +++ b/lib/posts/user.js @@ -0,0 +1,217 @@ +'use strict'; + +const async = require('async'); +const validator = require('validator'); +const _ = require('lodash'); +const db = require('../database'); +const user = require('../user'); +const topics = require('../topics'); +const groups = require('../groups'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +module.exports = function (Posts) { + Posts.getUserInfoForPosts = async function (uids, uid) { + const [userData, userSettings, signatureUids] = await Promise.all([getUserData(uids, uid), user.getMultipleUserSettings(uids), meta.config.disableSignatures ? [] : privileges.categories.filterUids('signature', 0, uids)]); + const uidsSignatureSet = new Set(signatureUids.map(uid => parseInt(uid, 10))); + const groupsMap = await getGroupsMap(userData); + userData.forEach((userData, index) => { + userData.signature = validator.escape(String(userData.signature || '')); + userData.fullname = userSettings[index].showfullname ? validator.escape(String(userData.fullname || '')) : undefined; + userData.selectedGroups = []; + if (meta.config.hideFullname) { + userData.fullname = undefined; + } + }); + const result = await Promise.all(userData.map(async userData => { + const [isMemberOfGroups, signature, customProfileInfo] = await Promise.all([checkGroupMembership(userData.uid, userData.groupTitleArray), parseSignature(userData, uid, uidsSignatureSet), plugins.hooks.fire('filter:posts.custom_profile_info', { + profile: [], + uid: userData.uid + })]); + if (isMemberOfGroups && userData.groupTitleArray) { + userData.groupTitleArray.forEach((userGroup, index) => { + if (isMemberOfGroups[index] && groupsMap[userGroup]) { + userData.selectedGroups.push(groupsMap[userGroup]); + } + }); + } + userData.signature = signature; + userData.custom_profile_info = customProfileInfo.profile; + return await plugins.hooks.fire('filter:posts.modifyUserInfo', userData); + })); + const hookResult = await plugins.hooks.fire('filter:posts.getUserInfoForPosts', { + users: result + }); + return hookResult.users; + }; + Posts.overrideGuestHandle = function (postData, handle) { + if (meta.config.allowGuestHandles && postData && postData.user && parseInt(postData.uid, 10) === 0 && handle) { + postData.user.username = validator.escape(String(handle)); + if (postData.user.hasOwnProperty('fullname')) { + postData.user.fullname = postData.user.username; + } + postData.user.displayname = postData.user.username; + } + }; + async function checkGroupMembership(uid, groupTitleArray) { + if (!Array.isArray(groupTitleArray) || !groupTitleArray.length) { + return null; + } + return await groups.isMemberOfGroups(uid, groupTitleArray); + } + async function parseSignature(userData, uid, signatureUids) { + if (!userData.signature || !signatureUids.has(userData.uid) || meta.config.disableSignatures) { + return ''; + } + const result = await Posts.parseSignature(userData, uid); + return result.userData.signature; + } + async function getGroupsMap(userData) { + const groupTitles = _.uniq(_.flatten(userData.map(u => u && u.groupTitleArray))); + const groupsMap = {}; + const groupsData = await groups.getGroupsData(groupTitles); + groupsData.forEach(group => { + if (group && group.userTitleEnabled && !group.hidden) { + groupsMap[group.name] = { + name: group.name, + slug: group.slug, + labelColor: group.labelColor, + textColor: group.textColor, + icon: group.icon, + userTitle: group.userTitle + }; + } + }); + return groupsMap; + } + async function getUserData(uids, uid) { + const fields = ['uid', 'username', 'fullname', 'userslug', 'reputation', 'postcount', 'topiccount', 'picture', 'signature', 'banned', 'banned:expire', 'status', 'lastonline', 'groupTitle', 'mutedUntil']; + const result = await plugins.hooks.fire('filter:posts.addUserFields', { + fields: fields, + uid: uid, + uids: uids + }); + return await user.getUsersFields(result.uids, _.uniq(result.fields)); + } + Posts.isOwner = async function (pids, uid) { + uid = parseInt(uid, 10); + const isArray = Array.isArray(pids); + pids = isArray ? pids : [pids]; + if (uid <= 0) { + return isArray ? pids.map(() => false) : false; + } + const postData = await Posts.getPostsFields(pids, ['uid']); + const result = postData.map(post => post && post.uid === uid); + return isArray ? result : result[0]; + }; + Posts.isModerator = async function (pids, uid) { + if (parseInt(uid, 10) <= 0) { + return pids.map(() => false); + } + const cids = await Posts.getCidsByPids(pids); + return await user.isModerator(uid, cids); + }; + Posts.changeOwner = async function (pids, toUid) { + const exists = await user.exists(toUid); + if (!exists) { + throw new Error('[[error:no-user]]'); + } + let postData = await Posts.getPostsFields(pids, ['pid', 'tid', 'uid', 'content', 'deleted', 'timestamp', 'upvotes', 'downvotes']); + postData = postData.filter(p => p.pid && p.uid !== parseInt(toUid, 10)); + pids = postData.map(p => p.pid); + const cids = await Posts.getCidsByPids(pids); + const bulkRemove = []; + const bulkAdd = []; + let repChange = 0; + const postsByUser = {}; + postData.forEach((post, i) => { + post.cid = cids[i]; + repChange += post.votes; + bulkRemove.push([`uid:${post.uid}:posts`, post.pid]); + bulkRemove.push([`cid:${post.cid}:uid:${post.uid}:pids`, post.pid]); + bulkRemove.push([`cid:${post.cid}:uid:${post.uid}:pids:votes`, post.pid]); + bulkAdd.push([`uid:${toUid}:posts`, post.timestamp, post.pid]); + bulkAdd.push([`cid:${post.cid}:uid:${toUid}:pids`, post.timestamp, post.pid]); + if (post.votes > 0 || post.votes < 0) { + bulkAdd.push([`cid:${post.cid}:uid:${toUid}:pids:votes`, post.votes, post.pid]); + } + postsByUser[post.uid] = postsByUser[post.uid] || []; + postsByUser[post.uid].push(post); + }); + await Promise.all([db.setObjectField(pids.map(pid => `post:${pid}`), 'uid', toUid), db.sortedSetRemoveBulk(bulkRemove), db.sortedSetAddBulk(bulkAdd), user.incrementUserReputationBy(toUid, repChange), handleMainPidOwnerChange(postData, toUid), updateTopicPosters(postData, toUid)]); + await Promise.all([user.updatePostCount(toUid), reduceCounters(postsByUser)]); + plugins.hooks.fire('action:post.changeOwner', { + posts: _.cloneDeep(postData), + toUid: toUid + }); + return postData; + }; + async function reduceCounters(postsByUser) { + await async.eachOfSeries(postsByUser, async (posts, uid) => { + const repChange = posts.reduce((acc, val) => acc + val.votes, 0); + await Promise.all([user.updatePostCount(uid), user.incrementUserReputationBy(uid, -repChange)]); + }); + } + async function updateTopicPosters(postData, toUid) { + const postsByTopic = _.groupBy(postData, p => parseInt(p.tid, 10)); + await async.eachOf(postsByTopic, async (posts, tid) => { + const postsByUser = _.groupBy(posts, p => parseInt(p.uid, 10)); + await db.sortedSetIncrBy(`tid:${tid}:posters`, posts.length, toUid); + await async.eachOf(postsByUser, async (posts, uid) => { + await db.sortedSetIncrBy(`tid:${tid}:posters`, -posts.length, uid); + }); + await db.sortedSetsRemoveRangeByScore([`tid:${tid}:posters`], '-inf', 0); + const posterCount = await db.sortedSetCard(`tid:${tid}:posters`); + await topics.setTopicField(tid, 'postercount', posterCount); + }); + } + async function handleMainPidOwnerChange(postData, toUid) { + const tids = _.uniq(postData.map(p => p.tid)); + const topicData = await topics.getTopicsFields(tids, ['tid', 'cid', 'deleted', 'title', 'uid', 'mainPid', 'timestamp']); + const tidToTopic = _.zipObject(tids, topicData); + const mainPosts = postData.filter(p => p.pid === tidToTopic[p.tid].mainPid); + if (!mainPosts.length) { + return; + } + const bulkAdd = []; + const bulkRemove = []; + const postsByUser = {}; + mainPosts.forEach(post => { + bulkRemove.push([`cid:${post.cid}:uid:${post.uid}:tids`, post.tid]); + bulkRemove.push([`uid:${post.uid}:topics`, post.tid]); + bulkAdd.push([`cid:${post.cid}:uid:${toUid}:tids`, tidToTopic[post.tid].timestamp, post.tid]); + bulkAdd.push([`uid:${toUid}:topics`, tidToTopic[post.tid].timestamp, post.tid]); + postsByUser[post.uid] = postsByUser[post.uid] || []; + postsByUser[post.uid].push(post); + }); + await Promise.all([db.setObjectField(mainPosts.map(p => `topic:${p.tid}`), 'uid', toUid), db.sortedSetRemoveBulk(bulkRemove), db.sortedSetAddBulk(bulkAdd), user.incrementUserFieldBy(toUid, 'topiccount', mainPosts.length), reduceTopicCounts(postsByUser)]); + const changedTopics = mainPosts.map(p => tidToTopic[p.tid]); + plugins.hooks.fire('action:topic.changeOwner', { + topics: _.cloneDeep(changedTopics), + toUid: toUid + }); + } + async function reduceTopicCounts(postsByUser) { + await async.eachSeries(Object.keys(postsByUser), async uid => { + const posts = postsByUser[uid]; + const exists = await user.exists(uid); + if (exists) { + await user.incrementUserFieldBy(uid, 'topiccount', -posts.length); + } + }); + } + Posts.filterPidsByUid = async function (pids, uids) { + if (!uids) { + return pids; + } + if (!Array.isArray(uids) || uids.length === 1) { + return await filterPidsBySingleUid(pids, uids); + } + const pidsArr = await Promise.all(uids.map(uid => Posts.filterPidsByUid(pids, uid))); + return _.union(...pidsArr); + }; + async function filterPidsBySingleUid(pids, uid) { + const isMembers = await db.isSortedSetMembers(`uid:${parseInt(uid, 10)}:posts`, pids); + return pids.filter((pid, index) => pid && isMembers[index]); + } +}; \ No newline at end of file diff --git a/lib/posts/votes.js b/lib/posts/votes.js new file mode 100644 index 0000000000..a8805da67b --- /dev/null +++ b/lib/posts/votes.js @@ -0,0 +1,250 @@ +'use strict'; + +const meta = require('../meta'); +const db = require('../database'); +const flags = require('../flags'); +const user = require('../user'); +const topics = require('../topics'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const translator = require('../translator'); +module.exports = function (Posts) { + const votesInProgress = {}; + Posts.upvote = async function (pid, uid) { + if (meta.config['reputation:disabled']) { + throw new Error('[[error:reputation-system-disabled]]'); + } + const canUpvote = await privileges.posts.can('posts:upvote', pid, uid); + if (!canUpvote) { + throw new Error('[[error:no-privileges]]'); + } + if (voteInProgress(pid, uid)) { + throw new Error('[[error:already-voting-for-this-post]]'); + } + putVoteInProgress(pid, uid); + try { + return await toggleVote('upvote', pid, uid); + } finally { + clearVoteProgress(pid, uid); + } + }; + Posts.downvote = async function (pid, uid) { + if (meta.config['reputation:disabled']) { + throw new Error('[[error:reputation-system-disabled]]'); + } + if (meta.config['downvote:disabled']) { + throw new Error('[[error:downvoting-disabled]]'); + } + const canDownvote = await privileges.posts.can('posts:downvote', pid, uid); + if (!canDownvote) { + throw new Error('[[error:no-privileges]]'); + } + if (voteInProgress(pid, uid)) { + throw new Error('[[error:already-voting-for-this-post]]'); + } + putVoteInProgress(pid, uid); + try { + return await toggleVote('downvote', pid, uid); + } finally { + clearVoteProgress(pid, uid); + } + }; + Posts.unvote = async function (pid, uid) { + if (voteInProgress(pid, uid)) { + throw new Error('[[error:already-voting-for-this-post]]'); + } + putVoteInProgress(pid, uid); + try { + const voteStatus = await Posts.hasVoted(pid, uid); + return await unvote(pid, uid, 'unvote', voteStatus); + } finally { + clearVoteProgress(pid, uid); + } + }; + Posts.hasVoted = async function (pid, uid) { + if (parseInt(uid, 10) <= 0) { + return { + upvoted: false, + downvoted: false + }; + } + const hasVoted = await db.isMemberOfSets([`pid:${pid}:upvote`, `pid:${pid}:downvote`], uid); + return { + upvoted: hasVoted[0], + downvoted: hasVoted[1] + }; + }; + Posts.getVoteStatusByPostIDs = async function (pids, uid) { + if (parseInt(uid, 10) <= 0) { + const data = pids.map(() => false); + return { + upvotes: data, + downvotes: data + }; + } + const upvoteSets = pids.map(pid => `pid:${pid}:upvote`); + const downvoteSets = pids.map(pid => `pid:${pid}:downvote`); + const data = await db.isMemberOfSets(upvoteSets.concat(downvoteSets), uid); + return { + upvotes: data.slice(0, pids.length), + downvotes: data.slice(pids.length, pids.length * 2) + }; + }; + Posts.getUpvotedUidsByPids = async function (pids) { + return await db.getSetsMembers(pids.map(pid => `pid:${pid}:upvote`)); + }; + function voteInProgress(pid, uid) { + return Array.isArray(votesInProgress[uid]) && votesInProgress[uid].includes(parseInt(pid, 10)); + } + function putVoteInProgress(pid, uid) { + votesInProgress[uid] = votesInProgress[uid] || []; + votesInProgress[uid].push(parseInt(pid, 10)); + } + function clearVoteProgress(pid, uid) { + if (Array.isArray(votesInProgress[uid])) { + const index = votesInProgress[uid].indexOf(parseInt(pid, 10)); + if (index !== -1) { + votesInProgress[uid].splice(index, 1); + } + } + } + async function toggleVote(type, pid, uid) { + const voteStatus = await Posts.hasVoted(pid, uid); + await unvote(pid, uid, type, voteStatus); + return await vote(type, false, pid, uid, voteStatus); + } + async function unvote(pid, uid, type, voteStatus) { + const owner = await Posts.getPostField(pid, 'uid'); + if (parseInt(uid, 10) === parseInt(owner, 10)) { + throw new Error('[[error:self-vote]]'); + } + if (type === 'downvote' || type === 'upvote') { + await checkVoteLimitation(pid, uid, type); + } + if (!voteStatus || !voteStatus.upvoted && !voteStatus.downvoted) { + return; + } + return await vote(voteStatus.upvoted ? 'downvote' : 'upvote', true, pid, uid, voteStatus); + } + async function checkVoteLimitation(pid, uid, type) { + const oneDay = 86400000; + const [reputation, isPrivileged, targetUid, votedPidsToday] = await Promise.all([user.getUserField(uid, 'reputation'), user.isPrivileged(uid), Posts.getPostField(pid, 'uid'), db.getSortedSetRevRangeByScore(`uid:${uid}:${type}`, 0, -1, '+inf', Date.now() - oneDay)]); + if (isPrivileged) { + return; + } + if (reputation < meta.config[`min:rep:${type}`]) { + throw new Error(`[[error:not-enough-reputation-to-${type}, ${meta.config[`min:rep:${type}`]}]]`); + } + const votesToday = meta.config[`${type}sPerDay`]; + if (votesToday && votedPidsToday.length >= votesToday) { + throw new Error(`[[error:too-many-${type}s-today, ${votesToday}]]`); + } + const voterPerUserToday = meta.config[`${type}sPerUserPerDay`]; + if (voterPerUserToday) { + const postData = await Posts.getPostsFields(votedPidsToday, ['uid']); + const targetUpVotes = postData.filter(p => p.uid === targetUid).length; + if (targetUpVotes >= voterPerUserToday) { + throw new Error(`[[error:too-many-${type}s-today-user, ${voterPerUserToday}]]`); + } + } + } + async function vote(type, unvote, pid, uid, voteStatus) { + uid = parseInt(uid, 10); + if (uid <= 0) { + throw new Error('[[error:not-logged-in]]'); + } + const now = Date.now(); + if (type === 'upvote' && !unvote) { + await db.sortedSetAdd(`uid:${uid}:upvote`, now, pid); + } else { + await db.sortedSetRemove(`uid:${uid}:upvote`, pid); + } + if (type === 'upvote' || unvote) { + await db.sortedSetRemove(`uid:${uid}:downvote`, pid); + } else { + await db.sortedSetAdd(`uid:${uid}:downvote`, now, pid); + } + const postData = await Posts.getPostFields(pid, ['pid', 'uid', 'tid']); + const newReputation = await user.incrementUserReputationBy(postData.uid, type === 'upvote' ? 1 : -1); + await adjustPostVotes(postData, uid, type, unvote); + await fireVoteHook(postData, uid, type, unvote, voteStatus); + return { + user: { + reputation: newReputation + }, + fromuid: uid, + post: postData, + upvote: type === 'upvote' && !unvote, + downvote: type === 'downvote' && !unvote + }; + } + async function fireVoteHook(postData, uid, type, unvote, voteStatus) { + let hook = type; + let current = voteStatus.upvoted ? 'upvote' : 'downvote'; + if (unvote) { + hook = 'unvote'; + } else { + current = 'unvote'; + } + plugins.hooks.fire(`action:post.${hook}`, { + pid: postData.pid, + uid: uid, + owner: postData.uid, + current: current + }); + } + async function adjustPostVotes(postData, uid, type, unvote) { + const notType = type === 'upvote' ? 'downvote' : 'upvote'; + if (unvote) { + await db.setRemove(`pid:${postData.pid}:${type}`, uid); + } else { + await db.setAdd(`pid:${postData.pid}:${type}`, uid); + } + await db.setRemove(`pid:${postData.pid}:${notType}`, uid); + const [upvotes, downvotes] = await Promise.all([db.setCount(`pid:${postData.pid}:upvote`), db.setCount(`pid:${postData.pid}:downvote`)]); + postData.upvotes = upvotes; + postData.downvotes = downvotes; + postData.votes = postData.upvotes - postData.downvotes; + await Posts.updatePostVoteCount(postData); + } + Posts.updatePostVoteCount = async function (postData) { + if (!postData || !postData.pid || !postData.tid) { + return; + } + const threshold = meta.config['flags:autoFlagOnDownvoteThreshold']; + if (threshold && postData.votes <= -threshold) { + const adminUid = await user.getFirstAdminUid(); + const reportMsg = await translator.translate(`[[flags:auto-flagged, ${-postData.votes}]]`); + const flagObj = await flags.create('post', postData.pid, adminUid, reportMsg, null, true); + await flags.notify(flagObj, adminUid, true); + } + await Promise.all([updateTopicVoteCount(postData), db.sortedSetAdd('posts:votes', postData.votes, postData.pid), Posts.setPostFields(postData.pid, { + upvotes: postData.upvotes, + downvotes: postData.downvotes + })]); + plugins.hooks.fire('action:post.updatePostVoteCount', { + post: postData + }); + }; + async function updateTopicVoteCount(postData) { + const topicData = await topics.getTopicFields(postData.tid, ['mainPid', 'cid', 'pinned']); + if (postData.uid) { + if (postData.votes !== 0) { + await db.sortedSetAdd(`cid:${topicData.cid}:uid:${postData.uid}:pids:votes`, postData.votes, postData.pid); + } else { + await db.sortedSetRemove(`cid:${topicData.cid}:uid:${postData.uid}:pids:votes`, postData.pid); + } + } + if (parseInt(topicData.mainPid, 10) !== parseInt(postData.pid, 10)) { + return await db.sortedSetAdd(`tid:${postData.tid}:posts:votes`, postData.votes, postData.pid); + } + const promises = [topics.setTopicFields(postData.tid, { + upvotes: postData.upvotes, + downvotes: postData.downvotes + }), db.sortedSetAdd('topics:votes', postData.votes, postData.tid)]; + if (!topicData.pinned) { + promises.push(db.sortedSetAdd(`cid:${topicData.cid}:tids:votes`, postData.votes, postData.tid)); + } + await Promise.all(promises); + } +}; \ No newline at end of file diff --git a/lib/prestart.js b/lib/prestart.js new file mode 100644 index 0000000000..ef9af67066 --- /dev/null +++ b/lib/prestart.js @@ -0,0 +1,106 @@ +'use strict'; + +const nconf = require('nconf'); +const url = require('url'); +const winston = require('winston'); +const path = require('path'); +const chalk = require('chalk'); +const pkg = require('../package.json'); +const { + paths +} = require('./constants'); +function setupWinston() { + if (!winston.format) { + return; + } + const formats = []; + if (nconf.get('log-colorize') !== 'false' && nconf.get('log-colorize') !== false) { + formats.push(winston.format.colorize()); + } + if (nconf.get('json-logging')) { + formats.push(winston.format.timestamp()); + formats.push(winston.format.json()); + } else { + const timestampFormat = winston.format(info => { + const dateString = `${new Date().toISOString()} [${nconf.get('port')}/${global.process.pid}]`; + info.level = `${dateString} - ${info.level}`; + return info; + }); + formats.push(timestampFormat()); + formats.push(winston.format.splat()); + formats.push(winston.format.simple()); + } + winston.configure({ + level: nconf.get('log-level') || (process.env.NODE_ENV === 'production' ? 'info' : 'verbose'), + format: winston.format.combine.apply(null, formats), + transports: [new winston.transports.Console({ + handleExceptions: true + })] + }); +} +function loadConfig(configFile) { + nconf.file({ + file: configFile + }); + nconf.defaults({ + base_dir: paths.baseDir, + themes_path: paths.nodeModules, + upload_path: 'public/uploads', + views_dir: path.join(paths.baseDir, 'build/public/templates'), + version: pkg.version, + isCluster: false, + isPrimary: true, + jobsDisabled: false, + fontawesome: { + pro: false, + styles: '*' + } + }); + const castAsBool = ['isCluster', 'isPrimary', 'jobsDisabled']; + nconf.stores.env.readOnly = false; + castAsBool.forEach(prop => { + const value = nconf.get(prop); + if (value !== undefined) { + nconf.set(prop, ['1', 1, 'true', true].includes(value)); + } + }); + nconf.stores.env.readOnly = true; + nconf.set('runJobs', nconf.get('isPrimary') && !nconf.get('jobsDisabled')); + nconf.set('themes_path', path.resolve(paths.baseDir, nconf.get('themes_path'))); + nconf.set('core_templates_path', path.join(paths.baseDir, 'src/views')); + nconf.set('upload_path', path.resolve(nconf.get('base_dir'), nconf.get('upload_path'))); + nconf.set('upload_url', '/assets/uploads'); + if (!nconf.get('sessionKey')) { + nconf.set('sessionKey', 'express.sid'); + } + if (nconf.get('url')) { + nconf.set('url', nconf.get('url').replace(/\/$/, '')); + nconf.set('url_parsed', url.parse(nconf.get('url'))); + const urlObject = url.parse(nconf.get('url')); + const relativePath = urlObject.pathname !== '/' ? urlObject.pathname.replace(/\/+$/, '') : ''; + nconf.set('base_url', `${urlObject.protocol}//${urlObject.host}`); + nconf.set('secure', urlObject.protocol === 'https:'); + nconf.set('use_port', !!urlObject.port); + nconf.set('relative_path', relativePath); + if (!nconf.get('asset_base_url')) { + nconf.set('asset_base_url', `${relativePath}/assets`); + } + nconf.set('port', nconf.get('PORT') || nconf.get('port') || urlObject.port || (nconf.get('PORT_ENV_VAR') ? nconf.get(nconf.get('PORT_ENV_VAR')) : false) || 4567); + const domain = nconf.get('cookieDomain') || urlObject.hostname; + const origins = nconf.get('socket.io:origins') || `${urlObject.protocol}//${domain}:*`; + nconf.set('socket.io:origins', origins); + } +} +function versionCheck() { + const version = process.version.slice(1); + const range = pkg.engines.node; + const semver = require('semver'); + const compatible = semver.satisfies(version, range); + if (!compatible) { + winston.warn('Your version of Node.js is too outdated for NodeBB. Please update your version of Node.js.'); + winston.warn(`Recommended ${chalk.green(range)}, ${chalk.yellow(version)} provided\n`); + } +} +exports.setupWinston = setupWinston; +exports.loadConfig = loadConfig; +exports.versionCheck = versionCheck; \ No newline at end of file diff --git a/lib/privileges/admin.js b/lib/privileges/admin.js new file mode 100644 index 0000000000..a1554082ce --- /dev/null +++ b/lib/privileges/admin.js @@ -0,0 +1,187 @@ +'use strict'; + +const _ = require('lodash'); +const user = require('../user'); +const groups = require('../groups'); +const helpers = require('./helpers'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const privsAdmin = module.exports; +const _privilegeMap = new Map([['admin:dashboard', { + label: '[[admin/manage/privileges:admin-dashboard]]', + type: 'admin' +}], ['admin:categories', { + label: '[[admin/manage/privileges:admin-categories]]', + type: 'admin' +}], ['admin:privileges', { + label: '[[admin/manage/privileges:admin-privileges]]', + type: 'admin' +}], ['admin:admins-mods', { + label: '[[admin/manage/privileges:admin-admins-mods]]', + type: 'admin' +}], ['admin:users', { + label: '[[admin/manage/privileges:admin-users]]', + type: 'admin' +}], ['admin:groups', { + label: '[[admin/manage/privileges:admin-groups]]', + type: 'admin' +}], ['admin:tags', { + label: '[[admin/manage/privileges:admin-tags]]', + type: 'admin' +}], ['admin:settings', { + label: '[[admin/manage/privileges:admin-settings]]', + type: 'admin' +}]]); +privsAdmin.init = async () => { + await plugins.hooks.fire('static:privileges.admin.init', { + privileges: _privilegeMap + }); + for (const [, value] of _privilegeMap) { + if (value && !value.type) { + value.type = 'other'; + } + } +}; +privsAdmin.getUserPrivilegeList = async () => await plugins.hooks.fire('filter:privileges.admin.list', Array.from(_privilegeMap.keys())); +privsAdmin.getGroupPrivilegeList = async () => await plugins.hooks.fire('filter:privileges.admin.groups.list', Array.from(_privilegeMap.keys()).map(privilege => `groups:${privilege}`)); +privsAdmin.getPrivilegeList = async () => { + const [user, group] = await Promise.all([privsAdmin.getUserPrivilegeList(), privsAdmin.getGroupPrivilegeList()]); + return user.concat(group); +}; +privsAdmin.routeMap = { + dashboard: 'admin:dashboard', + 'manage/categories': 'admin:categories', + 'manage/privileges': 'admin:privileges', + 'manage/admins-mods': 'admin:admins-mods', + 'manage/users': 'admin:users', + 'manage/groups': 'admin:groups', + 'manage/tags': 'admin:tags', + 'settings/tags': 'admin:tags', + 'extend/plugins': 'admin:settings', + 'extend/widgets': 'admin:settings', + 'extend/rewards': 'admin:settings', + 'category/uploadpicture': 'admin:categories', + uploadfavicon: 'admin:settings', + uploadTouchIcon: 'admin:settings', + uploadMaskableIcon: 'admin:settings', + uploadlogo: 'admin:settings', + uploadOgImage: 'admin:settings', + uploadDefaultAvatar: 'admin:settings' +}; +privsAdmin.routePrefixMap = { + 'dashboard/': 'admin:dashboard', + 'manage/categories/': 'admin:categories', + 'manage/privileges/': 'admin:privileges', + 'manage/groups/': 'admin:groups', + 'settings/': 'admin:settings', + 'appearance/': 'admin:settings', + 'plugins/': 'admin:settings' +}; +privsAdmin.socketMap = { + 'admin.rooms.getAll': 'admin:dashboard', + 'admin.analytics.get': 'admin:dashboard', + 'admin.categories.copySettingsFrom': 'admin:categories', + 'admin.categories.copyPrivilegesToChildren': 'admin:privileges', + 'admin.categories.copyPrivilegesFrom': 'admin:privileges', + 'admin.categories.copyPrivilegesToAllCategories': 'admin:privileges', + 'admin.user.makeAdmins': 'admin:admins-mods', + 'admin.user.removeAdmins': 'admin:admins-mods', + 'admin.user.loadGroups': 'admin:users', + 'admin.groups.join': 'admin:users', + 'admin.groups.leave': 'admin:users', + 'admin.user.resetLockouts': 'admin:users', + 'admin.user.validateEmail': 'admin:users', + 'admin.user.sendValidationEmail': 'admin:users', + 'admin.user.sendPasswordResetEmail': 'admin:users', + 'admin.user.forcePasswordReset': 'admin:users', + 'admin.user.invite': 'admin:users', + 'admin.tags.create': 'admin:tags', + 'admin.tags.rename': 'admin:tags', + 'admin.tags.deleteTags': 'admin:tags', + 'admin.getSearchDict': 'admin:settings', + 'admin.config.setMultiple': 'admin:settings', + 'admin.config.remove': 'admin:settings', + 'admin.themes.getInstalled': 'admin:settings', + 'admin.themes.set': 'admin:settings', + 'admin.reloadAllSessions': 'admin:settings', + 'admin.settings.get': 'admin:settings', + 'admin.settings.set': 'admin:settings' +}; +privsAdmin.resolve = path => { + if (privsAdmin.routeMap.hasOwnProperty(path)) { + return privsAdmin.routeMap[path]; + } + const found = Object.entries(privsAdmin.routePrefixMap).filter(entry => path.startsWith(entry[0])).sort((entry1, entry2) => entry2[0].length - entry1[0].length); + if (!found.length) { + return undefined; + } + return found[0][1]; +}; +privsAdmin.list = async function (uid) { + const privilegeLabels = Array.from(_privilegeMap.values()).map(data => data.label); + const userPrivilegeList = await privsAdmin.getUserPrivilegeList(); + const groupPrivilegeList = await privsAdmin.getGroupPrivilegeList(); + if (!(await user.isAdministrator(uid))) { + const idx = Array.from(_privilegeMap.keys()).indexOf('admin:privileges'); + privilegeLabels.splice(idx, 1); + userPrivilegeList.splice(idx, 1); + groupPrivilegeList.splice(idx, 1); + } + const labels = await utils.promiseParallel({ + users: plugins.hooks.fire('filter:privileges.admin.list_human', privilegeLabels.slice()), + groups: plugins.hooks.fire('filter:privileges.admin.groups.list_human', privilegeLabels.slice()) + }); + const keys = { + users: userPrivilegeList, + groups: groupPrivilegeList + }; + const payload = await utils.promiseParallel({ + labels, + labelData: Array.from(_privilegeMap.values()), + users: helpers.getUserPrivileges(0, keys.users), + groups: helpers.getGroupPrivileges(0, keys.groups) + }); + payload.keys = keys; + return payload; +}; +privsAdmin.get = async function (uid) { + const userPrivilegeList = await privsAdmin.getUserPrivilegeList(); + const [userPrivileges, isAdministrator] = await Promise.all([helpers.isAllowedTo(userPrivilegeList, uid, 0), user.isAdministrator(uid)]); + const combined = userPrivileges.map(allowed => allowed || isAdministrator); + const privData = _.zipObject(userPrivilegeList, combined); + privData.superadmin = isAdministrator; + return await plugins.hooks.fire('filter:privileges.admin.get', privData); +}; +privsAdmin.can = async function (privilege, uid) { + const [isUserAllowedTo, isAdministrator] = await Promise.all([helpers.isAllowedTo(privilege, uid, [0]), user.isAdministrator(uid)]); + return isAdministrator || isUserAllowedTo[0]; +}; +privsAdmin.canGroup = async function (privilege, groupName) { + return await groups.isMember(groupName, `cid:0:privileges:groups:${privilege}`); +}; +privsAdmin.give = async function (privileges, groupName) { + await helpers.giveOrRescind(groups.join, privileges, 0, groupName); + plugins.hooks.fire('action:privileges.admin.give', { + privileges: privileges, + groupNames: Array.isArray(groupName) ? groupName : [groupName] + }); +}; +privsAdmin.rescind = async function (privileges, groupName) { + await helpers.giveOrRescind(groups.leave, privileges, 0, groupName); + plugins.hooks.fire('action:privileges.admin.rescind', { + privileges: privileges, + groupNames: Array.isArray(groupName) ? groupName : [groupName] + }); +}; +privsAdmin.userPrivileges = async function (uid) { + const userPrivilegeList = await privsAdmin.getUserPrivilegeList(); + return await helpers.userOrGroupPrivileges(0, uid, userPrivilegeList); +}; +privsAdmin.groupPrivileges = async function (groupName) { + const groupPrivilegeList = await privsAdmin.getGroupPrivilegeList(); + return await helpers.userOrGroupPrivileges(0, groupName, groupPrivilegeList); +}; +privsAdmin.getUidsWithPrivilege = async function (privilege) { + const uidsByCid = await helpers.getUidsWithPrivilege([0], privilege); + return uidsByCid[0]; +}; \ No newline at end of file diff --git a/lib/privileges/categories.js b/lib/privileges/categories.js new file mode 100644 index 0000000000..889567f857 --- /dev/null +++ b/lib/privileges/categories.js @@ -0,0 +1,207 @@ +'use strict'; + +const _ = require('lodash'); +const categories = require('../categories'); +const user = require('../user'); +const groups = require('../groups'); +const helpers = require('./helpers'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const privsCategories = module.exports; +const _privilegeMap = new Map([['find', { + label: '[[admin/manage/privileges:find-category]]', + type: 'viewing' +}], ['read', { + label: '[[admin/manage/privileges:access-category]]', + type: 'viewing' +}], ['topics:read', { + label: '[[admin/manage/privileges:access-topics]]', + type: 'viewing' +}], ['topics:create', { + label: '[[admin/manage/privileges:create-topics]]', + type: 'posting' +}], ['topics:reply', { + label: '[[admin/manage/privileges:reply-to-topics]]', + type: 'posting' +}], ['topics:schedule', { + label: '[[admin/manage/privileges:schedule-topics]]', + type: 'posting' +}], ['topics:tag', { + label: '[[admin/manage/privileges:tag-topics]]', + type: 'posting' +}], ['posts:edit', { + label: '[[admin/manage/privileges:edit-posts]]', + type: 'posting' +}], ['posts:history', { + label: '[[admin/manage/privileges:view-edit-history]]', + type: 'posting' +}], ['posts:delete', { + label: '[[admin/manage/privileges:delete-posts]]', + type: 'posting' +}], ['posts:upvote', { + label: '[[admin/manage/privileges:upvote-posts]]', + type: 'posting' +}], ['posts:downvote', { + label: '[[admin/manage/privileges:downvote-posts]]', + type: 'posting' +}], ['topics:delete', { + label: '[[admin/manage/privileges:delete-topics]]', + type: 'posting' +}], ['posts:view_deleted', { + label: '[[admin/manage/privileges:view-deleted]]', + type: 'moderation' +}], ['purge', { + label: '[[admin/manage/privileges:purge]]', + type: 'moderation' +}], ['moderate', { + label: '[[admin/manage/privileges:moderate]]', + type: 'moderation' +}]]); +privsCategories.init = async () => { + privsCategories._coreSize = _privilegeMap.size; + await plugins.hooks.fire('static:privileges.categories.init', { + privileges: _privilegeMap + }); + for (const [, value] of _privilegeMap) { + if (value && !value.type) { + value.type = 'other'; + } + } +}; +privsCategories.getType = function (privilege) { + const priv = _privilegeMap.get(privilege); + return priv && priv.type ? priv.type : ''; +}; +privsCategories.getUserPrivilegeList = async () => await plugins.hooks.fire('filter:privileges.list', Array.from(_privilegeMap.keys())); +privsCategories.getGroupPrivilegeList = async () => await plugins.hooks.fire('filter:privileges.groups.list', Array.from(_privilegeMap.keys()).map(privilege => `groups:${privilege}`)); +privsCategories.getPrivilegeList = async () => { + const [user, group] = await Promise.all([privsCategories.getUserPrivilegeList(), privsCategories.getGroupPrivilegeList()]); + return user.concat(group); +}; +privsCategories.getPrivilegesByFilter = function (filter) { + return Array.from(_privilegeMap.entries()).filter(priv => priv[1] && (!filter || priv[1].type === filter)).map(priv => priv[0]); +}; +privsCategories.list = async function (cid) { + let labels = Array.from(_privilegeMap.values()).map(data => data.label); + labels = await utils.promiseParallel({ + users: plugins.hooks.fire('filter:privileges.list_human', labels.slice()), + groups: plugins.hooks.fire('filter:privileges.groups.list_human', labels.slice()) + }); + const keys = await utils.promiseParallel({ + users: privsCategories.getUserPrivilegeList(), + groups: privsCategories.getGroupPrivilegeList() + }); + const payload = await utils.promiseParallel({ + labels, + labelData: Array.from(_privilegeMap.values()), + users: helpers.getUserPrivileges(cid, keys.users), + groups: helpers.getGroupPrivileges(cid, keys.groups) + }); + payload.keys = keys; + payload.columnCountUserOther = payload.labels.users.length - privsCategories._coreSize; + payload.columnCountGroupOther = payload.labels.groups.length - privsCategories._coreSize; + return payload; +}; +privsCategories.get = async function (cid, uid) { + const privs = ['topics:create', 'topics:read', 'topics:schedule', 'topics:tag', 'read', 'posts:view_deleted']; + const [userPrivileges, isAdministrator, isModerator] = await Promise.all([helpers.isAllowedTo(privs, uid, cid), user.isAdministrator(uid), user.isModerator(uid, cid)]); + const combined = userPrivileges.map(allowed => allowed || isAdministrator); + const privData = _.zipObject(privs, combined); + const isAdminOrMod = isAdministrator || isModerator; + return await plugins.hooks.fire('filter:privileges.categories.get', { + ...privData, + cid: cid, + uid: uid, + editable: isAdminOrMod, + view_deleted: isAdminOrMod || privData['posts:view_deleted'], + isAdminOrMod: isAdminOrMod + }); +}; +privsCategories.isAdminOrMod = async function (cid, uid) { + if (parseInt(uid, 10) <= 0) { + return false; + } + const [isAdmin, isMod] = await Promise.all([user.isAdministrator(uid), user.isModerator(uid, cid)]); + return isAdmin || isMod; +}; +privsCategories.isUserAllowedTo = async function (privilege, cid, uid) { + if (Array.isArray(privilege) && !privilege.length || Array.isArray(cid) && !cid.length) { + return []; + } + if (!cid) { + return false; + } + const results = await helpers.isAllowedTo(privilege, uid, Array.isArray(cid) ? cid : [cid]); + if (Array.isArray(results) && results.length) { + return Array.isArray(cid) ? results : results[0]; + } + return false; +}; +privsCategories.can = async function (privilege, cid, uid) { + if (!cid) { + return false; + } + const [disabled, isAdmin, isAllowed] = await Promise.all([categories.getCategoryField(cid, 'disabled'), user.isAdministrator(uid), privsCategories.isUserAllowedTo(privilege, cid, uid)]); + return !disabled && (isAllowed || isAdmin); +}; +privsCategories.filterCids = async function (privilege, cids, uid) { + if (!Array.isArray(cids) || !cids.length) { + return []; + } + cids = _.uniq(cids); + const [categoryData, allowedTo, isAdmin] = await Promise.all([categories.getCategoriesFields(cids, ['disabled']), helpers.isAllowedTo(privilege, uid, cids), user.isAdministrator(uid)]); + return cids.filter((cid, index) => !!cid && !categoryData[index].disabled && (allowedTo[index] || isAdmin)); +}; +privsCategories.getBase = async function (privilege, cids, uid) { + return await utils.promiseParallel({ + categories: categories.getCategoriesFields(cids, ['disabled']), + allowedTo: helpers.isAllowedTo(privilege, uid, cids), + view_deleted: helpers.isAllowedTo('posts:view_deleted', uid, cids), + view_scheduled: helpers.isAllowedTo('topics:schedule', uid, cids), + isAdmin: user.isAdministrator(uid) + }); +}; +privsCategories.filterUids = async function (privilege, cid, uids) { + if (!uids.length) { + return []; + } + uids = _.uniq(uids); + const [allowedTo, isAdmins] = await Promise.all([helpers.isUsersAllowedTo(privilege, uids, cid), user.isAdministrator(uids)]); + return uids.filter((uid, index) => allowedTo[index] || isAdmins[index]); +}; +privsCategories.give = async function (privileges, cid, members) { + await helpers.giveOrRescind(groups.join, privileges, cid, members); + plugins.hooks.fire('action:privileges.categories.give', { + privileges: privileges, + cids: Array.isArray(cid) ? cid : [cid], + members: Array.isArray(members) ? members : [members] + }); +}; +privsCategories.rescind = async function (privileges, cid, members) { + await helpers.giveOrRescind(groups.leave, privileges, cid, members); + plugins.hooks.fire('action:privileges.categories.rescind', { + privileges: privileges, + cids: Array.isArray(cid) ? cid : [cid], + members: Array.isArray(members) ? members : [members] + }); +}; +privsCategories.canMoveAllTopics = async function (currentCid, targetCid, uid) { + const [isAdmin, isModerators] = await Promise.all([user.isAdministrator(uid), user.isModerator(uid, [currentCid, targetCid])]); + return isAdmin || !isModerators.includes(false); +}; +privsCategories.canPostTopic = async function (uid) { + let cids = await categories.getAllCidsFromSet('categories:cid'); + cids = await privsCategories.filterCids('topics:create', cids, uid); + return cids.length > 0; +}; +privsCategories.userPrivileges = async function (cid, uid) { + const userPrivilegeList = await privsCategories.getUserPrivilegeList(); + return await helpers.userOrGroupPrivileges(cid, uid, userPrivilegeList); +}; +privsCategories.groupPrivileges = async function (cid, groupName) { + const groupPrivilegeList = await privsCategories.getGroupPrivilegeList(); + return await helpers.userOrGroupPrivileges(cid, groupName, groupPrivilegeList); +}; +privsCategories.getUidsWithPrivilege = async function (cids, privilege) { + return await helpers.getUidsWithPrivilege(cids, privilege); +}; \ No newline at end of file diff --git a/lib/privileges/global.js b/lib/privileges/global.js new file mode 100644 index 0000000000..0ccb561f56 --- /dev/null +++ b/lib/privileges/global.js @@ -0,0 +1,150 @@ +'use strict'; + +const _ = require('lodash'); +const user = require('../user'); +const groups = require('../groups'); +const helpers = require('./helpers'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const privsGlobal = module.exports; +const _privilegeMap = new Map([['chat', { + label: '[[admin/manage/privileges:chat]]', + type: 'posting' +}], ['chat:privileged', { + label: '[[admin/manage/privileges:chat-with-privileged]]', + type: 'posting' +}], ['upload:post:image', { + label: '[[admin/manage/privileges:upload-images]]', + type: 'posting' +}], ['upload:post:file', { + label: '[[admin/manage/privileges:upload-files]]', + type: 'posting' +}], ['signature', { + label: '[[admin/manage/privileges:signature]]', + type: 'posting' +}], ['invite', { + label: '[[admin/manage/privileges:invite]]', + type: 'posting' +}], ['group:create', { + label: '[[admin/manage/privileges:allow-group-creation]]', + type: 'posting' +}], ['search:content', { + label: '[[admin/manage/privileges:search-content]]', + type: 'viewing' +}], ['search:users', { + label: '[[admin/manage/privileges:search-users]]', + type: 'viewing' +}], ['search:tags', { + label: '[[admin/manage/privileges:search-tags]]', + type: 'viewing' +}], ['view:users', { + label: '[[admin/manage/privileges:view-users]]', + type: 'viewing' +}], ['view:tags', { + label: '[[admin/manage/privileges:view-tags]]', + type: 'viewing' +}], ['view:groups', { + label: '[[admin/manage/privileges:view-groups]]', + type: 'viewing' +}], ['local:login', { + label: '[[admin/manage/privileges:allow-local-login]]', + type: 'viewing' +}], ['ban', { + label: '[[admin/manage/privileges:ban]]', + type: 'moderation' +}], ['mute', { + label: '[[admin/manage/privileges:mute]]', + type: 'moderation' +}], ['view:users:info', { + label: '[[admin/manage/privileges:view-users-info]]', + type: 'moderation' +}]]); +privsGlobal.init = async () => { + privsGlobal._coreSize = _privilegeMap.size; + await plugins.hooks.fire('static:privileges.global.init', { + privileges: _privilegeMap + }); + for (const [, value] of _privilegeMap) { + if (value && !value.type) { + value.type = 'other'; + } + } +}; +privsGlobal.getType = function (privilege) { + const priv = _privilegeMap.get(privilege); + return priv && priv.type ? priv.type : ''; +}; +privsGlobal.getUserPrivilegeList = async () => await plugins.hooks.fire('filter:privileges.global.list', Array.from(_privilegeMap.keys())); +privsGlobal.getGroupPrivilegeList = async () => await plugins.hooks.fire('filter:privileges.global.groups.list', Array.from(_privilegeMap.keys()).map(privilege => `groups:${privilege}`)); +privsGlobal.getPrivilegeList = async () => { + const [user, group] = await Promise.all([privsGlobal.getUserPrivilegeList(), privsGlobal.getGroupPrivilegeList()]); + return user.concat(group); +}; +privsGlobal.list = async function () { + async function getLabels() { + const labels = Array.from(_privilegeMap.values()).map(data => data.label); + return await utils.promiseParallel({ + users: plugins.hooks.fire('filter:privileges.global.list_human', labels.slice()), + groups: plugins.hooks.fire('filter:privileges.global.groups.list_human', labels.slice()) + }); + } + const keys = await utils.promiseParallel({ + users: privsGlobal.getUserPrivilegeList(), + groups: privsGlobal.getGroupPrivilegeList() + }); + const payload = await utils.promiseParallel({ + labels: getLabels(), + labelData: Array.from(_privilegeMap.values()), + users: helpers.getUserPrivileges(0, keys.users), + groups: helpers.getGroupPrivileges(0, keys.groups) + }); + payload.keys = keys; + payload.columnCountUserOther = keys.users.length - privsGlobal._coreSize; + payload.columnCountGroupOther = keys.groups.length - privsGlobal._coreSize; + return payload; +}; +privsGlobal.get = async function (uid) { + const userPrivilegeList = await privsGlobal.getUserPrivilegeList(); + const [userPrivileges, isAdministrator] = await Promise.all([helpers.isAllowedTo(userPrivilegeList, uid, 0), user.isAdministrator(uid)]); + const combined = userPrivileges.map(allowed => allowed || isAdministrator); + const privData = _.zipObject(userPrivilegeList, combined); + return await plugins.hooks.fire('filter:privileges.global.get', privData); +}; +privsGlobal.can = async function (privilege, uid) { + const isArray = Array.isArray(privilege); + const [isAdministrator, isUserAllowedTo] = await Promise.all([user.isAdministrator(uid), helpers.isAllowedTo(isArray ? privilege : [privilege], uid, 0)]); + return isArray ? isUserAllowedTo.map(allowed => isAdministrator || allowed) : isAdministrator || isUserAllowedTo[0]; +}; +privsGlobal.canGroup = async function (privilege, groupName) { + return await groups.isMember(groupName, `cid:0:privileges:groups:${privilege}`); +}; +privsGlobal.filterUids = async function (privilege, uids) { + const privCategories = require('./categories'); + return await privCategories.filterUids(privilege, 0, uids); +}; +privsGlobal.give = async function (privileges, groupName) { + await helpers.giveOrRescind(groups.join, privileges, 0, groupName); + plugins.hooks.fire('action:privileges.global.give', { + privileges: privileges, + groupNames: Array.isArray(groupName) ? groupName : [groupName] + }); +}; +privsGlobal.rescind = async function (privileges, groupName) { + await helpers.giveOrRescind(groups.leave, privileges, 0, groupName); + plugins.hooks.fire('action:privileges.global.rescind', { + privileges: privileges, + groupNames: Array.isArray(groupName) ? groupName : [groupName] + }); +}; +privsGlobal.userPrivileges = async function (uid) { + const userPrivilegeList = await privsGlobal.getUserPrivilegeList(); + return await helpers.userOrGroupPrivileges(0, uid, userPrivilegeList); +}; +privsGlobal.groupPrivileges = async function (groupName) { + const groupPrivilegeList = await privsGlobal.getGroupPrivilegeList(); + return await helpers.userOrGroupPrivileges(0, groupName, groupPrivilegeList); +}; +privsGlobal.getUidsWithPrivilege = async function (privilege) { + const uidsByCid = await helpers.getUidsWithPrivilege([0], privilege); + return uidsByCid[0]; +}; \ No newline at end of file diff --git a/lib/privileges/helpers.js b/lib/privileges/helpers.js new file mode 100644 index 0000000000..6d1dfb4603 --- /dev/null +++ b/lib/privileges/helpers.js @@ -0,0 +1,202 @@ +'use strict'; + +const _ = require('lodash'); +const validator = require('validator'); +const groups = require('../groups'); +const user = require('../user'); +const categories = require('../categories'); +const plugins = require('../plugins'); +const translator = require('../translator'); +const helpers = module.exports; +const uidToSystemGroup = { + 0: 'guests', + '-1': 'spiders' +}; +helpers.isUsersAllowedTo = async function (privilege, uids, cid) { + const [hasUserPrivilege, hasGroupPrivilege] = await Promise.all([groups.isMembers(uids, `cid:${cid}:privileges:${privilege}`), groups.isMembersOfGroupList(uids, `cid:${cid}:privileges:groups:${privilege}`)]); + const allowed = uids.map((uid, index) => hasUserPrivilege[index] || hasGroupPrivilege[index]); + const result = await plugins.hooks.fire('filter:privileges:isUsersAllowedTo', { + allowed: allowed, + privilege: privilege, + uids: uids, + cid: cid + }); + return result.allowed; +}; +helpers.isAllowedTo = async function (privilege, uidOrGroupName, cid) { + let allowed; + if (Array.isArray(privilege) && !Array.isArray(cid)) { + allowed = await isAllowedToPrivileges(privilege, uidOrGroupName, cid); + } else if (Array.isArray(cid) && !Array.isArray(privilege)) { + allowed = await isAllowedToCids(privilege, uidOrGroupName, cid); + } + if (allowed) { + ({ + allowed + } = await plugins.hooks.fire('filter:privileges:isAllowedTo', { + allowed: allowed, + privilege: privilege, + uid: uidOrGroupName, + cid: cid + })); + return allowed; + } + throw new Error('[[error:invalid-data]]'); +}; +async function isAllowedToCids(privilege, uidOrGroupName, cids) { + if (!privilege) { + return cids.map(() => false); + } + const groupKeys = cids.map(cid => `cid:${cid}:privileges:groups:${privilege}`); + if (isNaN(parseInt(uidOrGroupName, 10)) && (uidOrGroupName || '').length) { + return await checkIfAllowedGroup(uidOrGroupName, groupKeys); + } + if (parseInt(uidOrGroupName, 10) <= 0) { + return await isSystemGroupAllowedToCids(privilege, uidOrGroupName, cids); + } + const userKeys = cids.map(cid => `cid:${cid}:privileges:${privilege}`); + return await checkIfAllowedUser(uidOrGroupName, userKeys, groupKeys); +} +async function isAllowedToPrivileges(privileges, uidOrGroupName, cid) { + const groupKeys = privileges.map(privilege => `cid:${cid}:privileges:groups:${privilege}`); + if (isNaN(parseInt(uidOrGroupName, 10)) && (uidOrGroupName || '').length) { + return await checkIfAllowedGroup(uidOrGroupName, groupKeys); + } + if (parseInt(uidOrGroupName, 10) <= 0) { + return await isSystemGroupAllowedToPrivileges(privileges, uidOrGroupName, cid); + } + const userKeys = privileges.map(privilege => `cid:${cid}:privileges:${privilege}`); + return await checkIfAllowedUser(uidOrGroupName, userKeys, groupKeys); +} +async function checkIfAllowedUser(uid, userKeys, groupKeys) { + const [hasUserPrivilege, hasGroupPrivilege] = await Promise.all([groups.isMemberOfGroups(uid, userKeys), groups.isMemberOfGroupsList(uid, groupKeys)]); + return userKeys.map((key, index) => hasUserPrivilege[index] || hasGroupPrivilege[index]); +} +async function checkIfAllowedGroup(groupName, groupKeys) { + const sets = await Promise.all([groups.isMemberOfGroups(groupName, groupKeys), groups.isMemberOfGroups('registered-users', groupKeys)]); + return groupKeys.map((key, index) => sets[0][index] || sets[1][index]); +} +async function isSystemGroupAllowedToCids(privilege, uid, cids) { + const groupKeys = cids.map(cid => `cid:${cid}:privileges:groups:${privilege}`); + return await groups.isMemberOfGroups(uidToSystemGroup[uid], groupKeys); +} +async function isSystemGroupAllowedToPrivileges(privileges, uid, cid) { + const groupKeys = privileges.map(privilege => `cid:${cid}:privileges:groups:${privilege}`); + return await groups.isMemberOfGroups(uidToSystemGroup[uid], groupKeys); +} +helpers.getUserPrivileges = async function (cid, userPrivileges) { + let memberSets = await groups.getMembersOfGroups(userPrivileges.map(privilege => `cid:${cid}:privileges:${privilege}`)); + memberSets = memberSets.map(set => set.map(uid => parseInt(uid, 10))); + const members = _.uniq(_.flatten(memberSets)); + const memberData = await user.getUsersFields(members, ['picture', 'username', 'banned']); + memberData.forEach(member => { + member.privileges = {}; + for (let x = 0, numPrivs = userPrivileges.length; x < numPrivs; x += 1) { + member.privileges[userPrivileges[x]] = memberSets[x].includes(parseInt(member.uid, 10)); + } + const types = {}; + for (const [key] of Object.entries(member.privileges)) { + types[key] = getType(key); + } + member.types = types; + }); + return memberData; +}; +helpers.getGroupPrivileges = async function (cid, groupPrivileges) { + const [memberSets, allGroupNames] = await Promise.all([groups.getMembersOfGroups(groupPrivileges.map(privilege => `cid:${cid}:privileges:${privilege}`)), groups.getGroups('groups:createtime', 0, -1)]); + const uniqueGroups = _.uniq(_.flatten(memberSets)); + let groupNames = allGroupNames.filter(groupName => !groupName.includes(':privileges:') && uniqueGroups.includes(groupName)); + groupNames = groups.ephemeralGroups.concat(groupNames); + moveToFront(groupNames, groups.BANNED_USERS); + moveToFront(groupNames, 'Global Moderators'); + moveToFront(groupNames, 'unverified-users'); + moveToFront(groupNames, 'verified-users'); + moveToFront(groupNames, 'registered-users'); + const adminIndex = groupNames.indexOf('administrators'); + if (adminIndex !== -1) { + groupNames.splice(adminIndex, 1); + } + const groupData = await groups.getGroupsFields(groupNames, ['private', 'system']); + const memberData = groupNames.map((member, index) => { + const memberPrivs = {}; + for (let x = 0, numPrivs = groupPrivileges.length; x < numPrivs; x += 1) { + memberPrivs[groupPrivileges[x]] = memberSets[x].includes(member); + } + const types = {}; + for (const [key] of Object.entries(memberPrivs)) { + types[key] = getType(key); + } + return { + name: validator.escape(member), + nameEscaped: translator.escape(validator.escape(member)), + privileges: memberPrivs, + types: types, + isPrivate: groupData[index] && !!groupData[index].private, + isSystem: groupData[index] && !!groupData[index].system + }; + }); + return memberData; +}; +function getType(privilege) { + privilege = privilege.replace(/^groups:/, ''); + const global = require('./global'); + const categories = require('./categories'); + return global.getType(privilege) || categories.getType(privilege) || 'other'; +} +function moveToFront(groupNames, groupToMove) { + const index = groupNames.indexOf(groupToMove); + if (index !== -1) { + groupNames.splice(0, 0, groupNames.splice(index, 1)[0]); + } else { + groupNames.unshift(groupToMove); + } +} +helpers.giveOrRescind = async function (method, privileges, cids, members) { + members = Array.isArray(members) ? members : [members]; + cids = Array.isArray(cids) ? cids : [cids]; + for (const member of members) { + const groupKeys = []; + cids.forEach(cid => { + privileges.forEach(privilege => { + groupKeys.push(`cid:${cid}:privileges:${privilege}`); + }); + }); + await method(groupKeys, member); + } +}; +helpers.userOrGroupPrivileges = async function (cid, uidOrGroup, privilegeList) { + const groupNames = privilegeList.map(privilege => `cid:${cid}:privileges:${privilege}`); + const isMembers = await groups.isMemberOfGroups(uidOrGroup, groupNames); + return _.zipObject(privilegeList, isMembers); +}; +helpers.getUidsWithPrivilege = async (cids, privilege) => { + const disabled = (await categories.getCategoriesFields(cids, ['disabled'])).map(obj => obj.disabled); + const groupNames = cids.reduce((memo, cid) => { + memo.push(`cid:${cid}:privileges:${privilege}`); + memo.push(`cid:${cid}:privileges:groups:${privilege}`); + return memo; + }, []); + const memberSets = await groups.getMembersOfGroups(groupNames); + const sets = memberSets.reduce((memo, set, idx) => { + if (idx % 2) { + memo.groupNames.push(set); + } else { + memo.uids.push(set); + } + return memo; + }, { + groupNames: [], + uids: [] + }); + const uniqGroups = _.uniq(_.flatten(sets.groupNames)); + const groupUids = await groups.getMembersOfGroups(uniqGroups); + const map = _.zipObject(uniqGroups, groupUids); + const uidsByCid = cids.map((cid, index) => { + if (disabled[index]) { + return []; + } + return _.uniq(sets.uids[index].concat(_.flatten(sets.groupNames[index].map(g => map[g])))); + }); + return uidsByCid; +}; +require('../promisify')(helpers); \ No newline at end of file diff --git a/lib/privileges/index.js b/lib/privileges/index.js new file mode 100644 index 0000000000..d29e64b1e3 --- /dev/null +++ b/lib/privileges/index.js @@ -0,0 +1,15 @@ +'use strict'; + +const privileges = module.exports; +privileges.global = require('./global'); +privileges.admin = require('./admin'); +privileges.categories = require('./categories'); +privileges.topics = require('./topics'); +privileges.posts = require('./posts'); +privileges.users = require('./users'); +privileges.init = async () => { + await privileges.global.init(); + await privileges.admin.init(); + await privileges.categories.init(); +}; +require('../promisify')(privileges); \ No newline at end of file diff --git a/lib/privileges/posts.js b/lib/privileges/posts.js new file mode 100644 index 0000000000..4144ca0f14 --- /dev/null +++ b/lib/privileges/posts.js @@ -0,0 +1,213 @@ +'use strict'; + +const _ = require('lodash'); +const meta = require('../meta'); +const posts = require('../posts'); +const topics = require('../topics'); +const user = require('../user'); +const helpers = require('./helpers'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const privsCategories = require('./categories'); +const privsTopics = require('./topics'); +const privsPosts = module.exports; +privsPosts.get = async function (pids, uid) { + if (!Array.isArray(pids) || !pids.length) { + return []; + } + const cids = await posts.getCidsByPids(pids); + const uniqueCids = _.uniq(cids); + const results = await utils.promiseParallel({ + isAdmin: user.isAdministrator(uid), + isModerator: user.isModerator(uid, uniqueCids), + isOwner: posts.isOwner(pids, uid), + 'topics:read': helpers.isAllowedTo('topics:read', uid, uniqueCids), + read: helpers.isAllowedTo('read', uid, uniqueCids), + 'posts:edit': helpers.isAllowedTo('posts:edit', uid, uniqueCids), + 'posts:history': helpers.isAllowedTo('posts:history', uid, uniqueCids), + 'posts:view_deleted': helpers.isAllowedTo('posts:view_deleted', uid, uniqueCids) + }); + const isModerator = _.zipObject(uniqueCids, results.isModerator); + const privData = {}; + privData['topics:read'] = _.zipObject(uniqueCids, results['topics:read']); + privData.read = _.zipObject(uniqueCids, results.read); + privData['posts:edit'] = _.zipObject(uniqueCids, results['posts:edit']); + privData['posts:history'] = _.zipObject(uniqueCids, results['posts:history']); + privData['posts:view_deleted'] = _.zipObject(uniqueCids, results['posts:view_deleted']); + const privileges = cids.map((cid, i) => { + const isAdminOrMod = results.isAdmin || isModerator[cid]; + const editable = privData['posts:edit'][cid] && (results.isOwner[i] || results.isModerator) || results.isAdmin; + const viewDeletedPosts = results.isOwner[i] || privData['posts:view_deleted'][cid] || results.isAdmin; + const viewHistory = results.isOwner[i] || privData['posts:history'][cid] || results.isAdmin; + return { + editable: editable, + move: isAdminOrMod, + isAdminOrMod: isAdminOrMod, + 'topics:read': privData['topics:read'][cid] || results.isAdmin, + read: privData.read[cid] || results.isAdmin, + 'posts:history': viewHistory, + 'posts:view_deleted': viewDeletedPosts + }; + }); + return privileges; +}; +privsPosts.can = async function (privilege, pid, uid) { + const cid = await posts.getCidByPid(pid); + return await privsCategories.can(privilege, cid, uid); +}; +privsPosts.filter = async function (privilege, pids, uid) { + if (!Array.isArray(pids) || !pids.length) { + return []; + } + pids = _.uniq(pids); + const postData = await posts.getPostsFields(pids, ['uid', 'tid', 'deleted']); + const tids = _.uniq(postData.map(post => post && post.tid).filter(Boolean)); + const topicData = await topics.getTopicsFields(tids, ['deleted', 'scheduled', 'cid']); + const tidToTopic = _.zipObject(tids, topicData); + let cids = postData.map((post, index) => { + if (post) { + post.pid = pids[index]; + post.topic = tidToTopic[post.tid]; + } + return tidToTopic[post.tid] && tidToTopic[post.tid].cid; + }).filter(cid => parseInt(cid, 10)); + cids = _.uniq(cids); + const results = await privsCategories.getBase(privilege, cids, uid); + const allowedCids = cids.filter((cid, index) => !results.categories[index].disabled && (results.allowedTo[index] || results.isAdmin)); + const cidsSet = new Set(allowedCids); + const canViewDeleted = _.zipObject(cids, results.view_deleted); + const canViewScheduled = _.zipObject(cids, results.view_scheduled); + pids = postData.filter(post => post.topic && cidsSet.has(post.topic.cid) && (privsTopics.canViewDeletedScheduled({ + deleted: post.topic.deleted || post.deleted, + scheduled: post.topic.scheduled + }, {}, canViewDeleted[post.topic.cid], canViewScheduled[post.topic.cid]) || results.isAdmin)).map(post => post.pid); + const data = await plugins.hooks.fire('filter:privileges.posts.filter', { + privilege: privilege, + uid: uid, + pids: pids + }); + return data ? data.pids : null; +}; +privsPosts.canEdit = async function (pid, uid) { + const results = await utils.promiseParallel({ + isAdmin: user.isAdministrator(uid), + isMod: posts.isModerator([pid], uid), + owner: posts.isOwner(pid, uid), + edit: privsPosts.can('posts:edit', pid, uid), + postData: posts.getPostFields(pid, ['tid', 'timestamp', 'deleted', 'deleterUid']), + userData: user.getUserFields(uid, ['reputation']) + }); + results.isMod = results.isMod[0]; + if (results.isAdmin) { + return { + flag: true + }; + } + if (!results.isMod && meta.config.postEditDuration && Date.now() - results.postData.timestamp > meta.config.postEditDuration * 1000) { + return { + flag: false, + message: `[[error:post-edit-duration-expired, ${meta.config.postEditDuration}]]` + }; + } + if (!results.isMod && meta.config.newbiePostEditDuration > 0 && meta.config.newbieReputationThreshold > results.userData.reputation && Date.now() - results.postData.timestamp > meta.config.newbiePostEditDuration * 1000) { + return { + flag: false, + message: `[[error:post-edit-duration-expired, ${meta.config.newbiePostEditDuration}]]` + }; + } + const isLocked = await topics.isLocked(results.postData.tid); + if (!results.isMod && isLocked) { + return { + flag: false, + message: '[[error:topic-locked]]' + }; + } + if (!results.isMod && results.postData.deleted && parseInt(uid, 10) !== parseInt(results.postData.deleterUid, 10)) { + return { + flag: false, + message: '[[error:post-deleted]]' + }; + } + results.pid = parseInt(pid, 10); + results.uid = uid; + const result = await plugins.hooks.fire('filter:privileges.posts.edit', results); + return { + flag: result.edit && (result.owner || result.isMod), + message: '[[error:no-privileges]]' + }; +}; +privsPosts.canDelete = async function (pid, uid) { + const postData = await posts.getPostFields(pid, ['uid', 'tid', 'timestamp', 'deleterUid']); + const results = await utils.promiseParallel({ + isAdmin: user.isAdministrator(uid), + isMod: posts.isModerator([pid], uid), + isLocked: topics.isLocked(postData.tid), + isOwner: posts.isOwner(pid, uid), + 'posts:delete': privsPosts.can('posts:delete', pid, uid) + }); + results.isMod = results.isMod[0]; + if (results.isAdmin) { + return { + flag: true + }; + } + if (!results.isMod && results.isLocked) { + return { + flag: false, + message: '[[error:topic-locked]]' + }; + } + const { + postDeleteDuration + } = meta.config; + if (!results.isMod && postDeleteDuration && Date.now() - postData.timestamp > postDeleteDuration * 1000) { + return { + flag: false, + message: `[[error:post-delete-duration-expired, ${meta.config.postDeleteDuration}]]` + }; + } + const { + deleterUid + } = postData; + const flag = results['posts:delete'] && (results.isOwner && (deleterUid === 0 || deleterUid === postData.uid) || results.isMod); + return { + flag: flag, + message: '[[error:no-privileges]]' + }; +}; +privsPosts.canFlag = async function (pid, uid) { + const targetUid = await posts.getPostField(pid, 'uid'); + const [userReputation, isAdminOrModerator, targetPrivileged, reporterPrivileged] = await Promise.all([user.getUserField(uid, 'reputation'), isAdminOrMod(pid, uid), user.isPrivileged(targetUid), user.isPrivileged(uid)]); + const minimumReputation = meta.config['min:rep:flag']; + let canFlag = isAdminOrModerator || userReputation >= minimumReputation; + if (targetPrivileged && !reporterPrivileged) { + canFlag = false; + } + return { + flag: canFlag + }; +}; +privsPosts.canMove = async function (pid, uid) { + const isMain = await posts.isMain(pid); + if (isMain) { + throw new Error('[[error:cant-move-mainpost]]'); + } + return await isAdminOrMod(pid, uid); +}; +privsPosts.canPurge = async function (pid, uid) { + const cid = await posts.getCidByPid(pid); + const results = await utils.promiseParallel({ + purge: privsCategories.isUserAllowedTo('purge', cid, uid), + owner: posts.isOwner(pid, uid), + isAdmin: user.isAdministrator(uid), + isModerator: user.isModerator(uid, cid) + }); + return results.purge && (results.owner || results.isModerator) || results.isAdmin; +}; +async function isAdminOrMod(pid, uid) { + if (parseInt(uid, 10) <= 0) { + return false; + } + const cid = await posts.getCidByPid(pid); + return await privsCategories.isAdminOrMod(cid, uid); +} \ No newline at end of file diff --git a/lib/privileges/topics.js b/lib/privileges/topics.js new file mode 100644 index 0000000000..b473714f0a --- /dev/null +++ b/lib/privileges/topics.js @@ -0,0 +1,139 @@ +'use strict'; + +const _ = require('lodash'); +const meta = require('../meta'); +const topics = require('../topics'); +const user = require('../user'); +const helpers = require('./helpers'); +const categories = require('../categories'); +const plugins = require('../plugins'); +const privsCategories = require('./categories'); +const privsTopics = module.exports; +privsTopics.get = async function (tid, uid) { + uid = parseInt(uid, 10); + const privs = ['topics:reply', 'topics:read', 'topics:schedule', 'topics:tag', 'topics:delete', 'posts:edit', 'posts:history', 'posts:upvote', 'posts:downvote', 'posts:delete', 'posts:view_deleted', 'read', 'purge']; + const topicData = await topics.getTopicFields(tid, ['cid', 'uid', 'locked', 'deleted', 'scheduled']); + const [userPrivileges, isAdministrator, isModerator, disabled] = await Promise.all([helpers.isAllowedTo(privs, uid, topicData.cid), user.isAdministrator(uid), user.isModerator(uid, topicData.cid), categories.getCategoryField(topicData.cid, 'disabled')]); + const privData = _.zipObject(privs, userPrivileges); + const isOwner = uid > 0 && uid === topicData.uid; + const isAdminOrMod = isAdministrator || isModerator; + const editable = isAdminOrMod; + const deletable = privData['topics:delete'] && (isOwner || isModerator) || isAdministrator; + const mayReply = privsTopics.canViewDeletedScheduled(topicData, {}, false, privData['topics:schedule']); + return await plugins.hooks.fire('filter:privileges.topics.get', { + 'topics:reply': privData['topics:reply'] && (!topicData.locked && mayReply || isModerator) || isAdministrator, + 'topics:read': privData['topics:read'] || isAdministrator, + 'topics:schedule': privData['topics:schedule'] || isAdministrator, + 'topics:tag': privData['topics:tag'] || isAdministrator, + 'topics:delete': privData['topics:delete'] && (isOwner || isModerator) || isAdministrator, + 'posts:edit': privData['posts:edit'] && (!topicData.locked || isModerator) || isAdministrator, + 'posts:history': privData['posts:history'] || isAdministrator, + 'posts:upvote': privData['posts:upvote'] || isAdministrator, + 'posts:downvote': privData['posts:downvote'] || isAdministrator, + 'posts:delete': privData['posts:delete'] && (!topicData.locked || isModerator) || isAdministrator, + 'posts:view_deleted': privData['posts:view_deleted'] || isAdministrator, + read: privData.read || isAdministrator, + purge: privData.purge && (isOwner || isModerator) || isAdministrator, + view_thread_tools: editable || deletable, + editable: editable, + deletable: deletable, + view_deleted: isAdminOrMod || isOwner || privData['posts:view_deleted'], + view_scheduled: privData['topics:schedule'] || isAdministrator, + isAdminOrMod: isAdminOrMod, + disabled: disabled, + tid: tid, + uid: uid + }); +}; +privsTopics.can = async function (privilege, tid, uid) { + const cid = await topics.getTopicField(tid, 'cid'); + return await privsCategories.can(privilege, cid, uid); +}; +privsTopics.filterTids = async function (privilege, tids, uid) { + if (!Array.isArray(tids) || !tids.length) { + return []; + } + const topicsData = await topics.getTopicsFields(tids, ['tid', 'cid', 'deleted', 'scheduled']); + const cids = _.uniq(topicsData.map(topic => topic.cid)); + const results = await privsCategories.getBase(privilege, cids, uid); + const allowedCids = cids.filter((cid, index) => !results.categories[index].disabled && (results.allowedTo[index] || results.isAdmin)); + const cidsSet = new Set(allowedCids); + const canViewDeleted = _.zipObject(cids, results.view_deleted); + const canViewScheduled = _.zipObject(cids, results.view_scheduled); + tids = topicsData.filter(t => cidsSet.has(t.cid) && (results.isAdmin || privsTopics.canViewDeletedScheduled(t, {}, canViewDeleted[t.cid], canViewScheduled[t.cid]))).map(t => t.tid); + const data = await plugins.hooks.fire('filter:privileges.topics.filter', { + privilege: privilege, + uid: uid, + tids: tids + }); + return data ? data.tids : []; +}; +privsTopics.filterUids = async function (privilege, tid, uids) { + if (!Array.isArray(uids) || !uids.length) { + return []; + } + uids = _.uniq(uids); + const topicData = await topics.getTopicFields(tid, ['tid', 'cid', 'deleted', 'scheduled']); + const [disabled, allowedTo, isAdmins] = await Promise.all([categories.getCategoryField(topicData.cid, 'disabled'), helpers.isUsersAllowedTo(privilege, uids, topicData.cid), user.isAdministrator(uids)]); + if (topicData.scheduled) { + const canViewScheduled = await helpers.isUsersAllowedTo('topics:schedule', uids, topicData.cid); + uids = uids.filter((uid, index) => canViewScheduled[index]); + } + return uids.filter((uid, index) => !disabled && (allowedTo[index] && (topicData.scheduled || !topicData.deleted) || isAdmins[index])); +}; +privsTopics.canPurge = async function (tid, uid) { + const cid = await topics.getTopicField(tid, 'cid'); + const [purge, owner, isAdmin, isModerator] = await Promise.all([privsCategories.isUserAllowedTo('purge', cid, uid), topics.isOwner(tid, uid), user.isAdministrator(uid), user.isModerator(uid, cid)]); + return purge && (owner || isModerator) || isAdmin; +}; +privsTopics.canDelete = async function (tid, uid) { + const topicData = await topics.getTopicFields(tid, ['uid', 'cid', 'postcount', 'deleterUid']); + const [isModerator, isAdministrator, isOwner, allowedTo] = await Promise.all([user.isModerator(uid, topicData.cid), user.isAdministrator(uid), topics.isOwner(tid, uid), helpers.isAllowedTo('topics:delete', uid, [topicData.cid])]); + if (isAdministrator) { + return true; + } + const { + preventTopicDeleteAfterReplies + } = meta.config; + if (!isModerator && preventTopicDeleteAfterReplies && topicData.postcount - 1 >= preventTopicDeleteAfterReplies) { + const langKey = preventTopicDeleteAfterReplies > 1 ? `[[error:cant-delete-topic-has-replies, ${meta.config.preventTopicDeleteAfterReplies}]]` : '[[error:cant-delete-topic-has-reply]]'; + throw new Error(langKey); + } + const { + deleterUid + } = topicData; + return allowedTo[0] && (isOwner && (deleterUid === 0 || deleterUid === topicData.uid) || isModerator); +}; +privsTopics.canEdit = async function (tid, uid) { + return await privsTopics.isOwnerOrAdminOrMod(tid, uid); +}; +privsTopics.isOwnerOrAdminOrMod = async function (tid, uid) { + const [isOwner, isAdminOrMod] = await Promise.all([topics.isOwner(tid, uid), privsTopics.isAdminOrMod(tid, uid)]); + return isOwner || isAdminOrMod; +}; +privsTopics.isAdminOrMod = async function (tid, uid) { + if (parseInt(uid, 10) <= 0) { + return false; + } + const cid = await topics.getTopicField(tid, 'cid'); + return await privsCategories.isAdminOrMod(cid, uid); +}; +privsTopics.canViewDeletedScheduled = function (topic, privileges = {}, viewDeleted = false, viewScheduled = false) { + if (!topic) { + return false; + } + const { + deleted = false, + scheduled = false + } = topic; + const { + view_deleted = viewDeleted, + view_scheduled = viewScheduled + } = privileges; + if (scheduled) { + return view_scheduled; + } else if (deleted) { + return view_deleted; + } + return true; +}; \ No newline at end of file diff --git a/lib/privileges/users.js b/lib/privileges/users.js new file mode 100644 index 0000000000..da7bbbb800 --- /dev/null +++ b/lib/privileges/users.js @@ -0,0 +1,121 @@ +'use strict'; + +const _ = require('lodash'); +const user = require('../user'); +const meta = require('../meta'); +const groups = require('../groups'); +const plugins = require('../plugins'); +const helpers = require('./helpers'); +const privsUsers = module.exports; +privsUsers.isAdministrator = async function (uid) { + return await isGroupMember(uid, 'administrators'); +}; +privsUsers.isGlobalModerator = async function (uid) { + return await isGroupMember(uid, 'Global Moderators'); +}; +async function isGroupMember(uid, groupName) { + return await groups[Array.isArray(uid) ? 'isMembers' : 'isMember'](uid, groupName); +} +privsUsers.isModerator = async function (uid, cid) { + if (Array.isArray(cid)) { + return await isModeratorOfCategories(cid, uid); + } else if (Array.isArray(uid)) { + return await isModeratorsOfCategory(cid, uid); + } + return await isModeratorOfCategory(cid, uid); +}; +async function isModeratorOfCategories(cids, uid) { + if (parseInt(uid, 10) <= 0) { + return await filterIsModerator(cids, uid, cids.map(() => false)); + } + const isGlobalModerator = await privsUsers.isGlobalModerator(uid); + if (isGlobalModerator) { + return await filterIsModerator(cids, uid, cids.map(() => true)); + } + const uniqueCids = _.uniq(cids); + const isAllowed = await helpers.isAllowedTo('moderate', uid, uniqueCids); + const cidToIsAllowed = _.zipObject(uniqueCids, isAllowed); + const isModerator = cids.map(cid => cidToIsAllowed[cid]); + return await filterIsModerator(cids, uid, isModerator); +} +async function isModeratorsOfCategory(cid, uids) { + const [check1, check2, check3] = await Promise.all([privsUsers.isGlobalModerator(uids), groups.isMembers(uids, `cid:${cid}:privileges:moderate`), groups.isMembersOfGroupList(uids, `cid:${cid}:privileges:groups:moderate`)]); + const isModerator = uids.map((uid, idx) => check1[idx] || check2[idx] || check3[idx]); + return await filterIsModerator(cid, uids, isModerator); +} +async function isModeratorOfCategory(cid, uid) { + const result = await isModeratorOfCategories([cid], uid); + return result ? result[0] : false; +} +async function filterIsModerator(cid, uid, isModerator) { + const data = await plugins.hooks.fire('filter:user.isModerator', { + uid: uid, + cid: cid, + isModerator: isModerator + }); + if ((Array.isArray(uid) || Array.isArray(cid)) && !Array.isArray(data.isModerator)) { + throw new Error('filter:user.isModerator - i/o mismatch'); + } + return data.isModerator; +} +privsUsers.canEdit = async function (callerUid, uid) { + if (parseInt(callerUid, 10) === parseInt(uid, 10)) { + return true; + } + const [isAdmin, isGlobalMod, isTargetAdmin, isUserAllowedTo] = await Promise.all([privsUsers.isAdministrator(callerUid), privsUsers.isGlobalModerator(callerUid), privsUsers.isAdministrator(uid), helpers.isAllowedTo('admin:users', callerUid, [0])]); + const canManageUsers = isUserAllowedTo[0]; + const data = await plugins.hooks.fire('filter:user.canEdit', { + isAdmin: isAdmin, + isGlobalMod: isGlobalMod, + isTargetAdmin: isTargetAdmin, + canManageUsers: canManageUsers, + canEdit: isAdmin || (isGlobalMod || canManageUsers) && !isTargetAdmin, + callerUid: callerUid, + uid: uid + }); + return data.canEdit; +}; +privsUsers.canBanUser = async function (callerUid, uid) { + const privsGlobal = require('./global'); + const [canBan, isTargetAdmin] = await Promise.all([privsGlobal.can('ban', callerUid), privsUsers.isAdministrator(uid)]); + const data = await plugins.hooks.fire('filter:user.canBanUser', { + canBan: canBan && !isTargetAdmin, + callerUid: callerUid, + uid: uid + }); + return data.canBan; +}; +privsUsers.canMuteUser = async function (callerUid, uid) { + const privsGlobal = require('./global'); + const [canMute, isTargetAdmin] = await Promise.all([privsGlobal.can('mute', callerUid), privsUsers.isAdministrator(uid)]); + const data = await plugins.hooks.fire('filter:user.canMuteUser', { + canMute: canMute && !isTargetAdmin, + callerUid: callerUid, + uid: uid + }); + return data.canMute; +}; +privsUsers.canFlag = async function (callerUid, uid) { + const [userReputation, targetPrivileged, reporterPrivileged] = await Promise.all([user.getUserField(callerUid, 'reputation'), user.isPrivileged(uid), user.isPrivileged(callerUid)]); + const minimumReputation = meta.config['min:rep:flag']; + let canFlag = reporterPrivileged || userReputation >= minimumReputation; + if (targetPrivileged && !reporterPrivileged) { + canFlag = false; + } + return { + flag: canFlag + }; +}; +privsUsers.hasBanPrivilege = async uid => await hasGlobalPrivilege('ban', uid); +privsUsers.hasMutePrivilege = async uid => await hasGlobalPrivilege('mute', uid); +privsUsers.hasInvitePrivilege = async uid => await hasGlobalPrivilege('invite', uid); +async function hasGlobalPrivilege(privilege, uid) { + const privsGlobal = require('./global'); + const privilegeName = privilege.split('-').map(word => word.slice(0, 1).toUpperCase() + word.slice(1)).join(''); + let payload = { + uid + }; + payload[`can${privilegeName}`] = await privsGlobal.can(privilege, uid); + payload = await plugins.hooks.fire(`filter:user.has${privilegeName}Privilege`, payload); + return payload[`can${privilegeName}`]; +} \ No newline at end of file diff --git a/lib/promisify.js b/lib/promisify.js new file mode 100644 index 0000000000..861bc6c6e1 --- /dev/null +++ b/lib/promisify.js @@ -0,0 +1,53 @@ +'use strict'; + +const util = require('util'); +module.exports = function (theModule, ignoreKeys) { + ignoreKeys = ignoreKeys || []; + function isCallbackedFunction(func) { + if (typeof func !== 'function') { + return false; + } + const str = func.toString().split('\n')[0]; + return str.includes('callback)'); + } + function isAsyncFunction(fn) { + return fn && fn.constructor && fn.constructor.name === 'AsyncFunction'; + } + function promisifyRecursive(module) { + if (!module) { + return; + } + const keys = Object.keys(module); + keys.forEach(key => { + if (ignoreKeys.includes(key)) { + return; + } + if (isAsyncFunction(module[key])) { + module[key] = wrapCallback(module[key], util.callbackify(module[key])); + } else if (isCallbackedFunction(module[key])) { + module[key] = wrapPromise(module[key], util.promisify(module[key])); + } else if (typeof module[key] === 'object') { + promisifyRecursive(module[key]); + } + }); + } + function wrapCallback(origFn, callbackFn) { + return function wrapperCallback(...args) { + if (args.length && typeof args[args.length - 1] === 'function') { + const cb = args.pop(); + args.push((err, res) => res !== undefined ? cb(err, res) : cb(err)); + return callbackFn(...args); + } + return origFn(...args); + }; + } + function wrapPromise(origFn, promiseFn) { + return function wrapperPromise(...args) { + if (args.length && typeof args[args.length - 1] === 'function') { + return origFn(...args); + } + return promiseFn(...args); + }; + } + promisifyRecursive(theModule); +}; \ No newline at end of file diff --git a/lib/pubsub.js b/lib/pubsub.js new file mode 100644 index 0000000000..4ba7f10c92 --- /dev/null +++ b/lib/pubsub.js @@ -0,0 +1,65 @@ +'use strict'; + +const EventEmitter = require('events'); +const nconf = require('nconf'); +let real; +let noCluster; +let singleHost; +function get() { + if (real) { + return real; + } + let pubsub; + if (!nconf.get('isCluster')) { + if (noCluster) { + real = noCluster; + return real; + } + noCluster = new EventEmitter(); + noCluster.publish = noCluster.emit.bind(noCluster); + pubsub = noCluster; + } else if (nconf.get('singleHostCluster')) { + if (singleHost) { + real = singleHost; + return real; + } + singleHost = new EventEmitter(); + if (!process.send) { + singleHost.publish = singleHost.emit.bind(singleHost); + } else { + singleHost.publish = function (event, data) { + process.send({ + action: 'pubsub', + event: event, + data: data + }); + }; + process.on('message', message => { + if (message && typeof message === 'object' && message.action === 'pubsub') { + singleHost.emit(message.event, message.data); + } + }); + } + pubsub = singleHost; + } else if (nconf.get('redis')) { + pubsub = require('./database/redis/pubsub'); + } else { + throw new Error('[[error:redis-required-for-pubsub]]'); + } + real = pubsub; + return pubsub; +} +module.exports = { + publish: function (event, data) { + get().publish(event, data); + }, + on: function (event, callback) { + get().on(event, callback); + }, + removeAllListeners: function (event) { + get().removeAllListeners(event); + }, + reset: function () { + real = null; + } +}; \ No newline at end of file diff --git a/lib/request.js b/lib/request.js new file mode 100644 index 0000000000..c1ddf997e3 --- /dev/null +++ b/lib/request.js @@ -0,0 +1,71 @@ +'use strict'; + +const { + CookieJar +} = require('tough-cookie'); +const fetchCookie = require('fetch-cookie').default; +exports.jar = function () { + return new CookieJar(); +}; +async function call(url, method, { + body, + timeout, + jar, + ...config +} = {}) { + let fetchImpl = fetch; + if (jar) { + fetchImpl = fetchCookie(fetch, jar); + } + const jsonTest = /application\/([a-z]+\+)?json/; + const opts = { + ...config, + method, + headers: { + 'content-type': 'application/json', + ...config.headers + } + }; + if (timeout > 0) { + opts.signal = AbortSignal.timeout(timeout); + } + if (body && ['POST', 'PUT', 'PATCH', 'DEL', 'DELETE'].includes(method)) { + if (opts.headers['content-type'] && jsonTest.test(opts.headers['content-type'])) { + opts.body = JSON.stringify(body); + } else { + opts.body = body; + } + } + const response = await fetchImpl(url, opts); + const { + headers + } = response; + const contentType = headers.get('content-type'); + const isJSON = contentType && jsonTest.test(contentType); + let respBody = await response.text(); + if (isJSON && respBody) { + try { + respBody = JSON.parse(respBody); + } catch (err) { + throw new Error('invalid json in response body', url); + } + } + return { + body: respBody, + response: { + ok: response.ok, + status: response.status, + statusCode: response.status, + statusText: response.statusText, + headers: Object.fromEntries(response.headers.entries()) + } + }; +} +exports.get = async (url, config) => call(url, 'GET', config); +exports.head = async (url, config) => call(url, 'HEAD', config); +exports.del = async (url, config) => call(url, 'DELETE', config); +exports.delete = exports.del; +exports.options = async (url, config) => call(url, 'OPTIONS', config); +exports.post = async (url, config) => call(url, 'POST', config); +exports.put = async (url, config) => call(url, 'PUT', config); +exports.patch = async (url, config) => call(url, 'PATCH', config); \ No newline at end of file diff --git a/lib/rewards/admin.js b/lib/rewards/admin.js new file mode 100644 index 0000000000..3d6507256b --- /dev/null +++ b/lib/rewards/admin.js @@ -0,0 +1,60 @@ +'use strict'; + +const plugins = require('../plugins'); +const db = require('../database'); +const utils = require('../utils'); +const rewards = module.exports; +rewards.save = async function (data) { + await Promise.all(data.map(async (data, index) => { + if (!Object.keys(data.rewards).length) { + return; + } + const rewardsData = data.rewards; + delete data.rewards; + if (!parseInt(data.id, 10)) { + data.id = await db.incrObjectField('global', 'rewards:id'); + } + await rewards.delete(data); + await db.sortedSetAdd('rewards:list', index, data.id); + await db.setObject(`rewards:id:${data.id}`, data); + await db.setObject(`rewards:id:${data.id}:rewards`, rewardsData); + })); + await saveConditions(data); + return data; +}; +rewards.delete = async function (data) { + await Promise.all([db.sortedSetRemove('rewards:list', data.id), db.delete(`rewards:id:${data.id}`), db.delete(`rewards:id:${data.id}:rewards`)]); +}; +rewards.get = async function () { + return await utils.promiseParallel({ + active: getActiveRewards(), + conditions: plugins.hooks.fire('filter:rewards.conditions', []), + conditionals: plugins.hooks.fire('filter:rewards.conditionals', []), + rewards: plugins.hooks.fire('filter:rewards.rewards', []) + }); +}; +async function saveConditions(data) { + const rewardsPerCondition = {}; + await db.delete('conditions:active'); + const conditions = []; + data.forEach(reward => { + conditions.push(reward.condition); + rewardsPerCondition[reward.condition] = rewardsPerCondition[reward.condition] || []; + rewardsPerCondition[reward.condition].push(reward.id); + }); + await db.setAdd('conditions:active', conditions); + await Promise.all(Object.keys(rewardsPerCondition).map(c => db.setAdd(`condition:${c}:rewards`, rewardsPerCondition[c]))); +} +async function getActiveRewards() { + const rewardsList = await db.getSortedSetRange('rewards:list', 0, -1); + const rewardData = await Promise.all(rewardsList.map(async id => { + const [main, rewards] = await Promise.all([db.getObject(`rewards:id:${id}`), db.getObject(`rewards:id:${id}:rewards`)]); + if (main) { + main.disabled = main.disabled === 'true' || main.disabled === true; + main.rewards = rewards; + } + return main; + })); + return rewardData.filter(Boolean); +} +require('../promisify')(rewards); \ No newline at end of file diff --git a/lib/rewards/index.js b/lib/rewards/index.js new file mode 100644 index 0000000000..36dbb35dfe --- /dev/null +++ b/lib/rewards/index.js @@ -0,0 +1,76 @@ +'use strict'; + +const util = require('util'); +const db = require('../database'); +const plugins = require('../plugins'); +const rewards = module.exports; +rewards.checkConditionAndRewardUser = async function (params) { + const { + uid, + condition, + method + } = params; + const isActive = await isConditionActive(condition); + if (!isActive) { + return; + } + const ids = await getIDsByCondition(condition); + let rewardData = await getRewardDataByIDs(ids); + rewardData = rewardData.filter(r => r && !(r.disabled === 'true' || r.disabled === true)); + rewardData = await filterCompletedRewards(uid, rewardData); + if (!rewardData || !rewardData.length) { + return; + } + const eligible = await Promise.all(rewardData.map(reward => checkCondition(reward, method))); + const eligibleRewards = rewardData.filter((reward, index) => eligible[index]); + await giveRewards(uid, eligibleRewards); +}; +async function isConditionActive(condition) { + return await db.isSetMember('conditions:active', condition); +} +async function getIDsByCondition(condition) { + return await db.getSetMembers(`condition:${condition}:rewards`); +} +async function filterCompletedRewards(uid, rewards) { + const data = await db.getSortedSetRangeByScoreWithScores(`uid:${uid}:rewards`, 0, -1, 1, '+inf'); + const userRewards = {}; + data.forEach(obj => { + userRewards[obj.value] = parseInt(obj.score, 10); + }); + return rewards.filter(reward => { + if (!reward) { + return false; + } + const claimable = parseInt(reward.claimable, 10); + return claimable === 0 || !userRewards[reward.id] || userRewards[reward.id] < reward.claimable; + }); +} +async function getRewardDataByIDs(ids) { + return await db.getObjects(ids.map(id => `rewards:id:${id}`)); +} +async function getRewardsByRewardData(rewards) { + return await db.getObjects(rewards.map(reward => `rewards:id:${reward.id}:rewards`)); +} +async function checkCondition(reward, method) { + if (method.constructor && method.constructor.name !== 'AsyncFunction') { + method = util.promisify(method); + } + const value = await method(); + const bool = await plugins.hooks.fire(`filter:rewards.checkConditional:${reward.conditional}`, { + left: value, + right: reward.value + }); + return bool; +} +async function giveRewards(uid, rewards) { + const rewardData = await getRewardsByRewardData(rewards); + for (let i = 0; i < rewards.length; i++) { + await plugins.hooks.fire(`action:rewards.award:${rewards[i].rid}`, { + uid: uid, + rewardData: rewards[i], + reward: rewardData[i] + }); + await db.sortedSetIncrBy(`uid:${uid}:rewards`, 1, rewards[i].id); + } +} +require('../promisify')(rewards); \ No newline at end of file diff --git a/lib/routes/admin.js b/lib/routes/admin.js new file mode 100644 index 0000000000..9c67670afa --- /dev/null +++ b/lib/routes/admin.js @@ -0,0 +1,63 @@ +'use strict'; + +const helpers = require('./helpers'); +module.exports = function (app, name, middleware, controllers) { + const middlewares = [middleware.pluginHooks]; + helpers.setupAdminPageRoute(app, `/${name}`, middlewares, controllers.admin.routeIndex); + helpers.setupAdminPageRoute(app, `/${name}/dashboard`, middlewares, controllers.admin.dashboard.get); + helpers.setupAdminPageRoute(app, `/${name}/dashboard/logins`, middlewares, controllers.admin.dashboard.getLogins); + helpers.setupAdminPageRoute(app, `/${name}/dashboard/users`, middlewares, controllers.admin.dashboard.getUsers); + helpers.setupAdminPageRoute(app, `/${name}/dashboard/topics`, middlewares, controllers.admin.dashboard.getTopics); + helpers.setupAdminPageRoute(app, `/${name}/dashboard/searches`, middlewares, controllers.admin.dashboard.getSearches); + helpers.setupAdminPageRoute(app, `/${name}/manage/categories`, middlewares, controllers.admin.categories.getAll); + helpers.setupAdminPageRoute(app, `/${name}/manage/categories/:category_id`, middlewares, controllers.admin.categories.get); + helpers.setupAdminPageRoute(app, `/${name}/manage/categories/:category_id/analytics`, middlewares, controllers.admin.categories.getAnalytics); + helpers.setupAdminPageRoute(app, `/${name}/manage/privileges/:cid?`, middlewares, controllers.admin.privileges.get); + helpers.setupAdminPageRoute(app, `/${name}/manage/tags`, middlewares, controllers.admin.tags.get); + helpers.setupAdminPageRoute(app, `/${name}/manage/users`, middlewares, controllers.admin.users.index); + helpers.setupAdminPageRoute(app, `/${name}/manage/registration`, middlewares, controllers.admin.users.registrationQueue); + helpers.setupAdminPageRoute(app, `/${name}/manage/admins-mods`, middlewares, controllers.admin.adminsMods.get); + helpers.setupAdminPageRoute(app, `/${name}/manage/groups`, middlewares, controllers.admin.groups.list); + helpers.setupAdminPageRoute(app, `/${name}/manage/groups/:name`, middlewares, controllers.admin.groups.get); + helpers.setupAdminPageRoute(app, `/${name}/manage/uploads`, middlewares, controllers.admin.uploads.get); + helpers.setupAdminPageRoute(app, `/${name}/manage/digest`, middlewares, controllers.admin.digest.get); + helpers.setupAdminPageRoute(app, `/${name}/settings/email`, middlewares, controllers.admin.settings.email); + helpers.setupAdminPageRoute(app, `/${name}/settings/user`, middlewares, controllers.admin.settings.user); + helpers.setupAdminPageRoute(app, `/${name}/settings/post`, middlewares, controllers.admin.settings.post); + helpers.setupAdminPageRoute(app, `/${name}/settings/advanced`, middlewares, controllers.admin.settings.advanced); + helpers.setupAdminPageRoute(app, `/${name}/settings/navigation`, middlewares, controllers.admin.settings.navigation); + helpers.setupAdminPageRoute(app, `/${name}/settings/api`, middlewares, controllers.admin.settings.api); + helpers.setupAdminPageRoute(app, `/${name}/settings/:term?`, middlewares, controllers.admin.settings.get); + helpers.setupAdminPageRoute(app, `/${name}/appearance/:term?`, middlewares, controllers.admin.appearance.get); + helpers.setupAdminPageRoute(app, `/${name}/extend/plugins`, middlewares, controllers.admin.plugins.get); + helpers.setupAdminPageRoute(app, `/${name}/extend/widgets`, middlewares, controllers.admin.extend.widgets.get); + helpers.setupAdminPageRoute(app, `/${name}/extend/rewards`, middlewares, controllers.admin.extend.rewards.get); + helpers.setupAdminPageRoute(app, `/${name}/advanced/database`, middlewares, controllers.admin.database.get); + helpers.setupAdminPageRoute(app, `/${name}/advanced/events`, middlewares, controllers.admin.events.get); + helpers.setupAdminPageRoute(app, `/${name}/advanced/hooks`, middlewares, controllers.admin.hooks.get); + helpers.setupAdminPageRoute(app, `/${name}/advanced/logs`, middlewares, controllers.admin.logs.get); + helpers.setupAdminPageRoute(app, `/${name}/advanced/errors`, middlewares, controllers.admin.errors.get); + helpers.setupAdminPageRoute(app, `/${name}/advanced/errors/export`, middlewares, controllers.admin.errors.export); + helpers.setupAdminPageRoute(app, `/${name}/advanced/cache`, middlewares, controllers.admin.cache.get); + helpers.setupAdminPageRoute(app, `/${name}/development/logger`, middlewares, controllers.admin.logger.get); + helpers.setupAdminPageRoute(app, `/${name}/development/info`, middlewares, controllers.admin.info.get); + apiRoutes(app, name, middleware, controllers); +}; +function apiRoutes(router, name, middleware, controllers) { + router.get(`/api/${name}/config`, middleware.ensureLoggedIn, helpers.tryRoute(controllers.admin.getConfig)); + router.get(`/api/${name}/users/csv`, middleware.ensureLoggedIn, helpers.tryRoute(controllers.admin.users.getCSV)); + router.get(`/api/${name}/groups/:groupname/csv`, middleware.ensureLoggedIn, helpers.tryRoute(controllers.admin.groups.getCSV)); + router.get(`/api/${name}/analytics`, middleware.ensureLoggedIn, helpers.tryRoute(controllers.admin.dashboard.getAnalytics)); + router.get(`/api/${name}/advanced/cache/dump`, middleware.ensureLoggedIn, helpers.tryRoute(controllers.admin.cache.dump)); + const multipart = require('connect-multiparty'); + const multipartMiddleware = multipart(); + const middlewares = [multipartMiddleware, middleware.validateFiles, middleware.applyCSRF, middleware.ensureLoggedIn]; + router.post(`/api/${name}/category/uploadpicture`, middlewares, helpers.tryRoute(controllers.admin.uploads.uploadCategoryPicture)); + router.post(`/api/${name}/uploadfavicon`, middlewares, helpers.tryRoute(controllers.admin.uploads.uploadFavicon)); + router.post(`/api/${name}/uploadTouchIcon`, middlewares, helpers.tryRoute(controllers.admin.uploads.uploadTouchIcon)); + router.post(`/api/${name}/uploadMaskableIcon`, middlewares, helpers.tryRoute(controllers.admin.uploads.uploadMaskableIcon)); + router.post(`/api/${name}/uploadlogo`, middlewares, helpers.tryRoute(controllers.admin.uploads.uploadLogo)); + router.post(`/api/${name}/uploadOgImage`, middlewares, helpers.tryRoute(controllers.admin.uploads.uploadOgImage)); + router.post(`/api/${name}/upload/file`, middlewares, helpers.tryRoute(controllers.admin.uploads.uploadFile)); + router.post(`/api/${name}/uploadDefaultAvatar`, middlewares, helpers.tryRoute(controllers.admin.uploads.uploadDefaultAvatar)); +} \ No newline at end of file diff --git a/lib/routes/api.js b/lib/routes/api.js new file mode 100644 index 0000000000..5aa743e248 --- /dev/null +++ b/lib/routes/api.js @@ -0,0 +1,25 @@ +'use strict'; + +const express = require('express'); +const uploadsController = require('../controllers/uploads'); +const helpers = require('./helpers'); +module.exports = function (app, middleware, controllers) { + const middlewares = [middleware.autoLocale, middleware.authenticateRequest]; + const router = express.Router(); + app.use('/api', router); + router.get('/config', [...middlewares, middleware.applyCSRF], helpers.tryRoute(controllers.api.getConfig)); + router.get('/self', [...middlewares], helpers.tryRoute(controllers.user.getCurrentUser)); + router.get('/user/uid/:uid', [...middlewares, middleware.canViewUsers], helpers.tryRoute(controllers.user.getUserByUID)); + router.get('/user/username/:username', [...middlewares, middleware.canViewUsers], helpers.tryRoute(controllers.user.getUserByUsername)); + router.get('/user/email/:email', [...middlewares, middleware.canViewUsers], helpers.tryRoute(controllers.user.getUserByEmail)); + router.get('/categories/:cid/moderators', [...middlewares], helpers.tryRoute(controllers.api.getModerators)); + router.get('/recent/posts/:term?', [...middlewares], helpers.tryRoute(controllers.posts.getRecentPosts)); + router.get('/unread/total', [...middlewares, middleware.ensureLoggedIn], helpers.tryRoute(controllers.unread.unreadTotal)); + router.get('/topic/teaser/:topic_id', [...middlewares], helpers.tryRoute(controllers.topics.teaser)); + router.get('/topic/pagination/:topic_id', [...middlewares], helpers.tryRoute(controllers.topics.pagination)); + const multipart = require('connect-multiparty'); + const multipartMiddleware = multipart(); + const postMiddlewares = [middleware.maintenanceMode, multipartMiddleware, middleware.validateFiles, middleware.uploads.ratelimit, middleware.applyCSRF]; + router.post('/post/upload', postMiddlewares, helpers.tryRoute(uploadsController.uploadPost)); + router.post('/user/:userslug/uploadpicture', [...middlewares, ...postMiddlewares, middleware.exposeUid, middleware.ensureLoggedIn, middleware.canViewUsers, middleware.checkAccountPermissions], helpers.tryRoute(controllers.accounts.edit.uploadPicture)); +}; \ No newline at end of file diff --git a/lib/routes/authentication.js b/lib/routes/authentication.js new file mode 100644 index 0000000000..e97314942d --- /dev/null +++ b/lib/routes/authentication.js @@ -0,0 +1,154 @@ +'use strict'; + +const async = require('async'); +const passport = require('passport'); +const passportLocal = require('passport-local').Strategy; +const BearerStrategy = require('passport-http-bearer').Strategy; +const winston = require('winston'); +const controllers = require('../controllers'); +const helpers = require('../controllers/helpers'); +const plugins = require('../plugins'); +const api = require('../api'); +const { + generateToken +} = require('../middleware/csrf'); +let loginStrategies = []; +const Auth = module.exports; +Auth.initialize = function (app, middleware) { + app.use(passport.initialize()); + app.use(passport.session()); + app.use((req, res, next) => { + Auth.setAuthVars(req, res); + next(); + }); + Auth.app = app; + Auth.middleware = middleware; +}; +Auth.setAuthVars = function setAuthVars(req) { + const isSpider = req.isSpider(); + req.loggedIn = !isSpider && !!req.user; + if (req.user) { + req.uid = parseInt(req.user.uid, 10); + } else if (isSpider) { + req.uid = -1; + } else { + req.uid = 0; + } +}; +Auth.getLoginStrategies = function () { + return loginStrategies; +}; +Auth.verifyToken = async function (token, done) { + const tokenObj = await api.utils.tokens.get(token); + const uid = tokenObj ? tokenObj.uid : undefined; + if (uid !== undefined) { + if (parseInt(uid, 10) > 0) { + done(null, { + uid: uid + }); + } else { + done(null, { + master: true + }); + } + } else { + done(false); + } +}; +Auth.reloadRoutes = async function (params) { + loginStrategies.length = 0; + const { + router + } = params; + if (plugins.hooks.hasListeners('action:auth.overrideLogin')) { + winston.warn('[authentication] Login override detected, skipping local login strategy.'); + plugins.hooks.fire('action:auth.overrideLogin'); + } else { + passport.use(new passportLocal({ + passReqToCallback: true + }, controllers.authentication.localLogin)); + } + passport.use('core.api', new BearerStrategy({}, Auth.verifyToken)); + try { + loginStrategies = await plugins.hooks.fire('filter:auth.init', loginStrategies); + } catch (err) { + winston.error(`[authentication] ${err.stack}`); + } + loginStrategies = loginStrategies || []; + loginStrategies.forEach(strategy => { + if (strategy.url) { + router[strategy.urlMethod || 'get'](strategy.url, Auth.middleware.applyCSRF, async (req, res, next) => { + let opts = { + scope: strategy.scope, + prompt: strategy.prompt || undefined + }; + if (strategy.checkState !== false) { + req.session.ssoState = generateToken(req, true); + opts.state = req.session.ssoState; + } + if (req.query.next) { + req.session.next = req.query.next; + } + ({ + opts + } = await plugins.hooks.fire('filter:auth.options', { + req, + res, + opts + })); + passport.authenticate(strategy.name, opts)(req, res, next); + }); + } + router[strategy.callbackMethod || 'get'](strategy.callbackURL, (req, res, next) => { + if (strategy.checkState === false) { + return next(); + } + next(req.query.state !== req.session.ssoState ? new Error('[[error:csrf-invalid]]') : null); + }, (req, res, next) => { + req.session.registration = req.session.registration || {}; + req.session.registration.returnTo = req.session.next || req.session.returnTo; + passport.authenticate(strategy.name, (err, user) => { + if (err) { + if (req.session && req.session.registration) { + delete req.session.registration; + } + return next(err); + } + if (!user) { + if (req.session && req.session.registration) { + delete req.session.registration; + } + return helpers.redirect(res, strategy.failureUrl !== undefined ? strategy.failureUrl : '/login'); + } + res.locals.user = user; + res.locals.strategy = strategy; + next(); + })(req, res, next); + }, Auth.middleware.validateAuth, (req, res, next) => { + async.waterfall([async.apply(req.login.bind(req), res.locals.user, { + keepSessionInfo: true + }), async.apply(controllers.authentication.onSuccessfulLogin, req, res.locals.user.uid)], err => { + if (err) { + return next(err); + } + helpers.redirect(res, strategy.successUrl !== undefined ? strategy.successUrl : '/'); + }); + }); + }); + const multipart = require('connect-multiparty'); + const multipartMiddleware = multipart(); + const middlewares = [multipartMiddleware, Auth.middleware.applyCSRF, Auth.middleware.applyBlacklist]; + router.post('/register', middlewares, controllers.authentication.register); + router.post('/register/complete', middlewares, controllers.authentication.registerComplete); + router.post('/register/abort', middlewares, controllers.authentication.registerAbort); + router.post('/login', Auth.middleware.applyCSRF, Auth.middleware.applyBlacklist, controllers.authentication.login); + router.post('/logout', Auth.middleware.applyCSRF, controllers.authentication.logout); +}; +passport.serializeUser((user, done) => { + done(null, user.uid); +}); +passport.deserializeUser((uid, done) => { + done(null, { + uid: uid + }); +}); \ No newline at end of file diff --git a/lib/routes/debug.js b/lib/routes/debug.js new file mode 100644 index 0000000000..2c21dede4d --- /dev/null +++ b/lib/routes/debug.js @@ -0,0 +1,29 @@ +'use strict'; + +const express = require('express'); +const nconf = require('nconf'); +const fs = require('fs').promises; +const path = require('path'); +module.exports = function (app) { + const router = express.Router(); + router.get('/test', async (req, res) => { + res.redirect(404); + }); + router.get('/spec/:type', async (req, res, next) => { + const types = ['read', 'write']; + const { + type + } = req.params; + if (!types.includes(type)) { + return next(); + } + const handle = await fs.open(path.resolve(__dirname, '../../public/vendor/redoc/index.html'), 'r'); + let html = await handle.readFile({ + encoding: 'utf-8' + }); + await handle.close(); + html = html.replace('apiUrl', `${nconf.get('relative_path')}/assets/openapi/${type}.yaml`); + res.status(200).type('text/html').send(html); + }); + app.use(`${nconf.get('relative_path')}/debug`, router); +}; \ No newline at end of file diff --git a/lib/routes/feeds.js b/lib/routes/feeds.js new file mode 100644 index 0000000000..1130b4ed6b --- /dev/null +++ b/lib/routes/feeds.js @@ -0,0 +1,359 @@ +'use strict'; + +const rss = require('rss'); +const nconf = require('nconf'); +const validator = require('validator'); +const posts = require('../posts'); +const topics = require('../topics'); +const user = require('../user'); +const categories = require('../categories'); +const meta = require('../meta'); +const controllerHelpers = require('../controllers/helpers'); +const privileges = require('../privileges'); +const db = require('../database'); +const utils = require('../utils'); +const controllers404 = require('../controllers/404'); +const routeHelpers = require('./helpers'); +const terms = { + daily: 'day', + weekly: 'week', + monthly: 'month', + alltime: 'alltime' +}; +module.exports = function (app, middleware) { + app.get('/topic/:topic_id.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForTopic)); + app.get('/category/:category_id.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForCategory)); + app.get('/topics.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForTopics)); + app.get('/recent.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForRecent)); + app.get('/top.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForTop)); + app.get('/top/:term.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForTop)); + app.get('/popular.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForPopular)); + app.get('/popular/:term.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForPopular)); + app.get('/recentposts.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForRecentPosts)); + app.get('/category/:category_id/recentposts.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForCategoryRecentPosts)); + app.get('/user/:userslug/topics.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForUserTopics)); + app.get('/tags/:tag.rss', middleware.maintenanceMode, routeHelpers.tryRoute(generateForTag)); +}; +async function validateTokenIfRequiresLogin(requiresLogin, cid, req, res) { + const uid = parseInt(req.query.uid, 10) || 0; + const { + token + } = req.query; + if (!requiresLogin) { + return true; + } + if (uid <= 0 || !token) { + return controllerHelpers.notAllowed(req, res); + } + const userToken = await db.getObjectField(`user:${uid}`, 'rss_token'); + if (userToken !== token) { + await user.auth.logAttempt(uid, req.ip); + return controllerHelpers.notAllowed(req, res); + } + const userPrivileges = await privileges.categories.get(cid, uid); + if (!userPrivileges.read) { + return controllerHelpers.notAllowed(req, res); + } + return true; +} +async function generateForTopic(req, res, next) { + if (meta.config['feeds:disableRSS']) { + return next(); + } + const tid = req.params.topic_id; + const [userPrivileges, topic] = await Promise.all([privileges.topics.get(tid, req.uid), topics.getTopicData(tid)]); + if (!privileges.topics.canViewDeletedScheduled(topic, userPrivileges)) { + return next(); + } + if (await validateTokenIfRequiresLogin(!userPrivileges['topics:read'], topic.cid, req, res)) { + const topicData = await topics.getTopicWithPosts(topic, `tid:${tid}:posts`, req.uid || req.query.uid || 0, 0, 24, true); + topics.modifyPostsByPrivilege(topicData, userPrivileges); + const feed = new rss({ + title: utils.stripHTMLTags(topicData.title, utils.tags), + description: topicData.posts.length ? topicData.posts[0].content : '', + feed_url: `${nconf.get('url')}/topic/${tid}.rss`, + site_url: `${nconf.get('url')}/topic/${topicData.slug}`, + image_url: topicData.posts.length ? topicData.posts[0].picture : '', + author: topicData.posts.length ? topicData.posts[0].username : '', + ttl: 60 + }); + if (topicData.posts.length > 0) { + feed.pubDate = new Date(parseInt(topicData.posts[0].timestamp, 10)).toUTCString(); + } + const replies = topicData.posts.slice(1); + replies.forEach(postData => { + if (!postData.deleted) { + const dateStamp = new Date(parseInt(parseInt(postData.edited, 10) === 0 ? postData.timestamp : postData.edited, 10)).toUTCString(); + feed.item({ + title: `Reply to ${utils.stripHTMLTags(topicData.title, utils.tags)} on ${dateStamp}`, + description: postData.content, + url: `${nconf.get('url')}/post/${postData.pid}`, + author: postData.user ? postData.user.username : '', + date: dateStamp + }); + } + }); + sendFeed(feed, res); + } +} +async function generateForCategory(req, res, next) { + const cid = req.params.category_id; + if (meta.config['feeds:disableRSS'] || !parseInt(cid, 10)) { + return next(); + } + const uid = req.uid || req.query.uid || 0; + const [userPrivileges, category, tids] = await Promise.all([privileges.categories.get(cid, req.uid), categories.getCategoryData(cid), db.getSortedSetRevIntersect({ + sets: ['topics:tid', `cid:${cid}:tids:lastposttime`], + start: 0, + stop: 24, + weights: [1, 0] + })]); + if (!category || !category.name) { + return next(); + } + if (await validateTokenIfRequiresLogin(!userPrivileges.read, cid, req, res)) { + let topicsData = await topics.getTopicsByTids(tids, uid); + topicsData = await user.blocks.filter(uid, topicsData); + const feed = await generateTopicsFeed({ + uid: uid, + title: category.name, + description: category.description, + feed_url: `/category/${cid}.rss`, + site_url: `/category/${category.cid}` + }, topicsData, 'timestamp'); + sendFeed(feed, res); + } +} +async function generateForTopics(req, res, next) { + if (meta.config['feeds:disableRSS']) { + return next(); + } + const uid = await getUidFromToken(req); + await sendTopicsFeed({ + uid: uid, + title: 'Most recently created topics', + description: 'A list of topics that have been created recently', + feed_url: '/topics.rss', + useMainPost: true + }, 'topics:tid', res); +} +async function generateForRecent(req, res, next) { + await generateSorted({ + title: 'Recently Active Topics', + description: 'A list of topics that have been active within the past 24 hours', + feed_url: '/recent.rss', + site_url: '/recent', + sort: 'recent', + timestampField: 'lastposttime', + term: 'alltime' + }, req, res, next); +} +async function generateForTop(req, res, next) { + await generateSorted({ + title: 'Top Voted Topics', + description: 'A list of topics that have received the most votes', + feed_url: `/top/${req.params.term || 'daily'}.rss`, + site_url: `/top/${req.params.term || 'daily'}`, + sort: 'votes', + timestampField: 'timestamp', + term: 'day' + }, req, res, next); +} +async function generateForPopular(req, res, next) { + await generateSorted({ + title: 'Popular Topics', + description: 'A list of topics that are sorted by post count', + feed_url: `/popular/${req.params.term || 'daily'}.rss`, + site_url: `/popular/${req.params.term || 'daily'}`, + sort: 'posts', + timestampField: 'timestamp', + term: 'day' + }, req, res, next); +} +async function generateSorted(options, req, res, next) { + if (meta.config['feeds:disableRSS']) { + return next(); + } + const term = terms[req.params.term] || options.term; + const uid = await getUidFromToken(req); + const params = { + uid: uid, + start: 0, + stop: 19, + term: term, + sort: options.sort + }; + const { + cid + } = req.query; + if (cid) { + if (!(await privileges.categories.can('topics:read', cid, uid))) { + return controllerHelpers.notAllowed(req, res); + } + params.cids = [cid]; + } + const result = await topics.getSortedTopics(params); + const feed = await generateTopicsFeed({ + uid: uid, + title: options.title, + description: options.description, + feed_url: options.feed_url, + site_url: options.site_url + }, result.topics, options.timestampField); + sendFeed(feed, res); +} +async function sendTopicsFeed(options, set, res, timestampField) { + const start = options.hasOwnProperty('start') ? options.start : 0; + const stop = options.hasOwnProperty('stop') ? options.stop : 19; + const topicData = await topics.getTopicsFromSet(set, options.uid, start, stop); + const feed = await generateTopicsFeed(options, topicData.topics, timestampField); + sendFeed(feed, res); +} +async function generateTopicsFeed(feedOptions, feedTopics, timestampField) { + feedOptions.ttl = 60; + feedOptions.feed_url = nconf.get('url') + feedOptions.feed_url; + feedOptions.site_url = nconf.get('url') + feedOptions.site_url; + feedTopics = feedTopics.filter(Boolean); + const feed = new rss(feedOptions); + if (feedTopics.length > 0) { + feed.pubDate = new Date(feedTopics[0][timestampField]).toUTCString(); + } + async function addFeedItem(topicData) { + const feedItem = { + title: utils.stripHTMLTags(topicData.title, utils.tags), + url: `${nconf.get('url')}/topic/${topicData.slug}`, + date: new Date(topicData[timestampField]).toUTCString() + }; + if (topicData.deleted) { + return; + } + if (topicData.teaser && topicData.teaser.user && !feedOptions.useMainPost) { + feedItem.description = topicData.teaser.content; + feedItem.author = topicData.teaser.user.username; + feed.item(feedItem); + return; + } + const mainPost = await topics.getMainPost(topicData.tid, feedOptions.uid); + if (!mainPost) { + feed.item(feedItem); + return; + } + feedItem.description = mainPost.content; + feedItem.author = mainPost.user && mainPost.user.username; + feed.item(feedItem); + } + for (const topicData of feedTopics) { + await addFeedItem(topicData); + } + return feed; +} +async function generateForRecentPosts(req, res, next) { + if (meta.config['feeds:disableRSS']) { + return next(); + } + const page = parseInt(req.query.page, 10) || 1; + const postsPerPage = 20; + const start = Math.max(0, (page - 1) * postsPerPage); + const stop = start + postsPerPage - 1; + const postData = await posts.getRecentPosts(req.uid, start, stop, 'month'); + const feed = generateForPostsFeed({ + title: 'Recent Posts', + description: 'A list of recent posts', + feed_url: '/recentposts.rss', + site_url: '/recentposts' + }, postData); + sendFeed(feed, res); +} +async function generateForCategoryRecentPosts(req, res) { + if (meta.config['feeds:disableRSS']) { + return controllers404.handle404(req, res); + } + const cid = req.params.category_id; + const page = parseInt(req.query.page, 10) || 1; + const topicsPerPage = 20; + const start = Math.max(0, (page - 1) * topicsPerPage); + const stop = start + topicsPerPage - 1; + const [userPrivileges, category, postData] = await Promise.all([privileges.categories.get(cid, req.uid), categories.getCategoryData(cid), categories.getRecentReplies(cid, req.uid || req.query.uid || 0, start, stop)]); + if (!category) { + return controllers404.handle404(req, res); + } + if (await validateTokenIfRequiresLogin(!userPrivileges.read, cid, req, res)) { + const feed = generateForPostsFeed({ + title: `${category.name} Recent Posts`, + description: `A list of recent posts from ${category.name}`, + feed_url: `/category/${cid}/recentposts.rss`, + site_url: `/category/${cid}/recentposts` + }, postData); + sendFeed(feed, res); + } +} +function generateForPostsFeed(feedOptions, posts) { + feedOptions.ttl = 60; + feedOptions.feed_url = nconf.get('url') + feedOptions.feed_url; + feedOptions.site_url = nconf.get('url') + feedOptions.site_url; + const feed = new rss(feedOptions); + if (posts.length > 0) { + feed.pubDate = new Date(parseInt(posts[0].timestamp, 10)).toUTCString(); + } + posts.forEach(postData => { + feed.item({ + title: postData.topic ? postData.topic.title : '', + description: postData.content, + url: `${nconf.get('url')}/post/${postData.pid}`, + author: postData.user ? postData.user.username : '', + date: new Date(parseInt(postData.timestamp, 10)).toUTCString() + }); + }); + return feed; +} +async function generateForUserTopics(req, res, next) { + if (meta.config['feeds:disableRSS']) { + return next(); + } + const { + userslug + } = req.params; + const uid = await user.getUidByUserslug(userslug); + if (!uid) { + return next(); + } + const userData = await user.getUserFields(uid, ['uid', 'username']); + await sendTopicsFeed({ + uid: req.uid, + title: `Topics by ${userData.username}`, + description: `A list of topics that are posted by ${userData.username}`, + feed_url: `/user/${userslug}/topics.rss`, + site_url: `/user/${userslug}/topics` + }, `uid:${userData.uid}:topics`, res); +} +async function generateForTag(req, res) { + if (meta.config['feeds:disableRSS']) { + return controllers404.handle404(req, res); + } + const uid = await getUidFromToken(req); + const tag = validator.escape(String(req.params.tag)); + const page = parseInt(req.query.page, 10) || 1; + const topicsPerPage = meta.config.topicsPerPage || 20; + const start = Math.max(0, (page - 1) * topicsPerPage); + const stop = start + topicsPerPage - 1; + await sendTopicsFeed({ + uid: uid, + title: `Topics tagged with ${tag}`, + description: `A list of topics that have been tagged with ${tag}`, + feed_url: `/tags/${tag}.rss`, + site_url: `/tags/${tag}`, + start: start, + stop: stop + }, `tag:${tag}:topics`, res); +} +async function getUidFromToken(req) { + let token = null; + if (req.query.token && req.query.uid) { + token = await db.getObjectField(`user:${req.query.uid}`, 'rss_token'); + } + return token && token === req.query.token ? req.query.uid : req.uid; +} +function sendFeed(feed, res) { + const xml = feed.xml(); + res.type('xml').set('Content-Length', Buffer.byteLength(xml)).send(xml); +} \ No newline at end of file diff --git a/lib/routes/helpers.js b/lib/routes/helpers.js new file mode 100644 index 0000000000..c261fb3ae0 --- /dev/null +++ b/lib/routes/helpers.js @@ -0,0 +1,51 @@ +'use strict'; + +const helpers = module.exports; +const winston = require('winston'); +const middleware = require('../middleware'); +const controllerHelpers = require('../controllers/helpers'); +helpers.setupPageRoute = function (...args) { + const [router, name] = args; + let middlewares = args.length > 3 ? args[args.length - 2] : []; + const controller = args[args.length - 1]; + if (args.length === 5) { + winston.warn(`[helpers.setupPageRoute(${name})] passing \`middleware\` as the third param is deprecated, it can now be safely removed`); + } + middlewares = [middleware.autoLocale, middleware.applyBlacklist, middleware.authenticateRequest, middleware.redirectToHomeIfBanned, middleware.maintenanceMode, middleware.registrationComplete, middleware.pluginHooks, ...middlewares, middleware.pageView]; + router.get(name, middleware.busyCheck, middlewares, middleware.buildHeader, helpers.tryRoute(controller)); + router.get(`/api${name}`, middlewares, helpers.tryRoute(controller)); +}; +helpers.setupAdminPageRoute = function (...args) { + const [router, name] = args; + const middlewares = args.length > 3 ? args[args.length - 2] : []; + const controller = args[args.length - 1]; + if (args.length === 5) { + winston.warn(`[helpers.setupAdminPageRoute(${name})] passing \`middleware\` as the third param is deprecated, it can now be safely removed`); + } + router.get(name, middleware.autoLocale, middleware.admin.buildHeader, middlewares, helpers.tryRoute(controller)); + router.get(`/api${name}`, middlewares, helpers.tryRoute(controller)); +}; +helpers.setupApiRoute = function (...args) { + const [router, verb, name] = args; + let middlewares = args.length > 4 ? args[args.length - 2] : []; + const controller = args[args.length - 1]; + middlewares = [middleware.autoLocale, middleware.applyBlacklist, middleware.authenticateRequest, middleware.maintenanceMode, middleware.registrationComplete, middleware.pluginHooks, middleware.logApiUsage, middleware.handleMultipart, ...middlewares]; + router[verb](name, middlewares, helpers.tryRoute(controller, (err, res) => { + controllerHelpers.formatApiResponse(400, res, err); + })); +}; +helpers.tryRoute = function (controller, handler) { + if (controller && controller.constructor && controller.constructor.name === 'AsyncFunction') { + return async function (req, res, next) { + try { + await controller(req, res, next); + } catch (err) { + if (handler) { + return handler(err, res); + } + next(err); + } + }; + } + return controller; +}; \ No newline at end of file diff --git a/lib/routes/index.js b/lib/routes/index.js new file mode 100644 index 0000000000..6359d82667 --- /dev/null +++ b/lib/routes/index.js @@ -0,0 +1,187 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const path = require('path'); +const express = require('express'); +const meta = require('../meta'); +const controllers = require('../controllers'); +const controllerHelpers = require('../controllers/helpers'); +const plugins = require('../plugins'); +const authRoutes = require('./authentication'); +const writeRoutes = require('./write'); +const helpers = require('./helpers'); +const { + setupPageRoute +} = helpers; +const _mounts = { + user: require('./user'), + meta: require('./meta'), + api: require('./api'), + admin: require('./admin'), + feed: require('./feeds') +}; +_mounts.main = (app, middleware, controllers) => { + const loginRegisterMiddleware = [middleware.redirectToAccountIfLoggedIn]; + setupPageRoute(app, '/login', loginRegisterMiddleware, controllers.login); + setupPageRoute(app, '/register', loginRegisterMiddleware, controllers.register); + setupPageRoute(app, '/register/complete', [], controllers.registerInterstitial); + setupPageRoute(app, '/compose', [], controllers.composer.get); + setupPageRoute(app, '/confirm/:code', [], controllers.confirmEmail); + setupPageRoute(app, '/outgoing', [], controllers.outgoing); + setupPageRoute(app, '/search', [], controllers.search.search); + setupPageRoute(app, '/reset/:code?', [middleware.delayLoading], controllers.reset); + setupPageRoute(app, '/tos', [], controllers.termsOfUse); + setupPageRoute(app, '/email/unsubscribe/:token', [], controllers.accounts.settings.unsubscribe); + app.post('/email/unsubscribe/:token', controllers.accounts.settings.unsubscribePost); + app.post('/compose', middleware.applyCSRF, controllers.composer.post); +}; +_mounts.mod = (app, middleware, controllers) => { + setupPageRoute(app, '/flags', [], controllers.mods.flags.list); + setupPageRoute(app, '/flags/:flagId', [], controllers.mods.flags.detail); + setupPageRoute(app, '/post-queue/:id?', [], controllers.mods.postQueue); +}; +_mounts.globalMod = (app, middleware, controllers) => { + setupPageRoute(app, '/ip-blacklist', [], controllers.globalMods.ipBlacklist); + setupPageRoute(app, '/registration-queue', [], controllers.globalMods.registrationQueue); +}; +_mounts.topic = (app, name, middleware, controllers) => { + setupPageRoute(app, `/${name}/:topic_id/:slug/:post_index?`, [], controllers.topics.get); + setupPageRoute(app, `/${name}/:topic_id/:slug?`, [], controllers.topics.get); +}; +_mounts.post = (app, name, middleware, controllers) => { + const middlewares = [middleware.maintenanceMode, middleware.authenticateRequest, middleware.registrationComplete, middleware.pluginHooks]; + app.get(`/${name}/:pid`, middleware.busyCheck, middlewares, controllers.posts.redirectToPost); + app.get(`/api/${name}/:pid`, middlewares, controllers.posts.redirectToPost); +}; +_mounts.tags = (app, name, middleware, controllers) => { + setupPageRoute(app, `/${name}/:tag`, [middleware.privateTagListing], controllers.tags.getTag); + setupPageRoute(app, `/${name}`, [middleware.privateTagListing], controllers.tags.getTags); +}; +_mounts.categories = (app, name, middleware, controllers) => { + setupPageRoute(app, '/categories', [], controllers.categories.list); + setupPageRoute(app, '/popular', [], controllers.popular.get); + setupPageRoute(app, '/recent', [], controllers.recent.get); + setupPageRoute(app, '/top', [], controllers.top.get); + setupPageRoute(app, '/unread', [middleware.ensureLoggedIn], controllers.unread.get); +}; +_mounts.category = (app, name, middleware, controllers) => { + setupPageRoute(app, `/${name}/:category_id/:slug/:topic_index`, [], controllers.category.get); + setupPageRoute(app, `/${name}/:category_id/:slug?`, [], controllers.category.get); +}; +_mounts.users = (app, name, middleware, controllers) => { + const middlewares = [middleware.canViewUsers]; + setupPageRoute(app, `/${name}`, middlewares, controllers.users.index); +}; +_mounts.groups = (app, name, middleware, controllers) => { + const middlewares = [middleware.canViewGroups]; + setupPageRoute(app, `/${name}`, middlewares, controllers.groups.list); + setupPageRoute(app, `/${name}/:slug`, middlewares, controllers.groups.details); + setupPageRoute(app, `/${name}/:slug/members`, middlewares, controllers.groups.members); +}; +module.exports = async function (app, middleware) { + const router = express.Router(); + router.render = function (...args) { + app.render(...args); + }; + const remountable = ['admin', 'categories', 'category', 'topic', 'post', 'users', 'user', 'groups', 'tags']; + const { + mounts + } = await plugins.hooks.fire('filter:router.add', { + mounts: remountable.reduce((memo, mount) => { + memo[mount] = mount; + return memo; + }, {}) + }); + Object.keys(mounts).forEach(mount => { + if (!remountable.includes(mount)) { + delete mounts[mount]; + } else if (typeof mount !== 'string') { + mounts[mount] = mount; + } + }); + remountable.forEach(mount => { + if (!mounts.hasOwnProperty(mount)) { + mounts[mount] = mount; + } + }); + router.all('(/+api|/+api/*?)', middleware.prepareAPI); + router.all(`(/+api/admin|/+api/admin/*?${mounts.admin !== 'admin' ? `|/+api/${mounts.admin}|/+api/${mounts.admin}/*?` : ''})`, middleware.authenticateRequest, middleware.ensureLoggedIn, middleware.admin.checkPrivileges); + router.all(`(/+admin|/+admin/*?${mounts.admin !== 'admin' ? `|/+${mounts.admin}|/+${mounts.admin}/*?` : ''})`, middleware.ensureLoggedIn, middleware.applyCSRF, middleware.admin.checkPrivileges); + app.use(middleware.stripLeadingSlashes); + router.use('/', controllers.home.rewrite); + setupPageRoute(router, '/', [], controllers.home.pluginHook); + await plugins.reloadRoutes({ + router: router + }); + await authRoutes.reloadRoutes({ + router: router + }); + await writeRoutes.reload({ + router: router + }); + addCoreRoutes(app, router, middleware, mounts); + winston.info('[router] Routes added'); +}; +function addCoreRoutes(app, router, middleware, mounts) { + _mounts.meta(router, middleware, controllers); + _mounts.api(router, middleware, controllers); + _mounts.feed(router, middleware, controllers); + _mounts.main(router, middleware, controllers); + _mounts.mod(router, middleware, controllers); + _mounts.globalMod(router, middleware, controllers); + addRemountableRoutes(app, router, middleware, mounts); + const relativePath = nconf.get('relative_path'); + app.use(relativePath || '/', router); + if (process.env.NODE_ENV === 'development') { + require('./debug')(app, middleware, controllers); + } + app.use(middleware.privateUploads); + const statics = [{ + route: '/assets', + path: path.join(__dirname, '../../build/public') + }, { + route: '/assets', + path: path.join(__dirname, '../../public') + }]; + const staticOptions = { + maxAge: app.enabled('cache') ? 5184000000 : 0 + }; + if (path.resolve(__dirname, '../../public/uploads') !== nconf.get('upload_path')) { + statics.unshift({ + route: '/assets/uploads', + path: nconf.get('upload_path') + }); + } + statics.forEach(obj => { + app.use(relativePath + obj.route, middleware.addUploadHeaders, express.static(obj.path, staticOptions)); + }); + app.use(`${relativePath}/uploads`, (req, res) => { + res.redirect(`${relativePath}/assets/uploads${req.path}?${meta.config['cache-buster']}`); + }); + app.use(`${relativePath}/plugins`, (req, res) => { + res.redirect(`${relativePath}/assets/plugins${req.path}${req._parsedUrl.search || ''}`); + }); + app.use(`${relativePath}/assets/client-*.css`, middleware.buildSkinAsset); + app.use(`${relativePath}/assets/client-*-rtl.css`, middleware.buildSkinAsset); + app.use(controllers['404'].handle404); + app.use(controllers.errors.handleURIErrors); + app.use(controllers.errors.handleErrors); +} +function addRemountableRoutes(app, router, middleware, mounts) { + Object.keys(mounts).map(async mount => { + const original = mount; + mount = mounts[original]; + if (!mount) { + winston.warn(`[router] Not mounting /${original}`); + return; + } + if (mount !== original) { + winston.info(`[router] /${original} prefix re-mounted to /${mount}. Requests to /${original}/* will now redirect to /${mount}`); + router.use(new RegExp(`/(api/)?${original}`), (req, res) => { + controllerHelpers.redirect(res, `${nconf.get('relative_path')}/${mount}${req.path}`); + }); + } + _mounts[original](router, mount, middleware, controllers); + }); +} \ No newline at end of file diff --git a/lib/routes/meta.js b/lib/routes/meta.js new file mode 100644 index 0000000000..a25496319e --- /dev/null +++ b/lib/routes/meta.js @@ -0,0 +1,17 @@ +'use strict'; + +const path = require('path'); +const nconf = require('nconf'); +module.exports = function (app, middleware, controllers) { + app.get('/sitemap.xml', controllers.sitemap.render); + app.get('/sitemap/pages.xml', controllers.sitemap.getPages); + app.get('/sitemap/categories.xml', controllers.sitemap.getCategories); + app.get(/\/sitemap\/topics\.(\d+)\.xml/, controllers.sitemap.getTopicPage); + app.get('/robots.txt', controllers.robots); + app.get('/manifest.webmanifest', controllers.manifest); + app.get('/css/previews/:theme', controllers.admin.themes.get); + app.get('/osd.xml', controllers.osd.handle); + app.get('/service-worker.js', (req, res) => { + res.status(200).type('application/javascript').set('Service-Worker-Allowed', `${nconf.get('relative_path')}/`).sendFile(path.join(__dirname, '../../build/public/src/service-worker.js')); + }); +}; \ No newline at end of file diff --git a/lib/routes/user.js b/lib/routes/user.js new file mode 100644 index 0000000000..a89d849ed9 --- /dev/null +++ b/lib/routes/user.js @@ -0,0 +1,45 @@ +'use strict'; + +const helpers = require('./helpers'); +const { + setupPageRoute +} = helpers; +module.exports = function (app, name, middleware, controllers) { + const middlewares = [middleware.exposeUid, middleware.canViewUsers, middleware.buildAccountData]; + const accountMiddlewares = [...middlewares, middleware.ensureLoggedIn, middleware.checkAccountPermissions]; + setupPageRoute(app, '/me', [], middleware.redirectMeToUserslug); + setupPageRoute(app, '/me/*', [], middleware.redirectMeToUserslug); + setupPageRoute(app, '/uid/:uid*', [], middleware.redirectUidToUserslug); + setupPageRoute(app, `/${name}/:userslug`, middlewares, controllers.accounts.profile.get); + setupPageRoute(app, `/${name}/:userslug/following`, middlewares, controllers.accounts.follow.getFollowing); + setupPageRoute(app, `/${name}/:userslug/followers`, middlewares, controllers.accounts.follow.getFollowers); + setupPageRoute(app, `/${name}/:userslug/posts`, middlewares, controllers.accounts.posts.getPosts); + setupPageRoute(app, `/${name}/:userslug/topics`, middlewares, controllers.accounts.posts.getTopics); + setupPageRoute(app, `/${name}/:userslug/best`, middlewares, controllers.accounts.posts.getBestPosts); + setupPageRoute(app, `/${name}/:userslug/controversial`, middlewares, controllers.accounts.posts.getControversialPosts); + setupPageRoute(app, `/${name}/:userslug/groups`, middlewares, controllers.accounts.groups.get); + setupPageRoute(app, `/${name}/:userslug/categories`, accountMiddlewares, controllers.accounts.categories.get); + setupPageRoute(app, `/${name}/:userslug/tags`, accountMiddlewares, controllers.accounts.tags.get); + setupPageRoute(app, `/${name}/:userslug/bookmarks`, accountMiddlewares, controllers.accounts.posts.getBookmarks); + setupPageRoute(app, `/${name}/:userslug/watched`, accountMiddlewares, controllers.accounts.posts.getWatchedTopics); + setupPageRoute(app, `/${name}/:userslug/ignored`, accountMiddlewares, controllers.accounts.posts.getIgnoredTopics); + setupPageRoute(app, `/${name}/:userslug/upvoted`, accountMiddlewares, controllers.accounts.posts.getUpVotedPosts); + setupPageRoute(app, `/${name}/:userslug/downvoted`, accountMiddlewares, controllers.accounts.posts.getDownVotedPosts); + setupPageRoute(app, `/${name}/:userslug/edit`, accountMiddlewares, controllers.accounts.edit.get); + setupPageRoute(app, `/${name}/:userslug/edit/username`, accountMiddlewares, controllers.accounts.edit.username); + setupPageRoute(app, `/${name}/:userslug/edit/email`, accountMiddlewares, controllers.accounts.edit.email); + setupPageRoute(app, `/${name}/:userslug/edit/password`, accountMiddlewares, controllers.accounts.edit.password); + app.use('/.well-known/change-password', (req, res) => { + res.redirect('/me/edit/password'); + }); + setupPageRoute(app, `/${name}/:userslug/info`, accountMiddlewares, controllers.accounts.info.get); + setupPageRoute(app, `/${name}/:userslug/settings`, accountMiddlewares, controllers.accounts.settings.get); + setupPageRoute(app, `/${name}/:userslug/uploads`, accountMiddlewares, controllers.accounts.uploads.get); + setupPageRoute(app, `/${name}/:userslug/consent`, accountMiddlewares, controllers.accounts.consent.get); + setupPageRoute(app, `/${name}/:userslug/blocks`, accountMiddlewares, controllers.accounts.blocks.getBlocks); + setupPageRoute(app, `/${name}/:userslug/sessions`, accountMiddlewares, controllers.accounts.sessions.get); + setupPageRoute(app, '/notifications', [middleware.ensureLoggedIn], controllers.accounts.notifications.get); + setupPageRoute(app, `/${name}/:userslug/chats/:roomid?/:index?`, [middleware.exposeUid, middleware.canViewUsers], controllers.accounts.chats.get); + setupPageRoute(app, '/chats/:roomid?/:index?', [middleware.ensureLoggedIn], controllers.accounts.chats.redirectToChat); + setupPageRoute(app, `/message/:mid`, [middleware.ensureLoggedIn], controllers.accounts.chats.redirectToMessage); +}; \ No newline at end of file diff --git a/lib/routes/write/admin.js b/lib/routes/write/admin.js new file mode 100644 index 0000000000..6e46e6a19f --- /dev/null +++ b/lib/routes/write/admin.js @@ -0,0 +1,23 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn, middleware.admin.checkPrivileges]; + setupApiRoute(router, 'put', '/settings/:setting', [...middlewares, middleware.checkRequired.bind(null, ['value'])], controllers.write.admin.updateSetting); + setupApiRoute(router, 'get', '/analytics', [...middlewares], controllers.write.admin.getAnalyticsKeys); + setupApiRoute(router, 'get', '/analytics/:set', [...middlewares], controllers.write.admin.getAnalyticsData); + setupApiRoute(router, 'post', '/tokens', [...middlewares], controllers.write.admin.generateToken); + setupApiRoute(router, 'get', '/tokens/:token', [...middlewares], controllers.write.admin.getToken); + setupApiRoute(router, 'put', '/tokens/:token', [...middlewares], controllers.write.admin.updateToken); + setupApiRoute(router, 'delete', '/tokens/:token', [...middlewares], controllers.write.admin.deleteToken); + setupApiRoute(router, 'post', '/tokens/:token/roll', [...middlewares], controllers.write.admin.rollToken); + setupApiRoute(router, 'delete', '/chats/:roomId', [...middlewares, middleware.assert.room], controllers.write.admin.chats.deleteRoom); + setupApiRoute(router, 'get', '/groups', [...middlewares], controllers.write.admin.listGroups); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/categories.js b/lib/routes/write/categories.js new file mode 100644 index 0000000000..aab15519c5 --- /dev/null +++ b/lib/routes/write/categories.js @@ -0,0 +1,29 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn]; + setupApiRoute(router, 'get', '/', controllers.write.categories.list); + setupApiRoute(router, 'post', '/', [...middlewares, middleware.checkRequired.bind(null, ['name'])], controllers.write.categories.create); + setupApiRoute(router, 'get', '/:cid', [], controllers.write.categories.get); + setupApiRoute(router, 'put', '/:cid', [...middlewares], controllers.write.categories.update); + setupApiRoute(router, 'delete', '/:cid', [...middlewares], controllers.write.categories.delete); + setupApiRoute(router, 'get', '/:cid/count', [middleware.assert.category], controllers.write.categories.getTopicCount); + setupApiRoute(router, 'get', '/:cid/posts', [middleware.assert.category], controllers.write.categories.getPosts); + setupApiRoute(router, 'get', '/:cid/children', [middleware.assert.category], controllers.write.categories.getChildren); + setupApiRoute(router, 'get', '/:cid/topics', [middleware.assert.category], controllers.write.categories.getTopics); + setupApiRoute(router, 'put', '/:cid/watch', [...middlewares, middleware.assert.category], controllers.write.categories.setWatchState); + setupApiRoute(router, 'delete', '/:cid/watch', [...middlewares, middleware.assert.category], controllers.write.categories.setWatchState); + setupApiRoute(router, 'get', '/:cid/privileges', [...middlewares], controllers.write.categories.getPrivileges); + setupApiRoute(router, 'put', '/:cid/privileges/:privilege', [...middlewares, middleware.checkRequired.bind(null, ['member'])], controllers.write.categories.setPrivilege); + setupApiRoute(router, 'delete', '/:cid/privileges/:privilege', [...middlewares, middleware.checkRequired.bind(null, ['member'])], controllers.write.categories.setPrivilege); + setupApiRoute(router, 'put', '/:cid/moderator/:uid', [...middlewares], controllers.write.categories.setModerator); + setupApiRoute(router, 'delete', '/:cid/moderator/:uid', [...middlewares], controllers.write.categories.setModerator); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/chats.js b/lib/routes/write/chats.js new file mode 100644 index 0000000000..664ac91e4b --- /dev/null +++ b/lib/routes/write/chats.js @@ -0,0 +1,42 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn, middleware.canChat]; + setupApiRoute(router, 'get', '/', [...middlewares], controllers.write.chats.list); + setupApiRoute(router, 'post', '/', [...middlewares, middleware.checkRequired.bind(null, ['uids'])], controllers.write.chats.create); + setupApiRoute(router, 'get', '/unread', [...middlewares], controllers.write.chats.getUnread); + setupApiRoute(router, 'put', '/sort', [...middlewares, middleware.checkRequired.bind(null, ['roomIds', 'scores'])], controllers.write.chats.sortPublicRooms); + setupApiRoute(router, 'head', '/:roomId', [...middlewares, middleware.assert.room], controllers.write.chats.exists); + setupApiRoute(router, 'get', '/:roomId', [...middlewares, middleware.assert.room], controllers.write.chats.get); + setupApiRoute(router, 'post', '/:roomId', [...middlewares, middleware.assert.room, middleware.checkRequired.bind(null, ['message'])], controllers.write.chats.post); + setupApiRoute(router, 'put', '/:roomId', [...middlewares, middleware.assert.room], controllers.write.chats.update); + setupApiRoute(router, 'put', '/:roomId/state', [...middlewares, middleware.assert.room], controllers.write.chats.mark); + setupApiRoute(router, 'delete', '/:roomId/state', [...middlewares, middleware.assert.room], controllers.write.chats.mark); + setupApiRoute(router, 'put', '/:roomId/watch', [...middlewares, middleware.assert.room, middleware.checkRequired.bind(null, ['value'])], controllers.write.chats.watch); + setupApiRoute(router, 'delete', '/:roomId/watch', [...middlewares, middleware.assert.room], controllers.write.chats.watch); + setupApiRoute(router, 'put', '/:roomId/typing', [...middlewares, middleware.assert.room], controllers.write.chats.toggleTyping); + setupApiRoute(router, 'get', '/:roomId/users', [...middlewares, middleware.assert.room], controllers.write.chats.users); + setupApiRoute(router, 'post', '/:roomId/users', [...middlewares, middleware.assert.room, middleware.checkRequired.bind(null, ['uids'])], controllers.write.chats.invite); + setupApiRoute(router, 'delete', '/:roomId/users', [...middlewares, middleware.assert.room, middleware.checkRequired.bind(null, ['uids'])], controllers.write.chats.kick); + setupApiRoute(router, 'delete', '/:roomId/users/:uid', [...middlewares, middleware.assert.room, middleware.assert.user], controllers.write.chats.kickUser); + setupApiRoute(router, 'put', '/:roomId/owners/:uid', [...middlewares, middleware.assert.room, middleware.assert.user], controllers.write.chats.toggleOwner); + setupApiRoute(router, 'delete', '/:roomId/owners/:uid', [...middlewares, middleware.assert.room, middleware.assert.user], controllers.write.chats.toggleOwner); + setupApiRoute(router, 'get', '/:roomId/messages', [...middlewares, middleware.assert.room], controllers.write.chats.messages.list); + setupApiRoute(router, 'get', '/:roomId/messages/pinned', [...middlewares, middleware.assert.room], controllers.write.chats.messages.getPinned); + setupApiRoute(router, 'get', '/:roomId/messages/:mid', [...middlewares, middleware.assert.room, middleware.assert.message], controllers.write.chats.messages.get); + setupApiRoute(router, 'put', '/:roomId/messages/:mid', [...middlewares, middleware.assert.room, middleware.assert.message], controllers.write.chats.messages.edit); + setupApiRoute(router, 'post', '/:roomId/messages/:mid', [...middlewares, middleware.assert.room, middleware.assert.message], controllers.write.chats.messages.restore); + setupApiRoute(router, 'delete', '/:roomId/messages/:mid', [...middlewares, middleware.assert.room, middleware.assert.message], controllers.write.chats.messages.delete); + setupApiRoute(router, 'get', '/:roomId/messages/:mid/raw', [...middlewares, middleware.assert.room], controllers.write.chats.messages.getRaw); + setupApiRoute(router, 'get', '/:roomId/messages/:mid/ip', [...middlewares, middleware.assert.room], controllers.write.chats.messages.getIpAddress); + setupApiRoute(router, 'put', '/:roomId/messages/:mid/pin', [...middlewares, middleware.assert.room, middleware.assert.message], controllers.write.chats.messages.pin); + setupApiRoute(router, 'delete', '/:roomId/messages/:mid/pin', [...middlewares, middleware.assert.room, middleware.assert.message], controllers.write.chats.messages.unpin); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/files.js b/lib/routes/write/files.js new file mode 100644 index 0000000000..90ca977ed3 --- /dev/null +++ b/lib/routes/write/files.js @@ -0,0 +1,15 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn, middleware.admin.checkPrivileges]; + setupApiRoute(router, 'delete', '/', [...middlewares, middleware.checkRequired.bind(null, ['path']), middleware.assert.path], controllers.write.files.delete); + setupApiRoute(router, 'put', '/folder', [...middlewares, middleware.checkRequired.bind(null, ['path', 'folderName']), middleware.assert.path, middleware.assert.folderName], controllers.write.files.createFolder); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/flags.js b/lib/routes/write/flags.js new file mode 100644 index 0000000000..47e779190f --- /dev/null +++ b/lib/routes/write/flags.js @@ -0,0 +1,20 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn]; + setupApiRoute(router, 'post', '/', [...middlewares], controllers.write.flags.create); + setupApiRoute(router, 'get', '/:flagId', [...middlewares, middleware.assert.flag], controllers.write.flags.get); + setupApiRoute(router, 'put', '/:flagId', [...middlewares, middleware.assert.flag], controllers.write.flags.update); + setupApiRoute(router, 'delete', '/:flagId', [...middlewares, middleware.assert.flag], controllers.write.flags.delete); + setupApiRoute(router, 'delete', '/:flagId/report', middlewares, controllers.write.flags.rescind); + setupApiRoute(router, 'post', '/:flagId/notes', [...middlewares, middleware.assert.flag], controllers.write.flags.appendNote); + setupApiRoute(router, 'delete', '/:flagId/notes/:datetime', [...middlewares, middleware.assert.flag], controllers.write.flags.deleteNote); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/groups.js b/lib/routes/write/groups.js new file mode 100644 index 0000000000..e0e5ed97f2 --- /dev/null +++ b/lib/routes/write/groups.js @@ -0,0 +1,30 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn]; + setupApiRoute(router, 'get', '/', [], controllers.write.groups.list); + setupApiRoute(router, 'post', '/', [...middlewares, middleware.checkRequired.bind(null, ['name'])], controllers.write.groups.create); + setupApiRoute(router, 'head', '/:slug', [middleware.assert.group], controllers.write.groups.exists); + setupApiRoute(router, 'put', '/:slug', [...middlewares, middleware.assert.group], controllers.write.groups.update); + setupApiRoute(router, 'delete', '/:slug', [...middlewares, middleware.assert.group], controllers.write.groups.delete); + setupApiRoute(router, 'get', '/:slug/members', [...middlewares, middleware.assert.group], controllers.write.groups.listMembers); + setupApiRoute(router, 'put', '/:slug/membership/:uid', [...middlewares, middleware.assert.group], controllers.write.groups.join); + setupApiRoute(router, 'delete', '/:slug/membership/:uid', [...middlewares, middleware.assert.group], controllers.write.groups.leave); + setupApiRoute(router, 'put', '/:slug/ownership/:uid', [...middlewares, middleware.assert.group], controllers.write.groups.grant); + setupApiRoute(router, 'delete', '/:slug/ownership/:uid', [...middlewares, middleware.assert.group], controllers.write.groups.rescind); + setupApiRoute(router, 'get', '/:slug/pending', [...middlewares, middleware.assert.group], controllers.write.groups.getPending); + setupApiRoute(router, 'put', '/:slug/pending/:uid', [...middlewares, middleware.assert.group], controllers.write.groups.accept); + setupApiRoute(router, 'delete', '/:slug/pending/:uid', [...middlewares, middleware.assert.group], controllers.write.groups.reject); + setupApiRoute(router, 'get', '/:slug/invites', [...middlewares, middleware.assert.group], controllers.write.groups.getInvites); + setupApiRoute(router, 'post', '/:slug/invites/:uid', [...middlewares, middleware.assert.group], controllers.write.groups.issueInvite); + setupApiRoute(router, 'put', '/:slug/invites/:uid', [...middlewares, middleware.assert.group], controllers.write.groups.acceptInvite); + setupApiRoute(router, 'delete', '/:slug/invites/:uid', [...middlewares, middleware.assert.group], controllers.write.groups.rejectInvite); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/index.js b/lib/routes/write/index.js new file mode 100644 index 0000000000..251e47fead --- /dev/null +++ b/lib/routes/write/index.js @@ -0,0 +1,64 @@ +'use strict'; + +const winston = require('winston'); +const meta = require('../../meta'); +const plugins = require('../../plugins'); +const middleware = require('../../middleware'); +const writeControllers = require('../../controllers/write'); +const helpers = require('../../controllers/helpers'); +const { + setupApiRoute +} = require('../helpers'); +const Write = module.exports; +Write.reload = async params => { + const { + router + } = params; + let apiSettings = await meta.settings.get('core.api'); + plugins.hooks.register('core', { + hook: 'action:settings.set', + method: async data => { + if (data.plugin === 'core.api') { + apiSettings = await meta.settings.get('core.api'); + } + } + }); + router.use('/api/v3', (req, res, next) => { + if (apiSettings.requireHttps === 'on' && req.protocol !== 'https') { + res.set('Upgrade', 'TLS/1.0, HTTP/1.1'); + return helpers.formatApiResponse(426, res); + } + res.locals.isAPI = true; + next(); + }); + router.use('/api/v3/users', require('./users')()); + router.use('/api/v3/groups', require('./groups')()); + router.use('/api/v3/categories', require('./categories')()); + router.use('/api/v3/topics', require('./topics')()); + router.use('/api/v3/tags', require('./tags')()); + router.use('/api/v3/posts', require('./posts')()); + router.use('/api/v3/chats', require('./chats')()); + router.use('/api/v3/flags', require('./flags')()); + router.use('/api/v3/search', require('./search')()); + router.use('/api/v3/admin', require('./admin')()); + router.use('/api/v3/files', require('./files')()); + router.use('/api/v3/utilities', require('./utilities')()); + setupApiRoute(router, 'get', '/api/v3/ping', writeControllers.utilities.ping.get); + setupApiRoute(router, 'post', '/api/v3/ping', writeControllers.utilities.ping.post); + const pluginRouter = require('express').Router(); + await plugins.hooks.fire('static:api.routes', { + router: pluginRouter, + middleware, + helpers + }); + winston.info(`[api] Adding ${pluginRouter.stack.length} route(s) to \`api/v3/plugins\``); + router.use('/api/v3/plugins', pluginRouter); + router.use('/api/v3', (req, res) => { + helpers.formatApiResponse(404, res); + }); +}; +Write.cleanup = req => { + if (req && req.session) { + req.session.destroy(); + } +}; \ No newline at end of file diff --git a/lib/routes/write/posts.js b/lib/routes/write/posts.js new file mode 100644 index 0000000000..8eb1c99480 --- /dev/null +++ b/lib/routes/write/posts.js @@ -0,0 +1,34 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn, middleware.assert.post]; + setupApiRoute(router, 'get', '/:pid', [middleware.assert.post], controllers.write.posts.get); + setupApiRoute(router, 'put', '/:pid', [middleware.ensureLoggedIn, middleware.checkRequired.bind(null, ['content'])], controllers.write.posts.edit); + setupApiRoute(router, 'delete', '/:pid', middlewares, controllers.write.posts.purge); + setupApiRoute(router, 'get', '/:pid/index', [middleware.assert.post], controllers.write.posts.getIndex); + setupApiRoute(router, 'get', '/:pid/raw', [middleware.assert.post], controllers.write.posts.getRaw); + setupApiRoute(router, 'get', '/:pid/summary', [middleware.assert.post], controllers.write.posts.getSummary); + setupApiRoute(router, 'put', '/:pid/state', middlewares, controllers.write.posts.restore); + setupApiRoute(router, 'delete', '/:pid/state', middlewares, controllers.write.posts.delete); + setupApiRoute(router, 'put', '/:pid/move', [...middlewares, middleware.checkRequired.bind(null, ['tid'])], controllers.write.posts.move); + setupApiRoute(router, 'put', '/:pid/vote', [...middlewares, middleware.checkRequired.bind(null, ['delta'])], controllers.write.posts.vote); + setupApiRoute(router, 'delete', '/:pid/vote', middlewares, controllers.write.posts.unvote); + setupApiRoute(router, 'get', '/:pid/voters', [middleware.assert.post], controllers.write.posts.getVoters); + setupApiRoute(router, 'get', '/:pid/upvoters', [middleware.assert.post], controllers.write.posts.getUpvoters); + setupApiRoute(router, 'put', '/:pid/bookmark', middlewares, controllers.write.posts.bookmark); + setupApiRoute(router, 'delete', '/:pid/bookmark', middlewares, controllers.write.posts.unbookmark); + setupApiRoute(router, 'get', '/:pid/diffs', [middleware.assert.post], controllers.write.posts.getDiffs); + setupApiRoute(router, 'get', '/:pid/diffs/:since', [middleware.assert.post], controllers.write.posts.loadDiff); + setupApiRoute(router, 'put', '/:pid/diffs/:since', middlewares, controllers.write.posts.restoreDiff); + setupApiRoute(router, 'delete', '/:pid/diffs/:timestamp', middlewares, controllers.write.posts.deleteDiff); + setupApiRoute(router, 'get', '/:pid/replies', [middleware.assert.post], controllers.write.posts.getReplies); + router.all('/+byIndex/:index*?', [middleware.checkRequired.bind(null, ['tid'])], controllers.write.posts.redirectByIndex); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/search.js b/lib/routes/write/search.js new file mode 100644 index 0000000000..9fc8085fb6 --- /dev/null +++ b/lib/routes/write/search.js @@ -0,0 +1,16 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn]; + setupApiRoute(router, 'get', '/categories', [], controllers.write.search.categories); + setupApiRoute(router, 'get', '/chats/:roomId/users', [...middlewares, middleware.checkRequired.bind(null, ['query']), middleware.canChat, middleware.assert.room], controllers.write.search.roomUsers); + setupApiRoute(router, 'get', '/chats/:roomId/messages', [...middlewares, middleware.checkRequired.bind(null, ['query']), middleware.canChat, middleware.assert.room], controllers.write.search.roomMessages); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/tags.js b/lib/routes/write/tags.js new file mode 100644 index 0000000000..d0b03199a0 --- /dev/null +++ b/lib/routes/write/tags.js @@ -0,0 +1,15 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn]; + setupApiRoute(router, 'put', '/:tag/follow', [...middlewares], controllers.write.tags.follow); + setupApiRoute(router, 'delete', '/:tag/follow', [...middlewares], controllers.write.tags.unfollow); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/topics.js b/lib/routes/write/topics.js new file mode 100644 index 0000000000..815590ae30 --- /dev/null +++ b/lib/routes/write/topics.js @@ -0,0 +1,42 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + const middlewares = [middleware.ensureLoggedIn]; + const multipart = require('connect-multiparty'); + const multipartMiddleware = multipart(); + setupApiRoute(router, 'post', '/', [middleware.checkRequired.bind(null, ['cid', 'title', 'content'])], controllers.write.topics.create); + setupApiRoute(router, 'get', '/:tid', [], controllers.write.topics.get); + setupApiRoute(router, 'post', '/:tid', [middleware.checkRequired.bind(null, ['content']), middleware.assert.topic], controllers.write.topics.reply); + setupApiRoute(router, 'delete', '/:tid', [...middlewares], controllers.write.topics.purge); + setupApiRoute(router, 'put', '/:tid/state', [...middlewares], controllers.write.topics.restore); + setupApiRoute(router, 'delete', '/:tid/state', [...middlewares], controllers.write.topics.delete); + setupApiRoute(router, 'put', '/:tid/pin', [...middlewares, middleware.assert.topic], controllers.write.topics.pin); + setupApiRoute(router, 'delete', '/:tid/pin', [...middlewares], controllers.write.topics.unpin); + setupApiRoute(router, 'put', '/:tid/lock', [...middlewares], controllers.write.topics.lock); + setupApiRoute(router, 'delete', '/:tid/lock', [...middlewares], controllers.write.topics.unlock); + setupApiRoute(router, 'put', '/:tid/follow', [...middlewares, middleware.assert.topic], controllers.write.topics.follow); + setupApiRoute(router, 'delete', '/:tid/follow', [...middlewares, middleware.assert.topic], controllers.write.topics.unfollow); + setupApiRoute(router, 'put', '/:tid/ignore', [...middlewares, middleware.assert.topic], controllers.write.topics.ignore); + setupApiRoute(router, 'delete', '/:tid/ignore', [...middlewares, middleware.assert.topic], controllers.write.topics.unfollow); + setupApiRoute(router, 'put', '/:tid/tags', [...middlewares, middleware.checkRequired.bind(null, ['tags']), middleware.assert.topic], controllers.write.topics.updateTags); + setupApiRoute(router, 'patch', '/:tid/tags', [...middlewares, middleware.checkRequired.bind(null, ['tags']), middleware.assert.topic], controllers.write.topics.addTags); + setupApiRoute(router, 'delete', '/:tid/tags', [...middlewares, middleware.assert.topic], controllers.write.topics.deleteTags); + setupApiRoute(router, 'get', '/:tid/thumbs', [], controllers.write.topics.getThumbs); + setupApiRoute(router, 'post', '/:tid/thumbs', [multipartMiddleware, middleware.validateFiles, middleware.uploads.ratelimit, ...middlewares], controllers.write.topics.addThumb); + setupApiRoute(router, 'put', '/:tid/thumbs', [...middlewares, middleware.checkRequired.bind(null, ['tid'])], controllers.write.topics.migrateThumbs); + setupApiRoute(router, 'delete', '/:tid/thumbs', [...middlewares, middleware.checkRequired.bind(null, ['path'])], controllers.write.topics.deleteThumb); + setupApiRoute(router, 'put', '/:tid/thumbs/order', [...middlewares, middleware.checkRequired.bind(null, ['path', 'order'])], controllers.write.topics.reorderThumbs); + setupApiRoute(router, 'get', '/:tid/events', [middleware.assert.topic], controllers.write.topics.getEvents); + setupApiRoute(router, 'delete', '/:tid/events/:eventId', [middleware.assert.topic], controllers.write.topics.deleteEvent); + setupApiRoute(router, 'put', '/:tid/read', [...middlewares, middleware.assert.topic], controllers.write.topics.markRead); + setupApiRoute(router, 'delete', '/:tid/read', [...middlewares, middleware.assert.topic], controllers.write.topics.markUnread); + setupApiRoute(router, 'put', '/:tid/bump', [...middlewares, middleware.assert.topic], controllers.write.topics.bump); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/users.js b/lib/routes/write/users.js new file mode 100644 index 0000000000..e975e194c8 --- /dev/null +++ b/lib/routes/write/users.js @@ -0,0 +1,50 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +function guestRoutes() {} +function authenticatedRoutes() { + const middlewares = [middleware.ensureLoggedIn]; + setupApiRoute(router, 'post', '/', [...middlewares, middleware.checkRequired.bind(null, ['username'])], controllers.write.users.create); + setupApiRoute(router, 'delete', '/', [...middlewares, middleware.checkRequired.bind(null, ['uids'])], controllers.write.users.deleteMany); + setupApiRoute(router, 'head', '/:uid', [middleware.assert.user], controllers.write.users.exists); + setupApiRoute(router, 'get', '/:uid', [...middlewares, middleware.assert.user], controllers.write.users.get); + setupApiRoute(router, 'put', '/:uid', [...middlewares, middleware.assert.user], controllers.write.users.update); + setupApiRoute(router, 'delete', '/:uid', [...middlewares, middleware.assert.user], controllers.write.users.delete); + setupApiRoute(router, 'put', '/:uid/picture', [...middlewares, middleware.assert.user], controllers.write.users.changePicture); + setupApiRoute(router, 'delete', '/:uid/content', [...middlewares, middleware.assert.user], controllers.write.users.deleteContent); + setupApiRoute(router, 'delete', '/:uid/account', [...middlewares, middleware.assert.user], controllers.write.users.deleteAccount); + setupApiRoute(router, 'get', '/:uid/status', [], controllers.write.users.getStatus); + setupApiRoute(router, 'head', '/:uid/status/:status', [], controllers.write.users.checkStatus); + setupApiRoute(router, 'get', '/:uid/chat', [...middlewares], controllers.write.users.getPrivateRoomId); + setupApiRoute(router, 'put', '/:uid/settings', [...middlewares, middleware.checkRequired.bind(null, ['settings'])], controllers.write.users.updateSettings); + setupApiRoute(router, 'put', '/:uid/password', [...middlewares, middleware.checkRequired.bind(null, ['newPassword']), middleware.assert.user], controllers.write.users.changePassword); + setupApiRoute(router, 'put', '/:uid/follow', [...middlewares, middleware.assert.user], controllers.write.users.follow); + setupApiRoute(router, 'delete', '/:uid/follow', [...middlewares, middleware.assert.user], controllers.write.users.unfollow); + setupApiRoute(router, 'put', '/:uid/ban', [...middlewares, middleware.assert.user], controllers.write.users.ban); + setupApiRoute(router, 'delete', '/:uid/ban', [...middlewares, middleware.assert.user], controllers.write.users.unban); + setupApiRoute(router, 'put', '/:uid/mute', [...middlewares, middleware.assert.user], controllers.write.users.mute); + setupApiRoute(router, 'delete', '/:uid/mute', [...middlewares, middleware.assert.user], controllers.write.users.unmute); + setupApiRoute(router, 'post', '/:uid/tokens', [...middlewares, middleware.assert.user], controllers.write.users.generateToken); + setupApiRoute(router, 'delete', '/:uid/tokens/:token', [...middlewares, middleware.assert.user], controllers.write.users.deleteToken); + setupApiRoute(router, 'delete', '/:uid/sessions/:uuid', [...middlewares, middleware.assert.user], controllers.write.users.revokeSession); + setupApiRoute(router, 'post', '/:uid/invites', middlewares, controllers.write.users.invite); + setupApiRoute(router, 'get', '/:uid/invites/groups', [...middlewares, middleware.assert.user], controllers.write.users.getInviteGroups); + setupApiRoute(router, 'get', '/:uid/emails', [...middlewares, middleware.assert.user], controllers.write.users.listEmails); + setupApiRoute(router, 'post', '/:uid/emails', [...middlewares, middleware.assert.user], controllers.write.users.addEmail); + setupApiRoute(router, 'get', '/:uid/emails/:email', [...middlewares, middleware.assert.user], controllers.write.users.getEmail); + setupApiRoute(router, 'post', '/:uid/emails/:email/confirm', [...middlewares, middleware.assert.user], controllers.write.users.confirmEmail); + setupApiRoute(router, 'head', '/:uid/exports/:type', [...middlewares, middleware.assert.user, middleware.checkAccountPermissions], controllers.write.users.checkExportByType); + setupApiRoute(router, 'get', '/:uid/exports/:type', [...middlewares, middleware.assert.user, middleware.checkAccountPermissions], controllers.write.users.getExportByType); + setupApiRoute(router, 'post', '/:uid/exports/:type', [...middlewares, middleware.assert.user, middleware.checkAccountPermissions], controllers.write.users.generateExportsByType); + router.all('/+bySlug/:userslug*?', [], controllers.write.users.redirectBySlug); +} +module.exports = function () { + authenticatedRoutes(); + return router; +}; \ No newline at end of file diff --git a/lib/routes/write/utilities.js b/lib/routes/write/utilities.js new file mode 100644 index 0000000000..2626d9d645 --- /dev/null +++ b/lib/routes/write/utilities.js @@ -0,0 +1,13 @@ +'use strict'; + +const router = require('express').Router(); +const middleware = require('../../middleware'); +const controllers = require('../../controllers'); +const routeHelpers = require('../helpers'); +const { + setupApiRoute +} = routeHelpers; +module.exports = function () { + setupApiRoute(router, 'post', '/login', [middleware.checkRequired.bind(null, ['username', 'password'])], controllers.write.utilities.login); + return router; +}; \ No newline at end of file diff --git a/lib/search.js b/lib/search.js new file mode 100644 index 0000000000..af2efecd11 --- /dev/null +++ b/lib/search.js @@ -0,0 +1,302 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('./database'); +const batch = require('./batch'); +const posts = require('./posts'); +const topics = require('./topics'); +const categories = require('./categories'); +const user = require('./user'); +const plugins = require('./plugins'); +const privileges = require('./privileges'); +const utils = require('./utils'); +const search = module.exports; +search.search = async function (data) { + const start = process.hrtime(); + data.sortBy = data.sortBy || 'relevance'; + let result; + if (['posts', 'titles', 'titlesposts', 'bookmarks'].includes(data.searchIn)) { + result = await searchInContent(data); + } else if (data.searchIn === 'users') { + result = await user.search(data); + } else if (data.searchIn === 'categories') { + result = await categories.search(data); + } else if (data.searchIn === 'tags') { + result = await topics.searchAndLoadTags(data); + } else if (data.searchIn) { + result = await plugins.hooks.fire('filter:search.searchIn', { + data + }); + } else { + throw new Error('[[error:unknown-search-filter]]'); + } + result.time = (process.elapsedTimeSince(start) / 1000).toFixed(2); + return result; +}; +async function searchInContent(data) { + data.uid = data.uid || 0; + const [searchCids, searchUids] = await Promise.all([getSearchCids(data), getSearchUids(data)]); + async function doSearch(type, searchIn) { + if (searchIn.includes(data.searchIn)) { + const result = await plugins.hooks.fire('filter:search.query', { + index: type, + content: data.query, + matchWords: data.matchWords || 'all', + cid: searchCids, + uid: searchUids, + searchData: data, + ids: [] + }); + return Array.isArray(result) ? result : result.ids; + } + return []; + } + let pids = []; + let tids = []; + const inTopic = String(data.query || '').match(/^in:topic-([\d]+) /); + if (inTopic) { + const tid = inTopic[1]; + const cleanedTerm = data.query.replace(inTopic[0], ''); + pids = await topics.search(tid, cleanedTerm); + } else if (data.searchIn === 'bookmarks') { + pids = await searchInBookmarks(data, searchCids, searchUids); + } else { + [pids, tids] = await Promise.all([doSearch('post', ['posts', 'titlesposts']), doSearch('topic', ['titles', 'titlesposts'])]); + } + const mainPids = await topics.getMainPids(tids); + let allPids = mainPids.concat(pids).filter(Boolean); + allPids = await privileges.posts.filter('topics:read', allPids, data.uid); + allPids = await filterAndSort(allPids, data); + const metadata = await plugins.hooks.fire('filter:search.inContent', { + pids: allPids, + data: data + }); + if (data.returnIds) { + const mainPidsSet = new Set(mainPids); + const mainPidToTid = _.zipObject(mainPids, tids); + const pidsSet = new Set(pids); + const returnPids = allPids.filter(pid => pidsSet.has(pid)); + const returnTids = allPids.filter(pid => mainPidsSet.has(pid)).map(pid => mainPidToTid[pid]); + return { + pids: returnPids, + tids: returnTids + }; + } + const itemsPerPage = Math.min(data.itemsPerPage || 10, 100); + const returnData = { + posts: [], + matchCount: metadata.pids.length, + pageCount: Math.max(1, Math.ceil(parseInt(metadata.pids.length, 10) / itemsPerPage)) + }; + if (data.page) { + const start = Math.max(0, data.page - 1) * itemsPerPage; + metadata.pids = metadata.pids.slice(start, start + itemsPerPage); + } + returnData.posts = await posts.getPostSummaryByPids(metadata.pids, data.uid, {}); + await plugins.hooks.fire('filter:search.contentGetResult', { + result: returnData, + data: data + }); + delete metadata.pids; + delete metadata.data; + return Object.assign(returnData, metadata); +} +async function searchInBookmarks(data, searchCids, searchUids) { + const { + uid, + query, + matchWords + } = data; + const allPids = []; + await batch.processSortedSet(`uid:${uid}:bookmarks`, async pids => { + if (Array.isArray(searchCids) && searchCids.length) { + pids = await posts.filterPidsByCid(pids, searchCids); + } + if (Array.isArray(searchUids) && searchUids.length) { + pids = await posts.filterPidsByUid(pids, searchUids); + } + if (query) { + const tokens = String(query).split(' '); + const postData = await db.getObjectsFields(pids.map(pid => `post:${pid}`), ['content', 'tid']); + const tids = _.uniq(postData.map(p => p.tid)); + const topicData = await db.getObjectsFields(tids.map(tid => `topic:${tid}`), ['title']); + const tidToTopic = _.zipObject(tids, topicData); + pids = pids.filter((pid, i) => { + const content = String(postData[i].content); + const title = String(tidToTopic[postData[i].tid].title); + const method = matchWords === 'any' ? 'some' : 'every'; + return tokens[method](token => content.includes(token) || title.includes(token)); + }); + } + allPids.push(...pids); + }, { + batch: 500 + }); + return allPids; +} +async function filterAndSort(pids, data) { + if (data.sortBy === 'relevance' && !data.replies && !data.timeRange && !data.hasTags && data.searchIn !== 'bookmarks' && !plugins.hooks.hasListeners('filter:search.filterAndSort')) { + return pids; + } + let postsData = await getMatchedPosts(pids, data); + if (!postsData.length) { + return pids; + } + postsData = postsData.filter(Boolean); + postsData = filterByPostcount(postsData, data.replies, data.repliesFilter); + postsData = filterByTimerange(postsData, data.timeRange, data.timeFilter); + postsData = filterByTags(postsData, data.hasTags); + sortPosts(postsData, data); + const result = await plugins.hooks.fire('filter:search.filterAndSort', { + pids: pids, + posts: postsData, + data: data + }); + return result.posts.map(post => post && post.pid); +} +async function getMatchedPosts(pids, data) { + const postFields = ['pid', 'uid', 'tid', 'timestamp', 'deleted', 'upvotes', 'downvotes']; + let postsData = await posts.getPostsFields(pids, postFields); + postsData = postsData.filter(post => post && !post.deleted); + const uids = _.uniq(postsData.map(post => post.uid)); + const tids = _.uniq(postsData.map(post => post.tid)); + const [users, topics] = await Promise.all([getUsers(uids, data), getTopics(tids, data)]); + const tidToTopic = _.zipObject(tids, topics); + const uidToUser = _.zipObject(uids, users); + postsData.forEach(post => { + if (topics && tidToTopic[post.tid]) { + post.topic = tidToTopic[post.tid]; + if (post.topic && post.topic.category) { + post.category = post.topic.category; + } + } + if (uidToUser[post.uid]) { + post.user = uidToUser[post.uid]; + } + }); + return postsData.filter(post => post && post.topic && !post.topic.deleted); +} +async function getUsers(uids, data) { + if (data.sortBy.startsWith('user')) { + return user.getUsersFields(uids, ['username']); + } + return []; +} +async function getTopics(tids, data) { + const topicsData = await topics.getTopicsData(tids); + const cids = _.uniq(topicsData.map(topic => topic && topic.cid)); + const categories = await getCategories(cids, data); + const cidToCategory = _.zipObject(cids, categories); + topicsData.forEach(topic => { + if (topic && categories && cidToCategory[topic.cid]) { + topic.category = cidToCategory[topic.cid]; + } + if (topic && topic.tags) { + topic.tags = topic.tags.map(tag => tag.value); + } + }); + return topicsData; +} +async function getCategories(cids, data) { + const categoryFields = []; + if (data.sortBy.startsWith('category.')) { + categoryFields.push(data.sortBy.split('.')[1]); + } + if (!categoryFields.length) { + return null; + } + return await db.getObjectsFields(cids.map(cid => `category:${cid}`), categoryFields); +} +function filterByPostcount(posts, postCount, repliesFilter) { + postCount = parseInt(postCount, 10); + if (postCount) { + if (repliesFilter === 'atleast') { + posts = posts.filter(post => post.topic && post.topic.postcount >= postCount); + } else { + posts = posts.filter(post => post.topic && post.topic.postcount <= postCount); + } + } + return posts; +} +function filterByTimerange(posts, timeRange, timeFilter) { + timeRange = parseInt(timeRange, 10) * 1000; + if (timeRange) { + const time = Date.now() - timeRange; + if (timeFilter === 'newer') { + posts = posts.filter(post => post.timestamp >= time); + } else { + posts = posts.filter(post => post.timestamp <= time); + } + } + return posts; +} +function filterByTags(posts, hasTags) { + if (Array.isArray(hasTags) && hasTags.length) { + posts = posts.filter(post => { + let hasAllTags = false; + if (post && post.topic && Array.isArray(post.topic.tags) && post.topic.tags.length) { + hasAllTags = hasTags.every(tag => post.topic.tags.includes(tag)); + } + return hasAllTags; + }); + } + return posts; +} +function sortPosts(posts, data) { + if (!posts.length || data.sortBy === 'relevance') { + return; + } + data.sortDirection = data.sortDirection || 'desc'; + const direction = data.sortDirection === 'desc' ? 1 : -1; + const fields = data.sortBy.split('.'); + if (fields.length === 1) { + return posts.sort((p1, p2) => direction * (p2[fields[0]] - p1[fields[0]])); + } + const firstPost = posts[0]; + if (!fields || fields.length !== 2 || !firstPost[fields[0]] || !firstPost[fields[0]][fields[1]]) { + return; + } + const isNumeric = utils.isNumber(firstPost[fields[0]][fields[1]]); + if (isNumeric) { + posts.sort((p1, p2) => direction * (p2[fields[0]][fields[1]] - p1[fields[0]][fields[1]])); + } else { + posts.sort((p1, p2) => { + if (p1[fields[0]][fields[1]] > p2[fields[0]][fields[1]]) { + return direction; + } else if (p1[fields[0]][fields[1]] < p2[fields[0]][fields[1]]) { + return -direction; + } + return 0; + }); + } +} +async function getSearchCids(data) { + if (!Array.isArray(data.categories) || !data.categories.length) { + return []; + } + if (data.categories.includes('all')) { + return await categories.getCidsByPrivilege('categories:cid', data.uid, 'read'); + } + const [watchedCids, childrenCids] = await Promise.all([getWatchedCids(data), getChildrenCids(data)]); + return _.uniq(watchedCids.concat(childrenCids).concat(data.categories).filter(Boolean)); +} +async function getWatchedCids(data) { + if (!data.categories.includes('watched')) { + return []; + } + return await user.getWatchedCategories(data.uid); +} +async function getChildrenCids(data) { + if (!data.searchChildren) { + return []; + } + const childrenCids = await Promise.all(data.categories.map(cid => categories.getChildrenCids(cid))); + return await privileges.categories.filterCids('find', _.uniq(_.flatten(childrenCids)), data.uid); +} +async function getSearchUids(data) { + if (!data.postedBy) { + return []; + } + return await user.getUidsByUsernames(Array.isArray(data.postedBy) ? data.postedBy : [data.postedBy]); +} +require('./promisify')(search); \ No newline at end of file diff --git a/lib/settings.js b/lib/settings.js new file mode 100644 index 0000000000..a7df580b47 --- /dev/null +++ b/lib/settings.js @@ -0,0 +1,173 @@ +'use strict'; + +const meta = require('./meta'); +const pubsub = require('./pubsub'); +function expandObjBy(obj1, obj2) { + let changed = false; + if (!obj1 || !obj2) { + return changed; + } + for (const [key, val2] of Object.entries(obj2)) { + const val1 = obj1[key]; + const xorIsArray = Array.isArray(val1) !== Array.isArray(val2); + if (xorIsArray || !obj1.hasOwnProperty(key) || typeof val2 !== typeof val1) { + obj1[key] = val2; + changed = true; + } else if (typeof val2 === 'object' && !Array.isArray(val2)) { + if (expandObjBy(val1, val2)) { + changed = true; + } + } + } + return changed; +} +function trim(obj1, obj2) { + for (const [key, val1] of Object.entries(obj1)) { + if (!obj2.hasOwnProperty(key)) { + delete obj1[key]; + } else if (typeof val1 === 'object' && !Array.isArray(val1)) { + trim(val1, obj2[key]); + } + } +} +function mergeSettings(cfg, defCfg) { + if (typeof defCfg !== 'object') { + return; + } + if (typeof cfg._ !== 'object') { + cfg._ = defCfg; + } else { + expandObjBy(cfg._, defCfg); + trim(cfg._, defCfg); + } +} +function Settings(hash, version, defCfg, callback, forceUpdate, reset) { + this.hash = hash; + this.version = version || this.version; + this.defCfg = defCfg; + const self = this; + if (reset) { + this.reset(callback); + } else { + this.sync(function () { + this.checkStructure(callback, forceUpdate); + }); + } + pubsub.on(`action:settings.set.${hash}`, data => { + try { + self.cfg._ = JSON.parse(data._); + } catch (err) {} + }); +} +Settings.prototype.hash = ''; +Settings.prototype.defCfg = {}; +Settings.prototype.cfg = {}; +Settings.prototype.version = '0.0.0'; +Settings.prototype.sync = function (callback) { + const _this = this; + meta.settings.get(this.hash, (err, settings) => { + try { + if (settings._) { + settings._ = JSON.parse(settings._); + } + } catch (_error) {} + _this.cfg = settings; + if (typeof _this.cfg._ !== 'object') { + _this.cfg._ = _this.defCfg; + _this.persist(callback); + } else if (expandObjBy(_this.cfg._, _this.defCfg)) { + _this.persist(callback); + } else if (typeof callback === 'function') { + callback.apply(_this, err); + } + }); +}; +Settings.prototype.persist = function (callback) { + let conf = this.cfg._; + const _this = this; + if (typeof conf === 'object') { + conf = JSON.stringify(conf); + } + meta.settings.set(this.hash, this.createWrapper(this.cfg.v, conf), (...args) => { + if (typeof callback === 'function') { + callback.apply(_this, args || []); + } + }); + return this; +}; +Settings.prototype.get = function (key, def) { + let obj = this.cfg._; + const parts = (key || '').split('.'); + let part; + for (let i = 0; i < parts.length; i += 1) { + part = parts[i]; + if (part && obj != null) { + obj = obj[part]; + } + } + if (obj === undefined) { + if (def === undefined) { + def = this.defCfg; + for (let j = 0; j < parts.length; j += 1) { + part = parts[j]; + if (part && def != null) { + def = def[part]; + } + } + } + return def; + } + return obj; +}; +Settings.prototype.getWrapper = function () { + return this.cfg; +}; +Settings.prototype.createWrapper = function (version, settings) { + return { + v: version, + _: settings + }; +}; +Settings.prototype.createDefaultWrapper = function () { + return this.createWrapper(this.version, this.defCfg); +}; +Settings.prototype.set = function (key, val) { + let part; + let obj; + let parts; + this.cfg.v = this.version; + if (val == null || !key) { + this.cfg._ = val || key; + } else { + obj = this.cfg._; + parts = key.split('.'); + for (let i = 0, _len = parts.length - 1; i < _len; i += 1) { + part = parts[i]; + if (part) { + if (!obj.hasOwnProperty(part)) { + obj[part] = {}; + } + obj = obj[part]; + } + } + obj[parts[parts.length - 1]] = val; + } + return this; +}; +Settings.prototype.reset = function (callback) { + this.set(this.defCfg).persist(callback); + return this; +}; +Settings.prototype.checkStructure = function (callback, force) { + if (!force && this.cfg.v === this.version) { + if (typeof callback === 'function') { + callback(); + } + } else { + mergeSettings(this.cfg, this.defCfg); + this.cfg.v = this.version; + this.persist(callback); + } + return this; +}; +module.exports = Settings; \ No newline at end of file diff --git a/lib/sitemap.js b/lib/sitemap.js new file mode 100644 index 0000000000..911439b232 --- /dev/null +++ b/lib/sitemap.js @@ -0,0 +1,166 @@ +'use strict'; + +const { + SitemapStream, + streamToPromise +} = require('sitemap'); +const nconf = require('nconf'); +const db = require('./database'); +const categories = require('./categories'); +const topics = require('./topics'); +const privileges = require('./privileges'); +const meta = require('./meta'); +const plugins = require('./plugins'); +const utils = require('./utils'); +const sitemap = module.exports; +sitemap.maps = { + topics: [] +}; +sitemap.render = async function () { + const topicsPerPage = meta.config.sitemapTopics; + const returnData = { + url: nconf.get('url'), + topics: [] + }; + const [topicCount, categories, pages] = await Promise.all([db.getObjectField('global', 'topicCount'), getSitemapCategories(), getSitemapPages()]); + returnData.categories = categories.length > 0; + returnData.pages = pages.length > 0; + const numPages = Math.ceil(Math.max(0, topicCount / topicsPerPage)); + for (let x = 1; x <= numPages; x += 1) { + returnData.topics.push(x); + } + return returnData; +}; +async function getSitemapPages() { + const urls = [{ + url: '', + changefreq: 'weekly', + priority: 0.6 + }, { + url: `${nconf.get('relative_path')}/recent`, + changefreq: 'daily', + priority: 0.4 + }, { + url: `${nconf.get('relative_path')}/users`, + changefreq: 'daily', + priority: 0.4 + }, { + url: `${nconf.get('relative_path')}/groups`, + changefreq: 'daily', + priority: 0.4 + }]; + const data = await plugins.hooks.fire('filter:sitemap.getPages', { + urls: urls + }); + return data.urls; +} +sitemap.getPages = async function () { + if (sitemap.maps.pages && Date.now() < sitemap.maps.pagesCacheExpireTimestamp) { + return sitemap.maps.pages; + } + const urls = await getSitemapPages(); + if (!urls.length) { + sitemap.maps.pages = ''; + sitemap.maps.pagesCacheExpireTimestamp = Date.now() + 1000 * 60 * 60 * 24; + return sitemap.maps.pages; + } + sitemap.maps.pages = await urlsToSitemap(urls); + sitemap.maps.pagesCacheExpireTimestamp = Date.now() + 1000 * 60 * 60 * 24; + return sitemap.maps.pages; +}; +async function getSitemapCategories() { + const cids = await categories.getCidsByPrivilege('categories:cid', 0, 'find'); + const categoryData = await categories.getCategoriesFields(cids, ['slug']); + const data = await plugins.hooks.fire('filter:sitemap.getCategories', { + categories: categoryData + }); + return data.categories; +} +sitemap.getCategories = async function () { + if (sitemap.maps.categories && Date.now() < sitemap.maps.categoriesCacheExpireTimestamp) { + return sitemap.maps.categories; + } + const categoryUrls = []; + const categoriesData = await getSitemapCategories(); + categoriesData.forEach(category => { + if (category) { + categoryUrls.push({ + url: `${nconf.get('relative_path')}/category/${category.slug}`, + changefreq: 'weekly', + priority: 0.4 + }); + } + }); + if (!categoryUrls.length) { + sitemap.maps.categories = ''; + sitemap.maps.categoriesCacheExpireTimestamp = Date.now() + 1000 * 60 * 60 * 24; + return sitemap.maps.categories; + } + sitemap.maps.categories = await urlsToSitemap(categoryUrls); + sitemap.maps.categoriesCacheExpireTimestamp = Date.now() + 1000 * 60 * 60 * 24; + return sitemap.maps.categories; +}; +sitemap.getTopicPage = async function (page) { + if (parseInt(page, 10) <= 0) { + return; + } + const numTopics = meta.config.sitemapTopics; + const start = (parseInt(page, 10) - 1) * numTopics; + const stop = start + numTopics - 1; + if (sitemap.maps.topics[page - 1] && Date.now() < sitemap.maps.topics[page - 1].cacheExpireTimestamp) { + return sitemap.maps.topics[page - 1].sm; + } + const topicUrls = []; + let tids = await db.getSortedSetRange('topics:tid', start, stop); + tids = await privileges.topics.filterTids('topics:read', tids, 0); + const topicData = await topics.getTopicsFields(tids, ['tid', 'title', 'slug', 'lastposttime']); + const data = await plugins.hooks.fire('filter:sitemap.getCategories', { + page: page, + topics: topicData + }); + if (!data.topics.length) { + sitemap.maps.topics[page - 1] = { + sm: '', + cacheExpireTimestamp: Date.now() + 1000 * 60 * 60 * 24 + }; + return sitemap.maps.topics[page - 1].sm; + } + data.topics.forEach(topic => { + if (topic) { + topicUrls.push({ + url: `${nconf.get('relative_path')}/topic/${topic.slug}`, + lastmodISO: utils.toISOString(topic.lastposttime), + changefreq: 'daily', + priority: 0.6 + }); + } + }); + sitemap.maps.topics[page - 1] = { + sm: await urlsToSitemap(topicUrls), + cacheExpireTimestamp: Date.now() + 1000 * 60 * 60 * 24 + }; + return sitemap.maps.topics[page - 1].sm; +}; +async function urlsToSitemap(urls) { + if (!urls.length) { + return ''; + } + const smStream = new SitemapStream({ + hostname: nconf.get('url') + }); + urls.forEach(url => smStream.write(url)); + smStream.end(); + return (await streamToPromise(smStream)).toString(); +} +sitemap.clearCache = function () { + if (sitemap.maps.pages) { + sitemap.maps.pagesCacheExpireTimestamp = 0; + } + if (sitemap.maps.categories) { + sitemap.maps.categoriesCacheExpireTimestamp = 0; + } + sitemap.maps.topics.forEach(topicMap => { + topicMap.cacheExpireTimestamp = 0; + }); +}; +require('./promisify')(sitemap); \ No newline at end of file diff --git a/lib/slugify.js b/lib/slugify.js new file mode 100644 index 0000000000..9257bcdbc2 --- /dev/null +++ b/lib/slugify.js @@ -0,0 +1,3 @@ +'use strict'; + +module.exports = require('../public/src/modules/slugify'); \ No newline at end of file diff --git a/lib/social.js b/lib/social.js new file mode 100644 index 0000000000..bb3340076a --- /dev/null +++ b/lib/social.js @@ -0,0 +1,56 @@ +'use strict'; + +const _ = require('lodash'); +const plugins = require('./plugins'); +const db = require('./database'); +const meta = require('./meta'); +const social = module.exports; +social.postSharing = null; +social.getPostSharing = async function () { + if (social.postSharing) { + return _.cloneDeep(social.postSharing); + } + let networks = [{ + id: 'facebook', + name: 'Facebook', + class: 'fa-brands fa-facebook' + }, { + id: 'twitter', + name: 'X (Twitter)', + class: 'fa-brands fa-x-twitter' + }, { + id: 'whatsapp', + name: 'Whatsapp', + class: 'fa-brands fa-whatsapp' + }, { + id: 'telegram', + name: 'Telegram', + class: 'fa-brands fa-telegram' + }, { + id: 'linkedin', + name: 'LinkedIn', + class: 'fa-brands fa-linkedin' + }]; + networks = await plugins.hooks.fire('filter:social.posts', networks); + networks.forEach(network => { + network.activated = parseInt(meta.config[`post-sharing-${network.id}`], 10) === 1; + }); + social.postSharing = networks; + return _.cloneDeep(networks); +}; +social.getActivePostSharing = async function () { + const networks = await social.getPostSharing(); + return networks.filter(network => network && network.activated); +}; +social.setActivePostSharingNetworks = async function (networkIDs) { + social.postSharing = null; + if (!networkIDs.length) { + return; + } + const data = {}; + networkIDs.forEach(id => { + data[`post-sharing-${id}`] = 1; + }); + await db.setObject('config', data); +}; +require('./promisify')(social); \ No newline at end of file diff --git a/lib/socket.io/admin.js b/lib/socket.io/admin.js new file mode 100644 index 0000000000..2375c2f12e --- /dev/null +++ b/lib/socket.io/admin.js @@ -0,0 +1,109 @@ +'use strict'; + +const winston = require('winston'); +const meta = require('../meta'); +const user = require('../user'); +const events = require('../events'); +const db = require('../database'); +const privileges = require('../privileges'); +const websockets = require('./index'); +const batch = require('../batch'); +const index = require('./index'); +const getAdminSearchDict = require('../admin/search').getDictionary; +const SocketAdmin = module.exports; +SocketAdmin.user = require('./admin/user'); +SocketAdmin.categories = require('./admin/categories'); +SocketAdmin.settings = require('./admin/settings'); +SocketAdmin.tags = require('./admin/tags'); +SocketAdmin.rewards = require('./admin/rewards'); +SocketAdmin.navigation = require('./admin/navigation'); +SocketAdmin.rooms = require('./admin/rooms'); +SocketAdmin.themes = require('./admin/themes'); +SocketAdmin.plugins = require('./admin/plugins'); +SocketAdmin.widgets = require('./admin/widgets'); +SocketAdmin.config = require('./admin/config'); +SocketAdmin.settings = require('./admin/settings'); +SocketAdmin.email = require('./admin/email'); +SocketAdmin.analytics = require('./admin/analytics'); +SocketAdmin.logs = require('./admin/logs'); +SocketAdmin.errors = require('./admin/errors'); +SocketAdmin.digest = require('./admin/digest'); +SocketAdmin.cache = require('./admin/cache'); +SocketAdmin.before = async function (socket, method) { + const isAdmin = await user.isAdministrator(socket.uid); + if (isAdmin) { + return; + } + const privilegeSet = privileges.admin.socketMap.hasOwnProperty(method) ? privileges.admin.socketMap[method].split(';') : []; + const hasPrivilege = (await Promise.all(privilegeSet.map(async privilege => privileges.admin.can(privilege, socket.uid)))).some(Boolean); + if (privilegeSet.length && hasPrivilege) { + return; + } + winston.warn(`[socket.io] Call to admin method ( ${method} ) blocked (accessed by uid ${socket.uid})`); + throw new Error('[[error:no-privileges]]'); +}; +SocketAdmin.restart = async function (socket) { + await logRestart(socket); + meta.restart(); +}; +async function logRestart(socket) { + await events.log({ + type: 'restart', + uid: socket.uid, + ip: socket.ip + }); + await db.setObject('lastrestart', { + uid: socket.uid, + ip: socket.ip, + timestamp: Date.now() + }); +} +SocketAdmin.reload = async function (socket) { + await require('../meta/build').buildAll(); + await events.log({ + type: 'build', + uid: socket.uid, + ip: socket.ip + }); + await logRestart(socket); + meta.restart(); +}; +SocketAdmin.fireEvent = function (socket, data, callback) { + index.server.emit(data.name, data.payload || {}); + callback(); +}; +SocketAdmin.deleteEvents = function (socket, eids, callback) { + events.deleteEvents(eids, callback); +}; +SocketAdmin.deleteAllEvents = function (socket, data, callback) { + events.deleteAll(callback); +}; +SocketAdmin.getSearchDict = async function (socket) { + const settings = await user.getSettings(socket.uid); + const lang = settings.userLang || meta.config.defaultLang || 'en-GB'; + return await getAdminSearchDict(lang); +}; +SocketAdmin.deleteAllSessions = async function () { + await user.auth.deleteAllSessions(); +}; +SocketAdmin.reloadAllSessions = function (socket, data, callback) { + websockets.in(`uid_${socket.uid}`).emit('event:livereload'); + callback(); +}; +SocketAdmin.getServerTime = function (socket, data, callback) { + const now = new Date(); + callback(null, { + timestamp: now.getTime(), + offset: now.getTimezoneOffset() + }); +}; +SocketAdmin.clearSearchHistory = async function () { + const keys = await db.scan({ + match: 'searches:*' + }); + await batch.processArray(keys, db.deleteAll, { + batch: 500, + interval: 0 + }); +}; +require('../promisify')(SocketAdmin); \ No newline at end of file diff --git a/lib/socket.io/admin/analytics.js b/lib/socket.io/admin/analytics.js new file mode 100644 index 0000000000..e027aaa254 --- /dev/null +++ b/lib/socket.io/admin/analytics.js @@ -0,0 +1,32 @@ +'use strict'; + +const analytics = require('../../analytics'); +const utils = require('../../utils'); +const Analytics = module.exports; +Analytics.get = async function (socket, data) { + if (!data || !data.graph || !data.units) { + throw new Error('[[error:invalid-data]]'); + } + if (!data.amount) { + if (data.units === 'days') { + data.amount = 30; + } else { + data.amount = 24; + } + } + const getStats = data.units === 'days' ? analytics.getDailyStatsForSet : analytics.getHourlyStatsForSet; + if (data.graph === 'traffic') { + const result = await utils.promiseParallel({ + uniqueVisitors: getStats('analytics:uniquevisitors', data.until || Date.now(), data.amount), + pageviews: getStats('analytics:pageviews', data.until || Date.now(), data.amount), + pageviewsRegistered: getStats('analytics:pageviews:registered', data.until || Date.now(), data.amount), + pageviewsGuest: getStats('analytics:pageviews:guest', data.until || Date.now(), data.amount), + pageviewsBot: getStats('analytics:pageviews:bot', data.until || Date.now(), data.amount), + summary: analytics.getSummary() + }); + result.pastDay = result.pageviews.reduce((a, b) => parseInt(a, 10) + parseInt(b, 10)); + const last = result.pageviews.length - 1; + result.pageviews[last] = parseInt(result.pageviews[last], 10) + analytics.getUnwrittenPageviews(); + return result; + } +}; \ No newline at end of file diff --git a/lib/socket.io/admin/cache.js b/lib/socket.io/admin/cache.js new file mode 100644 index 0000000000..1fe0cf6867 --- /dev/null +++ b/lib/socket.io/admin/cache.js @@ -0,0 +1,31 @@ +'use strict'; + +const SocketCache = module.exports; +const db = require('../../database'); +const plugins = require('../../plugins'); +SocketCache.clear = async function (socket, data) { + let caches = { + post: require('../../posts/cache').getOrCreate(), + object: db.objectCache, + group: require('../../groups').cache, + local: require('../../cache') + }; + caches = await plugins.hooks.fire('filter:admin.cache.get', caches); + if (!caches[data.name]) { + return; + } + caches[data.name].reset(); +}; +SocketCache.toggle = async function (socket, data) { + let caches = { + post: require('../../posts/cache').getOrCreate(), + object: db.objectCache, + group: require('../../groups').cache, + local: require('../../cache') + }; + caches = await plugins.hooks.fire('filter:admin.cache.get', caches); + if (!caches[data.name]) { + return; + } + caches[data.name].enabled = data.enabled; +}; \ No newline at end of file diff --git a/lib/socket.io/admin/categories.js b/lib/socket.io/admin/categories.js new file mode 100644 index 0000000000..d4eb29d1cc --- /dev/null +++ b/lib/socket.io/admin/categories.js @@ -0,0 +1,33 @@ +'use strict'; + +const categories = require('../../categories'); +const Categories = module.exports; +Categories.getNames = async function () { + return await categories.getAllCategoryFields(['cid', 'name']); +}; +Categories.copyPrivilegesToChildren = async function (socket, data) { + const result = await categories.getChildren([data.cid], socket.uid); + const children = result[0]; + for (const child of children) { + await copyPrivilegesToChildrenRecursive(data.cid, child, data.group, data.filter); + } +}; +async function copyPrivilegesToChildrenRecursive(parentCid, category, group, filter) { + await categories.copyPrivilegesFrom(parentCid, category.cid, group, filter); + for (const child of category.children) { + await copyPrivilegesToChildrenRecursive(parentCid, child, group, filter); + } +} +Categories.copySettingsFrom = async function (socket, data) { + return await categories.copySettingsFrom(data.fromCid, data.toCid, data.copyParent); +}; +Categories.copyPrivilegesFrom = async function (socket, data) { + await categories.copyPrivilegesFrom(data.fromCid, data.toCid, data.group, data.filter); +}; +Categories.copyPrivilegesToAllCategories = async function (socket, data) { + let cids = await categories.getAllCidsFromSet('categories:cid'); + cids = cids.filter(cid => parseInt(cid, 10) !== parseInt(data.cid, 10)); + for (const toCid of cids) { + await categories.copyPrivilegesFrom(data.cid, toCid, data.group, data.filter); + } +}; \ No newline at end of file diff --git a/lib/socket.io/admin/config.js b/lib/socket.io/admin/config.js new file mode 100644 index 0000000000..491e97d6c3 --- /dev/null +++ b/lib/socket.io/admin/config.js @@ -0,0 +1,50 @@ +'use strict'; + +const meta = require('../../meta'); +const plugins = require('../../plugins'); +const logger = require('../../logger'); +const events = require('../../events'); +const index = require('../index'); +const Config = module.exports; +Config.set = async function (socket, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + const _data = {}; + _data[data.key] = data.value; + await Config.setMultiple(socket, _data); +}; +Config.setMultiple = async function (socket, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + const changes = {}; + const newData = meta.configs.serialize(data); + const oldData = meta.configs.serialize(meta.config); + Object.keys(newData).forEach(key => { + if (newData[key] !== oldData[key]) { + changes[key] = newData[key]; + changes[`${key}_old`] = meta.config[key]; + } + }); + await meta.configs.setMultiple(data); + for (const [key, value] of Object.entries(data)) { + const setting = { + key, + value + }; + plugins.hooks.fire('action:config.set', setting); + logger.monitorConfig({ + io: index.server + }, setting); + } + if (Object.keys(changes).length) { + changes.type = 'config-change'; + changes.uid = socket.uid; + changes.ip = socket.ip; + await events.log(changes); + } +}; +Config.remove = async function (socket, key) { + await meta.configs.remove(key); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/digest.js b/lib/socket.io/admin/digest.js new file mode 100644 index 0000000000..c58ca96628 --- /dev/null +++ b/lib/socket.io/admin/digest.js @@ -0,0 +1,24 @@ +'use strict'; + +const meta = require('../../meta'); +const userDigest = require('../../user/digest'); +const Digest = module.exports; +Digest.resend = async (socket, data) => { + const { + uid + } = data; + const interval = data.action.startsWith('resend-') ? data.action.slice(7) : await userDigest.getUsersInterval(uid); + if (!interval && meta.config.dailyDigestFreq === 'off') { + throw new Error('[[error:digest-not-enabled]]'); + } + if (uid) { + await userDigest.execute({ + interval: interval || meta.config.dailyDigestFreq, + subscribers: [uid] + }); + } else { + await userDigest.execute({ + interval: interval + }); + } +}; \ No newline at end of file diff --git a/lib/socket.io/admin/email.js b/lib/socket.io/admin/email.js new file mode 100644 index 0000000000..68e5481a67 --- /dev/null +++ b/lib/socket.io/admin/email.js @@ -0,0 +1,62 @@ +'use strict'; + +const meta = require('../../meta'); +const userDigest = require('../../user/digest'); +const userEmail = require('../../user/email'); +const notifications = require('../../notifications'); +const emailer = require('../../emailer'); +const utils = require('../../utils'); +const Email = module.exports; +Email.test = async function (socket, data) { + const payload = { + ...(data.payload || {}), + subject: '[[email:test-email.subject]]' + }; + switch (data.template) { + case 'digest': + await userDigest.execute({ + interval: 'month', + subscribers: [socket.uid] + }); + break; + case 'banned': + Object.assign(payload, { + username: 'test-user', + until: utils.toISOString(Date.now()), + reason: 'Test Reason' + }); + await emailer.send(data.template, socket.uid, payload); + break; + case 'verify-email': + case 'welcome': + await userEmail.sendValidationEmail(socket.uid, { + force: 1, + template: data.template, + subject: data.template === 'welcome' ? `[[email:welcome-to, ${meta.config.title || meta.config.browserTitle || 'NodeBB'}]]` : undefined + }); + break; + case 'notification': + { + const notification = await notifications.create({ + type: 'test', + bodyShort: '[[email:notif.test.short]]', + bodyLong: '[[email:notif.test.long]]', + nid: `uid:${socket.uid}:test`, + path: '/', + from: socket.uid + }); + await emailer.send('notification', socket.uid, { + path: notification.path, + subject: utils.stripHTMLTags(notification.subject || '[[notifications:new-notification]]'), + intro: utils.stripHTMLTags(notification.bodyShort), + body: notification.bodyLong || '', + notification, + showUnsubscribe: true + }); + break; + } + default: + await emailer.send(data.template, socket.uid, payload); + break; + } +}; \ No newline at end of file diff --git a/lib/socket.io/admin/errors.js b/lib/socket.io/admin/errors.js new file mode 100644 index 0000000000..445378617d --- /dev/null +++ b/lib/socket.io/admin/errors.js @@ -0,0 +1,7 @@ +'use strict'; + +const meta = require('../../meta'); +const Errors = module.exports; +Errors.clear = async function () { + await meta.errors.clear(); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/logs.js b/lib/socket.io/admin/logs.js new file mode 100644 index 0000000000..6f1dced634 --- /dev/null +++ b/lib/socket.io/admin/logs.js @@ -0,0 +1,10 @@ +'use strict'; + +const meta = require('../../meta'); +const Logs = module.exports; +Logs.get = async function () { + return await meta.logs.get(); +}; +Logs.clear = async function () { + await meta.logs.clear(); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/navigation.js b/lib/socket.io/admin/navigation.js new file mode 100644 index 0000000000..dbecb5156f --- /dev/null +++ b/lib/socket.io/admin/navigation.js @@ -0,0 +1,7 @@ +'use strict'; + +const navigationAdmin = require('../../navigation/admin'); +const SocketNavigation = module.exports; +SocketNavigation.save = async function (socket, data) { + await navigationAdmin.save(data); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/plugins.js b/lib/socket.io/admin/plugins.js new file mode 100644 index 0000000000..1a9b398058 --- /dev/null +++ b/lib/socket.io/admin/plugins.js @@ -0,0 +1,51 @@ +'use strict'; + +const nconf = require('nconf'); +const plugins = require('../../plugins'); +const events = require('../../events'); +const db = require('../../database'); +const postsCache = require('../../posts/cache'); +const { + pluginNamePattern +} = require('../../constants'); +const Plugins = module.exports; +Plugins.toggleActive = async function (socket, plugin_id) { + postsCache.reset(); + const data = await plugins.toggleActive(plugin_id); + await events.log({ + type: `plugin-${data.active ? 'activate' : 'deactivate'}`, + text: plugin_id, + uid: socket.uid + }); + return data; +}; +Plugins.toggleInstall = async function (socket, data) { + postsCache.reset(); + await plugins.checkWhitelist(data.id, data.version); + const pluginData = await plugins.toggleInstall(data.id, data.version); + await events.log({ + type: `plugin-${pluginData.installed ? 'install' : 'uninstall'}`, + text: data.id, + version: data.version, + uid: socket.uid + }); + return pluginData; +}; +Plugins.getActive = async function () { + return await plugins.getActive(); +}; +Plugins.orderActivePlugins = async function (socket, data) { + if (nconf.get('plugins:active')) { + throw new Error('[[error:plugins-set-in-configuration]]'); + } + data = data.filter(plugin => plugin && plugin.name); + data.forEach(plugin => { + if (!pluginNamePattern.test(plugin.name)) { + throw new Error('[[error:invalid-plugin-id]]'); + } + }); + await db.sortedSetAdd('plugins:active', data.map(p => p.order || 0), data.map(p => p.name)); +}; +Plugins.upgrade = async function (socket, data) { + return await plugins.upgrade(data.id, data.version); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/rewards.js b/lib/socket.io/admin/rewards.js new file mode 100644 index 0000000000..081270fd09 --- /dev/null +++ b/lib/socket.io/admin/rewards.js @@ -0,0 +1,10 @@ +'use strict'; + +const rewardsAdmin = require('../../rewards/admin'); +const SocketRewards = module.exports; +SocketRewards.save = async function (socket, data) { + return await rewardsAdmin.save(data); +}; +SocketRewards.delete = async function (socket, data) { + await rewardsAdmin.delete(data); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/rooms.js b/lib/socket.io/admin/rooms.js new file mode 100644 index 0000000000..8eefd4918c --- /dev/null +++ b/lib/socket.io/admin/rooms.js @@ -0,0 +1,125 @@ +'use strict'; + +const topics = require('../../topics'); +const io = require('..'); +const webserver = require('../../webserver'); +const totals = {}; +const SocketRooms = module.exports; +SocketRooms.totals = totals; +SocketRooms.getTotalGuestCount = async function () { + const s = await io.in('online_guests').fetchSockets(); + return s.length; +}; +SocketRooms.getAll = async function () { + const sockets = await io.server.fetchSockets(); + totals.onlineGuestCount = 0; + totals.onlineRegisteredCount = 0; + totals.socketCount = sockets.length; + totals.topTenTopics = []; + totals.users = { + categories: 0, + recent: 0, + unread: 0, + topics: 0, + category: 0 + }; + const userRooms = {}; + const topicData = {}; + for (const s of sockets) { + for (const key of s.rooms) { + if (key === 'online_guests') { + totals.onlineGuestCount += 1; + } else if (key === 'categories') { + totals.users.categories += 1; + } else if (key === 'recent_topics') { + totals.users.recent += 1; + } else if (key === 'unread_topics') { + totals.users.unread += 1; + } else if (key.startsWith('uid_')) { + userRooms[key] = 1; + } else if (key.startsWith('category_')) { + totals.users.category += 1; + } else { + const tid = key.match(/^topic_(\d+)/); + if (tid) { + totals.users.topics += 1; + topicData[tid[1]] = topicData[tid[1]] || { + count: 0 + }; + topicData[tid[1]].count += 1; + } + } + } + } + totals.onlineRegisteredCount = Object.keys(userRooms).length; + let topTenTopics = []; + Object.keys(topicData).forEach(tid => { + topTenTopics.push({ + tid: tid, + count: topicData[tid].count + }); + }); + topTenTopics = topTenTopics.sort((a, b) => b.count - a.count).slice(0, 10); + const topTenTids = topTenTopics.map(topic => topic.tid); + const titles = await topics.getTopicsFields(topTenTids, ['title']); + totals.topTenTopics = topTenTopics.map((topic, index) => { + topic.title = titles[index].title; + return topic; + }); + return totals; +}; +SocketRooms.getOnlineUserCount = function (io) { + let count = 0; + if (io) { + for (const [key] of io.sockets.adapter.rooms) { + if (key.startsWith('uid_')) { + count += 1; + } + } + } + return count; +}; +SocketRooms.getLocalStats = function () { + const Sockets = require('../index'); + const io = Sockets.server; + const socketData = { + onlineGuestCount: 0, + onlineRegisteredCount: 0, + socketCount: 0, + connectionCount: webserver.getConnectionCount(), + users: { + categories: 0, + recent: 0, + unread: 0, + topics: 0, + category: 0 + }, + topics: {} + }; + if (io && io.sockets) { + socketData.onlineGuestCount = Sockets.getCountInRoom('online_guests'); + socketData.onlineRegisteredCount = SocketRooms.getOnlineUserCount(io); + socketData.socketCount = io.sockets.sockets.size; + socketData.users.categories = Sockets.getCountInRoom('categories'); + socketData.users.recent = Sockets.getCountInRoom('recent_topics'); + socketData.users.unread = Sockets.getCountInRoom('unread_topics'); + let topTenTopics = []; + let tid; + for (const [room, clients] of io.sockets.adapter.rooms) { + tid = room.match(/^topic_(\d+)/); + if (tid) { + socketData.users.topics += clients.size; + topTenTopics.push({ + tid: tid[1], + count: clients.size + }); + } else if (room.match(/^category/)) { + socketData.users.category += clients.size; + } + } + topTenTopics = topTenTopics.sort((a, b) => b.count - a.count).slice(0, 10); + socketData.topics = topTenTopics; + } + return socketData; +}; +require('../../promisify')(SocketRooms); \ No newline at end of file diff --git a/lib/socket.io/admin/settings.js b/lib/socket.io/admin/settings.js new file mode 100644 index 0000000000..a1b9801bed --- /dev/null +++ b/lib/socket.io/admin/settings.js @@ -0,0 +1,20 @@ +'use strict'; + +const meta = require('../../meta'); +const events = require('../../events'); +const Settings = module.exports; +Settings.get = async function (socket, data) { + return await meta.settings.get(data.hash); +}; +Settings.set = async function (socket, data) { + await meta.settings.set(data.hash, data.values); + const eventData = data.values; + eventData.type = 'settings-change'; + eventData.uid = socket.uid; + eventData.ip = socket.ip; + eventData.hash = data.hash; + await events.log(eventData); +}; +Settings.clearSitemapCache = async function () { + require('../../sitemap').clearCache(); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/tags.js b/lib/socket.io/admin/tags.js new file mode 100644 index 0000000000..d50066b7b7 --- /dev/null +++ b/lib/socket.io/admin/tags.js @@ -0,0 +1,22 @@ +'use strict'; + +const topics = require('../../topics'); +const Tags = module.exports; +Tags.create = async function (socket, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + await topics.createEmptyTag(data.tag); +}; +Tags.rename = async function (socket, data) { + if (!Array.isArray(data)) { + throw new Error('[[error:invalid-data]]'); + } + await topics.renameTags(data); +}; +Tags.deleteTags = async function (socket, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + await topics.deleteTags(data.tags); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/themes.js b/lib/socket.io/admin/themes.js new file mode 100644 index 0000000000..e0f70a1dcf --- /dev/null +++ b/lib/socket.io/admin/themes.js @@ -0,0 +1,19 @@ +'use strict'; + +const meta = require('../../meta'); +const widgets = require('../../widgets'); +const Themes = module.exports; +Themes.getInstalled = async function () { + return await meta.themes.get(); +}; +Themes.set = async function (socket, data) { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + if (data.type === 'local') { + await widgets.saveLocationsOnThemeReset(); + } + data.ip = socket.ip; + data.uid = socket.uid; + await meta.themes.set(data); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/user.js b/lib/socket.io/admin/user.js new file mode 100644 index 0000000000..6ac37723dc --- /dev/null +++ b/lib/socket.io/admin/user.js @@ -0,0 +1,159 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +const groups = require('../../groups'); +const user = require('../../user'); +const events = require('../../events'); +const translator = require('../../translator'); +const utils = require('../../utils'); +const sockets = require('..'); +const User = module.exports; +User.makeAdmins = async function (socket, uids) { + if (!Array.isArray(uids)) { + throw new Error('[[error:invalid-data]]'); + } + const isMembersOfBanned = await groups.isMembers(uids, groups.BANNED_USERS); + if (isMembersOfBanned.includes(true)) { + throw new Error('[[error:cant-make-banned-users-admin]]'); + } + for (const uid of uids) { + await groups.join('administrators', uid); + await events.log({ + type: 'user-makeAdmin', + uid: socket.uid, + targetUid: uid, + ip: socket.ip + }); + } +}; +User.removeAdmins = async function (socket, uids) { + if (!Array.isArray(uids)) { + throw new Error('[[error:invalid-data]]'); + } + for (const uid of uids) { + const count = await groups.getMemberCount('administrators'); + if (count === 1) { + throw new Error('[[error:cant-remove-last-admin]]'); + } + await groups.leave('administrators', uid); + await events.log({ + type: 'user-removeAdmin', + uid: socket.uid, + targetUid: uid, + ip: socket.ip + }); + } +}; +User.resetLockouts = async function (socket, uids) { + if (!Array.isArray(uids)) { + throw new Error('[[error:invalid-data]]'); + } + await Promise.all(uids.map(uid => user.auth.resetLockout(uid))); +}; +User.validateEmail = async function (socket, uids) { + if (!Array.isArray(uids)) { + throw new Error('[[error:invalid-data]]'); + } + for (const uid of uids) { + const email = await user.email.getEmailForValidation(uid); + if (email) { + await user.setUserField(uid, 'email', email); + } + await user.email.confirmByUid(uid, socket.uid); + } +}; +User.sendValidationEmail = async function (socket, uids) { + if (!Array.isArray(uids)) { + throw new Error('[[error:invalid-data]]'); + } + const failed = []; + let errorLogged = false; + await async.eachLimit(uids, 50, async uid => { + const email = await user.email.getEmailForValidation(uid); + await user.email.sendValidationEmail(uid, { + force: true, + email: email + }).catch(err => { + if (!errorLogged) { + winston.error(`[user.create] Validation email failed to send\n[emailer.send] ${err.stack}`); + errorLogged = true; + } + failed.push(uid); + }); + }); + if (failed.length) { + throw Error(`Email sending failed for the following uids, check server logs for more info: ${failed.join(',')}`); + } +}; +User.sendPasswordResetEmail = async function (socket, uids) { + if (!Array.isArray(uids)) { + throw new Error('[[error:invalid-data]]'); + } + uids = uids.filter(uid => parseInt(uid, 10)); + await Promise.all(uids.map(async uid => { + const userData = await user.getUserFields(uid, ['email', 'username']); + if (!userData.email) { + throw new Error(`[[error:user-doesnt-have-email, ${userData.username}]]`); + } + await user.reset.send(userData.email); + })); +}; +User.forcePasswordReset = async function (socket, uids) { + if (!Array.isArray(uids)) { + throw new Error('[[error:invalid-data]]'); + } + uids = uids.filter(uid => parseInt(uid, 10)); + await db.setObjectField(uids.map(uid => `user:${uid}`), 'passwordExpiry', Date.now()); + await user.auth.revokeAllSessions(uids); + uids.forEach(uid => sockets.in(`uid_${uid}`).emit('event:logout')); +}; +User.restartJobs = async function () { + user.startJobs(); +}; +User.loadGroups = async function (socket, uids) { + const [userData, groupData] = await Promise.all([user.getUsersData(uids), groups.getUserGroupsFromSet('groups:createtime', uids)]); + userData.forEach((data, index) => { + data.groups = groupData[index].filter(group => !groups.isPrivilegeGroup(group.name)); + data.groups.forEach(group => { + group.nameEscaped = translator.escape(group.displayName); + }); + }); + return { + users: userData + }; +}; +User.setReputation = async function (socket, data) { + if (!data || !Array.isArray(data.uids) || !utils.isNumber(data.value)) { + throw new Error('[[error:invalid-data]]'); + } + await Promise.all([db.setObjectBulk(data.uids.map(uid => [`user:${uid}`, { + reputation: parseInt(data.value, 10) + }])), db.sortedSetAddBulk(data.uids.map(uid => ['users:reputation', data.value, uid]))]); +}; +User.exportUsersCSV = async function (socket, data) { + await events.log({ + type: 'exportUsersCSV', + uid: socket.uid, + ip: socket.ip + }); + setTimeout(async () => { + try { + await user.exportUsersCSV(data.fields); + if (socket.emit) { + socket.emit('event:export-users-csv'); + } + const notifications = require('../../notifications'); + const n = await notifications.create({ + bodyShort: '[[notifications:users-csv-exported]]', + path: '/api/admin/users/csv', + nid: 'users:csv:export', + from: socket.uid + }); + await notifications.push(n, [socket.uid]); + } catch (err) { + winston.error(err.stack); + } + }, 0); +}; \ No newline at end of file diff --git a/lib/socket.io/admin/widgets.js b/lib/socket.io/admin/widgets.js new file mode 100644 index 0000000000..bfc7852faa --- /dev/null +++ b/lib/socket.io/admin/widgets.js @@ -0,0 +1,10 @@ +'use strict'; + +const widgets = require('../../widgets'); +const Widgets = module.exports; +Widgets.set = async function (socket, data) { + if (!Array.isArray(data)) { + throw new Error('[[error:invalid-data]]'); + } + await widgets.setAreas(data); +}; \ No newline at end of file diff --git a/lib/socket.io/blacklist.js b/lib/socket.io/blacklist.js new file mode 100644 index 0000000000..a682e0fb68 --- /dev/null +++ b/lib/socket.io/blacklist.js @@ -0,0 +1,32 @@ +'use strict'; + +const user = require('../user'); +const meta = require('../meta'); +const events = require('../events'); +const SocketBlacklist = module.exports; +SocketBlacklist.validate = async function (socket, data) { + return meta.blacklist.validate(data.rules); +}; +SocketBlacklist.save = async function (socket, rules) { + await blacklist(socket, 'save', rules); +}; +SocketBlacklist.addRule = async function (socket, rule) { + await blacklist(socket, 'addRule', rule); +}; +async function blacklist(socket, method, rule) { + const isAdminOrGlobalMod = await user.isAdminOrGlobalMod(socket.uid); + if (!isAdminOrGlobalMod) { + throw new Error('[[error:no-privileges]]'); + } + if (socket.ip && rule.includes(socket.ip)) { + throw new Error('[[error:cant-blacklist-self-ip]]'); + } + await meta.blacklist[method](rule); + await events.log({ + type: `ip-blacklist-${method}`, + uid: socket.uid, + ip: socket.ip, + rule: rule + }); +} +require('../promisify')(SocketBlacklist); \ No newline at end of file diff --git a/lib/socket.io/categories.js b/lib/socket.io/categories.js new file mode 100644 index 0000000000..61a88f1e87 --- /dev/null +++ b/lib/socket.io/categories.js @@ -0,0 +1,114 @@ +'use strict'; + +const categories = require('../categories'); +const user = require('../user'); +const topics = require('../topics'); +const api = require('../api'); +const sockets = require('.'); +const SocketCategories = module.exports; +require('./categories/search')(SocketCategories); +SocketCategories.getRecentReplies = async function (socket, cid) { + sockets.warnDeprecated(socket, 'GET /api/v3/categories/:cid/posts'); + return await api.categories.getPosts(socket, { + cid + }); +}; +SocketCategories.get = async function (socket) { + sockets.warnDeprecated(socket, 'GET /api/v3/categories'); + const { + categories + } = await api.categories.list(socket); + return categories; +}; +SocketCategories.getWatchedCategories = async function (socket) { + sockets.warnDeprecated(socket); + const [categoriesData, ignoredCids] = await Promise.all([categories.getCategoriesByPrivilege('cid:0:children', socket.uid, 'find'), user.getIgnoredCategories(socket.uid)]); + return categoriesData.filter(category => category && !ignoredCids.includes(String(category.cid))); +}; +SocketCategories.loadMore = async function (socket, data) { + sockets.warnDeprecated(socket, 'GET /api/v3/categories/:cid/topics'); + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + data.query = data.query || {}; + const result = await api.categories.getTopics(socket, data); + result.template = { + category: true, + name: 'category' + }; + return result; +}; +SocketCategories.getTopicCount = async function (socket, cid) { + sockets.warnDeprecated(socket, 'GET /api/v3/categories/:cid'); + const { + count + } = await api.categories.getTopicCount(socket, { + cid + }); + return count; +}; +SocketCategories.getCategoriesByPrivilege = async function (socket, privilege) { + sockets.warnDeprecated(socket); + return await categories.getCategoriesByPrivilege('categories:cid', socket.uid, privilege); +}; +SocketCategories.getMoveCategories = async function (socket, data) { + sockets.warnDeprecated(socket); + return await SocketCategories.getSelectCategories(socket, data); +}; +SocketCategories.getSelectCategories = async function (socket) { + sockets.warnDeprecated(socket); + const [isAdmin, categoriesData] = await Promise.all([user.isAdministrator(socket.uid), categories.buildForSelect(socket.uid, 'find', ['disabled', 'link'])]); + return categoriesData.filter(category => category && (!category.disabled || isAdmin) && !category.link); +}; +SocketCategories.setWatchState = async function (socket, data) { + sockets.warnDeprecated(socket, 'PUT/DELETE /api/v3/categories/:cid/watch'); + if (!data || !data.cid || !data.state) { + throw new Error('[[error:invalid-data]]'); + } + data.state = categories.watchStates[data.state]; + await api.categories.setWatchState(socket, data); + return data.cid; +}; +SocketCategories.watch = async function (socket, data) { + sockets.warnDeprecated(socket); + return await ignoreOrWatch(user.watchCategory, socket, data); +}; +SocketCategories.ignore = async function (socket, data) { + sockets.warnDeprecated(socket); + return await ignoreOrWatch(user.ignoreCategory, socket, data); +}; +async function ignoreOrWatch(fn, socket, data) { + let targetUid = socket.uid; + const cids = Array.isArray(data.cid) ? data.cid.map(cid => parseInt(cid, 10)) : [parseInt(data.cid, 10)]; + if (data.hasOwnProperty('uid')) { + targetUid = data.uid; + } + await user.isAdminOrGlobalModOrSelf(socket.uid, targetUid); + const allCids = await categories.getAllCidsFromSet('categories:cid'); + const categoryData = await categories.getCategoriesFields(allCids, ['cid', 'parentCid']); + let cat; + do { + cat = categoryData.find(c => !cids.includes(c.cid) && cids.includes(c.parentCid)); + if (cat) { + cids.push(cat.cid); + } + } while (cat); + await fn(targetUid, cids); + await topics.pushUnreadCount(targetUid); + return cids; +} +SocketCategories.isModerator = async function (socket, cid) { + sockets.warnDeprecated(socket); + return await user.isModerator(socket.uid, cid); +}; +SocketCategories.loadMoreSubCategories = async function (socket, data) { + sockets.warnDeprecated(socket, `GET /api/v3/categories/:cid/children`); + if (!data || !data.cid || !(parseInt(data.start, 10) >= 0)) { + throw new Error('[[error:invalid-data]]'); + } + const { + categories: children + } = await api.categories.getChildren(socket, data); + return children; +}; +require('../promisify')(SocketCategories); \ No newline at end of file diff --git a/lib/socket.io/categories/search.js b/lib/socket.io/categories/search.js new file mode 100644 index 0000000000..d11993af52 --- /dev/null +++ b/lib/socket.io/categories/search.js @@ -0,0 +1,13 @@ +'use strict'; + +const sockets = require('..'); +const api = require('../../api'); +module.exports = function (SocketCategories) { + SocketCategories.categorySearch = async function (socket, data) { + sockets.warnDeprecated(socket, 'GET /api/v3/search/categories'); + const { + categories + } = await api.search.categories(socket, data); + return categories; + }; +}; \ No newline at end of file diff --git a/lib/socket.io/groups.js b/lib/socket.io/groups.js new file mode 100644 index 0000000000..014c3fa6b3 --- /dev/null +++ b/lib/socket.io/groups.js @@ -0,0 +1,104 @@ +'use strict'; + +const groups = require('../groups'); +const user = require('../user'); +const utils = require('../utils'); +const privileges = require('../privileges'); +const api = require('../api'); +const slugify = require('../slugify'); +const sockets = require('.'); +const SocketGroups = module.exports; +SocketGroups.before = async (socket, method, data) => { + if (!data) { + throw new Error('[[error:invalid-data]]'); + } +}; +SocketGroups.search = async (socket, data) => { + data.options = data.options || {}; + if (!data.query) { + const groupsPerPage = 15; + const groupData = await groups.getGroupsBySort(data.options.sort, 0, groupsPerPage - 1); + return groupData; + } + data.options.filterHidden = data.options.filterHidden || !(await user.isAdministrator(socket.uid)); + return await groups.search(data.query, data.options); +}; +SocketGroups.loadMore = async (socket, data) => { + sockets.warnDeprecated(socket, 'GET /api/v3/groups'); + if (!data.sort || !utils.isNumber(data.after) || parseInt(data.after, 10) < 0) { + throw new Error('[[error:invalid-data]]'); + } + return api.groups.list(socket, data); +}; +SocketGroups.searchMembers = async (socket, data) => { + sockets.warnDeprecated(socket, 'GET /api/v3/groups/:groupName/members'); + if (!data.groupName) { + throw new Error('[[error:invalid-data]]'); + } + data.slug = slugify(data.groupName); + delete data.groupName; + return api.groups.listMembers(socket, data); +}; +SocketGroups.loadMoreMembers = async (socket, data) => { + sockets.warnDeprecated(socket, 'GET /api/v3/groups/:groupName/members'); + if (!data.groupName || !utils.isNumber(data.after) || parseInt(data.after, 10) < 0) { + throw new Error('[[error:invalid-data]]'); + } + data.slug = slugify(data.groupName); + delete data.groupName; + return api.groups.listMembers(socket, data); +}; +SocketGroups.getChatGroups = async socket => { + sockets.warnDeprecated(socket, 'GET /api/v3/admin/groups'); + const isAdmin = await user.isAdministrator(socket.uid); + if (!isAdmin) { + throw new Error('[[error:no-privileges]]'); + } + const { + groups + } = await api.admin.listGroups(socket); + groups.sort((a, b) => b.system - a.system); + return groups.map(g => ({ + name: g.name, + displayName: g.displayName + })); +}; +SocketGroups.cover = {}; +SocketGroups.cover.update = async (socket, data) => { + if (!socket.uid) { + throw new Error('[[error:no-privileges]]'); + } + if (data.file || !data.imageData && !data.position) { + throw new Error('[[error:invalid-data]]'); + } + await canModifyGroup(socket.uid, data.groupName); + return await groups.updateCover(socket.uid, { + groupName: data.groupName, + imageData: data.imageData, + position: data.position + }); +}; +SocketGroups.cover.remove = async (socket, data) => { + if (!socket.uid) { + throw new Error('[[error:no-privileges]]'); + } + await canModifyGroup(socket.uid, data.groupName); + await groups.removeCover({ + groupName: data.groupName + }); +}; +async function canModifyGroup(uid, groupName) { + if (typeof groupName !== 'string') { + throw new Error('[[error:invalid-group-name]]'); + } + const results = await utils.promiseParallel({ + isOwner: groups.ownership.isOwner(uid, groupName), + system: groups.getGroupField(groupName, 'system'), + hasAdminPrivilege: privileges.admin.can('admin:groups', uid), + isGlobalMod: user.isGlobalModerator(uid) + }); + if (!(results.isOwner || results.hasAdminPrivilege || results.isGlobalMod && !results.system)) { + throw new Error('[[error:no-privileges]]'); + } +} +require('../promisify')(SocketGroups); \ No newline at end of file diff --git a/lib/socket.io/helpers.js b/lib/socket.io/helpers.js new file mode 100644 index 0000000000..474e83c59f --- /dev/null +++ b/lib/socket.io/helpers.js @@ -0,0 +1,192 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const websockets = require('./index'); +const user = require('../user'); +const posts = require('../posts'); +const topics = require('../topics'); +const categories = require('../categories'); +const privileges = require('../privileges'); +const notifications = require('../notifications'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const batch = require('../batch'); +const SocketHelpers = module.exports; +SocketHelpers.notifyNew = async function (uid, type, result) { + let uids = await user.getUidsFromSet('users:online', 0, -1); + uids = uids.filter(toUid => parseInt(toUid, 10) !== uid); + await batch.processArray(uids, async uids => { + await notifyUids(uid, uids, type, result); + }, { + interval: 1000 + }); +}; +async function notifyUids(uid, uids, type, result) { + const post = result.posts[0]; + const { + tid, + cid + } = post.topic; + uids = await privileges.topics.filterUids('topics:read', tid, uids); + const watchStateUids = uids; + const watchStates = await getWatchStates(watchStateUids, tid, cid); + const categoryWatchStates = _.zipObject(watchStateUids, watchStates.categoryWatchStates); + const topicFollowState = _.zipObject(watchStateUids, watchStates.topicFollowed); + uids = filterTidCidIgnorers(watchStateUids, watchStates); + uids = await user.blocks.filterUids(uid, uids); + uids = await user.blocks.filterUids(post.topic.uid, uids); + const data = await plugins.hooks.fire('filter:sockets.sendNewPostToUids', { + uidsTo: uids, + uidFrom: uid, + type: type, + post: post + }); + post.ip = undefined; + await Promise.all(data.uidsTo.map(async toUid => { + const copyResult = _.cloneDeep(result); + const postToUid = copyResult.posts[0]; + postToUid.categoryWatchState = categoryWatchStates[toUid]; + postToUid.topic.isFollowing = topicFollowState[toUid]; + await plugins.hooks.fire('filter:sockets.sendNewPostToUid', { + uid: toUid, + uidFrom: uid, + post: postToUid + }); + websockets.in(`uid_${toUid}`).emit('event:new_post', copyResult); + if (copyResult.topic && type === 'newTopic') { + await plugins.hooks.fire('filter:sockets.sendNewTopicToUid', { + uid: toUid, + uidFrom: uid, + topic: copyResult.topic + }); + websockets.in(`uid_${toUid}`).emit('event:new_topic', copyResult.topic); + } + })); +} +async function getWatchStates(uids, tid, cid) { + return await utils.promiseParallel({ + topicFollowed: db.isSetMembers(`tid:${tid}:followers`, uids), + topicIgnored: db.isSetMembers(`tid:${tid}:ignorers`, uids), + categoryWatchStates: categories.getUidsWatchStates(cid, uids) + }); +} +function filterTidCidIgnorers(uids, watchStates) { + return uids.filter((uid, index) => watchStates.topicFollowed[index] || !watchStates.topicIgnored[index] && watchStates.categoryWatchStates[index] !== categories.watchStates.ignoring); +} +SocketHelpers.sendNotificationToPostOwner = async function (pid, fromuid, command, notification) { + if (!pid || !fromuid || !notification) { + return; + } + fromuid = parseInt(fromuid, 10); + const postData = await posts.getPostFields(pid, ['tid', 'uid', 'content']); + const [canRead, isIgnoring] = await Promise.all([privileges.posts.can('topics:read', pid, postData.uid), topics.isIgnoring([postData.tid], postData.uid)]); + if (!canRead || isIgnoring[0] || !postData.uid || fromuid === postData.uid) { + return; + } + const [userData, topicTitle, postObj] = await Promise.all([user.getUserFields(fromuid, ['username']), topics.getTopicField(postData.tid, 'title'), posts.parsePost(postData)]); + const { + displayname + } = userData; + const title = utils.decodeHTMLEntities(topicTitle); + const titleEscaped = title.replace(/%/g, '%').replace(/,/g, ','); + const notifObj = await notifications.create({ + type: command, + bodyShort: `[[${notification}, ${displayname}, ${titleEscaped}]]`, + bodyLong: postObj.content, + pid: pid, + tid: postData.tid, + path: `/post/${pid}`, + nid: `${command}:post:${pid}:uid:${fromuid}`, + from: fromuid, + mergeId: `${notification}|${pid}`, + topicTitle: topicTitle + }); + notifications.push(notifObj, [postData.uid]); +}; +SocketHelpers.sendNotificationToTopicOwner = async function (tid, fromuid, command, notification) { + if (!tid || !fromuid || !notification) { + return; + } + fromuid = parseInt(fromuid, 10); + const [userData, topicData] = await Promise.all([user.getUserFields(fromuid, ['username']), topics.getTopicFields(tid, ['uid', 'slug', 'title'])]); + if (fromuid === topicData.uid) { + return; + } + const { + displayname + } = userData; + const ownerUid = topicData.uid; + const title = utils.decodeHTMLEntities(topicData.title); + const titleEscaped = title.replace(/%/g, '%').replace(/,/g, ','); + const notifObj = await notifications.create({ + bodyShort: `[[${notification}, ${displayname}, ${titleEscaped}]]`, + path: `/topic/${topicData.slug}`, + nid: `${command}:tid:${tid}:uid:${fromuid}`, + from: fromuid + }); + if (ownerUid) { + notifications.push(notifObj, [ownerUid]); + } +}; +SocketHelpers.upvote = async function (data, notification) { + if (!data || !data.post || !data.post.uid || !data.post.votes || !data.post.pid || !data.fromuid) { + return; + } + const { + votes + } = data.post; + const touid = data.post.uid; + const { + fromuid + } = data; + const { + pid + } = data.post; + const shouldNotify = { + all: function () { + return votes > 0; + }, + first: function () { + return votes === 1; + }, + everyTen: function () { + return votes > 0 && votes % 10 === 0; + }, + threshold: function () { + return [1, 5, 10, 25].includes(votes) || votes >= 50 && votes % 50 === 0; + }, + logarithmic: function () { + return votes > 1 && Math.log10(votes) % 1 === 0; + }, + disabled: function () { + return false; + } + }; + const settings = await user.getSettings(touid); + const should = shouldNotify[settings.upvoteNotifFreq] || shouldNotify.all; + if (should()) { + SocketHelpers.sendNotificationToPostOwner(pid, fromuid, 'upvote', notification); + } +}; +SocketHelpers.rescindUpvoteNotification = async function (pid, fromuid) { + await notifications.rescind(`upvote:post:${pid}:uid:${fromuid}`); + const uid = await posts.getPostField(pid, 'uid'); + const count = await user.notifications.getUnreadCount(uid); + websockets.in(`uid_${uid}`).emit('event:notifications.updateCount', count); +}; +SocketHelpers.emitToUids = async function (event, data, uids) { + uids.forEach(toUid => websockets.in(`uid_${toUid}`).emit(event, data)); +}; +SocketHelpers.removeSocketsFromRoomByUids = async function (uids, roomId) { + const sockets = _.flatten(await Promise.all(uids.map(uid => websockets.in(`uid_${uid}`).fetchSockets()))); + for (const s of sockets) { + if (s.rooms.has(`chat_room_${roomId}`)) { + websockets.in(s.id).socketsLeave(`chat_room_${roomId}`); + } + if (s.rooms.has(`chat_room_public_${roomId}`)) { + websockets.in(s.id).socketsLeave(`chat_room_public_${roomId}`); + } + } +}; +require('../promisify')(SocketHelpers); \ No newline at end of file diff --git a/lib/socket.io/index.js b/lib/socket.io/index.js new file mode 100644 index 0000000000..8ac09aeeed --- /dev/null +++ b/lib/socket.io/index.js @@ -0,0 +1,303 @@ +'use strict'; + +const _ = require('lodash'); +const os = require('os'); +const nconf = require('nconf'); +const winston = require('winston'); +const util = require('util'); +const validator = require('validator'); +const cookieParser = require('cookie-parser')(nconf.get('secret')); +const db = require('../database'); +const user = require('../user'); +const logger = require('../logger'); +const plugins = require('../plugins'); +const ratelimit = require('../middleware/ratelimit'); +const blacklist = require('../meta/blacklist'); +const als = require('../als'); +const apiHelpers = require('../api/helpers'); +const Namespaces = Object.create(null); +const Sockets = module.exports; +Sockets.init = async function (server) { + requireModules(); + const SocketIO = require('socket.io').Server; + const io = new SocketIO({ + path: `${nconf.get('relative_path')}/socket.io` + }); + if (nconf.get('isCluster')) { + if (nconf.get('redis')) { + const adapter = await require('../database/redis').socketAdapter(); + io.adapter(adapter); + } else { + winston.warn('clustering detected, you should setup redis!'); + } + } + io.on('connection', onConnection); + const opts = { + transports: nconf.get('socket.io:transports') || ['polling', 'websocket'], + cookie: false, + allowRequest: (req, callback) => { + authorize(req, err => { + if (err) { + return callback(err); + } + const csrf = require('../middleware/csrf'); + const isValid = csrf.isRequestValid({ + session: req.session || {}, + query: req._query, + headers: req.headers + }); + callback(null, isValid); + }); + } + }; + if (process.env.NODE_ENV !== 'development' || nconf.get('socket.io:cors')) { + const origins = nconf.get('socket.io:origins'); + opts.cors = nconf.get('socket.io:cors') || { + origin: origins, + methods: ['GET', 'POST'], + allowedHeaders: ['content-type'] + }; + winston.info(`[socket.io] Restricting access to origin: ${origins}`); + } + io.listen(server, opts); + Sockets.server = io; +}; +function onConnection(socket) { + socket.uid = socket.request.uid; + socket.data.uid = socket.uid; + socket.ip = (socket.request.headers['x-forwarded-for'] || socket.request.connection.remoteAddress || '').split(',')[0]; + socket.request.ip = socket.ip; + logger.io_one(socket, socket.uid); + onConnect(socket); + socket.onAny((event, ...args) => { + const payload = { + event: event, + ...deserializePayload(args) + }; + als.run({ + uid: socket.uid, + req: apiHelpers.buildReqObject(socket, payload), + socket: { + ...payload + } + }, onMessage, socket, payload); + }); + socket.on('disconnecting', () => { + for (const room of socket.rooms) { + if (room && room.match(/^chat_room_\d+$/)) { + Sockets.server.in(room).emit('event:chats.typing', { + roomId: room.split('_').pop(), + uid: socket.uid, + username: '', + typing: false + }); + } + } + }); + socket.on('disconnect', () => { + onDisconnect(socket); + }); +} +function onDisconnect(socket) { + require('./uploads').clear(socket.id); + plugins.hooks.fire('action:sockets.disconnect', { + socket: socket + }); +} +async function onConnect(socket) { + try { + await validateSession(socket, '[[error:invalid-session]]'); + } catch (e) { + if (e.message === '[[error:invalid-session]]') { + socket.emit('event:invalid_session'); + } + return; + } + if (socket.uid > 0) { + socket.join(`uid_${socket.uid}`); + socket.join('online_users'); + } else if (socket.uid === 0) { + socket.join('online_guests'); + } + socket.join(`sess_${socket.request.signedCookies[nconf.get('sessionKey')]}`); + socket.emit('checkSession', socket.uid); + socket.emit('setHostname', os.hostname()); + plugins.hooks.fire('action:sockets.connect', { + socket: socket + }); +} +function deserializePayload(payload) { + if (!Array.isArray(payload) || !payload.length) { + winston.warn('[socket.io] Empty payload'); + return {}; + } + const params = typeof payload[0] === 'function' ? {} : payload[0]; + const callback = typeof payload[payload.length - 1] === 'function' ? payload[payload.length - 1] : function () {}; + return { + params, + callback + }; +} +async function onMessage(socket, payload) { + const { + event, + params, + callback + } = payload; + try { + if (!event) { + return winston.warn('[socket.io] Empty method name'); + } + if (typeof event !== 'string') { + const escapedName = validator.escape(typeof event); + return callback({ + message: `[[error:invalid-event, ${escapedName}]]` + }); + } + const parts = event.split('.'); + const namespace = parts[0]; + const methodToCall = parts.reduce((prev, cur) => { + if (prev !== null && prev[cur] && (!prev.hasOwnProperty || prev.hasOwnProperty(cur))) { + return prev[cur]; + } + return null; + }, Namespaces); + if (!methodToCall || typeof methodToCall !== 'function') { + if (process.env.NODE_ENV === 'development') { + winston.warn(`[socket.io] Unrecognized message: ${event}`); + } + const escapedName = validator.escape(String(event)); + return callback({ + message: `[[error:invalid-event, ${escapedName}]]` + }); + } + socket.previousEvents = socket.previousEvents || []; + socket.previousEvents.push(event); + if (socket.previousEvents.length > 20) { + socket.previousEvents.shift(); + } + if (!event.startsWith('admin.') && ratelimit.isFlooding(socket)) { + winston.warn(`[socket.io] Too many emits! Disconnecting uid : ${socket.uid}. Events : ${socket.previousEvents}`); + return socket.disconnect(); + } + await blacklist.test(socket.ip); + await checkMaintenance(socket); + await validateSession(socket, '[[error:revalidate-failure]]'); + if (Namespaces[namespace].before) { + await Namespaces[namespace].before(socket, event, params); + } + if (methodToCall.constructor && methodToCall.constructor.name === 'AsyncFunction') { + const result = await methodToCall(socket, params); + callback(null, result); + } else { + methodToCall(socket, params, (err, result) => { + callback(err ? { + message: err.message + } : null, result); + }); + } + } catch (err) { + winston.debug(`${event}\n${err.stack ? err.stack : err.message}`); + callback({ + message: err.message + }); + } +} +function requireModules() { + const modules = ['admin', 'categories', 'groups', 'meta', 'modules', 'notifications', 'plugins', 'posts', 'topics', 'user', 'blacklist', 'uploads']; + modules.forEach(module => { + Namespaces[module] = require(`./${module}`); + }); +} +async function checkMaintenance(socket) { + const meta = require('../meta'); + if (!meta.config.maintenanceMode) { + return; + } + const isAdmin = await user.isAdministrator(socket.uid); + if (isAdmin) { + return; + } + const validator = require('validator'); + throw new Error(`[[pages:maintenance.text, ${validator.escape(String(meta.config.title || 'NodeBB'))}]]`); +} +async function validateSession(socket, errorMsg) { + const req = socket.request; + const { + sessionId + } = await plugins.hooks.fire('filter:sockets.sessionId', { + sessionId: req.signedCookies ? req.signedCookies[nconf.get('sessionKey')] : null, + request: req + }); + if (!sessionId) { + return; + } + const sessionData = await db.sessionStoreGet(sessionId); + if (!sessionData) { + throw new Error(errorMsg); + } + await plugins.hooks.fire('static:sockets.validateSession', { + req: req, + socket: socket, + session: sessionData + }); +} +const cookieParserAsync = util.promisify((req, callback) => cookieParser(req, {}, err => callback(err))); +async function authorize(request, callback) { + if (!request) { + return callback(new Error('[[error:not-authorized]]')); + } + await cookieParserAsync(request); + const { + sessionId + } = await plugins.hooks.fire('filter:sockets.sessionId', { + sessionId: request.signedCookies ? request.signedCookies[nconf.get('sessionKey')] : null, + request: request + }); + const sessionData = await db.sessionStoreGet(sessionId); + request.session = sessionData; + let uid = 0; + if (sessionData && sessionData.passport && sessionData.passport.user) { + uid = parseInt(sessionData.passport.user, 10); + } + request.uid = uid; + callback(null, uid); +} +Sockets.in = function (room) { + return Sockets.server && Sockets.server.in(room); +}; +Sockets.getUserSocketCount = function (uid) { + return Sockets.getCountInRoom(`uid_${uid}`); +}; +Sockets.getCountInRoom = function (room) { + if (!Sockets.server) { + return 0; + } + const roomMap = Sockets.server.sockets.adapter.rooms.get(room); + return roomMap ? roomMap.size : 0; +}; +Sockets.getUidsInRoom = async function (room) { + if (!Sockets.server) { + return []; + } + const ioRoom = Sockets.server.in(room); + const uids = []; + if (ioRoom) { + const sockets = await ioRoom.fetchSockets(); + for (const s of sockets) { + if (s && s.data && s.data.uid > 0) { + uids.push(s.data.uid); + } + } + } + return _.uniq(uids); +}; +Sockets.warnDeprecated = (socket, replacement) => { + if (socket.previousEvents && socket.emit) { + socket.emit('event:deprecated_call', { + eventName: socket.previousEvents[socket.previousEvents.length - 1], + replacement: replacement + }); + } + winston.warn(['[deprecated]', `${new Error('-').stack.split('\n').slice(2, 5).join('\n')}`, ` ${replacement ? `use ${replacement}` : 'there is no replacement for this call.'}`].join('\n')); +}; \ No newline at end of file diff --git a/lib/socket.io/meta.js b/lib/socket.io/meta.js new file mode 100644 index 0000000000..c62622bbdc --- /dev/null +++ b/lib/socket.io/meta.js @@ -0,0 +1,54 @@ +'use strict'; + +const os = require('os'); +const user = require('../user'); +const meta = require('../meta'); +const topics = require('../topics'); +const SocketMeta = module.exports; +SocketMeta.rooms = {}; +SocketMeta.reconnected = function (socket, data, callback) { + callback = callback || function () {}; + if (socket.uid) { + topics.pushUnreadCount(socket.uid); + user.notifications.pushCount(socket.uid); + } + callback(null, { + 'cache-buster': meta.config['cache-buster'], + hostname: os.hostname() + }); +}; +SocketMeta.rooms.enter = async function (socket, data) { + if (!socket.uid) { + return; + } + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + if (data.enter) { + data.enter = data.enter.toString(); + } + if (data.enter && data.enter.startsWith('uid_') && data.enter !== `uid_${socket.uid}`) { + throw new Error('[[error:not-allowed]]'); + } + if (data.enter && data.enter.startsWith('chat_')) { + throw new Error('[[error:not-allowed]]'); + } + leaveCurrentRoom(socket); + if (data.enter) { + socket.join(data.enter); + socket.currentRoom = data.enter; + } +}; +SocketMeta.rooms.leaveCurrent = async function (socket) { + if (!socket.uid || !socket.currentRoom) { + return; + } + leaveCurrentRoom(socket); +}; +function leaveCurrentRoom(socket) { + if (socket.currentRoom) { + socket.leave(socket.currentRoom); + socket.currentRoom = ''; + } +} +require('../promisify')(SocketMeta); \ No newline at end of file diff --git a/lib/socket.io/modules.js b/lib/socket.io/modules.js new file mode 100644 index 0000000000..3f0e9bdbc9 --- /dev/null +++ b/lib/socket.io/modules.js @@ -0,0 +1,175 @@ +'use strict'; + +const Messaging = require('../messaging'); +const utils = require('../utils'); +const user = require('../user'); +const groups = require('../groups'); +const api = require('../api'); +const sockets = require('.'); +const SocketModules = module.exports; +SocketModules.chats = {}; +SocketModules.settings = {}; +SocketModules.chats.getRaw = async function (socket, data) { + sockets.warnDeprecated(socket, 'GET /api/v3/chats/:roomId/messages/:mid/raw'); + if (!data || !data.hasOwnProperty('mid')) { + throw new Error('[[error:invalid-data]]'); + } + const roomId = await Messaging.getMessageField(data.mid, 'roomId'); + const { + content + } = await api.chats.getRawMessage(socket, { + mid: data.mid, + roomId + }); + return content; +}; +SocketModules.chats.isDnD = async function (socket, uid) { + sockets.warnDeprecated(socket, 'GET /api/v3/users/:uid/status OR HEAD /api/v3/users/:uid/status/:status'); + const { + status + } = await api.users.getStatus(socket, { + uid + }); + return status === 'dnd'; +}; +SocketModules.chats.canMessage = async function (socket, roomId) { + sockets.warnDeprecated(socket); + await Messaging.canMessageRoom(socket.uid, roomId); +}; +SocketModules.chats.markAllRead = async function (socket) { + sockets.warnDeprecated(socket); + await Messaging.markAllRead(socket.uid); + Messaging.pushUnreadCount(socket.uid); +}; +SocketModules.chats.getRecentChats = async function (socket, data) { + sockets.warnDeprecated(socket, 'GET /api/v3/chats'); + if (!data || !utils.isNumber(data.after) || !utils.isNumber(data.uid)) { + throw new Error('[[error:invalid-data]]'); + } + const start = parseInt(data.after, 10); + const stop = start + 9; + const { + uid + } = data; + return api.chats.list(socket, { + uid, + start, + stop + }); +}; +SocketModules.chats.hasPrivateChat = async function (socket, uid) { + sockets.warnDeprecated(socket, 'GET /api/v3/users/:uid/chat'); + if (socket.uid <= 0 || uid <= 0) { + throw new Error('[[error:invalid-data]]'); + } + const { + roomId + } = await api.users.getPrivateRoomId(socket, { + uid + }); + return roomId; +}; +SocketModules.chats.getIP = async function (socket, mid) { + sockets.warnDeprecated(socket, 'GET /api/v3/chats/:roomId/messages/:mid/ip'); + const { + ip + } = await api.chats.getIpAddress(socket, { + mid + }); + return ip; +}; +SocketModules.chats.getUnreadCount = async function (socket) { + sockets.warnDeprecated(socket, 'GET /api/v3/chats/unread'); + const { + count + } = await api.chats.getUnread(socket); + return count; +}; +SocketModules.chats.enter = async function (socket, roomIds) { + await joinLeave(socket, roomIds, 'join'); +}; +SocketModules.chats.leave = async function (socket, roomIds) { + await joinLeave(socket, roomIds, 'leave'); +}; +SocketModules.chats.enterPublic = async function (socket, roomIds) { + await joinLeave(socket, roomIds, 'join', 'chat_room_public'); +}; +SocketModules.chats.leavePublic = async function (socket, roomIds) { + await joinLeave(socket, roomIds, 'leave', 'chat_room_public'); +}; +async function joinLeave(socket, roomIds, method, prefix = 'chat_room') { + if (!(socket.uid > 0)) { + throw new Error('[[error:not-allowed]]'); + } + if (!Array.isArray(roomIds)) { + roomIds = [roomIds]; + } + if (roomIds.length) { + const [isAdmin, inRooms, roomData] = await Promise.all([user.isAdministrator(socket.uid), Messaging.isUserInRoom(socket.uid, roomIds), Messaging.getRoomsData(roomIds, ['public', 'groups'])]); + await Promise.all(roomIds.map(async (roomId, idx) => { + const isPublic = roomData[idx] && roomData[idx].public; + const roomGroups = roomData[idx] && roomData[idx].groups; + if (isAdmin || inRooms[idx] && (!isPublic || !roomGroups.length || (await groups.isMemberOfAny(socket.uid, roomGroups)))) { + socket[method](`${prefix}_${roomId}`); + } + })); + } +} +SocketModules.chats.sortPublicRooms = async function (socket, data) { + sockets.warnDeprecated(socket, 'PUT /api/v3/chats/sort'); + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + await api.chats.sortPublicRooms(socket, data); +}; +SocketModules.chats.searchMembers = async function (socket, data) { + sockets.warnDeprecated(socket, 'GET /api/v3/search/chats/:roomId/users?query='); + if (!data || !data.roomId) { + throw new Error('[[error:invalid-data]]'); + } + data.query = data.username; + delete data.username; + return await api.search.roomUsers(socket, data); +}; +SocketModules.chats.toggleOwner = async (socket, data) => { + sockets.warnDeprecated(socket, 'PUT/DELETE /api/v3/chats/:roomId/owners/:uid'); + if (!data || !data.uid || !data.roomId) { + throw new Error('[[error:invalid-data]]'); + } + await api.chats.toggleOwner(socket, data); +}; +SocketModules.chats.setNotificationSetting = async (socket, data) => { + sockets.warnDeprecated(socket, 'PUT/DELETE /api/v3/chats/:roomId/watch'); + if (!data || !utils.isNumber(data.value) || !data.roomId) { + throw new Error('[[error:invalid-data]]'); + } + await api.chats.watch(socket, data); +}; +SocketModules.chats.searchMessages = async (socket, data) => { + sockets.warnDeprecated(socket, 'GET /api/v3/search/chats/:roomId/messages'); + if (!data || !utils.isNumber(data.roomId) || !data.content) { + throw new Error('[[error:invalid-data]]'); + } + data.query = data.content; + delete data.content; + return await api.search.roomMessages(socket, data); +}; +SocketModules.chats.loadPinnedMessages = async (socket, data) => { + sockets.warnDeprecated(socket, 'GET /api/v3/chats/:roomId/messages/pinned'); + if (!data || !data.roomId || !utils.isNumber(data.start)) { + throw new Error('[[error:invalid-data]]'); + } + const { + messages + } = await api.chats.getPinnedMessages(socket, data); + return messages; +}; +SocketModules.chats.typing = async (socket, data) => { + sockets.warnDeprecated(socket, 'PUT /api/v3/chats/:roomId/typing'); + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + delete data.username; + await api.chats.toggleTyping(socket, data); +}; +require('../promisify')(SocketModules); \ No newline at end of file diff --git a/lib/socket.io/notifications.js b/lib/socket.io/notifications.js new file mode 100644 index 0000000000..7d27435559 --- /dev/null +++ b/lib/socket.io/notifications.js @@ -0,0 +1,33 @@ +'use strict'; + +const user = require('../user'); +const notifications = require('../notifications'); +const SocketNotifs = module.exports; +SocketNotifs.get = async function (socket, data) { + if (data && Array.isArray(data.nids) && socket.uid) { + return await user.notifications.getNotifications(data.nids, socket.uid); + } + return await user.notifications.get(socket.uid); +}; +SocketNotifs.getCount = async function (socket) { + return await user.notifications.getUnreadCount(socket.uid); +}; +SocketNotifs.deleteAll = async function (socket) { + if (!socket.uid) { + throw new Error('[[error:no-privileges]]'); + } + await user.notifications.deleteAll(socket.uid); +}; +SocketNotifs.markRead = async function (socket, nid) { + await notifications.markRead(nid, socket.uid); + user.notifications.pushCount(socket.uid); +}; +SocketNotifs.markUnread = async function (socket, nid) { + await notifications.markUnread(nid, socket.uid); + user.notifications.pushCount(socket.uid); +}; +SocketNotifs.markAllRead = async function (socket) { + await notifications.markAllRead(socket.uid); + user.notifications.pushCount(socket.uid); +}; +require('../promisify')(SocketNotifs); \ No newline at end of file diff --git a/lib/socket.io/plugins.js b/lib/socket.io/plugins.js new file mode 100644 index 0000000000..483ac975b5 --- /dev/null +++ b/lib/socket.io/plugins.js @@ -0,0 +1,4 @@ +'use strict'; + +const SocketPlugins = {}; +module.exports = SocketPlugins; \ No newline at end of file diff --git a/lib/socket.io/posts.js b/lib/socket.io/posts.js new file mode 100644 index 0000000000..c14c3d6646 --- /dev/null +++ b/lib/socket.io/posts.js @@ -0,0 +1,171 @@ +'use strict'; + +const validator = require('validator'); +const db = require('../database'); +const posts = require('../posts'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const meta = require('../meta'); +const topics = require('../topics'); +const notifications = require('../notifications'); +const utils = require('../utils'); +const events = require('../events'); +const translator = require('../translator'); +const api = require('../api'); +const sockets = require('.'); +const SocketPosts = module.exports; +require('./posts/votes')(SocketPosts); +require('./posts/tools')(SocketPosts); +SocketPosts.getRawPost = async function (socket, pid) { + sockets.warnDeprecated(socket, 'GET /api/v3/posts/:pid/raw'); + return await api.posts.getRaw(socket, { + pid + }); +}; +SocketPosts.getPostSummaryByIndex = async function (socket, data) { + sockets.warnDeprecated(socket, 'GET /api/v3/posts/byIndex/:index/summary?tid=:tid'); + if (data.index < 0) { + data.index = 0; + } + let pid; + if (data.index === 0) { + pid = await topics.getTopicField(data.tid, 'mainPid'); + } else { + pid = await db.getSortedSetRange(`tid:${data.tid}:posts`, data.index - 1, data.index - 1); + } + pid = Array.isArray(pid) ? pid[0] : pid; + if (!pid) { + return 0; + } + return await api.posts.getSummary(socket, { + pid + }); +}; +SocketPosts.getPostTimestampByIndex = async function (socket, data) { + if (data.index < 0) { + data.index = 0; + } + let pid; + if (data.index === 0) { + pid = await topics.getTopicField(data.tid, 'mainPid'); + } else { + pid = await db.getSortedSetRange(`tid:${data.tid}:posts`, data.index - 1, data.index - 1); + } + pid = Array.isArray(pid) ? pid[0] : pid; + const topicPrivileges = await privileges.topics.get(data.tid, socket.uid); + if (!topicPrivileges['topics:read']) { + throw new Error('[[error:no-privileges]]'); + } + return await posts.getPostField(pid, 'timestamp'); +}; +SocketPosts.getPostSummaryByPid = async function (socket, data) { + sockets.warnDeprecated(socket, 'GET /api/v3/posts/:pid/summary'); + const { + pid + } = data; + return await api.posts.getSummary(socket, { + pid + }); +}; +SocketPosts.getCategory = async function (socket, pid) { + return await posts.getCidByPid(pid); +}; +SocketPosts.getPidIndex = async function (socket, data) { + sockets.warnDeprecated(socket, 'GET /api/v3/posts/:pid/index'); + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + return await api.posts.getIndex(socket, { + pid: data.pid, + sort: data.topicPostSort + }); +}; +SocketPosts.getReplies = async function (socket, pid) { + sockets.warnDeprecated(socket, 'GET /api/v3/posts/:pid/replies'); + if (!utils.isNumber(pid)) { + throw new Error('[[error:invalid-data]]'); + } + return await api.posts.getReplies(socket, { + pid + }); +}; +SocketPosts.accept = async function (socket, data) { + await canEditQueue(socket, data, 'accept'); + const result = await posts.submitFromQueue(data.id); + if (result && socket.uid !== parseInt(result.uid, 10)) { + await sendQueueNotification('post-queue-accepted', result.uid, `/post/${result.pid}`); + } + await logQueueEvent(socket, result, 'accept'); +}; +SocketPosts.reject = async function (socket, data) { + await canEditQueue(socket, data, 'reject'); + const result = await posts.removeFromQueue(data.id); + if (result && socket.uid !== parseInt(result.uid, 10)) { + await sendQueueNotification('post-queue-rejected', result.uid, '/'); + } + await logQueueEvent(socket, result, 'reject'); +}; +async function logQueueEvent(socket, result, type) { + const eventData = { + type: `post-queue-${result.type}-${type}`, + uid: socket.uid, + ip: socket.ip, + content: result.data.content, + targetUid: result.uid + }; + if (result.type === 'topic') { + eventData.cid = result.data.cid; + eventData.title = result.data.title; + } else { + eventData.tid = result.data.tid; + } + if (result.pid) { + eventData.pid = result.pid; + } + await events.log(eventData); +} +SocketPosts.notify = async function (socket, data) { + await canEditQueue(socket, data, 'notify'); + const result = await posts.getFromQueue(data.id); + if (result) { + await sendQueueNotification('post-queue-notify', result.uid, `/post-queue/${data.id}`, validator.escape(String(data.message))); + } +}; +async function canEditQueue(socket, data, action) { + const [canEditQueue, queuedPost] = await Promise.all([posts.canEditQueue(socket.uid, data, action), posts.getFromQueue(data.id)]); + if (!queuedPost) { + throw new Error('[[error:no-post]]'); + } + if (!canEditQueue) { + throw new Error('[[error:no-privileges]]'); + } +} +async function sendQueueNotification(type, targetUid, path, notificationText) { + const bodyShort = notificationText ? translator.compile(`notifications:${type}`, notificationText) : translator.compile(`notifications:${type}`); + const notifData = { + type: type, + nid: `${type}-${targetUid}-${path}`, + bodyShort: bodyShort, + path: path + }; + if (parseInt(meta.config.postQueueNotificationUid, 10) > 0) { + notifData.from = meta.config.postQueueNotificationUid; + } + const notifObj = await notifications.create(notifData); + await notifications.push(notifObj, [targetUid]); +} +SocketPosts.editQueuedContent = async function (socket, data) { + if (!data || !data.id || !data.content && !data.title && !data.cid) { + throw new Error('[[error:invalid-data]]'); + } + await posts.editQueuedContent(socket.uid, data); + if (data.content) { + return await plugins.hooks.fire('filter:parse.post', { + postData: data + }); + } + return { + postData: data + }; +}; +require('../promisify')(SocketPosts); \ No newline at end of file diff --git a/lib/socket.io/posts/tools.js b/lib/socket.io/posts/tools.js new file mode 100644 index 0000000000..0440c91399 --- /dev/null +++ b/lib/socket.io/posts/tools.js @@ -0,0 +1,93 @@ +'use strict'; + +const nconf = require('nconf'); +const db = require('../../database'); +const posts = require('../../posts'); +const flags = require('../../flags'); +const events = require('../../events'); +const privileges = require('../../privileges'); +const plugins = require('../../plugins'); +const social = require('../../social'); +const user = require('../../user'); +const utils = require('../../utils'); +module.exports = function (SocketPosts) { + SocketPosts.loadPostTools = async function (socket, data) { + if (!data || !data.pid) { + throw new Error('[[error:invalid-data]]'); + } + const cid = await posts.getCidByPid(data.pid); + const results = await utils.promiseParallel({ + posts: posts.getPostFields(data.pid, ['deleted', 'bookmarks', 'uid', 'ip', 'flagId']), + isAdmin: user.isAdministrator(socket.uid), + isGlobalMod: user.isGlobalModerator(socket.uid), + isModerator: user.isModerator(socket.uid, cid), + canEdit: privileges.posts.canEdit(data.pid, socket.uid), + canDelete: privileges.posts.canDelete(data.pid, socket.uid), + canPurge: privileges.posts.canPurge(data.pid, socket.uid), + canFlag: privileges.posts.canFlag(data.pid, socket.uid), + canViewHistory: privileges.posts.can('posts:history', data.pid, socket.uid), + flagged: flags.exists('post', data.pid, socket.uid), + bookmarked: posts.hasBookmarked(data.pid, socket.uid), + postSharing: social.getActivePostSharing(), + history: posts.diffs.exists(data.pid), + canViewInfo: privileges.global.can('view:users:info', socket.uid) + }); + const postData = results.posts; + postData.absolute_url = `${nconf.get('url')}/post/${data.pid}`; + postData.bookmarked = results.bookmarked; + postData.selfPost = socket.uid && socket.uid === postData.uid; + postData.display_edit_tools = results.canEdit.flag; + postData.display_delete_tools = results.canDelete.flag; + postData.display_purge_tools = results.canPurge; + postData.display_flag_tools = socket.uid && results.canFlag.flag; + postData.display_moderator_tools = postData.display_edit_tools || postData.display_delete_tools; + postData.display_move_tools = results.isAdmin || results.isModerator; + postData.display_change_owner_tools = results.isAdmin || results.isModerator; + postData.display_ip_ban = (results.isAdmin || results.isGlobalMod) && !postData.selfPost; + postData.display_history = results.history && results.canViewHistory; + postData.flags = { + flagId: parseInt(results.posts.flagId, 10) || null, + can: results.canFlag.flag, + exists: !!results.posts.flagId, + flagged: results.flagged, + state: await db.getObjectField(`flag:${postData.flagId}`, 'state') + }; + if (!results.isAdmin && !results.canViewInfo) { + postData.ip = undefined; + } + const { + tools + } = await plugins.hooks.fire('filter:post.tools', { + pid: data.pid, + post: postData, + uid: socket.uid, + tools: [] + }); + postData.tools = tools; + return results; + }; + SocketPosts.changeOwner = async function (socket, data) { + if (!data || !Array.isArray(data.pids) || !data.toUid) { + throw new Error('[[error:invalid-data]]'); + } + const isAdminOrGlobalMod = await user.isAdminOrGlobalMod(socket.uid); + if (!isAdminOrGlobalMod) { + throw new Error('[[error:no-privileges]]'); + } + const postData = await posts.changeOwner(data.pids, data.toUid); + const logs = postData.map(({ + pid, + uid, + cid + }) => events.log({ + type: 'post-change-owner', + uid: socket.uid, + ip: socket.ip, + targetUid: data.toUid, + pid: pid, + originalUid: uid, + cid: cid + })); + await Promise.all(logs); + }; +}; \ No newline at end of file diff --git a/lib/socket.io/posts/votes.js b/lib/socket.io/posts/votes.js new file mode 100644 index 0000000000..4b48b7d466 --- /dev/null +++ b/lib/socket.io/posts/votes.js @@ -0,0 +1,24 @@ +'use strict'; + +const api = require('../../api'); +const sockets = require('../index'); +module.exports = function (SocketPosts) { + SocketPosts.getVoters = async function (socket, data) { + if (!data || !data.pid) { + throw new Error('[[error:invalid-data]]'); + } + sockets.warnDeprecated(socket, 'GET /api/v3/posts/:pid/voters'); + return await api.posts.getVoters(socket, { + pid: data.pid + }); + }; + SocketPosts.getUpvoters = async function (socket, pids) { + if (!Array.isArray(pids)) { + throw new Error('[[error:invalid-data]]'); + } + sockets.warnDeprecated(socket, 'GET /api/v3/posts/:pid/upvoters'); + return await api.posts.getUpvoters(socket, { + pid: pids[0] + }); + }; +}; \ No newline at end of file diff --git a/lib/socket.io/topics.js b/lib/socket.io/topics.js new file mode 100644 index 0000000000..c8cd35df92 --- /dev/null +++ b/lib/socket.io/topics.js @@ -0,0 +1,115 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const posts = require('../posts'); +const topics = require('../topics'); +const user = require('../user'); +const meta = require('../meta'); +const privileges = require('../privileges'); +const cache = require('../cache'); +const events = require('../events'); +const SocketTopics = module.exports; +require('./topics/unread')(SocketTopics); +require('./topics/move')(SocketTopics); +require('./topics/tools')(SocketTopics); +require('./topics/infinitescroll')(SocketTopics); +require('./topics/tags')(SocketTopics); +require('./topics/merge')(SocketTopics); +SocketTopics.postcount = async function (socket, tid) { + const canRead = await privileges.topics.can('topics:read', tid, socket.uid); + if (!canRead) { + throw new Error('[[no-privileges]]'); + } + return await topics.getTopicField(tid, 'postcount'); +}; +SocketTopics.bookmark = async function (socket, data) { + if (!socket.uid || !data) { + throw new Error('[[error:invalid-data]]'); + } + const postcount = await topics.getTopicField(data.tid, 'postcount'); + if (data.index > meta.config.bookmarkThreshold && postcount > meta.config.bookmarkThreshold) { + const currentIndex = await db.sortedSetScore(`tid:${data.tid}:bookmarks`, socket.uid); + if (!currentIndex || data.index > currentIndex && data.index <= postcount || currentIndex > postcount) { + await topics.setUserBookmark(data.tid, socket.uid, data.index); + } + } +}; +SocketTopics.createTopicFromPosts = async function (socket, data) { + if (!socket.uid) { + throw new Error('[[error:not-logged-in]]'); + } + if (!data || !data.title || !data.pids || !Array.isArray(data.pids)) { + throw new Error('[[error:invalid-data]]'); + } + const result = await topics.createTopicFromPosts(socket.uid, data.title, data.pids, data.fromTid, data.cid); + await events.log({ + type: `topic-fork`, + uid: socket.uid, + ip: socket.ip, + pids: String(data.pids), + fromTid: data.fromTid, + toTid: result.tid + }); + return result; +}; +SocketTopics.isFollowed = async function (socket, tid) { + const isFollowing = await topics.isFollowing([tid], socket.uid); + return isFollowing[0]; +}; +SocketTopics.isModerator = async function (socket, tid) { + const cid = await topics.getTopicField(tid, 'cid'); + return await user.isModerator(socket.uid, cid); +}; +SocketTopics.getMyNextPostIndex = async function (socket, data) { + if (!data || !data.tid || !data.index || !data.sort) { + throw new Error('[[error:invalid-data]]'); + } + async function getTopicPids(index) { + const topicSet = data.sort === 'most_votes' ? `tid:${data.tid}:posts:votes` : `tid:${data.tid}:posts`; + const reverse = data.sort === 'newest_to_oldest' || data.sort === 'most_votes'; + const cacheKey = `np:s:${topicSet}:r:${String(reverse)}:tid:${data.tid}:pids`; + const topicPids = cache.get(cacheKey); + if (topicPids) { + return topicPids.slice(index - 1); + } + const pids = await db[reverse ? 'getSortedSetRevRange' : 'getSortedSetRange'](topicSet, 0, -1); + cache.set(cacheKey, pids, 30000); + return pids.slice(index - 1); + } + async function getUserPids() { + const cid = await topics.getTopicField(data.tid, 'cid'); + const cacheKey = `np:cid:${cid}:uid:${socket.uid}:pids`; + const userPids = cache.get(cacheKey); + if (userPids) { + return userPids; + } + const pids = await db.getSortedSetRange(`cid:${cid}:uid:${socket.uid}:pids`, 0, -1); + cache.set(cacheKey, pids, 30000); + return pids; + } + const postCountInTopic = await db.sortedSetScore(`tid:${data.tid}:posters`, socket.uid); + if (postCountInTopic <= 0) { + return 0; + } + const [topicPids, userPidsInCategory] = await Promise.all([getTopicPids(data.index), getUserPids()]); + const userPidsInTopic = _.intersection(topicPids, userPidsInCategory); + if (!userPidsInTopic.length) { + if (postCountInTopic > 0) { + const wrapIndex = await SocketTopics.getMyNextPostIndex(socket, { + ...data, + index: 1 + }); + return wrapIndex; + } + return 0; + } + return await posts.getPidIndex(userPidsInTopic[0], data.tid, data.sort); +}; +SocketTopics.getPostCountInTopic = async function (socket, tid) { + if (!socket.uid || !tid) { + return 0; + } + return await db.sortedSetScore(`tid:${tid}:posters`, socket.uid); +}; +require('../promisify')(SocketTopics); \ No newline at end of file diff --git a/lib/socket.io/topics/infinitescroll.js b/lib/socket.io/topics/infinitescroll.js new file mode 100644 index 0000000000..73c8f4ceae --- /dev/null +++ b/lib/socket.io/topics/infinitescroll.js @@ -0,0 +1,36 @@ +'use strict'; + +const topics = require('../../topics'); +const privileges = require('../../privileges'); +const meta = require('../../meta'); +const utils = require('../../utils'); +const social = require('../../social'); +module.exports = function (SocketTopics) { + SocketTopics.loadMore = async function (socket, data) { + if (!data || !data.tid || !utils.isNumber(data.after) || parseInt(data.after, 10) < 0) { + throw new Error('[[error:invalid-data]]'); + } + const [userPrivileges, topicData] = await Promise.all([privileges.topics.get(data.tid, socket.uid), topics.getTopicData(data.tid)]); + if (!userPrivileges['topics:read'] || !privileges.topics.canViewDeletedScheduled(topicData, userPrivileges)) { + throw new Error('[[error:no-privileges]]'); + } + const set = data.topicPostSort === 'most_votes' ? `tid:${data.tid}:posts:votes` : `tid:${data.tid}:posts`; + const reverse = data.topicPostSort === 'newest_to_oldest' || data.topicPostSort === 'most_votes'; + let start = Math.max(0, parseInt(data.after, 10)); + const infScrollPostsPerPage = Math.max(0, Math.min(meta.config.postsPerPage || 20, parseInt(data.count, 10) || meta.config.postsPerPage || 20)); + if (parseInt(data.direction, 10) === -1) { + start -= infScrollPostsPerPage; + } + let stop = start + infScrollPostsPerPage - 1; + start = Math.max(0, start); + stop = Math.max(0, stop); + const [posts, postSharing] = await Promise.all([topics.getTopicPosts(topicData, set, start, stop, socket.uid, reverse), social.getActivePostSharing()]); + topicData.posts = posts; + topicData.privileges = userPrivileges; + topicData.postSharing = postSharing; + topicData['reputation:disabled'] = meta.config['reputation:disabled'] === 1; + topicData['downvote:disabled'] = meta.config['downvote:disabled'] === 1; + topics.modifyPostsByPrivilege(topicData, userPrivileges); + return topicData; + }; +}; \ No newline at end of file diff --git a/lib/socket.io/topics/merge.js b/lib/socket.io/topics/merge.js new file mode 100644 index 0000000000..f082092e69 --- /dev/null +++ b/lib/socket.io/topics/merge.js @@ -0,0 +1,28 @@ +'use strict'; + +const topics = require('../../topics'); +const privileges = require('../../privileges'); +const events = require('../../events'); +module.exports = function (SocketTopics) { + SocketTopics.merge = async function (socket, data) { + if (!data || !Array.isArray(data.tids)) { + throw new Error('[[error:invalid-data]]'); + } + const allowed = await Promise.all(data.tids.map(tid => privileges.topics.isAdminOrMod(tid, socket.uid))); + if (allowed.includes(false)) { + throw new Error('[[error:no-privileges]]'); + } + if (data.options && data.options.mainTid && !data.tids.includes(data.options.mainTid)) { + throw new Error('[[error:invalid-data]]'); + } + const mergeIntoTid = await topics.merge(data.tids, socket.uid, data.options); + await events.log({ + type: `topic-merge`, + uid: socket.uid, + ip: socket.ip, + mergeIntoTid: mergeIntoTid, + tids: String(data.tids) + }); + return mergeIntoTid; + }; +}; \ No newline at end of file diff --git a/lib/socket.io/topics/move.js b/lib/socket.io/topics/move.js new file mode 100644 index 0000000000..11a1f50013 --- /dev/null +++ b/lib/socket.io/topics/move.js @@ -0,0 +1,70 @@ +'use strict'; + +const async = require('async'); +const user = require('../../user'); +const topics = require('../../topics'); +const categories = require('../../categories'); +const privileges = require('../../privileges'); +const socketHelpers = require('../helpers'); +const events = require('../../events'); +module.exports = function (SocketTopics) { + SocketTopics.move = async function (socket, data) { + if (!data || !Array.isArray(data.tids) || !data.cid) { + throw new Error('[[error:invalid-data]]'); + } + const canMove = await privileges.categories.isAdminOrMod(data.cid, socket.uid); + if (!canMove) { + throw new Error('[[error:no-privileges]]'); + } + const uids = await user.getUidsFromSet('users:online', 0, -1); + const cids = [parseInt(data.cid, 10)]; + await async.eachLimit(data.tids, 10, async tid => { + const canMove = await privileges.topics.isAdminOrMod(tid, socket.uid); + if (!canMove) { + throw new Error('[[error:no-privileges]]'); + } + const topicData = await topics.getTopicFields(tid, ['tid', 'cid', 'slug', 'deleted']); + if (!cids.includes(topicData.cid)) { + cids.push(topicData.cid); + } + data.uid = socket.uid; + await topics.tools.move(tid, data); + const notifyUids = await privileges.categories.filterUids('topics:read', topicData.cid, uids); + socketHelpers.emitToUids('event:topic_moved', topicData, notifyUids); + if (!topicData.deleted) { + socketHelpers.sendNotificationToTopicOwner(tid, socket.uid, 'move', 'notifications:moved-your-topic'); + } + await events.log({ + type: `topic-move`, + uid: socket.uid, + ip: socket.ip, + tid: tid, + fromCid: topicData.cid, + toCid: data.cid + }); + }); + await categories.onTopicsMoved(cids); + }; + SocketTopics.moveAll = async function (socket, data) { + if (!data || !data.cid || !data.currentCid) { + throw new Error('[[error:invalid-data]]'); + } + const canMove = await privileges.categories.canMoveAllTopics(data.currentCid, data.cid, socket.uid); + if (!canMove) { + throw new Error('[[error:no-privileges]]'); + } + const tids = await categories.getAllTopicIds(data.currentCid, 0, -1); + data.uid = socket.uid; + await async.eachLimit(tids, 50, async tid => { + await topics.tools.move(tid, data); + }); + await categories.onTopicsMoved([data.currentCid, data.cid]); + await events.log({ + type: `topic-move-all`, + uid: socket.uid, + ip: socket.ip, + fromCid: data.currentCid, + toCid: data.cid + }); + }; +}; \ No newline at end of file diff --git a/lib/socket.io/topics/tags.js b/lib/socket.io/topics/tags.js new file mode 100644 index 0000000000..4b983cc445 --- /dev/null +++ b/lib/socket.io/topics/tags.js @@ -0,0 +1,95 @@ +'use strict'; + +const meta = require('../../meta'); +const user = require('../../user'); +const topics = require('../../topics'); +const categories = require('../../categories'); +const privileges = require('../../privileges'); +const utils = require('../../utils'); +module.exports = function (SocketTopics) { + SocketTopics.isTagAllowed = async function (socket, data) { + if (!data || !utils.isNumber(data.cid) || !data.tag) { + throw new Error('[[error:invalid-data]]'); + } + const systemTags = (meta.config.systemTags || '').split(','); + const [tagWhitelist, isPrivileged] = await Promise.all([categories.getTagWhitelist([data.cid]), user.isPrivileged(socket.uid)]); + return isPrivileged || !systemTags.includes(data.tag) && (!tagWhitelist[0].length || tagWhitelist[0].includes(data.tag)); + }; + SocketTopics.canRemoveTag = async function (socket, data) { + if (!data || !data.tag) { + throw new Error('[[error:invalid-data]]'); + } + const systemTags = (meta.config.systemTags || '').split(','); + const isPrivileged = await user.isPrivileged(socket.uid); + return isPrivileged || !systemTags.includes(String(data.tag).trim()); + }; + SocketTopics.autocompleteTags = async function (socket, data) { + if (data.cid) { + const canRead = await privileges.categories.can('topics:read', data.cid, socket.uid); + if (!canRead) { + throw new Error('[[error:no-privileges]]'); + } + } + data.cids = await categories.getCidsByPrivilege('categories:cid', socket.uid, 'topics:read'); + const result = await topics.autocompleteTags(data); + return result.map(tag => tag.value); + }; + SocketTopics.searchTags = async function (socket, data) { + const result = await searchTags(socket.uid, topics.searchTags, data); + return result.map(tag => tag.value); + }; + SocketTopics.searchAndLoadTags = async function (socket, data) { + return await searchTags(socket.uid, topics.searchAndLoadTags, data); + }; + async function searchTags(uid, method, data) { + const allowed = await privileges.global.can('search:tags', uid); + if (!allowed) { + throw new Error('[[error:no-privileges]]'); + } + if (data.cid) { + const canRead = await privileges.categories.can('topics:read', data.cid, uid); + if (!canRead) { + throw new Error('[[error:no-privileges]]'); + } + } + data.cids = await categories.getCidsByPrivilege('categories:cid', uid, 'topics:read'); + return await method(data); + } + SocketTopics.tagFilterSearch = async function (socket, data) { + let cids = []; + if (Array.isArray(data.cids)) { + cids = await privileges.categories.filterCids('topics:read', data.cids, socket.uid); + } else { + cids = await categories.getCidsByPrivilege('categories:cid', socket.uid, 'topics:read'); + } + let tags = []; + if (data.query) { + const allowed = await privileges.global.can('search:tags', socket.uid); + if (!allowed) { + throw new Error('[[error:no-privileges]]'); + } + tags = await topics.searchTags({ + query: data.query, + cid: cids.length === 1 ? cids[0] : null, + cids: cids + }); + topics.getTagData(tags); + } else { + tags = await topics.getCategoryTagsData(cids, 0, 39); + } + return tags.filter(t => t.score > 0); + }; + SocketTopics.loadMoreTags = async function (socket, data) { + if (!data || !utils.isNumber(data.after)) { + throw new Error('[[error:invalid-data]]'); + } + const start = parseInt(data.after, 10); + const stop = start + 99; + const cids = await categories.getCidsByPrivilege('categories:cid', socket.uid, 'topics:read'); + const tags = await topics.getCategoryTagsData(cids, start, stop); + return { + tags: tags.filter(Boolean), + nextStart: stop + 1 + }; + }; +}; \ No newline at end of file diff --git a/lib/socket.io/topics/tools.js b/lib/socket.io/topics/tools.js new file mode 100644 index 0000000000..89d8f4ba43 --- /dev/null +++ b/lib/socket.io/topics/tools.js @@ -0,0 +1,36 @@ +'use strict'; + +const topics = require('../../topics'); +const privileges = require('../../privileges'); +const plugins = require('../../plugins'); +module.exports = function (SocketTopics) { + SocketTopics.loadTopicTools = async function (socket, data) { + if (!socket.uid) { + throw new Error('[[error:no-privileges]]'); + } + if (!data) { + throw new Error('[[error:invalid-data]]'); + } + const [topicData, userPrivileges] = await Promise.all([topics.getTopicData(data.tid), privileges.topics.get(data.tid, socket.uid)]); + if (!topicData) { + throw new Error('[[error:no-topic]]'); + } + if (!userPrivileges['topics:read']) { + throw new Error('[[error:no-privileges]]'); + } + topicData.privileges = userPrivileges; + const result = await plugins.hooks.fire('filter:topic.thread_tools', { + topic: topicData, + uid: socket.uid, + tools: [] + }); + result.topic.thread_tools = result.tools; + return result.topic; + }; + SocketTopics.orderPinnedTopics = async function (socket, data) { + if (!data || !data.tid) { + throw new Error('[[error:invalid-data]]'); + } + await topics.tools.orderPinnedTopics(socket.uid, data); + }; +}; \ No newline at end of file diff --git a/lib/socket.io/topics/unread.js b/lib/socket.io/topics/unread.js new file mode 100644 index 0000000000..79cef6977c --- /dev/null +++ b/lib/socket.io/topics/unread.js @@ -0,0 +1,58 @@ +'use strict'; + +const topics = require('../../topics'); +const api = require('../../api'); +const sockets = require('..'); +module.exports = function (SocketTopics) { + SocketTopics.markAsRead = async function (socket, tids) { + sockets.warnDeprecated(socket, 'PUT /api/v3/topics/:tid/read'); + if (!Array.isArray(tids) || socket.uid <= 0) { + throw new Error('[[error:invalid-data]]'); + } + await Promise.all(tids.map(async tid => api.topics.markRead(socket, { + tid + }))); + }; + SocketTopics.markTopicNotificationsRead = async function (socket, tids) { + if (!Array.isArray(tids) || !socket.uid) { + throw new Error('[[error:invalid-data]]'); + } + await topics.markTopicNotificationsRead(tids, socket.uid); + }; + SocketTopics.markAllRead = async function (socket) { + if (socket.uid <= 0) { + throw new Error('[[error:invalid-uid]]'); + } + await topics.markAllRead(socket.uid); + topics.pushUnreadCount(socket.uid); + }; + SocketTopics.markCategoryTopicsRead = async function (socket, cid) { + const tids = await topics.getUnreadTids({ + cid: cid, + uid: socket.uid, + filter: '' + }); + await SocketTopics.markAsRead(socket, tids); + }; + SocketTopics.markUnread = async function (socket, tid) { + sockets.warnDeprecated(socket, 'DELETE /api/v3/topics/:tid/read'); + if (!tid || socket.uid <= 0) { + throw new Error('[[error:invalid-data]]'); + } + await api.topics.markUnread(socket, { + tid + }); + }; + SocketTopics.markAsUnreadForAll = async function (socket, tids) { + sockets.warnDeprecated(socket, 'PUT /api/v3/topics/:tid/bump'); + if (!Array.isArray(tids)) { + throw new Error('[[error:invalid-tid]]'); + } + if (socket.uid <= 0) { + throw new Error('[[error:no-privileges]]'); + } + await Promise.all(tids.map(async tid => api.topics.bump(socket, { + tid + }))); + }; +}; \ No newline at end of file diff --git a/lib/socket.io/uploads.js b/lib/socket.io/uploads.js new file mode 100644 index 0000000000..e664772736 --- /dev/null +++ b/lib/socket.io/uploads.js @@ -0,0 +1,58 @@ +'use strict'; + +const socketUser = require('./user'); +const socketGroup = require('./groups'); +const image = require('../image'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const inProgress = {}; +const uploads = module.exports; +uploads.upload = async function (socket, data) { + if (!socket.uid || !data || !data.chunk || !data.params || !data.params.method) { + throw new Error('[[error:invalid-data]]'); + } + const { + method + } = data.params; + const defaultMaxSize = method === 'user.uploadCroppedPicture' ? meta.config.maximumProfileImageSize : meta.config.maximumCoverImageSize; + const { + methods, + maxSize + } = await plugins.hooks.fire('filter:uploads.upload', { + methods: { + 'user.uploadCroppedPicture': socketUser.uploadCroppedPicture, + 'user.updateCover': socketUser.updateCover, + 'groups.cover.update': socketGroup.cover.update + }, + maxSize: defaultMaxSize, + data: data + }); + if (!methods.hasOwnProperty(data.params.method)) { + throw new Error('[[error:invalid-data]]'); + } + inProgress[socket.id] = inProgress[socket.id] || Object.create(null); + const socketUploads = inProgress[socket.id]; + socketUploads[method] = socketUploads[method] || { + imageData: '' + }; + socketUploads[method].imageData += data.chunk; + try { + const size = image.sizeFromBase64(socketUploads[method].imageData); + if (size > maxSize * 1024) { + throw new Error(`[[error:file-too-big, ${maxSize}]]`); + } + if (socketUploads[method].imageData.length < data.params.size) { + return; + } + data.params.imageData = socketUploads[method].imageData; + const result = await methods[method](socket, data.params); + delete socketUploads[method]; + return result; + } catch (err) { + delete inProgress[socket.id]; + throw err; + } +}; +uploads.clear = function (sid) { + delete inProgress[sid]; +}; \ No newline at end of file diff --git a/lib/socket.io/user.js b/lib/socket.io/user.js new file mode 100644 index 0000000000..85e8e54166 --- /dev/null +++ b/lib/socket.io/user.js @@ -0,0 +1,178 @@ +'use strict'; + +const util = require('util'); +const winston = require('winston'); +const sleep = util.promisify(setTimeout); +const user = require('../user'); +const topics = require('../topics'); +const messaging = require('../messaging'); +const plugins = require('../plugins'); +const meta = require('../meta'); +const events = require('../events'); +const emailer = require('../emailer'); +const db = require('../database'); +const userController = require('../controllers/user'); +const privileges = require('../privileges'); +const utils = require('../utils'); +const SocketUser = module.exports; +require('./user/profile')(SocketUser); +require('./user/status')(SocketUser); +require('./user/picture')(SocketUser); +require('./user/registration')(SocketUser); +SocketUser.reset = {}; +SocketUser.reset.send = async function (socket, email) { + if (!email) { + throw new Error('[[error:invalid-data]]'); + } + if (meta.config['password:disableEdit']) { + throw new Error('[[error:no-privileges]]'); + } + async function logEvent(text) { + await events.log({ + type: 'password-reset', + text: text, + ip: socket.ip, + uid: socket.uid, + email: email + }); + } + try { + await user.reset.send(email); + await logEvent('[[success:success]]'); + await sleep(2500 + (Math.random() * 500 - 250)); + } catch (err) { + await logEvent(err.message); + await sleep(2500 + (Math.random() * 500 - 250)); + const internalErrors = ['[[error:invalid-email]]']; + if (!internalErrors.includes(err.message)) { + throw err; + } + } +}; +SocketUser.reset.commit = async function (socket, data) { + if (!data || !data.code || !data.password) { + throw new Error('[[error:invalid-data]]'); + } + const [uid] = await Promise.all([db.getObjectField('reset:uid', data.code), user.reset.commit(data.code, data.password), plugins.hooks.fire('action:password.reset', { + uid: socket.uid + })]); + await events.log({ + type: 'password-reset', + uid: uid, + ip: socket.ip + }); + const username = await user.getUserField(uid, 'username'); + const now = new Date(); + const parsedDate = `${now.getFullYear()}/${now.getMonth() + 1}/${now.getDate()}`; + emailer.send('reset_notify', uid, { + username: username, + date: parsedDate, + subject: '[[email:reset.notify.subject]]' + }).catch(err => winston.error(`[emailer.send] ${err.stack}`)); +}; +SocketUser.isFollowing = async function (socket, data) { + if (!socket.uid || !data.uid) { + return false; + } + return await user.isFollowing(socket.uid, data.uid); +}; +SocketUser.getUnreadCount = async function (socket) { + if (!socket.uid) { + return 0; + } + return await topics.getTotalUnread(socket.uid, ''); +}; +SocketUser.getUnreadChatCount = async function (socket) { + if (!socket.uid) { + return 0; + } + return await messaging.getUnreadCount(socket.uid); +}; +SocketUser.getUnreadCounts = async function (socket) { + if (!socket.uid) { + return {}; + } + const results = await utils.promiseParallel({ + unreadCounts: topics.getUnreadTids({ + uid: socket.uid, + count: true + }), + unreadChatCount: messaging.getUnreadCount(socket.uid), + unreadNotificationCount: user.notifications.getUnreadCount(socket.uid) + }); + results.unreadTopicCount = results.unreadCounts['']; + results.unreadNewTopicCount = results.unreadCounts.new; + results.unreadWatchedTopicCount = results.unreadCounts.watched; + results.unreadUnrepliedTopicCount = results.unreadCounts.unreplied; + return results; +}; +SocketUser.getUserByUID = async function (socket, uid) { + return await userController.getUserDataByField(socket.uid, 'uid', uid); +}; +SocketUser.getUserByUsername = async function (socket, username) { + return await userController.getUserDataByField(socket.uid, 'username', username); +}; +SocketUser.getUserByEmail = async function (socket, email) { + return await userController.getUserDataByField(socket.uid, 'email', email); +}; +SocketUser.setModerationNote = async function (socket, data) { + if (!socket.uid || !data || !data.uid || !data.note) { + throw new Error('[[error:invalid-data]]'); + } + const noteData = { + uid: socket.uid, + note: data.note, + timestamp: Date.now() + }; + let canEdit = await privileges.users.canEdit(socket.uid, data.uid); + if (!canEdit) { + canEdit = await user.isModeratorOfAnyCategory(socket.uid); + } + if (!canEdit) { + throw new Error('[[error:no-privileges]]'); + } + await user.appendModerationNote({ + uid: data.uid, + noteData + }); + return await user.getModerationNotes(data.uid, 0, 0); +}; +SocketUser.editModerationNote = async function (socket, data) { + if (!socket.uid || !data || !data.uid || !data.note || !data.id) { + throw new Error('[[error:invalid-data]]'); + } + const noteData = { + note: data.note, + timestamp: data.id + }; + let canEdit = await privileges.users.canEdit(socket.uid, data.uid); + if (!canEdit) { + canEdit = await user.isModeratorOfAnyCategory(socket.uid); + } + if (!canEdit) { + throw new Error('[[error:no-privileges]]'); + } + await user.setModerationNote({ + uid: data.uid, + noteData + }); + return await user.getModerationNotesByIds(data.uid, [data.id]); +}; +SocketUser.deleteUpload = async function (socket, data) { + if (!data || !data.name || !data.uid) { + throw new Error('[[error:invalid-data]]'); + } + await user.deleteUpload(socket.uid, data.uid, data.name); +}; +SocketUser.gdpr = {}; +SocketUser.gdpr.consent = async function (socket) { + await user.setUserField(socket.uid, 'gdpr_consent', 1); +}; +SocketUser.gdpr.check = async function (socket, data) { + const isAdmin = await user.isAdministrator(socket.uid); + if (!isAdmin) { + data.uid = socket.uid; + } + return await db.getObjectField(`user:${data.uid}`, 'gdpr_consent'); +}; +require('../promisify')(SocketUser); \ No newline at end of file diff --git a/lib/socket.io/user/picture.js b/lib/socket.io/user/picture.js new file mode 100644 index 0000000000..12a9421c0f --- /dev/null +++ b/lib/socket.io/user/picture.js @@ -0,0 +1,50 @@ +'use strict'; + +const user = require('../../user'); +const plugins = require('../../plugins'); +module.exports = function (SocketUser) { + SocketUser.removeUploadedPicture = async function (socket, data) { + if (!socket.uid || !data || !data.uid) { + throw new Error('[[error:invalid-data]]'); + } + await user.isAdminOrSelf(socket.uid, data.uid); + const userData = await user.removeProfileImage(data.uid); + plugins.hooks.fire('action:user.removeUploadedPicture', { + callerUid: socket.uid, + uid: data.uid, + user: userData + }); + }; + SocketUser.getProfilePictures = async function (socket, data) { + if (!data || !data.uid) { + throw new Error('[[error:invalid-data]]'); + } + const [list, userObj] = await Promise.all([plugins.hooks.fire('filter:user.listPictures', { + uid: data.uid, + pictures: [] + }), user.getUserData(data.uid)]); + if (userObj.uploadedpicture) { + list.pictures.push({ + type: 'uploaded', + url: userObj.uploadedpicture, + text: '[[user:uploaded-picture]]' + }); + } + list.pictures = list.pictures.map(({ + type, + url, + text + }) => ({ + type, + username: text, + picture: url + })); + list.pictures.unshift({ + type: 'default', + 'icon:text': userObj['icon:text'], + 'icon:bgColor': userObj['icon:bgColor'], + username: '[[user:default-picture]]' + }); + return list.pictures; + }; +}; \ No newline at end of file diff --git a/lib/socket.io/user/profile.js b/lib/socket.io/user/profile.js new file mode 100644 index 0000000000..9acfb5f3ee --- /dev/null +++ b/lib/socket.io/user/profile.js @@ -0,0 +1,54 @@ +'use strict'; + +const user = require('../../user'); +const privileges = require('../../privileges'); +const plugins = require('../../plugins'); +module.exports = function (SocketUser) { + SocketUser.updateCover = async function (socket, data) { + if (!socket.uid) { + throw new Error('[[error:no-privileges]]'); + } + await user.isAdminOrGlobalModOrSelf(socket.uid, data.uid); + await user.checkMinReputation(socket.uid, data.uid, 'min:rep:cover-picture'); + return await user.updateCoverPicture(data); + }; + SocketUser.uploadCroppedPicture = async function (socket, data) { + if (!socket.uid || !(await privileges.users.canEdit(socket.uid, data.uid))) { + throw new Error('[[error:no-privileges]]'); + } + await user.checkMinReputation(socket.uid, data.uid, 'min:rep:profile-picture'); + data.callerUid = socket.uid; + return await user.uploadCroppedPicture(data); + }; + SocketUser.removeCover = async function (socket, data) { + if (!socket.uid) { + throw new Error('[[error:no-privileges]]'); + } + await user.isAdminOrGlobalModOrSelf(socket.uid, data.uid); + const userData = await user.getUserFields(data.uid, ['cover:url']); + await user.removeCoverPicture(data); + plugins.hooks.fire('action:user.removeCoverPicture', { + callerUid: socket.uid, + uid: data.uid, + user: userData + }); + }; + SocketUser.toggleBlock = async function (socket, data) { + const isBlocked = await user.blocks.is(data.blockeeUid, data.blockerUid); + const { + action, + blockerUid, + blockeeUid + } = data; + if (action !== 'block' && action !== 'unblock') { + throw new Error('[[error:unknow-block-action]]'); + } + await user.blocks.can(socket.uid, blockerUid, blockeeUid, action); + if (data.action === 'block') { + await user.blocks.add(blockeeUid, blockerUid); + } else if (data.action === 'unblock') { + await user.blocks.remove(blockeeUid, blockerUid); + } + return !isBlocked; + }; +}; \ No newline at end of file diff --git a/lib/socket.io/user/registration.js b/lib/socket.io/user/registration.js new file mode 100644 index 0000000000..7d4e2fb12c --- /dev/null +++ b/lib/socket.io/user/registration.js @@ -0,0 +1,40 @@ +'use strict'; + +const user = require('../../user'); +const events = require('../../events'); +module.exports = function (SocketUser) { + SocketUser.acceptRegistration = async function (socket, data) { + const isAdminOrGlobalMod = await user.isAdminOrGlobalMod(socket.uid); + if (!isAdminOrGlobalMod) { + throw new Error('[[error:no-privileges]]'); + } + const uid = await user.acceptRegistration(data.username); + await events.log({ + type: 'registration-approved', + uid: socket.uid, + ip: socket.ip, + targetUid: uid + }); + return uid; + }; + SocketUser.rejectRegistration = async function (socket, data) { + const isAdminOrGlobalMod = await user.isAdminOrGlobalMod(socket.uid); + if (!isAdminOrGlobalMod) { + throw new Error('[[error:no-privileges]]'); + } + await user.rejectRegistration(data.username); + await events.log({ + type: 'registration-rejected', + uid: socket.uid, + ip: socket.ip, + username: data.username + }); + }; + SocketUser.deleteInvitation = async function (socket, data) { + const isAdminOrGlobalMod = await user.isAdminOrGlobalMod(socket.uid); + if (!isAdminOrGlobalMod) { + throw new Error('[[error:no-privileges]]'); + } + await user.deleteInvitation(data.invitedBy, data.email); + }; +}; \ No newline at end of file diff --git a/lib/socket.io/user/status.js b/lib/socket.io/user/status.js new file mode 100644 index 0000000000..3aa09db02d --- /dev/null +++ b/lib/socket.io/user/status.js @@ -0,0 +1,38 @@ +'use strict'; + +const user = require('../../user'); +const websockets = require('../index'); +module.exports = function (SocketUser) { + SocketUser.checkStatus = async function (socket, uid) { + if (!socket.uid) { + throw new Error('[[error:invalid-uid]]'); + } + const userData = await user.getUserFields(uid, ['lastonline', 'status']); + return user.getStatus(userData); + }; + SocketUser.setStatus = async function (socket, status) { + if (socket.uid <= 0) { + throw new Error('[[error:invalid-uid]]'); + } + const allowedStatus = ['online', 'offline', 'dnd', 'away']; + if (!allowedStatus.includes(status)) { + throw new Error('[[error:invalid-user-status]]'); + } + const userData = { + status: status + }; + if (status !== 'offline') { + userData.lastonline = Date.now(); + } + await user.setUserFields(socket.uid, userData); + if (status !== 'offline') { + await user.updateOnlineUsers(socket.uid); + } + const eventData = { + uid: socket.uid, + status: status + }; + websockets.server.emit('event:user_status_change', eventData); + return eventData; + }; +}; \ No newline at end of file diff --git a/lib/start.js b/lib/start.js new file mode 100644 index 0000000000..11f6d36835 --- /dev/null +++ b/lib/start.js @@ -0,0 +1,128 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const start = module.exports; +start.start = async function () { + printStartupInfo(); + addProcessHandlers(); + try { + const db = require('./database'); + await db.init(); + await db.checkCompatibility(); + const meta = require('./meta'); + await meta.configs.init(); + if (nconf.get('runJobs')) { + await runUpgrades(); + } + if (nconf.get('dep-check') === undefined || nconf.get('dep-check') !== false) { + await meta.dependencies.check(); + } else { + winston.warn('[init] Dependency checking skipped!'); + } + await db.initSessionStore(); + const webserver = require('./webserver'); + const sockets = require('./socket.io'); + await sockets.init(webserver.server); + if (nconf.get('runJobs')) { + require('./notifications').startJobs(); + require('./user').startJobs(); + require('./plugins').startJobs(); + require('./topics').scheduled.startJobs(); + await db.delete('locks'); + } + await webserver.listen(); + if (process.send) { + process.send({ + action: 'listening' + }); + } + } catch (err) { + switch (err.message) { + case 'dependencies-out-of-date': + winston.error('One or more of NodeBB\'s dependent packages are out-of-date. Please run the following command to update them:'); + winston.error(' ./nodebb upgrade'); + break; + case 'dependencies-missing': + winston.error('One or more of NodeBB\'s dependent packages are missing. Please run the following command to update them:'); + winston.error(' ./nodebb upgrade'); + break; + default: + winston.error(err.stack); + break; + } + process.exit(); + } +}; +async function runUpgrades() { + const upgrade = require('./upgrade'); + try { + await upgrade.check(); + } catch (err) { + if (err && err.message === 'schema-out-of-date') { + await upgrade.run(); + } else { + throw err; + } + } +} +function printStartupInfo() { + if (nconf.get('isPrimary')) { + winston.info('Initializing NodeBB v%s %s', nconf.get('version'), nconf.get('url')); + const host = nconf.get(`${nconf.get('database')}:host`); + const storeLocation = host ? `at ${host}${!host.includes('/') ? `:${nconf.get(`${nconf.get('database')}:port`)}` : ''}` : ''; + winston.verbose('* using %s store %s', nconf.get('database'), storeLocation); + winston.verbose('* using themes stored in: %s', nconf.get('themes_path')); + } +} +function addProcessHandlers() { + ['SIGTERM', 'SIGINT', 'SIGQUIT'].forEach(signal => { + process.on(signal, () => shutdown()); + }); + process.on('SIGHUP', restart); + process.on('uncaughtException', err => { + winston.error(err.stack); + require('./meta').js.killMinifier(); + shutdown(1); + }); + process.on('message', msg => { + if (msg && Array.isArray(msg.compiling)) { + if (msg.compiling.includes('tpl')) { + const benchpressjs = require('benchpressjs'); + benchpressjs.flush(); + } else if (msg.compiling.includes('lang')) { + const translator = require('./translator'); + translator.flush(); + } + } + }); +} +function restart() { + if (process.send) { + winston.info('[app] Restarting...'); + process.send({ + action: 'restart' + }); + } else { + winston.error('[app] Could not restart server. Shutting down.'); + shutdown(1); + } +} +async function shutdown(code) { + winston.info('[app] Shutdown (SIGTERM/SIGINT/SIGQUIT) Initialised.'); + try { + await require('./webserver').destroy(); + winston.info('[app] Web server closed to connections.'); + await require('./analytics').writeData(); + winston.info('[app] Live analytics saved.'); + const db = require('./database'); + await db.delete('locks'); + await db.close(); + winston.info('[app] Database connection closed.'); + winston.info('[app] Shutdown complete.'); + process.exit(code || 0); + } catch (err) { + winston.error(err.stack); + return process.exit(code || 0); + } +} \ No newline at end of file diff --git a/lib/topics/bookmarks.js b/lib/topics/bookmarks.js new file mode 100644 index 0000000000..610fc24bf4 --- /dev/null +++ b/lib/topics/bookmarks.js @@ -0,0 +1,56 @@ +'use strict'; + +const async = require('async'); +const db = require('../database'); +const user = require('../user'); +module.exports = function (Topics) { + Topics.getUserBookmark = async function (tid, uid) { + if (parseInt(uid, 10) <= 0) { + return null; + } + return await db.sortedSetScore(`tid:${tid}:bookmarks`, uid); + }; + Topics.getUserBookmarks = async function (tids, uid) { + if (parseInt(uid, 10) <= 0) { + return tids.map(() => null); + } + return await db.sortedSetsScore(tids.map(tid => `tid:${tid}:bookmarks`), uid); + }; + Topics.setUserBookmark = async function (tid, uid, index) { + if (parseInt(uid, 10) <= 0) { + return; + } + await db.sortedSetAdd(`tid:${tid}:bookmarks`, index, uid); + }; + Topics.getTopicBookmarks = async function (tid) { + return await db.getSortedSetRangeWithScores(`tid:${tid}:bookmarks`, 0, -1); + }; + Topics.updateTopicBookmarks = async function (tid, pids) { + const maxIndex = await Topics.getPostCount(tid); + const indices = await db.sortedSetRanks(`tid:${tid}:posts`, pids); + const postIndices = indices.map(i => i === null ? 0 : i + 1); + const minIndex = Math.min(...postIndices); + const bookmarks = await Topics.getTopicBookmarks(tid); + const uidData = bookmarks.map(b => ({ + uid: b.value, + bookmark: parseInt(b.score, 10) + })).filter(data => data.bookmark >= minIndex); + await async.eachLimit(uidData, 50, async data => { + let bookmark = Math.min(data.bookmark, maxIndex); + postIndices.forEach(i => { + if (i < data.bookmark) { + bookmark -= 1; + } + }); + bookmark = Math.min(bookmark, maxIndex - pids.length); + if (bookmark === data.bookmark) { + return; + } + const settings = await user.getSettings(data.uid); + if (settings.topicPostSort === 'most_votes') { + return; + } + await Topics.setUserBookmark(tid, data.uid, bookmark); + }); + }; +}; \ No newline at end of file diff --git a/lib/topics/create.js b/lib/topics/create.js new file mode 100644 index 0000000000..ea132a457a --- /dev/null +++ b/lib/topics/create.js @@ -0,0 +1,251 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const utils = require('../utils'); +const slugify = require('../slugify'); +const plugins = require('../plugins'); +const analytics = require('../analytics'); +const user = require('../user'); +const meta = require('../meta'); +const posts = require('../posts'); +const privileges = require('../privileges'); +const categories = require('../categories'); +const translator = require('../translator'); +module.exports = function (Topics) { + Topics.create = async function (data) { + const timestamp = data.timestamp || Date.now(); + const tid = await db.incrObjectField('global', 'nextTid'); + let topicData = { + tid: tid, + uid: data.uid, + cid: data.cid, + mainPid: 0, + title: data.title, + slug: `${tid}/${slugify(data.title) || 'topic'}`, + timestamp: timestamp, + lastposttime: 0, + postcount: 0, + viewcount: 0 + }; + if (Array.isArray(data.tags) && data.tags.length) { + topicData.tags = data.tags.join(','); + } + const result = await plugins.hooks.fire('filter:topic.create', { + topic: topicData, + data: data + }); + topicData = result.topic; + await db.setObject(`topic:${topicData.tid}`, topicData); + const timestampedSortedSetKeys = ['topics:tid', `cid:${topicData.cid}:tids`, `cid:${topicData.cid}:tids:create`, `cid:${topicData.cid}:uid:${topicData.uid}:tids`]; + const scheduled = timestamp > Date.now(); + if (scheduled) { + timestampedSortedSetKeys.push('topics:scheduled'); + } + await Promise.all([db.sortedSetsAdd(timestampedSortedSetKeys, timestamp, topicData.tid), db.sortedSetsAdd(['topics:views', 'topics:posts', 'topics:votes', `cid:${topicData.cid}:tids:votes`, `cid:${topicData.cid}:tids:posts`, `cid:${topicData.cid}:tids:views`], 0, topicData.tid), user.addTopicIdToUser(topicData.uid, topicData.tid, timestamp), db.incrObjectField(`category:${topicData.cid}`, 'topic_count'), db.incrObjectField('global', 'topicCount'), Topics.createTags(data.tags, topicData.tid, timestamp), scheduled ? Promise.resolve() : categories.updateRecentTid(topicData.cid, topicData.tid)]); + if (scheduled) { + await Topics.scheduled.pin(tid, topicData); + } + plugins.hooks.fire('action:topic.save', { + topic: _.clone(topicData), + data: data + }); + return topicData.tid; + }; + Topics.post = async function (data) { + data = await plugins.hooks.fire('filter:topic.post', data); + const { + uid + } = data; + const [categoryExists, canCreate, canTag, isAdmin] = await Promise.all([categories.exists(data.cid), privileges.categories.can('topics:create', data.cid, uid), privileges.categories.can('topics:tag', data.cid, uid), privileges.users.isAdministrator(uid)]); + data.title = String(data.title).trim(); + data.tags = data.tags || []; + data.content = String(data.content || '').trimEnd(); + if (!isAdmin) { + Topics.checkTitle(data.title); + } + await Topics.validateTags(data.tags, data.cid, uid); + data.tags = await Topics.filterTags(data.tags, data.cid); + if (!data.fromQueue && !isAdmin) { + Topics.checkContent(data.content); + if (!(await posts.canUserPostContentWithLinks(uid, data.content))) { + throw new Error(`[[error:not-enough-reputation-to-post-links, ${meta.config['min:rep:post-links']}]]`); + } + } + if (!categoryExists) { + throw new Error('[[error:no-category]]'); + } + if (!canCreate || !canTag && data.tags.length) { + throw new Error('[[error:no-privileges]]'); + } + await guestHandleValid(data); + if (!data.fromQueue) { + await user.isReadyToPost(uid, data.cid); + } + const tid = await Topics.create(data); + let postData = data; + postData.tid = tid; + postData.ip = data.req ? data.req.ip : null; + postData.isMain = true; + postData = await posts.create(postData); + postData = await onNewPost(postData, data); + const [settings, topics] = await Promise.all([user.getSettings(uid), Topics.getTopicsByTids([postData.tid], uid)]); + if (!Array.isArray(topics) || !topics.length) { + throw new Error('[[error:no-topic]]'); + } + if (uid > 0 && settings.followTopicsOnCreate) { + await Topics.follow(postData.tid, uid); + } + const topicData = topics[0]; + topicData.unreplied = true; + topicData.mainPost = postData; + topicData.index = 0; + postData.index = 0; + if (topicData.scheduled) { + await Topics.delete(tid); + } + analytics.increment(['topics', `topics:byCid:${topicData.cid}`]); + plugins.hooks.fire('action:topic.post', { + topic: topicData, + post: postData, + data: data + }); + if (parseInt(uid, 10) && !topicData.scheduled) { + user.notifications.sendTopicNotificationToFollowers(uid, topicData, postData); + Topics.notifyTagFollowers(postData, uid); + categories.notifyCategoryFollowers(postData, uid); + } + return { + topicData: topicData, + postData: postData + }; + }; + Topics.reply = async function (data) { + data = await plugins.hooks.fire('filter:topic.reply', data); + const { + tid + } = data; + const { + uid + } = data; + const [topicData, isAdmin] = await Promise.all([Topics.getTopicData(tid), privileges.users.isAdministrator(uid)]); + await canReply(data, topicData); + data.cid = topicData.cid; + await guestHandleValid(data); + data.content = String(data.content || '').trimEnd(); + if (!data.fromQueue && !isAdmin) { + await user.isReadyToPost(uid, data.cid); + Topics.checkContent(data.content); + if (!(await posts.canUserPostContentWithLinks(uid, data.content))) { + throw new Error(`[[error:not-enough-reputation-to-post-links, ${meta.config['min:rep:post-links']}]]`); + } + } + if (topicData.scheduled) { + data.timestamp = topicData.lastposttime + 1; + } + data.ip = data.req ? data.req.ip : null; + let postData = await posts.create(data); + postData = await onNewPost(postData, data); + const settings = await user.getSettings(uid); + if (uid > 0 && settings.followTopicsOnReply) { + await Topics.follow(postData.tid, uid); + } + if (parseInt(uid, 10)) { + user.setUserField(uid, 'lastonline', Date.now()); + } + if (parseInt(uid, 10) || meta.config.allowGuestReplyNotifications) { + const { + displayname + } = postData.user; + Topics.notifyFollowers(postData, uid, { + type: 'new-reply', + bodyShort: translator.compile('notifications:user-posted-to', displayname, postData.topic.title), + nid: `new_post:tid:${postData.topic.tid}:pid:${postData.pid}:uid:${uid}`, + mergeId: `notifications:user-posted-to|${postData.topic.tid}` + }); + } + analytics.increment(['posts', `posts:byCid:${data.cid}`]); + plugins.hooks.fire('action:topic.reply', { + post: _.clone(postData), + data: data + }); + return postData; + }; + async function onNewPost(postData, data) { + const { + tid, + uid + } = postData; + await Topics.markAsRead([tid], uid); + const [userInfo, topicInfo] = await Promise.all([posts.getUserInfoForPosts([postData.uid], uid), Topics.getTopicFields(tid, ['tid', 'uid', 'title', 'slug', 'cid', 'postcount', 'mainPid', 'scheduled', 'tags']), Topics.addParentPosts([postData]), Topics.syncBacklinks(postData), posts.parsePost(postData)]); + postData.user = userInfo[0]; + postData.topic = topicInfo; + postData.index = topicInfo.postcount - 1; + posts.overrideGuestHandle(postData, data.handle); + postData.votes = 0; + postData.bookmarked = false; + postData.display_edit_tools = true; + postData.display_delete_tools = true; + postData.display_moderator_tools = true; + postData.display_move_tools = true; + postData.selfPost = false; + postData.timestampISO = utils.toISOString(postData.timestamp); + postData.topic.title = String(postData.topic.title); + return postData; + } + Topics.checkTitle = function (title) { + check(title, meta.config.minimumTitleLength, meta.config.maximumTitleLength, 'title-too-short', 'title-too-long'); + }; + Topics.checkContent = function (content) { + check(content, meta.config.minimumPostLength, meta.config.maximumPostLength, 'content-too-short', 'content-too-long'); + }; + function check(item, min, max, minError, maxError) { + if (typeof item === 'string') { + item = utils.stripHTMLTags(item).trim(); + } + if (item === null || item === undefined || item.length < parseInt(min, 10)) { + throw new Error(`[[error:${minError}, ${min}]]`); + } else if (item.length > parseInt(max, 10)) { + throw new Error(`[[error:${maxError}, ${max}]]`); + } + } + async function guestHandleValid(data) { + if (meta.config.allowGuestHandles && parseInt(data.uid, 10) === 0 && data.handle) { + if (data.handle.length > meta.config.maximumUsernameLength) { + throw new Error('[[error:guest-handle-invalid]]'); + } + const exists = await user.existsBySlug(slugify(data.handle)); + if (exists) { + throw new Error('[[error:username-taken]]'); + } + } + } + async function canReply(data, topicData) { + if (!topicData) { + throw new Error('[[error:no-topic]]'); + } + const { + tid, + uid + } = data; + const { + cid, + deleted, + locked, + scheduled + } = topicData; + const [canReply, canSchedule, isAdminOrMod] = await Promise.all([privileges.topics.can('topics:reply', tid, uid), privileges.topics.can('topics:schedule', tid, uid), privileges.categories.isAdminOrMod(cid, uid)]); + if (locked && !isAdminOrMod) { + throw new Error('[[error:topic-locked]]'); + } + if (!scheduled && deleted && !isAdminOrMod) { + throw new Error('[[error:topic-deleted]]'); + } + if (scheduled && !canSchedule) { + throw new Error('[[error:no-privileges]]'); + } + if (!canReply) { + throw new Error('[[error:no-privileges]]'); + } + } +}; \ No newline at end of file diff --git a/lib/topics/data.js b/lib/topics/data.js new file mode 100644 index 0000000000..e0967b14a8 --- /dev/null +++ b/lib/topics/data.js @@ -0,0 +1,111 @@ +'use strict'; + +const validator = require('validator'); +const db = require('../database'); +const categories = require('../categories'); +const utils = require('../utils'); +const translator = require('../translator'); +const plugins = require('../plugins'); +const intFields = ['tid', 'cid', 'uid', 'mainPid', 'postcount', 'viewcount', 'postercount', 'deleted', 'locked', 'pinned', 'pinExpiry', 'timestamp', 'upvotes', 'downvotes', 'lastposttime', 'deleterUid']; +module.exports = function (Topics) { + Topics.getTopicsFields = async function (tids, fields) { + if (!Array.isArray(tids) || !tids.length) { + return []; + } + if (fields.includes('scheduled') && !fields.includes('timestamp')) { + fields.push('timestamp'); + } + const keys = tids.map(tid => `topic:${tid}`); + const topics = await db.getObjects(keys, fields); + const result = await plugins.hooks.fire('filter:topic.getFields', { + tids: tids, + topics: topics, + fields: fields, + keys: keys + }); + result.topics.forEach(topic => modifyTopic(topic, fields)); + return result.topics; + }; + Topics.getTopicField = async function (tid, field) { + const topic = await Topics.getTopicFields(tid, [field]); + return topic ? topic[field] : null; + }; + Topics.getTopicFields = async function (tid, fields) { + const topics = await Topics.getTopicsFields([tid], fields); + return topics ? topics[0] : null; + }; + Topics.getTopicData = async function (tid) { + const topics = await Topics.getTopicsFields([tid], []); + return topics && topics.length ? topics[0] : null; + }; + Topics.getTopicsData = async function (tids) { + return await Topics.getTopicsFields(tids, []); + }; + Topics.getCategoryData = async function (tid) { + const cid = await Topics.getTopicField(tid, 'cid'); + return await categories.getCategoryData(cid); + }; + Topics.setTopicField = async function (tid, field, value) { + await db.setObjectField(`topic:${tid}`, field, value); + }; + Topics.setTopicFields = async function (tid, data) { + await db.setObject(`topic:${tid}`, data); + }; + Topics.deleteTopicField = async function (tid, field) { + await db.deleteObjectField(`topic:${tid}`, field); + }; + Topics.deleteTopicFields = async function (tid, fields) { + await db.deleteObjectFields(`topic:${tid}`, fields); + }; +}; +function escapeTitle(topicData) { + if (topicData) { + if (topicData.title) { + topicData.title = translator.escape(validator.escape(topicData.title)); + } + if (topicData.titleRaw) { + topicData.titleRaw = translator.escape(topicData.titleRaw); + } + } +} +function modifyTopic(topic, fields) { + if (!topic) { + return; + } + db.parseIntFields(topic, intFields, fields); + if (topic.hasOwnProperty('title')) { + topic.titleRaw = topic.title; + topic.title = String(topic.title); + } + escapeTitle(topic); + if (topic.hasOwnProperty('timestamp')) { + topic.timestampISO = utils.toISOString(topic.timestamp); + if (!fields.length || fields.includes('scheduled')) { + topic.scheduled = topic.timestamp > Date.now(); + } + } + if (topic.hasOwnProperty('lastposttime')) { + topic.lastposttimeISO = utils.toISOString(topic.lastposttime); + } + if (topic.hasOwnProperty('pinExpiry')) { + topic.pinExpiryISO = utils.toISOString(topic.pinExpiry); + } + if (topic.hasOwnProperty('upvotes') && topic.hasOwnProperty('downvotes')) { + topic.votes = topic.upvotes - topic.downvotes; + } + if (fields.includes('teaserPid') || !fields.length) { + topic.teaserPid = topic.teaserPid || null; + } + if (fields.includes('tags') || !fields.length) { + const tags = String(topic.tags || ''); + topic.tags = tags.split(',').filter(Boolean).map(tag => { + const escaped = validator.escape(String(tag)); + return { + value: tag, + valueEscaped: escaped, + valueEncoded: encodeURIComponent(escaped), + class: escaped.replace(/\s/g, '-') + }; + }); + } +} \ No newline at end of file diff --git a/lib/topics/delete.js b/lib/topics/delete.js new file mode 100644 index 0000000000..ef668ac397 --- /dev/null +++ b/lib/topics/delete.js @@ -0,0 +1,88 @@ +'use strict'; + +const db = require('../database'); +const user = require('../user'); +const posts = require('../posts'); +const categories = require('../categories'); +const flags = require('../flags'); +const plugins = require('../plugins'); +const batch = require('../batch'); +module.exports = function (Topics) { + Topics.delete = async function (tid, uid) { + const [cid, pids] = await Promise.all([Topics.getTopicField(tid, 'cid'), Topics.getPids(tid)]); + await Promise.all([db.sortedSetRemove(`cid:${cid}:pids`, pids), resolveTopicPostFlags(pids, uid), Topics.setTopicFields(tid, { + deleted: 1, + deleterUid: uid, + deletedTimestamp: Date.now() + })]); + await categories.updateRecentTidForCid(cid); + }; + async function resolveTopicPostFlags(pids, uid) { + await batch.processArray(pids, async pids => { + const postData = await posts.getPostsFields(pids, ['pid', 'flagId']); + const flaggedPosts = postData.filter(p => p && parseInt(p.flagId, 10)); + await Promise.all(flaggedPosts.map(p => flags.update(p.flagId, uid, { + state: 'resolved' + }))); + }, { + batch: 500 + }); + } + async function addTopicPidsToCid(tid, cid) { + const pids = await Topics.getPids(tid); + let postData = await posts.getPostsFields(pids, ['pid', 'timestamp', 'deleted']); + postData = postData.filter(post => post && !post.deleted); + const pidsToAdd = postData.map(post => post.pid); + const scores = postData.map(post => post.timestamp); + await db.sortedSetAdd(`cid:${cid}:pids`, scores, pidsToAdd); + } + Topics.restore = async function (tid) { + const cid = await Topics.getTopicField(tid, 'cid'); + await Promise.all([Topics.deleteTopicFields(tid, ['deleterUid', 'deletedTimestamp']), addTopicPidsToCid(tid, cid)]); + await Topics.setTopicField(tid, 'deleted', 0); + await categories.updateRecentTidForCid(cid); + }; + Topics.purgePostsAndTopic = async function (tid, uid) { + const mainPid = await Topics.getTopicField(tid, 'mainPid'); + await batch.processSortedSet(`tid:${tid}:posts`, async pids => { + await posts.purge(pids, uid); + }, { + alwaysStartAt: 0, + batch: 500 + }); + await posts.purge(mainPid, uid); + await Topics.purge(tid, uid); + }; + Topics.purge = async function (tid, uid) { + const [deletedTopic, tags] = await Promise.all([Topics.getTopicData(tid), Topics.getTopicTags(tid)]); + if (!deletedTopic) { + return; + } + deletedTopic.tags = tags; + await deleteFromFollowersIgnorers(tid); + await Promise.all([db.deleteAll([`tid:${tid}:followers`, `tid:${tid}:ignorers`, `tid:${tid}:posts`, `tid:${tid}:posts:votes`, `tid:${tid}:bookmarks`, `tid:${tid}:posters`]), db.sortedSetsRemove(['topics:tid', 'topics:recent', 'topics:posts', 'topics:views', 'topics:votes', 'topics:scheduled'], tid), deleteTopicFromCategoryAndUser(tid), Topics.deleteTopicTags(tid), Topics.events.purge(tid), Topics.thumbs.deleteAll(tid), reduceCounters(tid)]); + plugins.hooks.fire('action:topic.purge', { + topic: deletedTopic, + uid: uid + }); + await db.delete(`topic:${tid}`); + }; + async function deleteFromFollowersIgnorers(tid) { + const [followers, ignorers] = await Promise.all([db.getSetMembers(`tid:${tid}:followers`), db.getSetMembers(`tid:${tid}:ignorers`)]); + const followerKeys = followers.map(uid => `uid:${uid}:followed_tids`); + const ignorerKeys = ignorers.map(uid => `uid:${uid}ignored_tids`); + await db.sortedSetsRemove(followerKeys.concat(ignorerKeys), tid); + } + async function deleteTopicFromCategoryAndUser(tid) { + const topicData = await Topics.getTopicFields(tid, ['cid', 'uid']); + await Promise.all([db.sortedSetsRemove([`cid:${topicData.cid}:tids`, `cid:${topicData.cid}:tids:pinned`, `cid:${topicData.cid}:tids:create`, `cid:${topicData.cid}:tids:posts`, `cid:${topicData.cid}:tids:lastposttime`, `cid:${topicData.cid}:tids:votes`, `cid:${topicData.cid}:tids:views`, `cid:${topicData.cid}:recent_tids`, `cid:${topicData.cid}:uid:${topicData.uid}:tids`, `uid:${topicData.uid}:topics`], tid), user.decrementUserFieldBy(topicData.uid, 'topiccount', 1)]); + await categories.updateRecentTidForCid(topicData.cid); + } + async function reduceCounters(tid) { + const incr = -1; + await db.incrObjectFieldBy('global', 'topicCount', incr); + const topicData = await Topics.getTopicFields(tid, ['cid', 'postcount']); + const postCountChange = incr * topicData.postcount; + await Promise.all([db.incrObjectFieldBy('global', 'postCount', postCountChange), db.incrObjectFieldBy(`category:${topicData.cid}`, 'post_count', postCountChange), db.incrObjectFieldBy(`category:${topicData.cid}`, 'topic_count', incr)]); + } +}; \ No newline at end of file diff --git a/lib/topics/events.js b/lib/topics/events.js new file mode 100644 index 0000000000..4ed0dbe86c --- /dev/null +++ b/lib/topics/events.js @@ -0,0 +1,214 @@ +'use strict'; + +const _ = require('lodash'); +const nconf = require('nconf'); +const db = require('../database'); +const meta = require('../meta'); +const user = require('../user'); +const posts = require('../posts'); +const categories = require('../categories'); +const plugins = require('../plugins'); +const translator = require('../translator'); +const privileges = require('../privileges'); +const utils = require('../utils'); +const helpers = require('../helpers'); +const relative_path = nconf.get('relative_path'); +const Events = module.exports; +Events._types = { + pin: { + icon: 'fa-thumb-tack', + translation: async (event, language) => translateSimple(event, language, 'topic:user-pinned-topic') + }, + unpin: { + icon: 'fa-thumb-tack fa-rotate-90', + translation: async (event, language) => translateSimple(event, language, 'topic:user-unpinned-topic') + }, + lock: { + icon: 'fa-lock', + translation: async (event, language) => translateSimple(event, language, 'topic:user-locked-topic') + }, + unlock: { + icon: 'fa-unlock', + translation: async (event, language) => translateSimple(event, language, 'topic:user-unlocked-topic') + }, + delete: { + icon: 'fa-trash', + translation: async (event, language) => translateSimple(event, language, 'topic:user-deleted-topic') + }, + restore: { + icon: 'fa-trash-o', + translation: async (event, language) => translateSimple(event, language, 'topic:user-restored-topic') + }, + move: { + icon: 'fa-arrow-circle-right', + translation: async (event, language) => translateEventArgs(event, language, 'topic:user-moved-topic-from', renderUser(event), `${event.fromCategory.name}`, renderTimeago(event)) + }, + 'post-queue': { + icon: 'fa-history', + translation: async (event, language) => translateEventArgs(event, language, 'topic:user-queued-post', renderUser(event), `${relative_path}${event.href}`, renderTimeago(event)) + }, + backlink: { + icon: 'fa-link', + translation: async (event, language) => translateEventArgs(event, language, 'topic:user-referenced-topic', renderUser(event), `${relative_path}${event.href}`, renderTimeago(event)) + }, + fork: { + icon: 'fa-code-fork', + translation: async (event, language) => translateEventArgs(event, language, 'topic:user-forked-topic', renderUser(event), `${relative_path}${event.href}`, renderTimeago(event)) + } +}; +Events.init = async () => { + const { + types + } = await plugins.hooks.fire('filter:topicEvents.init', { + types: Events._types + }); + Events._types = types; +}; +async function translateEventArgs(event, language, prefix, ...args) { + const key = getTranslationKey(event, prefix); + const compiled = translator.compile.apply(null, [key, ...args]); + return utils.decodeHTMLEntities(await translator.translate(compiled, language)); +} +async function translateSimple(event, language, prefix) { + return await translateEventArgs(event, language, prefix, renderUser(event), renderTimeago(event)); +} +Events.translateSimple = translateSimple; +Events.translateEventArgs = translateEventArgs; +function getTranslationKey(event, prefix) { + const cutoffMs = 1000 * 60 * 60 * 24 * Math.max(0, parseInt(meta.config.timeagoCutoff, 10)); + let translationSuffix = 'ago'; + if (cutoffMs > 0 && Date.now() - event.timestamp > cutoffMs) { + translationSuffix = 'on'; + } + return `${prefix}-${translationSuffix}`; +} +function renderUser(event) { + if (!event.user || event.user.system) { + return '[[global:system-user]]'; + } + return `${helpers.buildAvatar(event.user, '16px', true)} ${event.user.username}`; +} +function renderTimeago(event) { + return ``; +} +Events.get = async (tid, uid, reverse = false) => { + if (!tid) { + return []; + } + let eventIds = await db.getSortedSetRangeWithScores(`topic:${tid}:events`, 0, -1); + const keys = eventIds.map(obj => `topicEvent:${obj.value}`); + const timestamps = eventIds.map(obj => obj.score); + eventIds = eventIds.map(obj => obj.value); + let events = await db.getObjects(keys); + events.forEach((e, idx) => { + e.timestamp = timestamps[idx]; + }); + await addEventsFromPostQueue(tid, uid, events); + events = await modifyEvent({ + uid, + events + }); + if (reverse) { + events.reverse(); + } + return events; +}; +async function getUserInfo(uids) { + uids = uids.filter((uid, idx) => !isNaN(parseInt(uid, 10)) && uids.indexOf(uid) === idx); + const userData = await user.getUsersFields(uids, ['picture', 'username', 'userslug']); + const userMap = userData.reduce((memo, cur) => memo.set(cur.uid, cur), new Map()); + userMap.set('system', { + system: true + }); + return userMap; +} +async function getCategoryInfo(cids) { + const uniqCids = _.uniq(cids); + const catData = await categories.getCategoriesFields(uniqCids, ['name', 'slug', 'icon', 'color', 'bgColor']); + return _.zipObject(uniqCids, catData); +} +async function addEventsFromPostQueue(tid, uid, events) { + const isPrivileged = await user.isPrivileged(uid); + if (isPrivileged) { + const queuedPosts = await posts.getQueuedPosts({ + tid + }, { + metadata: false + }); + events.push(...queuedPosts.map(item => ({ + type: 'post-queue', + href: `/post-queue/${item.id}`, + timestamp: item.data.timestamp || Date.now(), + uid: item.data.uid + }))); + } +} +async function modifyEvent({ + uid, + events +}) { + const [users, fromCategories, userSettings] = await Promise.all([getUserInfo(events.map(event => event.uid).filter(Boolean)), getCategoryInfo(events.map(event => event.fromCid).filter(Boolean)), user.getSettings(uid)]); + if (meta.config.topicBacklinks !== 1) { + events = events.filter(event => event.type !== 'backlink'); + } else { + const backlinkPids = events.filter(e => e.type === 'backlink').map(e => e.href.split('/').pop()); + const pids = await privileges.posts.filter('topics:read', backlinkPids, uid); + events = events.filter(e => e.type !== 'backlink' || pids.includes(e.href.split('/').pop())); + } + events = events.filter(event => Events._types.hasOwnProperty(event.type)); + events.forEach(event => { + event.timestampISO = utils.toISOString(event.timestamp); + if (event.hasOwnProperty('uid')) { + event.user = users.get(event.uid === 'system' ? 'system' : parseInt(event.uid, 10)); + } + if (event.hasOwnProperty('fromCid')) { + event.fromCategory = fromCategories[event.fromCid]; + } + Object.assign(event, Events._types[event.type]); + }); + await Promise.all(events.map(async event => { + if (Events._types[event.type].translation) { + event.text = await Events._types[event.type].translation(event, userSettings.userLang); + } + })); + events.sort((a, b) => a.timestamp - b.timestamp); + return events; +} +Events.log = async (tid, payload) => { + const topics = require('.'); + const { + type + } = payload; + const timestamp = payload.timestamp || Date.now(); + if (!Events._types.hasOwnProperty(type)) { + throw new Error(`[[error:topic-event-unrecognized, ${type}]]`); + } else if (!(await topics.exists(tid))) { + throw new Error('[[error:no-topic]]'); + } + const eventId = await db.incrObjectField('global', 'nextTopicEventId'); + payload.id = eventId; + await Promise.all([db.setObject(`topicEvent:${eventId}`, payload), db.sortedSetAdd(`topic:${tid}:events`, timestamp, eventId)]); + payload.timestamp = timestamp; + let events = await modifyEvent({ + uid: payload.uid, + events: [payload] + }); + ({ + events + } = await plugins.hooks.fire('filter:topic.events.log', { + events + })); + return events; +}; +Events.purge = async (tid, eventIds = []) => { + if (eventIds.length) { + const isTopicEvent = await db.isSortedSetMembers(`topic:${tid}:events`, eventIds); + eventIds = eventIds.filter((id, index) => isTopicEvent[index]); + await Promise.all([db.sortedSetRemove(`topic:${tid}:events`, eventIds), db.deleteAll(eventIds.map(id => `topicEvent:${id}`))]); + } else { + const keys = [`topic:${tid}:events`]; + const eventIds = await db.getSortedSetRange(keys[0], 0, -1); + keys.push(...eventIds.map(id => `topicEvent:${id}`)); + await db.deleteAll(keys); + } +}; \ No newline at end of file diff --git a/lib/topics/follow.js b/lib/topics/follow.js new file mode 100644 index 0000000000..e805908e5a --- /dev/null +++ b/lib/topics/follow.js @@ -0,0 +1,158 @@ +'use strict'; + +const db = require('../database'); +const notifications = require('../notifications'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const utils = require('../utils'); +module.exports = function (Topics) { + Topics.toggleFollow = async function (tid, uid) { + const exists = await Topics.exists(tid); + if (!exists) { + throw new Error('[[error:no-topic]]'); + } + const isFollowing = await Topics.isFollowing([tid], uid); + if (isFollowing[0]) { + await Topics.unfollow(tid, uid); + } else { + await Topics.follow(tid, uid); + } + return !isFollowing[0]; + }; + Topics.follow = async function (tid, uid) { + await setWatching(follow, unignore, 'action:topic.follow', tid, uid); + }; + Topics.unfollow = async function (tid, uid) { + await setWatching(unfollow, unignore, 'action:topic.unfollow', tid, uid); + }; + Topics.ignore = async function (tid, uid) { + await setWatching(ignore, unfollow, 'action:topic.ignore', tid, uid); + }; + async function setWatching(method1, method2, hook, tid, uid) { + if (!(parseInt(uid, 10) > 0)) { + throw new Error('[[error:not-logged-in]]'); + } + const exists = await Topics.exists(tid); + if (!exists) { + throw new Error('[[error:no-topic]]'); + } + await method1(tid, uid); + await method2(tid, uid); + plugins.hooks.fire(hook, { + uid: uid, + tid: tid + }); + } + async function follow(tid, uid) { + await addToSets(`tid:${tid}:followers`, `uid:${uid}:followed_tids`, tid, uid); + } + async function unfollow(tid, uid) { + await removeFromSets(`tid:${tid}:followers`, `uid:${uid}:followed_tids`, tid, uid); + } + async function ignore(tid, uid) { + await addToSets(`tid:${tid}:ignorers`, `uid:${uid}:ignored_tids`, tid, uid); + } + async function unignore(tid, uid) { + await removeFromSets(`tid:${tid}:ignorers`, `uid:${uid}:ignored_tids`, tid, uid); + } + async function addToSets(set1, set2, tid, uid) { + await db.setAdd(set1, uid); + await db.sortedSetAdd(set2, Date.now(), tid); + } + async function removeFromSets(set1, set2, tid, uid) { + await db.setRemove(set1, uid); + await db.sortedSetRemove(set2, tid); + } + Topics.isFollowing = async function (tids, uid) { + return await isIgnoringOrFollowing('followers', tids, uid); + }; + Topics.isIgnoring = async function (tids, uid) { + return await isIgnoringOrFollowing('ignorers', tids, uid); + }; + Topics.getFollowData = async function (tids, uid) { + if (!Array.isArray(tids)) { + return; + } + if (parseInt(uid, 10) <= 0) { + return tids.map(() => ({ + following: false, + ignoring: false + })); + } + const keys = []; + tids.forEach(tid => keys.push(`tid:${tid}:followers`, `tid:${tid}:ignorers`)); + const data = await db.isMemberOfSets(keys, uid); + const followData = []; + for (let i = 0; i < data.length; i += 2) { + followData.push({ + following: data[i], + ignoring: data[i + 1] + }); + } + return followData; + }; + async function isIgnoringOrFollowing(set, tids, uid) { + if (!Array.isArray(tids)) { + return; + } + if (parseInt(uid, 10) <= 0) { + return tids.map(() => false); + } + const keys = tids.map(tid => `tid:${tid}:${set}`); + return await db.isMemberOfSets(keys, uid); + } + Topics.getFollowers = async function (tid) { + return await db.getSetMembers(`tid:${tid}:followers`); + }; + Topics.getIgnorers = async function (tid) { + return await db.getSetMembers(`tid:${tid}:ignorers`); + }; + Topics.filterIgnoringUids = async function (tid, uids) { + const isIgnoring = await db.isSetMembers(`tid:${tid}:ignorers`, uids); + const readingUids = uids.filter((uid, index) => uid && !isIgnoring[index]); + return readingUids; + }; + Topics.filterWatchedTids = async function (tids, uid) { + if (parseInt(uid, 10) <= 0) { + return []; + } + const scores = await db.sortedSetScores(`uid:${uid}:followed_tids`, tids); + return tids.filter((tid, index) => tid && !!scores[index]); + }; + Topics.filterNotIgnoredTids = async function (tids, uid) { + if (parseInt(uid, 10) <= 0) { + return tids; + } + const scores = await db.sortedSetScores(`uid:${uid}:ignored_tids`, tids); + return tids.filter((tid, index) => tid && !scores[index]); + }; + Topics.notifyFollowers = async function (postData, exceptUid, notifData) { + notifData = notifData || {}; + let followers = await Topics.getFollowers(postData.topic.tid); + const index = followers.indexOf(String(exceptUid)); + if (index !== -1) { + followers.splice(index, 1); + } + followers = await privileges.topics.filterUids('topics:read', postData.topic.tid, followers); + if (!followers.length) { + return; + } + let { + title + } = postData.topic; + if (title) { + title = utils.decodeHTMLEntities(title); + } + const notification = await notifications.create({ + subject: title, + bodyLong: postData.content, + pid: postData.pid, + path: `/post/${postData.pid}`, + tid: postData.topic.tid, + from: exceptUid, + topicTitle: title, + ...notifData + }); + notifications.push(notification, followers); + }; +}; \ No newline at end of file diff --git a/lib/topics/fork.js b/lib/topics/fork.js new file mode 100644 index 0000000000..513a40ec9a --- /dev/null +++ b/lib/topics/fork.js @@ -0,0 +1,123 @@ +'use strict'; + +const db = require('../database'); +const posts = require('../posts'); +const categories = require('../categories'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const meta = require('../meta'); +module.exports = function (Topics) { + Topics.createTopicFromPosts = async function (uid, title, pids, fromTid, cid) { + if (title) { + title = title.trim(); + } + if (title.length < meta.config.minimumTitleLength) { + throw new Error(`[[error:title-too-short, ${meta.config.minimumTitleLength}]]`); + } else if (title.length > meta.config.maximumTitleLength) { + throw new Error(`[[error:title-too-long, ${meta.config.maximumTitleLength}]]`); + } + if (!pids || !pids.length) { + throw new Error('[[error:invalid-pid]]'); + } + pids.sort((a, b) => a - b); + const mainPid = pids[0]; + if (!cid) { + cid = await posts.getCidByPid(mainPid); + } + const [postData, isAdminOrMod] = await Promise.all([posts.getPostData(mainPid), privileges.categories.isAdminOrMod(cid, uid)]); + if (!isAdminOrMod) { + throw new Error('[[error:no-privileges]]'); + } + const scheduled = postData.timestamp > Date.now(); + const params = { + uid: postData.uid, + title: title, + cid: cid, + timestamp: scheduled && postData.timestamp + }; + const result = await plugins.hooks.fire('filter:topic.fork', { + params: params, + tid: postData.tid + }); + const tid = await Topics.create(result.params); + await Topics.updateTopicBookmarks(fromTid, pids); + for (const pid of pids) { + const canEdit = await privileges.posts.canEdit(pid, uid); + if (!canEdit.flag) { + throw new Error(canEdit.message); + } + await Topics.movePostToTopic(uid, pid, tid, scheduled); + } + await Topics.updateLastPostTime(tid, scheduled ? postData.timestamp + 1 : Date.now()); + await Promise.all([Topics.setTopicFields(tid, { + upvotes: postData.upvotes, + downvotes: postData.downvotes, + forkedFromTid: fromTid, + forkerUid: uid, + forkTimestamp: Date.now() + }), db.sortedSetsAdd(['topics:votes', `cid:${cid}:tids:votes`], postData.votes, tid), Topics.events.log(fromTid, { + type: 'fork', + uid, + href: `/topic/${tid}` + })]); + plugins.hooks.fire('action:topic.fork', { + tid: tid, + fromTid: fromTid, + uid: uid + }); + return await Topics.getTopicData(tid); + }; + Topics.movePostToTopic = async function (callerUid, pid, tid, forceScheduled = false) { + tid = parseInt(tid, 10); + const topicData = await Topics.getTopicFields(tid, ['tid', 'scheduled']); + if (!topicData.tid) { + throw new Error('[[error:no-topic]]'); + } + if (!forceScheduled && topicData.scheduled) { + throw new Error('[[error:cant-move-posts-to-scheduled]]'); + } + const postData = await posts.getPostFields(pid, ['tid', 'uid', 'timestamp', 'upvotes', 'downvotes']); + if (!postData || !postData.tid) { + throw new Error('[[error:no-post]]'); + } + const isSourceTopicScheduled = await Topics.getTopicField(postData.tid, 'scheduled'); + if (!forceScheduled && isSourceTopicScheduled) { + throw new Error('[[error:cant-move-from-scheduled-to-existing]]'); + } + if (postData.tid === tid) { + throw new Error('[[error:cant-move-to-same-topic]]'); + } + postData.pid = pid; + await Topics.removePostFromTopic(postData.tid, postData); + await Promise.all([updateCategory(postData, tid), posts.setPostField(pid, 'tid', tid), Topics.addPostToTopic(tid, postData)]); + await Promise.all([Topics.updateLastPostTimeFromLastPid(tid), Topics.updateLastPostTimeFromLastPid(postData.tid)]); + plugins.hooks.fire('action:post.move', { + uid: callerUid, + post: postData, + tid: tid + }); + }; + async function updateCategory(postData, toTid) { + const topicData = await Topics.getTopicsFields([postData.tid, toTid], ['cid', 'pinned']); + if (!topicData[0].cid || !topicData[1].cid) { + return; + } + if (!topicData[0].pinned) { + await db.sortedSetIncrBy(`cid:${topicData[0].cid}:tids:posts`, -1, postData.tid); + } + if (!topicData[1].pinned) { + await db.sortedSetIncrBy(`cid:${topicData[1].cid}:tids:posts`, 1, toTid); + } + if (topicData[0].cid === topicData[1].cid) { + await categories.updateRecentTidForCid(topicData[0].cid); + return; + } + const removeFrom = [`cid:${topicData[0].cid}:pids`, `cid:${topicData[0].cid}:uid:${postData.uid}:pids`, `cid:${topicData[0].cid}:uid:${postData.uid}:pids:votes`]; + const tasks = [db.incrObjectFieldBy(`category:${topicData[0].cid}`, 'post_count', -1), db.incrObjectFieldBy(`category:${topicData[1].cid}`, 'post_count', 1), db.sortedSetRemove(removeFrom, postData.pid), db.sortedSetAdd(`cid:${topicData[1].cid}:pids`, postData.timestamp, postData.pid), db.sortedSetAdd(`cid:${topicData[1].cid}:uid:${postData.uid}:pids`, postData.timestamp, postData.pid)]; + if (postData.votes > 0 || postData.votes < 0) { + tasks.push(db.sortedSetAdd(`cid:${topicData[1].cid}:uid:${postData.uid}:pids:votes`, postData.votes, postData.pid)); + } + await Promise.all(tasks); + await Promise.all([categories.updateRecentTidForCid(topicData[0].cid), categories.updateRecentTidForCid(topicData[1].cid)]); + } +}; \ No newline at end of file diff --git a/lib/topics/index.js b/lib/topics/index.js new file mode 100644 index 0000000000..51fa4318be --- /dev/null +++ b/lib/topics/index.js @@ -0,0 +1,236 @@ +'use strict'; + +const _ = require('lodash'); +const validator = require('validator'); +const db = require('../database'); +const posts = require('../posts'); +const utils = require('../utils'); +const plugins = require('../plugins'); +const meta = require('../meta'); +const user = require('../user'); +const categories = require('../categories'); +const privileges = require('../privileges'); +const social = require('../social'); +const Topics = module.exports; +require('./data')(Topics); +require('./create')(Topics); +require('./delete')(Topics); +require('./sorted')(Topics); +require('./unread')(Topics); +require('./recent')(Topics); +require('./user')(Topics); +require('./fork')(Topics); +require('./posts')(Topics); +require('./follow')(Topics); +require('./tags')(Topics); +require('./teaser')(Topics); +Topics.scheduled = require('./scheduled'); +require('./suggested')(Topics); +require('./tools')(Topics); +Topics.thumbs = require('./thumbs'); +require('./bookmarks')(Topics); +require('./merge')(Topics); +Topics.events = require('./events'); +Topics.exists = async function (tids) { + return await db.exists(Array.isArray(tids) ? tids.map(tid => `topic:${tid}`) : `topic:${tids}`); +}; +Topics.getTopicsFromSet = async function (set, uid, start, stop) { + const tids = await db.getSortedSetRevRange(set, start, stop); + const topics = await Topics.getTopics(tids, uid); + Topics.calculateTopicIndices(topics, start); + return { + topics: topics, + nextStart: stop + 1 + }; +}; +Topics.getTopics = async function (tids, options) { + let uid = options; + if (typeof options === 'object') { + uid = options.uid; + } + tids = await privileges.topics.filterTids('topics:read', tids, uid); + return await Topics.getTopicsByTids(tids, options); +}; +Topics.getTopicsByTids = async function (tids, options) { + if (!Array.isArray(tids) || !tids.length) { + return []; + } + let uid = options; + if (typeof options === 'object') { + uid = options.uid; + } + async function loadTopics() { + const topics = await Topics.getTopicsData(tids); + const uids = _.uniq(topics.map(t => t && t.uid && t.uid.toString()).filter(v => utils.isNumber(v))); + const cids = _.uniq(topics.map(t => t && t.cid && t.cid.toString()).filter(v => utils.isNumber(v))); + const guestTopics = topics.filter(t => t && t.uid === 0); + async function loadGuestHandles() { + const mainPids = guestTopics.map(t => t.mainPid); + const postData = await posts.getPostsFields(mainPids, ['handle']); + return postData.map(p => p.handle); + } + async function loadShowfullnameSettings() { + if (meta.config.hideFullname) { + return uids.map(() => ({ + showfullname: false + })); + } + const data = await db.getObjectsFields(uids.map(uid => `user:${uid}:settings`), ['showfullname']); + data.forEach(settings => { + settings.showfullname = parseInt(settings.showfullname, 10) === 1; + }); + return data; + } + const [teasers, users, userSettings, categoriesData, guestHandles, thumbs] = await Promise.all([Topics.getTeasers(topics, options), user.getUsersFields(uids, ['uid', 'username', 'fullname', 'userslug', 'reputation', 'postcount', 'picture', 'signature', 'banned', 'status']), loadShowfullnameSettings(), categories.getCategoriesFields(cids, ['cid', 'name', 'slug', 'icon', 'backgroundImage', 'imageClass', 'bgColor', 'color', 'disabled']), loadGuestHandles(), Topics.thumbs.load(topics)]); + users.forEach((userObj, idx) => { + if (!userSettings[idx].showfullname) { + userObj.fullname = undefined; + } + }); + return { + topics, + teasers, + usersMap: _.zipObject(uids, users), + categoriesMap: _.zipObject(cids, categoriesData), + tidToGuestHandle: _.zipObject(guestTopics.map(t => t.tid), guestHandles), + thumbs + }; + } + const [result, hasRead, followData, bookmarks, callerSettings] = await Promise.all([loadTopics(), Topics.hasReadTopics(tids, uid), Topics.getFollowData(tids, uid), Topics.getUserBookmarks(tids, uid), user.getSettings(uid)]); + const sortNewToOld = callerSettings.topicPostSort === 'newest_to_oldest'; + result.topics.forEach((topic, i) => { + if (topic) { + topic.thumbs = result.thumbs[i]; + topic.category = result.categoriesMap[topic.cid]; + topic.user = topic.uid ? result.usersMap[topic.uid] : { + ...result.usersMap[topic.uid] + }; + if (result.tidToGuestHandle[topic.tid]) { + topic.user.username = validator.escape(result.tidToGuestHandle[topic.tid]); + topic.user.displayname = topic.user.username; + } + topic.teaser = result.teasers[i] || null; + topic.isOwner = topic.uid === parseInt(uid, 10); + topic.ignored = followData[i].ignoring; + topic.followed = followData[i].following; + topic.unread = parseInt(uid, 10) <= 0 || !hasRead[i] && !topic.ignored; + topic.bookmark = bookmarks[i] && (sortNewToOld ? Math.max(1, topic.postcount + 2 - bookmarks[i]) : Math.min(topic.postcount, bookmarks[i] + 1)); + topic.unreplied = !topic.teaser; + topic.icons = []; + } + }); + const filteredTopics = result.topics.filter(topic => topic && topic.category && !topic.category.disabled); + const hookResult = await plugins.hooks.fire('filter:topics.get', { + topics: filteredTopics, + uid: uid + }); + return hookResult.topics; +}; +Topics.getTopicWithPosts = async function (topicData, set, uid, start, stop, reverse) { + const [posts, category, tagWhitelist, threadTools, followData, bookmark, postSharing, deleter, merger, forker, related, thumbs, events] = await Promise.all([Topics.getTopicPosts(topicData, set, start, stop, uid, reverse), categories.getCategoryData(topicData.cid), categories.getTagWhitelist([topicData.cid]), plugins.hooks.fire('filter:topic.thread_tools', { + topic: topicData, + uid: uid, + tools: [] + }), Topics.getFollowData([topicData.tid], uid), Topics.getUserBookmark(topicData.tid, uid), social.getActivePostSharing(), getDeleter(topicData), getMerger(topicData), getForker(topicData), Topics.getRelatedTopics(topicData, uid), Topics.thumbs.load([topicData]), Topics.events.get(topicData.tid, uid, reverse)]); + topicData.thumbs = thumbs[0]; + topicData.posts = posts; + topicData.events = events; + topicData.posts.forEach(p => { + p.events = events.filter(event => event.timestamp >= p.eventStart && event.timestamp < p.eventEnd); + p.eventStart = undefined; + p.eventEnd = undefined; + }); + topicData.category = category; + topicData.tagWhitelist = tagWhitelist[0]; + topicData.minTags = category.minTags; + topicData.maxTags = category.maxTags; + topicData.thread_tools = threadTools.tools; + topicData.isFollowing = followData[0].following; + topicData.isNotFollowing = !followData[0].following && !followData[0].ignoring; + topicData.isIgnoring = followData[0].ignoring; + topicData.bookmark = bookmark; + topicData.postSharing = postSharing; + topicData.deleter = deleter; + if (deleter) { + topicData.deletedTimestampISO = utils.toISOString(topicData.deletedTimestamp); + } + topicData.merger = merger; + if (merger) { + topicData.mergedTimestampISO = utils.toISOString(topicData.mergedTimestamp); + } + topicData.forker = forker; + if (forker) { + topicData.forkTimestampISO = utils.toISOString(topicData.forkTimestamp); + } + topicData.related = related || []; + topicData.unreplied = topicData.postcount === 1; + topicData.icons = []; + const result = await plugins.hooks.fire('filter:topic.get', { + topic: topicData, + uid: uid + }); + return result.topic; +}; +async function getDeleter(topicData) { + if (!parseInt(topicData.deleterUid, 10)) { + return null; + } + return await user.getUserFields(topicData.deleterUid, ['username', 'userslug', 'picture']); +} +async function getMerger(topicData) { + if (!parseInt(topicData.mergerUid, 10)) { + return null; + } + const [merger, mergedIntoTitle] = await Promise.all([user.getUserFields(topicData.mergerUid, ['username', 'userslug', 'picture']), Topics.getTopicField(topicData.mergeIntoTid, 'title')]); + merger.mergedIntoTitle = mergedIntoTitle; + return merger; +} +async function getForker(topicData) { + if (!parseInt(topicData.forkerUid, 10)) { + return null; + } + const [forker, forkedFromTitle] = await Promise.all([user.getUserFields(topicData.forkerUid, ['username', 'userslug', 'picture']), Topics.getTopicField(topicData.forkedFromTid, 'title')]); + forker.forkedFromTitle = forkedFromTitle; + return forker; +} +Topics.getMainPost = async function (tid, uid) { + const mainPosts = await Topics.getMainPosts([tid], uid); + return Array.isArray(mainPosts) && mainPosts.length ? mainPosts[0] : null; +}; +Topics.getMainPids = async function (tids) { + if (!Array.isArray(tids) || !tids.length) { + return []; + } + const topicData = await Topics.getTopicsFields(tids, ['mainPid']); + return topicData.map(topic => topic && topic.mainPid); +}; +Topics.getMainPosts = async function (tids, uid) { + const mainPids = await Topics.getMainPids(tids); + return await getMainPosts(mainPids, uid); +}; +async function getMainPosts(mainPids, uid) { + let postData = await posts.getPostsByPids(mainPids, uid); + postData = await user.blocks.filter(uid, postData); + postData.forEach(post => { + if (post) { + post.index = 0; + } + }); + return await Topics.addPostData(postData, uid); +} +Topics.isLocked = async function (tid) { + const locked = await Topics.getTopicField(tid, 'locked'); + return locked === 1; +}; +Topics.search = async function (tid, term) { + if (!tid || !term) { + throw new Error('[[error:invalid-data]]'); + } + const result = await plugins.hooks.fire('filter:topic.search', { + tid: tid, + term: term, + ids: [] + }); + return Array.isArray(result) ? result : result.ids; +}; +require('../promisify')(Topics); \ No newline at end of file diff --git a/lib/topics/merge.js b/lib/topics/merge.js new file mode 100644 index 0000000000..ec2c015b79 --- /dev/null +++ b/lib/topics/merge.js @@ -0,0 +1,64 @@ +'use strict'; + +const plugins = require('../plugins'); +const posts = require('../posts'); +module.exports = function (Topics) { + Topics.merge = async function (tids, uid, options) { + options = options || {}; + const topicsData = await Topics.getTopicsFields(tids, ['scheduled']); + if (topicsData.some(t => t.scheduled)) { + throw new Error('[[error:cant-merge-scheduled]]'); + } + const oldestTid = findOldestTopic(tids); + let mergeIntoTid = oldestTid; + if (options.mainTid) { + mergeIntoTid = options.mainTid; + } else if (options.newTopicTitle) { + mergeIntoTid = await createNewTopic(options.newTopicTitle, oldestTid); + } + const otherTids = tids.sort((a, b) => a - b).filter(tid => tid && parseInt(tid, 10) !== parseInt(mergeIntoTid, 10)); + for (const tid of otherTids) { + const pids = await Topics.getPids(tid); + for (const pid of pids) { + await Topics.movePostToTopic(uid, pid, mergeIntoTid); + } + await Topics.setTopicField(tid, 'mainPid', 0); + await Topics.delete(tid, uid); + await Topics.setTopicFields(tid, { + mergeIntoTid: mergeIntoTid, + mergerUid: uid, + mergedTimestamp: Date.now() + }); + } + await Promise.all([posts.updateQueuedPostsTopic(mergeIntoTid, otherTids), updateViewCount(mergeIntoTid, tids)]); + plugins.hooks.fire('action:topic.merge', { + uid: uid, + tids: tids, + mergeIntoTid: mergeIntoTid, + otherTids: otherTids + }); + return mergeIntoTid; + }; + async function createNewTopic(title, oldestTid) { + const topicData = await Topics.getTopicFields(oldestTid, ['uid', 'cid']); + const params = { + uid: topicData.uid, + cid: topicData.cid, + title: title + }; + const result = await plugins.hooks.fire('filter:topic.mergeCreateNewTopic', { + oldestTid: oldestTid, + params: params + }); + const tid = await Topics.create(result.params); + return tid; + } + async function updateViewCount(mergeIntoTid, tids) { + const topicData = await Topics.getTopicsFields(tids, ['viewcount']); + const totalViewCount = topicData.reduce((count, topic) => count + parseInt(topic.viewcount, 10), 0); + await Topics.setTopicField(mergeIntoTid, 'viewcount', totalViewCount); + } + function findOldestTopic(tids) { + return Math.min.apply(null, tids); + } +}; \ No newline at end of file diff --git a/lib/topics/posts.js b/lib/topics/posts.js new file mode 100644 index 0000000000..1e2b93a0d1 --- /dev/null +++ b/lib/topics/posts.js @@ -0,0 +1,353 @@ +'use strict'; + +const _ = require('lodash'); +const validator = require('validator'); +const nconf = require('nconf'); +const db = require('../database'); +const user = require('../user'); +const posts = require('../posts'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const backlinkRegex = new RegExp(`(?:${nconf.get('url').replace('/', '\\/')}|\b|\\s)\\/topic\\/(\\d+)(?:\\/\\w+)?`, 'g'); +module.exports = function (Topics) { + Topics.onNewPostMade = async function (postData) { + await Topics.updateLastPostTime(postData.tid, postData.timestamp); + await Topics.addPostToTopic(postData.tid, postData); + }; + Topics.getTopicPosts = async function (topicData, set, start, stop, uid, reverse) { + if (!topicData) { + return []; + } + let repliesStart = start; + let repliesStop = stop; + if (stop > 0) { + repliesStop -= 1; + if (start > 0) { + repliesStart -= 1; + } + } + let pids = []; + if (start !== 0 || stop !== 0) { + pids = await posts.getPidsFromSet(set, repliesStart, repliesStop, reverse); + } + if (!pids.length && !topicData.mainPid) { + return []; + } + if (topicData.mainPid && start === 0) { + pids.unshift(topicData.mainPid); + } + let postData = await posts.getPostsByPids(pids, uid); + if (!postData.length) { + return []; + } + let replies = postData; + if (topicData.mainPid && start === 0) { + postData[0].index = 0; + replies = postData.slice(1); + } + Topics.calculatePostIndices(replies, repliesStart); + await addEventStartEnd(postData, set, reverse, topicData); + const allPosts = postData.slice(); + postData = await user.blocks.filter(uid, postData); + if (allPosts.length !== postData.length) { + const includedPids = new Set(postData.map(p => p.pid)); + allPosts.reverse().forEach((p, index) => { + if (!includedPids.has(p.pid) && allPosts[index + 1] && !reverse) { + allPosts[index + 1].eventEnd = p.eventEnd; + } + }); + } + const result = await plugins.hooks.fire('filter:topic.getPosts', { + topic: topicData, + uid: uid, + posts: await Topics.addPostData(postData, uid) + }); + return result.posts; + }; + async function addEventStartEnd(postData, set, reverse, topicData) { + if (!postData.length) { + return; + } + postData.forEach((p, index) => { + if (p && p.index === 0 && reverse) { + p.eventStart = topicData.lastposttime; + p.eventEnd = Date.now(); + } else if (p && postData[index + 1]) { + p.eventStart = reverse ? postData[index + 1].timestamp : p.timestamp; + p.eventEnd = reverse ? p.timestamp : postData[index + 1].timestamp; + } + }); + const lastPost = postData[postData.length - 1]; + if (lastPost) { + lastPost.eventStart = reverse ? topicData.timestamp : lastPost.timestamp; + lastPost.eventEnd = reverse ? lastPost.timestamp : Date.now(); + if (lastPost.index) { + const nextPost = await db[reverse ? 'getSortedSetRevRangeWithScores' : 'getSortedSetRangeWithScores'](set, lastPost.index, lastPost.index); + if (reverse) { + lastPost.eventStart = nextPost.length ? nextPost[0].score : lastPost.eventStart; + } else { + lastPost.eventEnd = nextPost.length ? nextPost[0].score : lastPost.eventEnd; + } + } + } + } + Topics.addPostData = async function (postData, uid) { + if (!Array.isArray(postData) || !postData.length) { + return []; + } + const pids = postData.map(post => post && post.pid); + async function getPostUserData(field, method) { + const uids = _.uniq(postData.filter(p => p && parseInt(p[field], 10) >= 0).map(p => p[field])); + const userData = await method(uids); + return _.zipObject(uids, userData); + } + const [bookmarks, voteData, userData, editors, replies] = await Promise.all([posts.hasBookmarked(pids, uid), posts.getVoteStatusByPostIDs(pids, uid), getPostUserData('uid', async uids => await posts.getUserInfoForPosts(uids, uid)), getPostUserData('editor', async uids => await user.getUsersFields(uids, ['uid', 'username', 'userslug'])), getPostReplies(postData, uid), Topics.addParentPosts(postData)]); + postData.forEach((postObj, i) => { + if (postObj) { + postObj.user = postObj.uid ? userData[postObj.uid] : { + ...userData[postObj.uid] + }; + postObj.editor = postObj.editor ? editors[postObj.editor] : null; + postObj.bookmarked = bookmarks[i]; + postObj.upvoted = voteData.upvotes[i]; + postObj.downvoted = voteData.downvotes[i]; + postObj.votes = postObj.votes || 0; + postObj.replies = replies[i]; + postObj.selfPost = parseInt(uid, 10) > 0 && parseInt(uid, 10) === postObj.uid; + if (meta.config.allowGuestHandles && postObj.uid === 0 && postObj.handle) { + postObj.user.username = validator.escape(String(postObj.handle)); + postObj.user.displayname = postObj.user.username; + } + } + }); + const result = await plugins.hooks.fire('filter:topics.addPostData', { + posts: postData, + uid: uid + }); + return result.posts; + }; + Topics.modifyPostsByPrivilege = function (topicData, topicPrivileges) { + const loggedIn = parseInt(topicPrivileges.uid, 10) > 0; + topicData.posts.forEach(post => { + if (post) { + post.topicOwnerPost = parseInt(topicData.uid, 10) === parseInt(post.uid, 10); + post.display_edit_tools = topicPrivileges.isAdminOrMod || post.selfPost && topicPrivileges['posts:edit']; + post.display_delete_tools = topicPrivileges.isAdminOrMod || post.selfPost && topicPrivileges['posts:delete']; + post.display_moderator_tools = post.display_edit_tools || post.display_delete_tools; + post.display_move_tools = topicPrivileges.isAdminOrMod && post.index !== 0; + post.display_post_menu = topicPrivileges.isAdminOrMod || post.selfPost && !topicData.locked && !post.deleted || post.selfPost && post.deleted && parseInt(post.deleterUid, 10) === parseInt(topicPrivileges.uid, 10) || (loggedIn || topicData.postSharing.length) && !post.deleted; + post.ip = topicPrivileges.isAdminOrMod ? post.ip : undefined; + posts.modifyPostByPrivilege(post, topicPrivileges); + } + }); + }; + Topics.addParentPosts = async function (postData) { + let parentPids = postData.map(postObj => postObj && postObj.hasOwnProperty('toPid') ? parseInt(postObj.toPid, 10) : null).filter(Boolean); + if (!parentPids.length) { + return; + } + parentPids = _.uniq(parentPids); + const parentPosts = await posts.getPostsFields(parentPids, ['uid']); + const parentUids = _.uniq(parentPosts.map(postObj => postObj && postObj.uid)); + const userData = await user.getUsersFields(parentUids, ['username']); + const usersMap = _.zipObject(parentUids, userData); + const parents = {}; + parentPosts.forEach((post, i) => { + if (usersMap[post.uid]) { + parents[parentPids[i]] = { + username: usersMap[post.uid].username, + displayname: usersMap[post.uid].displayname + }; + } + }); + postData.forEach(post => { + if (parents[post.toPid]) { + post.parent = parents[post.toPid]; + } + }); + }; + Topics.calculatePostIndices = function (posts, start) { + posts.forEach((post, index) => { + if (post) { + post.index = start + index + 1; + } + }); + }; + Topics.getLatestUndeletedPid = async function (tid) { + const pid = await Topics.getLatestUndeletedReply(tid); + if (pid) { + return pid; + } + const mainPid = await Topics.getTopicField(tid, 'mainPid'); + const mainPost = await posts.getPostFields(mainPid, ['pid', 'deleted']); + return mainPost.pid && !mainPost.deleted ? mainPost.pid : null; + }; + Topics.getLatestUndeletedReply = async function (tid) { + let isDeleted = false; + let index = 0; + do { + const pids = await db.getSortedSetRevRange(`tid:${tid}:posts`, index, index); + if (!pids.length) { + return null; + } + isDeleted = await posts.getPostField(pids[0], 'deleted'); + if (!isDeleted) { + return parseInt(pids[0], 10); + } + index += 1; + } while (isDeleted); + }; + Topics.addPostToTopic = async function (tid, postData) { + const mainPid = await Topics.getTopicField(tid, 'mainPid'); + if (!parseInt(mainPid, 10)) { + await Topics.setTopicField(tid, 'mainPid', postData.pid); + } else { + const upvotes = parseInt(postData.upvotes, 10) || 0; + const downvotes = parseInt(postData.downvotes, 10) || 0; + const votes = upvotes - downvotes; + await db.sortedSetsAdd([`tid:${tid}:posts`, `tid:${tid}:posts:votes`], [postData.timestamp, votes], postData.pid); + } + await Topics.increasePostCount(tid); + await db.sortedSetIncrBy(`tid:${tid}:posters`, 1, postData.uid); + const posterCount = await db.sortedSetCard(`tid:${tid}:posters`); + await Topics.setTopicField(tid, 'postercount', posterCount); + await Topics.updateTeaser(tid); + }; + Topics.removePostFromTopic = async function (tid, postData) { + await db.sortedSetsRemove([`tid:${tid}:posts`, `tid:${tid}:posts:votes`], postData.pid); + await Topics.decreasePostCount(tid); + await db.sortedSetIncrBy(`tid:${tid}:posters`, -1, postData.uid); + await db.sortedSetsRemoveRangeByScore([`tid:${tid}:posters`], '-inf', 0); + const posterCount = await db.sortedSetCard(`tid:${tid}:posters`); + await Topics.setTopicField(tid, 'postercount', posterCount); + await Topics.updateTeaser(tid); + }; + Topics.getPids = async function (tid) { + let [mainPid, pids] = await Promise.all([Topics.getTopicField(tid, 'mainPid'), db.getSortedSetRange(`tid:${tid}:posts`, 0, -1)]); + if (parseInt(mainPid, 10)) { + pids = [mainPid].concat(pids); + } + return pids; + }; + Topics.increasePostCount = async function (tid) { + incrementFieldAndUpdateSortedSet(tid, 'postcount', 1, 'topics:posts'); + }; + Topics.decreasePostCount = async function (tid) { + incrementFieldAndUpdateSortedSet(tid, 'postcount', -1, 'topics:posts'); + }; + Topics.increaseViewCount = async function (tid) { + const cid = await Topics.getTopicField(tid, 'cid'); + incrementFieldAndUpdateSortedSet(tid, 'viewcount', 1, ['topics:views', `cid:${cid}:tids:views`]); + }; + async function incrementFieldAndUpdateSortedSet(tid, field, by, set) { + const value = await db.incrObjectFieldBy(`topic:${tid}`, field, by); + await db[Array.isArray(set) ? 'sortedSetsAdd' : 'sortedSetAdd'](set, value, tid); + } + Topics.getTitleByPid = async function (pid) { + return await Topics.getTopicFieldByPid('title', pid); + }; + Topics.getTopicFieldByPid = async function (field, pid) { + const tid = await posts.getPostField(pid, 'tid'); + return await Topics.getTopicField(tid, field); + }; + Topics.getTopicDataByPid = async function (pid) { + const tid = await posts.getPostField(pid, 'tid'); + return await Topics.getTopicData(tid); + }; + Topics.getPostCount = async function (tid) { + return await db.getObjectField(`topic:${tid}`, 'postcount'); + }; + async function getPostReplies(postData, callerUid) { + const pids = postData.map(p => p && p.pid); + const keys = pids.map(pid => `pid:${pid}:replies`); + const [arrayOfReplyPids, userSettings] = await Promise.all([db.getSortedSetsMembers(keys), user.getSettings(callerUid)]); + const uniquePids = _.uniq(_.flatten(arrayOfReplyPids)); + let replyData = await posts.getPostsFields(uniquePids, ['pid', 'uid', 'timestamp']); + const result = await plugins.hooks.fire('filter:topics.getPostReplies', { + uid: callerUid, + replies: replyData + }); + replyData = await user.blocks.filter(callerUid, result.replies); + const uids = replyData.map(replyData => replyData && replyData.uid); + const uniqueUids = _.uniq(uids); + const userData = await user.getUsersWithFields(uniqueUids, ['uid', 'username', 'userslug', 'picture'], callerUid); + const uidMap = _.zipObject(uniqueUids, userData); + const pidMap = _.zipObject(replyData.map(r => r.pid), replyData); + const postDataMap = _.zipObject(pids, postData); + const returnData = await Promise.all(arrayOfReplyPids.map(async (replyPids, idx) => { + const currentPost = postData[idx]; + replyPids = replyPids.filter(pid => pidMap[pid]); + const uidsUsed = {}; + const currentData = { + hasMore: false, + hasSingleImmediateReply: false, + users: [], + text: replyPids.length > 1 ? `[[topic:replies-to-this-post, ${replyPids.length}]]` : '[[topic:one-reply-to-this-post]]', + count: replyPids.length, + timestampISO: replyPids.length ? utils.toISOString(pidMap[replyPids[0]].timestamp) : undefined + }; + replyPids.sort((a, b) => parseInt(a, 10) - parseInt(b, 10)); + replyPids.forEach(replyPid => { + const replyData = pidMap[replyPid]; + if (!uidsUsed[replyData.uid] && currentData.users.length < 6) { + currentData.users.push(uidMap[replyData.uid]); + uidsUsed[replyData.uid] = true; + } + }); + if (currentData.users.length > 5) { + currentData.users.pop(); + currentData.hasMore = true; + } + if (replyPids.length === 1) { + const currentIndex = currentPost ? currentPost.index : null; + const replyPid = replyPids[0]; + let replyPost = postDataMap[replyPid]; + if (!replyPost) { + const tid = await posts.getPostField(replyPid, 'tid'); + replyPost = { + index: await posts.getPidIndex(replyPid, tid, userSettings.topicPostSort), + tid: tid + }; + } + currentData.hasSingleImmediateReply = currentPost && currentPost.tid === replyPost.tid && Math.abs(currentIndex - replyPost.index) === 1; + } + return currentData; + })); + return returnData; + } + Topics.syncBacklinks = async postData => { + if (!postData) { + throw new Error('[[error:invalid-data]]'); + } + let { + content + } = postData; + content = content.split('\n').filter(line => !line.trim().startsWith('>')).join('\n'); + const matches = [...content.matchAll(backlinkRegex)]; + if (!matches) { + return 0; + } + const { + pid, + uid, + tid + } = postData; + let add = _.uniq(matches.map(match => match[1]).map(tid => parseInt(tid, 10))); + const now = Date.now(); + const topicsExist = await Topics.exists(add); + const current = (await db.getSortedSetMembers(`pid:${pid}:backlinks`)).map(tid => parseInt(tid, 10)); + const remove = current.filter(tid => !add.includes(tid)); + add = add.filter((_tid, idx) => topicsExist[idx] && !current.includes(_tid) && tid !== _tid); + await db.sortedSetRemove(`pid:${pid}:backlinks`, remove); + await db.sortedSetAdd(`pid:${pid}:backlinks`, add.map(() => now), add); + await Promise.all(add.map(async tid => { + await Topics.events.log(tid, { + uid, + type: 'backlink', + href: `/post/${pid}` + }); + })); + return add.length + (current - remove); + }; +}; \ No newline at end of file diff --git a/lib/topics/recent.js b/lib/topics/recent.js new file mode 100644 index 0000000000..c15c9ffc99 --- /dev/null +++ b/lib/topics/recent.js @@ -0,0 +1,77 @@ +'use strict'; + +const db = require('../database'); +const plugins = require('../plugins'); +const posts = require('../posts'); +module.exports = function (Topics) { + const terms = { + day: 86400000, + week: 604800000, + month: 2592000000, + year: 31104000000 + }; + Topics.getRecentTopics = async function (cid, uid, start, stop, filter) { + return await Topics.getSortedTopics({ + cids: cid, + uid: uid, + start: start, + stop: stop, + filter: filter, + sort: 'recent' + }); + }; + Topics.getLatestTopics = async function (options) { + const tids = await Topics.getLatestTidsFromSet('topics:recent', options.start, options.stop, options.term); + const topics = await Topics.getTopics(tids, options); + return { + topics: topics, + nextStart: options.stop + 1 + }; + }; + Topics.getSinceFromTerm = function (term) { + if (terms.hasOwnProperty(term)) { + return terms[term]; + } + return terms.day; + }; + Topics.getLatestTidsFromSet = async function (set, start, stop, term) { + const since = Topics.getSinceFromTerm(term); + const count = parseInt(stop, 10) === -1 ? stop : stop - start + 1; + return await db.getSortedSetRevRangeByScore(set, start, count, '+inf', Date.now() - since); + }; + Topics.updateLastPostTimeFromLastPid = async function (tid) { + const pid = await Topics.getLatestUndeletedPid(tid); + if (!pid) { + return; + } + const timestamp = await posts.getPostField(pid, 'timestamp'); + if (!timestamp) { + return; + } + await Topics.updateLastPostTime(tid, timestamp); + }; + Topics.updateLastPostTime = async function (tid, lastposttime) { + await Topics.setTopicField(tid, 'lastposttime', lastposttime); + const topicData = await Topics.getTopicFields(tid, ['cid', 'deleted', 'pinned']); + await db.sortedSetAdd(`cid:${topicData.cid}:tids:lastposttime`, lastposttime, tid); + await Topics.updateRecent(tid, lastposttime); + if (!topicData.pinned) { + await db.sortedSetAdd(`cid:${topicData.cid}:tids`, lastposttime, tid); + } + }; + Topics.updateRecent = async function (tid, timestamp) { + let data = { + tid: tid, + timestamp: timestamp + }; + if (plugins.hooks.hasListeners('filter:topics.updateRecent')) { + data = await plugins.hooks.fire('filter:topics.updateRecent', { + tid: tid, + timestamp: timestamp + }); + } + if (data && data.tid && data.timestamp) { + await db.sortedSetAdd('topics:recent', data.timestamp, data.tid); + } + }; +}; \ No newline at end of file diff --git a/lib/topics/scheduled.js b/lib/topics/scheduled.js new file mode 100644 index 0000000000..4606ecce7b --- /dev/null +++ b/lib/topics/scheduled.js @@ -0,0 +1,94 @@ +'use strict'; + +const _ = require('lodash'); +const winston = require('winston'); +const { + CronJob +} = require('cron'); +const db = require('../database'); +const posts = require('../posts'); +const socketHelpers = require('../socket.io/helpers'); +const topics = require('./index'); +const groups = require('../groups'); +const user = require('../user'); +const Scheduled = module.exports; +Scheduled.startJobs = function () { + winston.verbose('[scheduled topics] Starting jobs.'); + new CronJob('*/1 * * * *', Scheduled.handleExpired, null, true); +}; +Scheduled.handleExpired = async function () { + const now = Date.now(); + const tids = await db.getSortedSetRangeByScore('topics:scheduled', 0, -1, '-inf', now); + if (!tids.length) { + return; + } + await postTids(tids); + await db.sortedSetsRemoveRangeByScore([`topics:scheduled`], '-inf', now); +}; +async function postTids(tids) { + let topicsData = await topics.getTopicsData(tids); + topicsData = topicsData.filter(topicData => Boolean(topicData)); + const uids = _.uniq(topicsData.map(topicData => topicData.uid)).filter(uid => uid); + await Promise.all([].concat(topicsData.map(topicData => topics.restore(topicData.tid)), topicsData.map(topicData => topics.updateLastPostTimeFromLastPid(topicData.tid)))); + await Promise.all([].concat(sendNotifications(uids, topicsData), updateUserLastposttimes(uids, topicsData), updateGroupPosts(uids, topicsData), ...topicsData.map(topicData => unpin(topicData.tid, topicData)))); +} +Scheduled.pin = async function (tid, topicData) { + return Promise.all([topics.setTopicField(tid, 'pinned', 1), db.sortedSetAdd(`cid:${topicData.cid}:tids:pinned`, Date.now(), tid), db.sortedSetsRemove([`cid:${topicData.cid}:tids`, `cid:${topicData.cid}:tids:create`, `cid:${topicData.cid}:tids:posts`, `cid:${topicData.cid}:tids:votes`, `cid:${topicData.cid}:tids:views`], tid)]); +}; +Scheduled.reschedule = async function ({ + cid, + tid, + timestamp, + uid +}) { + if (timestamp < Date.now()) { + await postTids([tid]); + } else { + const mainPid = await topics.getTopicField(tid, 'mainPid'); + await Promise.all([db.sortedSetsAdd(['topics:scheduled', `uid:${uid}:topics`, 'topics:tid', `cid:${cid}:uid:${uid}:tids`], timestamp, tid), posts.setPostField(mainPid, 'timestamp', timestamp), db.sortedSetsAdd(['posts:pid', `uid:${uid}:posts`, `cid:${cid}:uid:${uid}:pids`], timestamp, mainPid), shiftPostTimes(tid, timestamp)]); + await topics.updateLastPostTimeFromLastPid(tid); + } +}; +function unpin(tid, topicData) { + return [topics.setTopicField(tid, 'pinned', 0), topics.deleteTopicField(tid, 'pinExpiry'), db.sortedSetRemove(`cid:${topicData.cid}:tids:pinned`, tid), db.sortedSetAddBulk([[`cid:${topicData.cid}:tids`, topicData.lastposttime, tid], [`cid:${topicData.cid}:tids:create`, topicData.timestamp, tid], [`cid:${topicData.cid}:tids:posts`, topicData.postcount, tid], [`cid:${topicData.cid}:tids:votes`, parseInt(topicData.votes, 10) || 0, tid], [`cid:${topicData.cid}:tids:views`, topicData.viewcount, tid]])]; +} +async function sendNotifications(uids, topicsData) { + const userData = await user.getUsersData(uids); + const uidToUserData = Object.fromEntries(uids.map((uid, idx) => [uid, userData[idx]])); + const postsData = await posts.getPostsData(topicsData.map(t => t && t.mainPid)); + postsData.forEach((postData, idx) => { + if (postData) { + postData.user = uidToUserData[topicsData[idx].uid]; + postData.topic = topicsData[idx]; + } + }); + return Promise.all(topicsData.map((t, idx) => user.notifications.sendTopicNotificationToFollowers(t.uid, t, postsData[idx])).concat(topicsData.map((t, idx) => socketHelpers.notifyNew(t.uid, 'newTopic', { + posts: [postsData[idx]], + topic: t + })))); +} +async function updateUserLastposttimes(uids, topicsData) { + const lastposttimes = (await user.getUsersFields(uids, ['lastposttime'])).map(u => u.lastposttime); + let tstampByUid = {}; + topicsData.forEach(tD => { + tstampByUid[tD.uid] = tstampByUid[tD.uid] ? tstampByUid[tD.uid].concat(tD.lastposttime) : [tD.lastposttime]; + }); + tstampByUid = Object.fromEntries(Object.entries(tstampByUid).map(uidTimestamp => [uidTimestamp[0], Math.max(...uidTimestamp[1])])); + const uidsToUpdate = uids.filter((uid, idx) => tstampByUid[uid] > lastposttimes[idx]); + return Promise.all(uidsToUpdate.map(uid => user.setUserField(uid, 'lastposttime', tstampByUid[uid]))); +} +async function updateGroupPosts(uids, topicsData) { + const postsData = await posts.getPostsData(topicsData.map(t => t && t.mainPid)); + await Promise.all(postsData.map(async (post, i) => { + if (topicsData[i]) { + post.cid = topicsData[i].cid; + await groups.onNewPostMade(post); + } + })); +} +async function shiftPostTimes(tid, timestamp) { + const pids = await posts.getPidsFromSet(`tid:${tid}:posts`, 0, -1, false); + return db.setObjectBulk(pids.map((pid, idx) => [`post:${pid}`, { + timestamp: timestamp + idx + 1 + }])); +} \ No newline at end of file diff --git a/lib/topics/sorted.js b/lib/topics/sorted.js new file mode 100644 index 0000000000..12734c9211 --- /dev/null +++ b/lib/topics/sorted.js @@ -0,0 +1,245 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const privileges = require('../privileges'); +const user = require('../user'); +const categories = require('../categories'); +const meta = require('../meta'); +const plugins = require('../plugins'); +module.exports = function (Topics) { + Topics.getSortedTopics = async function (params) { + const data = { + nextStart: 0, + topicCount: 0, + topics: [] + }; + params.term = params.term || 'alltime'; + params.sort = params.sort || 'recent'; + params.query = params.query || {}; + if (params.hasOwnProperty('cids') && params.cids && !Array.isArray(params.cids)) { + params.cids = [params.cids]; + } + params.tags = params.tags || []; + if (params.tags && !Array.isArray(params.tags)) { + params.tags = [params.tags]; + } + data.tids = await getTids(params); + data.tids = await sortTids(data.tids, params); + data.tids = await filterTids(data.tids.slice(0, meta.config.recentMaxTopics), params); + data.topicCount = data.tids.length; + data.topics = await getTopics(data.tids, params); + data.nextStart = params.stop + 1; + return data; + }; + async function getTids(params) { + if (plugins.hooks.hasListeners('filter:topics.getSortedTids')) { + const result = await plugins.hooks.fire('filter:topics.getSortedTids', { + params: params, + tids: [] + }); + return result.tids; + } + let tids = []; + if (params.term !== 'alltime') { + if (params.sort === 'posts') { + tids = await getTidsWithMostPostsInTerm(params.cids, params.uid, params.term); + } else { + tids = await Topics.getLatestTidsFromSet('topics:tid', 0, -1, params.term); + } + if (params.filter === 'watched') { + tids = await Topics.filterWatchedTids(tids, params.uid); + } + } else if (params.filter === 'watched') { + tids = await getWatchedTopics(params); + } else if (params.cids) { + tids = await getCidTids(params); + } else if (params.tags.length) { + tids = await getTagTids(params); + } else { + const method = params.sort === 'old' ? 'getSortedSetRange' : 'getSortedSetRevRange'; + tids = await db[method](sortToSet(params.sort), 0, meta.config.recentMaxTopics - 1); + } + return tids; + } + function sortToSet(sort) { + const map = { + recent: 'topics:recent', + old: 'topics:recent', + create: 'topics:tid', + posts: 'topics:posts', + votes: 'topics:votes', + views: 'topics:views' + }; + if (map.hasOwnProperty(sort)) { + return map[sort]; + } + return 'topics:recent'; + } + async function getTidsWithMostPostsInTerm(cids, uid, term) { + if (Array.isArray(cids)) { + cids = await privileges.categories.filterCids('topics:read', cids, uid); + } else { + cids = await categories.getCidsByPrivilege('categories:cid', uid, 'topics:read'); + } + const pids = await db.getSortedSetRevRangeByScore(cids.map(cid => `cid:${cid}:pids`), 0, 1000, '+inf', Date.now() - Topics.getSinceFromTerm(term)); + const postObjs = await db.getObjectsFields(pids.map(pid => `post:${pid}`), ['tid']); + const tidToCount = {}; + postObjs.forEach(post => { + tidToCount[post.tid] = tidToCount[post.tid] || 0; + tidToCount[post.tid] += 1; + }); + return _.uniq(postObjs.map(post => String(post.tid))).sort((t1, t2) => tidToCount[t2] - tidToCount[t1]); + } + async function getWatchedTopics(params) { + const sortSet = ['recent', 'old'].includes(params.sort) ? 'topics:recent' : `topics:${params.sort}`; + const method = params.sort === 'old' ? 'getSortedSetIntersect' : 'getSortedSetRevIntersect'; + return await db[method]({ + sets: [sortSet, `uid:${params.uid}:followed_tids`], + weights: [1, 0], + start: 0, + stop: meta.config.recentMaxTopics - 1 + }); + } + async function getTagTids(params) { + const sets = [sortToSet(params.sort), ...params.tags.map(tag => `tag:${tag}:topics`)]; + const method = params.sort === 'old' ? 'getSortedSetIntersect' : 'getSortedSetRevIntersect'; + return await db[method]({ + sets: sets, + start: 0, + stop: meta.config.recentMaxTopics - 1, + weights: sets.map((s, index) => index ? 0 : 1) + }); + } + async function getCidTids(params) { + if (params.tags.length) { + return _.intersection(...(await Promise.all(params.tags.map(async tag => { + const sets = params.cids.map(cid => `cid:${cid}:tag:${tag}:topics`); + return await db.getSortedSetRevRange(sets, 0, -1); + })))); + } + const sets = []; + const pinnedSets = []; + params.cids.forEach(cid => { + if (params.sort === 'recent' || params.sort === 'old') { + sets.push(`cid:${cid}:tids`); + } else { + sets.push(`cid:${cid}:tids${params.sort ? `:${params.sort}` : ''}`); + } + pinnedSets.push(`cid:${cid}:tids:pinned`); + }); + let pinnedTids = await db.getSortedSetRevRange(pinnedSets, 0, -1); + pinnedTids = await Topics.tools.checkPinExpiry(pinnedTids); + const method = params.sort === 'old' ? 'getSortedSetRange' : 'getSortedSetRevRange'; + const tids = await db[method](sets, 0, meta.config.recentMaxTopics - 1); + return pinnedTids.concat(tids); + } + async function sortTids(tids, params) { + if (params.term === 'alltime' && !params.cids && !params.tags.length && params.filter !== 'watched' && !params.floatPinned) { + return tids; + } + if (params.sort === 'posts' && params.term !== 'alltime') { + return tids; + } + const { + sortMap, + fields + } = await plugins.hooks.fire('filter:topics.sortOptions', { + params, + fields: ['tid', 'timestamp', 'lastposttime', 'upvotes', 'downvotes', 'postcount', 'pinned'], + sortMap: { + recent: sortRecent, + old: sortOld, + create: sortCreate, + posts: sortPopular, + votes: sortVotes, + views: sortViews + } + }); + const topicData = await Topics.getTopicsFields(tids, fields); + const sortFn = sortMap.hasOwnProperty(params.sort) && sortMap[params.sort] ? sortMap[params.sort] : sortRecent; + if (params.floatPinned) { + floatPinned(topicData, sortFn); + } else { + topicData.sort(sortFn); + } + return topicData.map(topic => topic && topic.tid); + } + function floatPinned(topicData, sortFn) { + topicData.sort((a, b) => a.pinned !== b.pinned ? b.pinned - a.pinned : sortFn(a, b)); + } + function sortRecent(a, b) { + return b.lastposttime - a.lastposttime; + } + function sortOld(a, b) { + return a.lastposttime - b.lastposttime; + } + function sortCreate(a, b) { + return b.timestamp - a.timestamp; + } + function sortVotes(a, b) { + if (a.votes !== b.votes) { + return b.votes - a.votes; + } + return b.postcount - a.postcount; + } + function sortPopular(a, b) { + if (a.postcount !== b.postcount) { + return b.postcount - a.postcount; + } + return b.viewcount - a.viewcount; + } + function sortViews(a, b) { + return b.viewcount - a.viewcount; + } + async function filterTids(tids, params) { + const { + filter + } = params; + const { + uid + } = params; + if (filter === 'new') { + tids = await Topics.filterNewTids(tids, uid); + } else if (filter === 'unreplied') { + tids = await Topics.filterUnrepliedTids(tids); + } else { + tids = await Topics.filterNotIgnoredTids(tids, uid); + } + tids = await privileges.topics.filterTids('topics:read', tids, uid); + let topicData = await Topics.getTopicsFields(tids, ['uid', 'tid', 'cid', 'tags']); + const topicCids = _.uniq(topicData.map(topic => topic.cid)).filter(Boolean); + async function getIgnoredCids() { + if (params.cids || filter === 'watched' || meta.config.disableRecentCategoryFilter) { + return []; + } + return await categories.isIgnored(topicCids, uid); + } + const [ignoredCids, filtered] = await Promise.all([getIgnoredCids(), user.blocks.filter(uid, topicData)]); + const isCidIgnored = _.zipObject(topicCids, ignoredCids); + topicData = filtered; + const cids = params.cids && params.cids.map(String); + const { + tags + } = params; + tids = topicData.filter(t => t && t.cid && !isCidIgnored[t.cid] && (!cids || cids.includes(String(t.cid))) && (!tags.length || tags.every(tag => t.tags.find(topicTag => topicTag.value === tag)))).map(t => t.tid); + const result = await plugins.hooks.fire('filter:topics.filterSortedTids', { + tids: tids, + params: params + }); + return result.tids; + } + async function getTopics(tids, params) { + tids = tids.slice(params.start, params.stop !== -1 ? params.stop + 1 : undefined); + const topicData = await Topics.getTopicsByTids(tids, params); + Topics.calculateTopicIndices(topicData, params.start); + return topicData; + } + Topics.calculateTopicIndices = function (topicData, start) { + topicData.forEach((topic, index) => { + if (topic) { + topic.index = start + index; + } + }); + }; +}; \ No newline at end of file diff --git a/lib/topics/suggested.js b/lib/topics/suggested.js new file mode 100644 index 0000000000..9569783230 --- /dev/null +++ b/lib/topics/suggested.js @@ -0,0 +1,64 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const user = require('../user'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +module.exports = function (Topics) { + Topics.getSuggestedTopics = async function (tid, uid, start, stop, cutoff = 0) { + let tids; + if (!tid) { + return []; + } + tid = String(tid); + cutoff = cutoff === 0 ? cutoff : cutoff * 2592000000; + const { + cid, + title, + tags + } = await Topics.getTopicFields(tid, ['cid', 'title', 'tags']); + const [tagTids, searchTids] = await Promise.all([getTidsWithSameTags(tid, tags.map(t => t.value), cutoff), getSearchTids(tid, title, cid, cutoff)]); + tids = _.uniq(tagTids.concat(searchTids)); + let categoryTids = []; + if (stop !== -1 && tids.length < stop - start + 1) { + categoryTids = await getCategoryTids(tid, cid, cutoff); + } + tids = _.shuffle(_.uniq(tids.concat(categoryTids))); + tids = await privileges.topics.filterTids('topics:read', tids, uid); + let topicData = await Topics.getTopicsByTids(tids, uid); + topicData = topicData.filter(topic => topic && String(topic.tid) !== tid); + topicData = await user.blocks.filter(uid, topicData); + topicData = topicData.slice(start, stop !== -1 ? stop + 1 : undefined).sort((t1, t2) => t2.timestamp - t1.timestamp); + Topics.calculateTopicIndices(topicData, start); + return topicData; + }; + async function getTidsWithSameTags(tid, tags, cutoff) { + let tids = cutoff === 0 ? await db.getSortedSetRevRange(tags.map(tag => `tag:${tag}:topics`), 0, -1) : await db.getSortedSetRevRangeByScore(tags.map(tag => `tag:${tag}:topics`), 0, -1, '+inf', Date.now() - cutoff); + tids = tids.filter(_tid => _tid !== tid); + return _.shuffle(_.uniq(tids)).slice(0, 10); + } + async function getSearchTids(tid, title, cid, cutoff) { + let { + ids: tids + } = await plugins.hooks.fire('filter:search.query', { + index: 'topic', + content: title, + matchWords: 'any', + cid: [cid], + limit: 20, + ids: [] + }); + tids = tids.filter(_tid => String(_tid) !== tid); + if (cutoff) { + const topicData = await Topics.getTopicsFields(tids, ['tid', 'timestamp']); + const now = Date.now(); + tids = topicData.filter(t => t && t.timestamp > now - cutoff).map(t => t.tid); + } + return _.shuffle(tids).slice(0, 10).map(String); + } + async function getCategoryTids(tid, cid, cutoff) { + const tids = cutoff === 0 ? await db.getSortedSetRevRange(`cid:${cid}:tids:lastposttime`, 0, 9) : await db.getSortedSetRevRangeByScore(`cid:${cid}:tids:lastposttime`, 0, 10, '+inf', Date.now() - cutoff); + return _.shuffle(tids.filter(_tid => _tid !== tid)); + } +}; \ No newline at end of file diff --git a/lib/topics/tags.js b/lib/topics/tags.js new file mode 100644 index 0000000000..2e3b1ee484 --- /dev/null +++ b/lib/topics/tags.js @@ -0,0 +1,553 @@ +'use strict'; + +const async = require('async'); +const validator = require('validator'); +const _ = require('lodash'); +const db = require('../database'); +const meta = require('../meta'); +const user = require('../user'); +const categories = require('../categories'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const notifications = require('../notifications'); +const translator = require('../translator'); +const utils = require('../utils'); +const batch = require('../batch'); +const cache = require('../cache'); +module.exports = function (Topics) { + Topics.createTags = async function (tags, tid, timestamp) { + if (!Array.isArray(tags) || !tags.length) { + return; + } + const cid = await Topics.getTopicField(tid, 'cid'); + const topicSets = tags.map(tag => `tag:${tag}:topics`).concat(tags.map(tag => `cid:${cid}:tag:${tag}:topics`)); + await db.sortedSetsAdd(topicSets, timestamp, tid); + await Topics.updateCategoryTagsCount([cid], tags); + await Promise.all(tags.map(updateTagCount)); + }; + Topics.filterTags = async function (tags, cid) { + const result = await plugins.hooks.fire('filter:tags.filter', { + tags: tags, + cid: cid + }); + tags = _.uniq(result.tags).map(tag => utils.cleanUpTag(tag, meta.config.maximumTagLength)).filter(tag => tag && tag.length >= (meta.config.minimumTagLength || 3)); + return await filterCategoryTags(tags, cid); + }; + Topics.updateCategoryTagsCount = async function (cids, tags) { + await Promise.all(cids.map(async cid => { + const counts = await db.sortedSetsCard(tags.map(tag => `cid:${cid}:tag:${tag}:topics`)); + const tagToCount = _.zipObject(tags, counts); + const set = `cid:${cid}:tags`; + const bulkAdd = tags.filter(tag => tagToCount[tag] > 0).map(tag => [set, tagToCount[tag], tag]); + const bulkRemove = tags.filter(tag => tagToCount[tag] <= 0).map(tag => [set, tag]); + await Promise.all([db.sortedSetAddBulk(bulkAdd), db.sortedSetRemoveBulk(bulkRemove)]); + })); + await db.sortedSetsRemoveRangeByScore(cids.map(cid => `cid:${cid}:tags`), '-inf', 0); + }; + Topics.validateTags = async function (tags, cid, uid, tid = null) { + if (!Array.isArray(tags)) { + throw new Error('[[error:invalid-data]]'); + } + tags = _.uniq(tags); + const [categoryData, isPrivileged, currentTags] = await Promise.all([categories.getCategoryFields(cid, ['minTags', 'maxTags']), user.isPrivileged(uid), tid ? Topics.getTopicTags(tid) : []]); + if (tags.length < parseInt(categoryData.minTags, 10)) { + throw new Error(`[[error:not-enough-tags, ${categoryData.minTags}]]`); + } else if (tags.length > parseInt(categoryData.maxTags, 10)) { + throw new Error(`[[error:too-many-tags, ${categoryData.maxTags}]]`); + } + const addedTags = tags.filter(tag => !currentTags.includes(tag)); + const removedTags = currentTags.filter(tag => !tags.includes(tag)); + const systemTags = (meta.config.systemTags || '').split(','); + if (!isPrivileged && systemTags.length && addedTags.length && addedTags.some(tag => systemTags.includes(tag))) { + throw new Error('[[error:cant-use-system-tag]]'); + } + if (!isPrivileged && systemTags.length && removedTags.length && removedTags.some(tag => systemTags.includes(tag))) { + throw new Error('[[error:cant-remove-system-tag]]'); + } + }; + async function filterCategoryTags(tags, cid) { + const tagWhitelist = await categories.getTagWhitelist([cid]); + if (!Array.isArray(tagWhitelist[0]) || !tagWhitelist[0].length) { + return tags; + } + const whitelistSet = new Set(tagWhitelist[0]); + return tags.filter(tag => whitelistSet.has(tag)); + } + Topics.createEmptyTag = async function (tag) { + if (!tag) { + throw new Error('[[error:invalid-tag]]'); + } + if (tag.length < (meta.config.minimumTagLength || 3)) { + throw new Error('[[error:tag-too-short]]'); + } + const isMember = await db.isSortedSetMember('tags:topic:count', tag); + if (!isMember) { + await db.sortedSetAdd('tags:topic:count', 0, tag); + cache.del('tags:topic:count'); + } + const allCids = await categories.getAllCidsFromSet('categories:cid'); + const isMembers = await db.isMemberOfSortedSets(allCids.map(cid => `cid:${cid}:tags`), tag); + const bulkAdd = allCids.filter((cid, index) => !isMembers[index]).map(cid => [`cid:${cid}:tags`, 0, tag]); + await db.sortedSetAddBulk(bulkAdd); + }; + Topics.renameTags = async function (data) { + for (const tagData of data) { + await renameTag(tagData.value, tagData.newName); + } + }; + async function renameTag(tag, newTagName) { + if (!newTagName || tag === newTagName) { + return; + } + newTagName = utils.cleanUpTag(newTagName, meta.config.maximumTagLength); + await Topics.createEmptyTag(newTagName); + const allCids = {}; + await batch.processSortedSet(`tag:${tag}:topics`, async tids => { + const topicData = await Topics.getTopicsFields(tids, ['tid', 'cid', 'tags']); + const cids = topicData.map(t => t.cid); + topicData.forEach(t => { + allCids[t.cid] = true; + }); + const scores = await db.sortedSetScores(`tag:${tag}:topics`, tids); + await db.sortedSetAdd(`tag:${newTagName}:topics`, scores, tids); + await db.sortedSetRemove(`tag:${tag}:topics`, tids); + await db.sortedSetAddBulk(topicData.map((t, index) => [`cid:${t.cid}:tag:${newTagName}:topics`, scores[index], t.tid])); + await db.sortedSetRemove(cids.map(cid => `cid:${cid}:tag:${tag}:topics`), tids); + topicData.forEach(topic => { + topic.tags = topic.tags.map(tagItem => tagItem.value); + const index = topic.tags.indexOf(tag); + if (index !== -1) { + topic.tags.splice(index, 1, newTagName); + } + }); + await db.setObjectBulk(topicData.map(t => [`topic:${t.tid}`, { + tags: t.tags.join(',') + }])); + }, {}); + const followers = await db.getSortedSetRangeWithScores(`tag:${tag}:followers`, 0, -1); + if (followers.length) { + const userKeys = followers.map(item => `uid:${item.value}:followed_tags`); + const scores = await db.sortedSetsScore(userKeys, tag); + await db.sortedSetsRemove(userKeys, tag); + await db.sortedSetsAdd(userKeys, scores, newTagName); + await db.sortedSetAdd(`tag:${newTagName}:followers`, followers.map(item => item.score), followers.map(item => item.value)); + } + await Topics.deleteTag(tag); + await updateTagCount(newTagName); + await Topics.updateCategoryTagsCount(Object.keys(allCids), [newTagName]); + } + async function updateTagCount(tag) { + const count = await Topics.getTagTopicCount(tag); + await db.sortedSetAdd('tags:topic:count', count || 0, tag); + cache.del('tags:topic:count'); + } + Topics.getTagTids = async function (tag, start, stop) { + const tids = await db.getSortedSetRevRange(`tag:${tag}:topics`, start, stop); + const payload = await plugins.hooks.fire('filter:topics.getTagTids', { + tag, + start, + stop, + tids + }); + return payload.tids; + }; + Topics.getTagTidsByCids = async function (tag, cids, start, stop) { + const keys = cids.map(cid => `cid:${cid}:tag:${tag}:topics`); + const tids = await db.getSortedSetRevRange(keys, start, stop); + const payload = await plugins.hooks.fire('filter:topics.getTagTidsByCids', { + tag, + cids, + start, + stop, + tids + }); + return payload.tids; + }; + Topics.getTagTopicCount = async function (tag, cids = []) { + let count = 0; + if (cids.length) { + count = await db.sortedSetsCardSum(cids.map(cid => `cid:${cid}:tag:${tag}:topics`)); + } else { + count = await db.sortedSetCard(`tag:${tag}:topics`); + } + const payload = await plugins.hooks.fire('filter:topics.getTagTopicCount', { + tag, + count, + cids + }); + return payload.count; + }; + Topics.deleteTags = async function (tags) { + if (!Array.isArray(tags) || !tags.length) { + return; + } + await Promise.all([removeTagsFromTopics(tags), removeTagsFromUsers(tags)]); + const keys = tags.map(tag => `tag:${tag}:topics`); + await db.deleteAll(keys); + await db.sortedSetRemove('tags:topic:count', tags); + cache.del('tags:topic:count'); + const cids = await categories.getAllCidsFromSet('categories:cid'); + await db.sortedSetRemove(cids.map(cid => `cid:${cid}:tags`), tags); + const deleteKeys = []; + tags.forEach(tag => { + deleteKeys.push(`tag:${tag}`); + deleteKeys.push(`tag:${tag}:followers`); + cids.forEach(cid => { + deleteKeys.push(`cid:${cid}:tag:${tag}:topics`); + }); + }); + await db.deleteAll(deleteKeys); + }; + async function removeTagsFromTopics(tags) { + await async.eachLimit(tags, 50, async tag => { + const tids = await db.getSortedSetRange(`tag:${tag}:topics`, 0, -1); + if (!tids.length) { + return; + } + let topicsTags = await Topics.getTopicsTags(tids); + topicsTags = topicsTags.map(topicTags => topicTags.filter(topicTag => topicTag && topicTag !== tag)); + await db.setObjectBulk(tids.map((tid, index) => [`topic:${tid}`, { + tags: topicsTags[index].join(',') + }])); + }); + } + async function removeTagsFromUsers(tags) { + await async.eachLimit(tags, 50, async tag => { + const uids = await db.getSortedSetRange(`tag:${tag}:followers`, 0, -1); + await db.sortedSetsRemove(uids.map(uid => `uid:${uid}:followed_tags`), tag); + }); + } + Topics.deleteTag = async function (tag) { + await Topics.deleteTags([tag]); + }; + Topics.getTags = async function (start, stop) { + return await getFromSet('tags:topic:count', start, stop); + }; + Topics.getCategoryTags = async function (cids, start, stop) { + if (Array.isArray(cids)) { + return await db.getSortedSetRevUnion({ + sets: cids.map(cid => `cid:${cid}:tags`), + start, + stop + }); + } + return await db.getSortedSetRevRange(`cid:${cids}:tags`, start, stop); + }; + Topics.getCategoryTagsData = async function (cids, start, stop) { + return await getFromSet(Array.isArray(cids) ? cids.map(cid => `cid:${cid}:tags`) : `cid:${cids}:tags`, start, stop); + }; + async function getFromSet(set, start, stop) { + let tags; + if (Array.isArray(set)) { + tags = await db.getSortedSetRevUnion({ + sets: set, + start, + stop, + withScores: true + }); + } else { + tags = await db.getSortedSetRevRangeWithScores(set, start, stop); + } + const payload = await plugins.hooks.fire('filter:tags.getAll', { + tags: tags + }); + return Topics.getTagData(payload.tags); + } + Topics.getTagData = function (tags) { + if (!tags || !tags.length) { + return []; + } + tags.forEach(tag => { + tag.valueEscaped = validator.escape(String(tag.value)); + tag.valueEncoded = encodeURIComponent(tag.valueEscaped); + tag.class = tag.valueEscaped.replace(/\s/g, '-'); + }); + return tags; + }; + Topics.getTopicTags = async function (tid) { + const data = await Topics.getTopicsTags([tid]); + return data && data[0]; + }; + Topics.getTopicsTags = async function (tids) { + const topicTagData = await Topics.getTopicsFields(tids, ['tags']); + return tids.map((tid, i) => topicTagData[i].tags.map(tagData => tagData.value)); + }; + Topics.getTopicTagsObjects = async function (tid) { + const data = await Topics.getTopicsTagsObjects([tid]); + return Array.isArray(data) && data.length ? data[0] : []; + }; + Topics.getTopicsTagsObjects = async function (tids) { + const topicTags = await Topics.getTopicsTags(tids); + const uniqueTopicTags = _.uniq(_.flatten(topicTags)); + const tags = uniqueTopicTags.map(tag => ({ + value: tag + })); + const tagData = Topics.getTagData(tags); + const tagDataMap = _.zipObject(uniqueTopicTags, tagData); + topicTags.forEach((tags, index) => { + if (Array.isArray(tags)) { + topicTags[index] = tags.map(tag => tagDataMap[tag]); + } + }); + return topicTags; + }; + Topics.addTags = async function (tags, tids) { + const topicData = await Topics.getTopicsFields(tids, ['tid', 'cid', 'timestamp', 'tags']); + const bulkAdd = []; + const bulkSet = []; + topicData.forEach(t => { + const topicTags = t.tags.map(tagItem => tagItem.value); + tags.forEach(tag => { + bulkAdd.push([`tag:${tag}:topics`, t.timestamp, t.tid]); + bulkAdd.push([`cid:${t.cid}:tag:${tag}:topics`, t.timestamp, t.tid]); + if (!topicTags.includes(tag)) { + topicTags.push(tag); + } + }); + bulkSet.push([`topic:${t.tid}`, { + tags: topicTags.join(',') + }]); + }); + await Promise.all([db.sortedSetAddBulk(bulkAdd), db.setObjectBulk(bulkSet)]); + await Promise.all(tags.map(updateTagCount)); + await Topics.updateCategoryTagsCount(_.uniq(topicData.map(t => t.cid)), tags); + }; + Topics.removeTags = async function (tags, tids) { + const topicData = await Topics.getTopicsFields(tids, ['tid', 'cid', 'tags']); + const bulkRemove = []; + const bulkSet = []; + topicData.forEach(t => { + const topicTags = t.tags.map(tagItem => tagItem.value); + tags.forEach(tag => { + bulkRemove.push([`tag:${tag}:topics`, t.tid]); + bulkRemove.push([`cid:${t.cid}:tag:${tag}:topics`, t.tid]); + if (topicTags.includes(tag)) { + topicTags.splice(topicTags.indexOf(tag), 1); + } + }); + bulkSet.push([`topic:${t.tid}`, { + tags: topicTags.join(',') + }]); + }); + await Promise.all([db.sortedSetRemoveBulk(bulkRemove), db.setObjectBulk(bulkSet)]); + await Promise.all(tags.map(updateTagCount)); + await Topics.updateCategoryTagsCount(_.uniq(topicData.map(t => t.cid)), tags); + }; + Topics.updateTopicTags = async function (tid, tags) { + await Topics.deleteTopicTags(tid); + const cid = await Topics.getTopicField(tid, 'cid'); + tags = await Topics.filterTags(tags, cid); + await Topics.addTags(tags, [tid]); + plugins.hooks.fire('action:topic.updateTags', { + tags, + tid + }); + }; + Topics.deleteTopicTags = async function (tid) { + const topicData = await Topics.getTopicFields(tid, ['cid', 'tags']); + const { + cid + } = topicData; + const tags = topicData.tags.map(tagItem => tagItem.value); + await db.deleteObjectField(`topic:${tid}`, 'tags'); + const sets = tags.map(tag => `tag:${tag}:topics`).concat(tags.map(tag => `cid:${cid}:tag:${tag}:topics`)); + await db.sortedSetsRemove(sets, tid); + await Topics.updateCategoryTagsCount([cid], tags); + await Promise.all(tags.map(updateTagCount)); + }; + Topics.searchTags = async function (data) { + if (!data || !data.query) { + return []; + } + let result; + if (plugins.hooks.hasListeners('filter:topics.searchTags')) { + result = await plugins.hooks.fire('filter:topics.searchTags', { + data: data + }); + } else { + result = await findMatches(data); + } + result = await plugins.hooks.fire('filter:tags.search', { + data: data, + matches: result.matches + }); + return result.matches; + }; + Topics.autocompleteTags = async function (data) { + if (!data || !data.query) { + return []; + } + let result; + if (plugins.hooks.hasListeners('filter:topics.autocompleteTags')) { + result = await plugins.hooks.fire('filter:topics.autocompleteTags', { + data: data + }); + } else { + result = await findMatches(data); + } + return result.matches; + }; + async function getAllTags() { + const cached = cache.get('tags:topic:count'); + if (cached !== undefined) { + return cached; + } + const tags = await db.getSortedSetRevRangeWithScores('tags:topic:count', 0, -1); + cache.set('tags:topic:count', tags); + return tags; + } + async function findMatches(data) { + let { + query + } = data; + let tagWhitelist = []; + if (parseInt(data.cid, 10)) { + tagWhitelist = await categories.getTagWhitelist([data.cid]); + } + let tags = []; + if (Array.isArray(tagWhitelist[0]) && tagWhitelist[0].length) { + const scores = await db.sortedSetScores(`cid:${data.cid}:tags`, tagWhitelist[0]); + tags = tagWhitelist[0].map((tag, index) => ({ + value: tag, + score: scores[index] + })); + } else if (data.cids) { + tags = await db.getSortedSetRevUnion({ + sets: data.cids.map(cid => `cid:${cid}:tags`), + start: 0, + stop: -1, + withScores: true + }); + } else { + tags = await getAllTags(); + } + query = query.toLowerCase(); + const matches = []; + for (let i = 0; i < tags.length; i += 1) { + if (tags[i].value && tags[i].value.toLowerCase().startsWith(query)) { + matches.push(tags[i]); + if (matches.length > 39) { + break; + } + } + } + matches.sort((a, b) => { + if (a.value < b.value) { + return -1; + } else if (a.value > b.value) { + return 1; + } + return 0; + }); + return { + matches: matches + }; + } + Topics.searchAndLoadTags = async function (data) { + const searchResult = { + tags: [], + matchCount: 0, + pageCount: 1 + }; + if (!data || !data.query || !data.query.length) { + return searchResult; + } + const tags = await Topics.searchTags(data); + const tagData = Topics.getTagData(tags.map(tag => ({ + value: tag.value + }))); + tagData.forEach((tag, index) => { + tag.score = tags[index].score; + }); + tagData.sort((a, b) => b.score - a.score); + searchResult.tags = tagData; + searchResult.matchCount = tagData.length; + searchResult.pageCount = 1; + return searchResult; + }; + Topics.getRelatedTopics = async function (topicData, uid) { + if (plugins.hooks.hasListeners('filter:topic.getRelatedTopics')) { + const result = await plugins.hooks.fire('filter:topic.getRelatedTopics', { + topic: topicData, + uid: uid, + topics: [] + }); + return result.topics; + } + let maximumTopics = meta.config.maximumRelatedTopics; + if (maximumTopics === 0 || !topicData.tags || !topicData.tags.length) { + return []; + } + maximumTopics = maximumTopics || 5; + let tids = await Promise.all(topicData.tags.map(tag => Topics.getTagTids(tag.value, 0, 5))); + tids = _.shuffle(_.uniq(_.flatten(tids))).slice(0, maximumTopics); + const topics = await Topics.getTopics(tids, uid); + return topics.filter(t => t && !t.deleted && parseInt(t.uid, 10) !== parseInt(uid, 10)); + }; + Topics.isFollowingTag = async function (tag, uid) { + return await db.isSortedSetMember(`tag:${tag}:followers`, uid); + }; + Topics.getTagFollowers = async function (tag, start = 0, stop = -1) { + return await db.getSortedSetRange(`tag:${tag}:followers`, start, stop); + }; + Topics.followTag = async (tag, uid) => { + if (!(parseInt(uid, 10) > 0)) { + throw new Error('[[error:not-logged-in]]'); + } + const now = Date.now(); + await db.sortedSetAddBulk([[`tag:${tag}:followers`, now, uid], [`uid:${uid}:followed_tags`, now, tag]]); + plugins.hooks.fire('action:tags.follow', { + tag, + uid + }); + }; + Topics.unfollowTag = async (tag, uid) => { + if (!(parseInt(uid, 10) > 0)) { + throw new Error('[[error:not-logged-in]]'); + } + await db.sortedSetRemoveBulk([[`tag:${tag}:followers`, uid], [`uid:${uid}:followed_tags`, tag]]); + plugins.hooks.fire('action:tags.unfollow', { + tag, + uid + }); + }; + Topics.notifyTagFollowers = async function (postData, exceptUid) { + let { + tags + } = postData.topic; + if (!tags.length) { + return; + } + tags = tags.map(tag => tag.value); + const [followersOfPoster, allFollowers] = await Promise.all([db.getSortedSetRange(`followers:${exceptUid}`, 0, -1), db.getSortedSetRange(tags.map(tag => `tag:${tag}:followers`), 0, -1)]); + const followerSet = new Set(followersOfPoster); + let followers = _.uniq(allFollowers).filter(uid => !followerSet.has(uid) && uid !== String(exceptUid)); + followers = await privileges.topics.filterUids('topics:read', postData.topic.tid, followers); + if (!followers.length) { + return; + } + const { + displayname + } = postData.user; + const notifBase = 'notifications:user-posted-topic-with-tag'; + let bodyShort = translator.compile(notifBase, displayname, tags[0]); + if (tags.length === 2) { + bodyShort = translator.compile(`${notifBase}-dual`, displayname, tags[0], tags[1]); + } else if (tags.length === 3) { + bodyShort = translator.compile(`${notifBase}-triple`, displayname, tags[0], tags[1], tags[2]); + } else if (tags.length > 3) { + bodyShort = translator.compile(`${notifBase}-multiple`, displayname, tags.join(', ')); + } + const notification = await notifications.create({ + type: 'new-topic-with-tag', + nid: `new_topic:tid:${postData.topic.tid}:uid:${exceptUid}`, + bodyShort: bodyShort, + bodyLong: postData.content, + pid: postData.pid, + path: `/post/${postData.pid}`, + tid: postData.topic.tid, + from: exceptUid + }); + notifications.push(notification, followers); + }; +}; \ No newline at end of file diff --git a/lib/topics/teaser.js b/lib/topics/teaser.js new file mode 100644 index 0000000000..6f81ddf1fb --- /dev/null +++ b/lib/topics/teaser.js @@ -0,0 +1,154 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const meta = require('../meta'); +const user = require('../user'); +const posts = require('../posts'); +const plugins = require('../plugins'); +const utils = require('../utils'); +module.exports = function (Topics) { + Topics.getTeasers = async function (topics, options) { + if (!Array.isArray(topics) || !topics.length) { + return []; + } + let uid = options; + let { + teaserPost + } = meta.config; + if (typeof options === 'object') { + uid = options.uid; + teaserPost = options.teaserPost || meta.config.teaserPost; + } + const counts = []; + const teaserPids = []; + const tidToPost = {}; + topics.forEach(topic => { + counts.push(topic && topic.postcount); + if (topic) { + if (topic.teaserPid === 'null') { + delete topic.teaserPid; + } + if (teaserPost === 'first') { + teaserPids.push(topic.mainPid); + } else if (teaserPost === 'last-post') { + teaserPids.push(topic.teaserPid || topic.mainPid); + } else { + teaserPids.push(topic.teaserPid); + } + } + }); + const [allPostData, callerSettings] = await Promise.all([posts.getPostsFields(teaserPids, ['pid', 'uid', 'timestamp', 'tid', 'content']), user.getSettings(uid)]); + let postData = allPostData.filter(post => post && post.pid); + postData = await handleBlocks(uid, postData); + postData = postData.filter(Boolean); + const uids = _.uniq(postData.map(post => post.uid)); + const sortNewToOld = callerSettings.topicPostSort === 'newest_to_oldest'; + const usersData = await user.getUsersFields(uids, ['uid', 'username', 'userslug', 'picture']); + const users = {}; + usersData.forEach(user => { + users[user.uid] = user; + }); + postData.forEach(post => { + if (!users.hasOwnProperty(post.uid)) { + post.uid = 0; + } + post.user = users[post.uid]; + post.timestampISO = utils.toISOString(post.timestamp); + tidToPost[post.tid] = post; + }); + await Promise.all(postData.map(p => posts.parsePost(p))); + const { + tags + } = await plugins.hooks.fire('filter:teasers.configureStripTags', { + tags: utils.stripTags.slice(0) + }); + const teasers = topics.map((topic, index) => { + if (!topic) { + return null; + } + if (tidToPost[topic.tid]) { + tidToPost[topic.tid].index = calcTeaserIndex(teaserPost, counts[index], sortNewToOld); + if (tidToPost[topic.tid].content) { + tidToPost[topic.tid].content = utils.stripHTMLTags(replaceImgWithAltText(tidToPost[topic.tid].content), tags); + } + } + return tidToPost[topic.tid]; + }); + const result = await plugins.hooks.fire('filter:teasers.get', { + teasers: teasers, + uid: uid + }); + return result.teasers; + }; + function calcTeaserIndex(teaserPost, postCountInTopic, sortNewToOld) { + if (teaserPost === 'first') { + return 1; + } + if (sortNewToOld) { + return Math.min(2, postCountInTopic); + } + return postCountInTopic; + } + function replaceImgWithAltText(str) { + return String(str).replace(/]*>/gi, '$1'); + } + async function handleBlocks(uid, teasers) { + const blockedUids = await user.blocks.list(uid); + if (!blockedUids.length) { + return teasers; + } + return await Promise.all(teasers.map(async postData => { + if (blockedUids.includes(parseInt(postData.uid, 10))) { + return await getPreviousNonBlockedPost(postData, blockedUids); + } + return postData; + })); + } + async function getPreviousNonBlockedPost(postData, blockedUids) { + let isBlocked = false; + let prevPost = postData; + const postsPerIteration = 5; + let start = 0; + let stop = start + postsPerIteration - 1; + let checkedAllReplies = false; + function checkBlocked(post) { + const isPostBlocked = blockedUids.includes(parseInt(post.uid, 10)); + prevPost = !isPostBlocked ? post : prevPost; + return isPostBlocked; + } + do { + let pids = await db.getSortedSetRevRange(`tid:${postData.tid}:posts`, start, stop); + if (!pids.length) { + checkedAllReplies = true; + const mainPid = await Topics.getTopicField(postData.tid, 'mainPid'); + pids = [mainPid]; + } + const prevPosts = await posts.getPostsFields(pids, ['pid', 'uid', 'timestamp', 'tid', 'content']); + isBlocked = prevPosts.every(checkBlocked); + start += postsPerIteration; + stop = start + postsPerIteration - 1; + } while (isBlocked && prevPost && prevPost.pid && !checkedAllReplies); + return prevPost; + } + Topics.getTeasersByTids = async function (tids, uid) { + if (!Array.isArray(tids) || !tids.length) { + return []; + } + const topics = await Topics.getTopicsFields(tids, ['tid', 'postcount', 'teaserPid', 'mainPid']); + return await Topics.getTeasers(topics, uid); + }; + Topics.getTeaser = async function (tid, uid) { + const teasers = await Topics.getTeasersByTids([tid], uid); + return Array.isArray(teasers) && teasers.length ? teasers[0] : null; + }; + Topics.updateTeaser = async function (tid) { + let pid = await Topics.getLatestUndeletedReply(tid); + pid = pid || null; + if (pid) { + await Topics.setTopicField(tid, 'teaserPid', pid); + } else { + await Topics.deleteTopicField(tid, 'teaserPid'); + } + }; +}; \ No newline at end of file diff --git a/lib/topics/thumbs.js b/lib/topics/thumbs.js new file mode 100644 index 0000000000..8fd46212c0 --- /dev/null +++ b/lib/topics/thumbs.js @@ -0,0 +1,138 @@ +'use strict'; + +const _ = require('lodash'); +const nconf = require('nconf'); +const path = require('path'); +const validator = require('validator'); +const db = require('../database'); +const file = require('../file'); +const plugins = require('../plugins'); +const posts = require('../posts'); +const meta = require('../meta'); +const cache = require('../cache'); +const Thumbs = module.exports; +Thumbs.exists = async function (id, path) { + const isDraft = validator.isUUID(String(id)); + const set = `${isDraft ? 'draft' : 'topic'}:${id}:thumbs`; + return db.isSortedSetMember(set, path); +}; +Thumbs.load = async function (topicData) { + const topicsWithThumbs = topicData.filter(t => t && parseInt(t.numThumbs, 10) > 0); + const tidsWithThumbs = topicsWithThumbs.map(t => t.tid); + const thumbs = await Thumbs.get(tidsWithThumbs); + const tidToThumbs = _.zipObject(tidsWithThumbs, thumbs); + return topicData.map(t => t && t.tid ? tidToThumbs[t.tid] || [] : []); +}; +Thumbs.get = async function (tids) { + let singular = false; + if (!Array.isArray(tids)) { + tids = [tids]; + singular = true; + } + if (!meta.config.allowTopicsThumbnail || !tids.length) { + return singular ? [] : tids.map(() => []); + } + const hasTimestampPrefix = /^\d+-/; + const upload_url = nconf.get('relative_path') + nconf.get('upload_url'); + const sets = tids.map(tid => `${validator.isUUID(String(tid)) ? 'draft' : 'topic'}:${tid}:thumbs`); + const thumbs = await Promise.all(sets.map(getThumbs)); + let response = thumbs.map((thumbSet, idx) => thumbSet.map(thumb => ({ + id: tids[idx], + name: (() => { + const name = path.basename(thumb); + return hasTimestampPrefix.test(name) ? name.slice(14) : name; + })(), + path: thumb, + url: thumb.startsWith('http') ? thumb : path.posix.join(upload_url, thumb.replace(/\\/g, '/')) + }))); + ({ + thumbs: response + } = await plugins.hooks.fire('filter:topics.getThumbs', { + tids, + thumbs: response + })); + return singular ? response.pop() : response; +}; +async function getThumbs(set) { + const cached = cache.get(set); + if (cached !== undefined) { + return cached.slice(); + } + const thumbs = await db.getSortedSetRange(set, 0, -1); + cache.set(set, thumbs); + return thumbs.slice(); +} +Thumbs.associate = async function ({ + id, + path, + score +}) { + const isDraft = validator.isUUID(String(id)); + const isLocal = !path.startsWith('http'); + const set = `${isDraft ? 'draft' : 'topic'}:${id}:thumbs`; + const numThumbs = await db.sortedSetCard(set); + if (isLocal) { + path = path.replace(nconf.get('upload_path'), ''); + } + const topics = require('.'); + await db.sortedSetAdd(set, isFinite(score) ? score : numThumbs, path); + if (!isDraft) { + const numThumbs = await db.sortedSetCard(set); + await topics.setTopicField(id, 'numThumbs', numThumbs); + } + cache.del(set); + if (!isDraft && isLocal) { + const mainPid = (await topics.getMainPids([id]))[0]; + await posts.uploads.associate(mainPid, path.slice(1)); + } +}; +Thumbs.migrate = async function (uuid, id) { + const set = `draft:${uuid}:thumbs`; + const thumbs = await db.getSortedSetRangeWithScores(set, 0, -1); + await Promise.all(thumbs.map(async thumb => await Thumbs.associate({ + id, + path: thumb.value, + score: thumb.score + }))); + await db.delete(set); + cache.del(set); +}; +Thumbs.delete = async function (id, relativePaths) { + const isDraft = validator.isUUID(String(id)); + const set = `${isDraft ? 'draft' : 'topic'}:${id}:thumbs`; + if (typeof relativePaths === 'string') { + relativePaths = [relativePaths]; + } else if (!Array.isArray(relativePaths)) { + throw new Error('[[error:invalid-data]]'); + } + const absolutePaths = relativePaths.map(relativePath => path.join(nconf.get('upload_path'), relativePath)); + const [associated, existsOnDisk] = await Promise.all([db.isSortedSetMembers(set, relativePaths), Promise.all(absolutePaths.map(async absolutePath => file.exists(absolutePath)))]); + const toRemove = []; + const toDelete = []; + relativePaths.forEach((relativePath, idx) => { + if (associated[idx]) { + toRemove.push(relativePath); + } + if (existsOnDisk[idx]) { + toDelete.push(absolutePaths[idx]); + } + }); + await db.sortedSetRemove(set, toRemove); + if (isDraft && toDelete.length) { + await Promise.all(toDelete.map(async absolutePath => file.delete(absolutePath))); + } + if (toRemove.length && !isDraft) { + const topics = require('.'); + const mainPid = (await topics.getMainPids([id]))[0]; + await Promise.all([db.incrObjectFieldBy(`topic:${id}`, 'numThumbs', -toRemove.length), Promise.all(toRemove.map(async relativePath => posts.uploads.dissociate(mainPid, relativePath.slice(1))))]); + } + if (toRemove.length) { + cache.del(set); + } +}; +Thumbs.deleteAll = async id => { + const isDraft = validator.isUUID(String(id)); + const set = `${isDraft ? 'draft' : 'topic'}:${id}:thumbs`; + const thumbs = await db.getSortedSetRange(set, 0, -1); + await Thumbs.delete(id, thumbs); +}; \ No newline at end of file diff --git a/lib/topics/tools.js b/lib/topics/tools.js new file mode 100644 index 0000000000..789ba0aae9 --- /dev/null +++ b/lib/topics/tools.js @@ -0,0 +1,252 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const topics = require('.'); +const categories = require('../categories'); +const user = require('../user'); +const plugins = require('../plugins'); +const privileges = require('../privileges'); +const utils = require('../utils'); +module.exports = function (Topics) { + const topicTools = {}; + Topics.tools = topicTools; + topicTools.delete = async function (tid, uid) { + return await toggleDelete(tid, uid, true); + }; + topicTools.restore = async function (tid, uid) { + return await toggleDelete(tid, uid, false); + }; + async function toggleDelete(tid, uid, isDelete) { + const topicData = await Topics.getTopicData(tid); + if (!topicData) { + throw new Error('[[error:no-topic]]'); + } + if (topicData.scheduled) { + throw new Error('[[error:invalid-data]]'); + } + const canDelete = await privileges.topics.canDelete(tid, uid); + const hook = isDelete ? 'delete' : 'restore'; + const data = await plugins.hooks.fire(`filter:topic.${hook}`, { + topicData: topicData, + uid: uid, + isDelete: isDelete, + canDelete: canDelete, + canRestore: canDelete + }); + if (!data.canDelete && data.isDelete || !data.canRestore && !data.isDelete) { + throw new Error('[[error:no-privileges]]'); + } + if (data.topicData.deleted && data.isDelete) { + throw new Error('[[error:topic-already-deleted]]'); + } else if (!data.topicData.deleted && !data.isDelete) { + throw new Error('[[error:topic-already-restored]]'); + } + if (data.isDelete) { + await Topics.delete(data.topicData.tid, data.uid); + } else { + await Topics.restore(data.topicData.tid); + } + const events = await Topics.events.log(tid, { + type: isDelete ? 'delete' : 'restore', + uid + }); + data.topicData.deleted = data.isDelete ? 1 : 0; + if (data.isDelete) { + plugins.hooks.fire('action:topic.delete', { + topic: data.topicData, + uid: data.uid + }); + } else { + plugins.hooks.fire('action:topic.restore', { + topic: data.topicData, + uid: data.uid + }); + } + const userData = await user.getUserFields(data.uid, ['username', 'userslug']); + return { + tid: data.topicData.tid, + cid: data.topicData.cid, + isDelete: data.isDelete, + uid: data.uid, + user: userData, + events + }; + } + topicTools.purge = async function (tid, uid) { + const topicData = await Topics.getTopicData(tid); + if (!topicData) { + throw new Error('[[error:no-topic]]'); + } + const canPurge = await privileges.topics.canPurge(tid, uid); + if (!canPurge) { + throw new Error('[[error:no-privileges]]'); + } + await Topics.purgePostsAndTopic(tid, uid); + return { + tid: tid, + cid: topicData.cid, + uid: uid + }; + }; + topicTools.lock = async function (tid, uid) { + return await toggleLock(tid, uid, true); + }; + topicTools.unlock = async function (tid, uid) { + return await toggleLock(tid, uid, false); + }; + async function toggleLock(tid, uid, lock) { + const topicData = await Topics.getTopicFields(tid, ['tid', 'uid', 'cid']); + if (!topicData || !topicData.cid) { + throw new Error('[[error:no-topic]]'); + } + const isAdminOrMod = await privileges.categories.isAdminOrMod(topicData.cid, uid); + if (!isAdminOrMod) { + throw new Error('[[error:no-privileges]]'); + } + await Topics.setTopicField(tid, 'locked', lock ? 1 : 0); + topicData.events = await Topics.events.log(tid, { + type: lock ? 'lock' : 'unlock', + uid + }); + topicData.isLocked = lock; + topicData.locked = lock; + plugins.hooks.fire('action:topic.lock', { + topic: _.clone(topicData), + uid: uid + }); + return topicData; + } + topicTools.pin = async function (tid, uid) { + return await togglePin(tid, uid, true); + }; + topicTools.unpin = async function (tid, uid) { + return await togglePin(tid, uid, false); + }; + topicTools.setPinExpiry = async (tid, expiry, uid) => { + if (isNaN(parseInt(expiry, 10)) || expiry <= Date.now()) { + throw new Error('[[error:invalid-data]]'); + } + const topicData = await Topics.getTopicFields(tid, ['tid', 'uid', 'cid']); + const isAdminOrMod = await privileges.categories.isAdminOrMod(topicData.cid, uid); + if (!isAdminOrMod) { + throw new Error('[[error:no-privileges]]'); + } + await Topics.setTopicField(tid, 'pinExpiry', expiry); + plugins.hooks.fire('action:topic.setPinExpiry', { + topic: _.clone(topicData), + uid: uid + }); + }; + topicTools.checkPinExpiry = async tids => { + const expiry = (await topics.getTopicsFields(tids, ['pinExpiry'])).map(obj => obj.pinExpiry); + const now = Date.now(); + tids = await Promise.all(tids.map(async (tid, idx) => { + if (expiry[idx] && parseInt(expiry[idx], 10) <= now) { + await togglePin(tid, 'system', false); + return null; + } + return tid; + })); + return tids.filter(Boolean); + }; + async function togglePin(tid, uid, pin) { + const topicData = await Topics.getTopicData(tid); + if (!topicData) { + throw new Error('[[error:no-topic]]'); + } + if (topicData.scheduled) { + throw new Error('[[error:cant-pin-scheduled]]'); + } + if (uid !== 'system' && !(await privileges.topics.isAdminOrMod(tid, uid))) { + throw new Error('[[error:no-privileges]]'); + } + const promises = [Topics.setTopicField(tid, 'pinned', pin ? 1 : 0), Topics.events.log(tid, { + type: pin ? 'pin' : 'unpin', + uid + })]; + if (pin) { + promises.push(db.sortedSetAdd(`cid:${topicData.cid}:tids:pinned`, Date.now(), tid)); + promises.push(db.sortedSetsRemove([`cid:${topicData.cid}:tids`, `cid:${topicData.cid}:tids:create`, `cid:${topicData.cid}:tids:posts`, `cid:${topicData.cid}:tids:votes`, `cid:${topicData.cid}:tids:views`], tid)); + } else { + promises.push(db.sortedSetRemove(`cid:${topicData.cid}:tids:pinned`, tid)); + promises.push(Topics.deleteTopicField(tid, 'pinExpiry')); + promises.push(db.sortedSetAddBulk([[`cid:${topicData.cid}:tids`, topicData.lastposttime, tid], [`cid:${topicData.cid}:tids:create`, topicData.timestamp, tid], [`cid:${topicData.cid}:tids:posts`, topicData.postcount, tid], [`cid:${topicData.cid}:tids:votes`, parseInt(topicData.votes, 10) || 0, tid], [`cid:${topicData.cid}:tids:views`, topicData.viewcount, tid]])); + topicData.pinExpiry = undefined; + topicData.pinExpiryISO = undefined; + } + const results = await Promise.all(promises); + topicData.isPinned = pin; + topicData.pinned = pin; + topicData.events = results[1]; + plugins.hooks.fire('action:topic.pin', { + topic: _.clone(topicData), + uid + }); + return topicData; + } + topicTools.orderPinnedTopics = async function (uid, data) { + const { + tid, + order + } = data; + const cid = await Topics.getTopicField(tid, 'cid'); + if (!cid || !tid || !utils.isNumber(order) || order < 0) { + throw new Error('[[error:invalid-data]]'); + } + const isAdminOrMod = await privileges.categories.isAdminOrMod(cid, uid); + if (!isAdminOrMod) { + throw new Error('[[error:no-privileges]]'); + } + const pinnedTids = await db.getSortedSetRange(`cid:${cid}:tids:pinned`, 0, -1); + const currentIndex = pinnedTids.indexOf(String(tid)); + if (currentIndex === -1) { + return; + } + const newOrder = pinnedTids.length - order - 1; + if (pinnedTids.length > 1) { + pinnedTids.splice(Math.max(0, newOrder), 0, pinnedTids.splice(currentIndex, 1)[0]); + } + await db.sortedSetAdd(`cid:${cid}:tids:pinned`, pinnedTids.map((tid, index) => index), pinnedTids); + }; + topicTools.move = async function (tid, data) { + const cid = parseInt(data.cid, 10); + const topicData = await Topics.getTopicData(tid); + if (!topicData) { + throw new Error('[[error:no-topic]]'); + } + if (cid === topicData.cid) { + throw new Error('[[error:cant-move-topic-to-same-category]]'); + } + const tags = await Topics.getTopicTags(tid); + await db.sortedSetsRemove([`cid:${topicData.cid}:tids`, `cid:${topicData.cid}:tids:create`, `cid:${topicData.cid}:tids:pinned`, `cid:${topicData.cid}:tids:posts`, `cid:${topicData.cid}:tids:votes`, `cid:${topicData.cid}:tids:views`, `cid:${topicData.cid}:tids:lastposttime`, `cid:${topicData.cid}:recent_tids`, `cid:${topicData.cid}:uid:${topicData.uid}:tids`, ...tags.map(tag => `cid:${topicData.cid}:tag:${tag}:topics`)], tid); + topicData.postcount = topicData.postcount || 0; + const votes = topicData.upvotes - topicData.downvotes; + const bulk = [[`cid:${cid}:tids:lastposttime`, topicData.lastposttime, tid], [`cid:${cid}:uid:${topicData.uid}:tids`, topicData.timestamp, tid], ...tags.map(tag => [`cid:${cid}:tag:${tag}:topics`, topicData.timestamp, tid])]; + if (topicData.pinned) { + bulk.push([`cid:${cid}:tids:pinned`, Date.now(), tid]); + } else { + bulk.push([`cid:${cid}:tids`, topicData.lastposttime, tid]); + bulk.push([`cid:${cid}:tids:create`, topicData.timestamp, tid]); + bulk.push([`cid:${cid}:tids:posts`, topicData.postcount, tid]); + bulk.push([`cid:${cid}:tids:votes`, votes, tid]); + bulk.push([`cid:${cid}:tids:views`, topicData.viewcount, tid]); + } + await db.sortedSetAddBulk(bulk); + const oldCid = topicData.cid; + await categories.moveRecentReplies(tid, oldCid, cid); + await Promise.all([Topics.setTopicFields(tid, { + cid: cid, + oldCid: oldCid + }), Topics.updateCategoryTagsCount([oldCid, cid], tags), Topics.events.log(tid, { + type: 'move', + uid: data.uid, + fromCid: oldCid + })]); + const hookData = _.clone(data); + hookData.fromCid = oldCid; + hookData.toCid = cid; + hookData.tid = tid; + plugins.hooks.fire('action:topic.move', hookData); + }; +}; \ No newline at end of file diff --git a/lib/topics/unread.js b/lib/topics/unread.js new file mode 100644 index 0000000000..da0d12a14b --- /dev/null +++ b/lib/topics/unread.js @@ -0,0 +1,351 @@ +'use strict'; + +const async = require('async'); +const _ = require('lodash'); +const db = require('../database'); +const user = require('../user'); +const posts = require('../posts'); +const notifications = require('../notifications'); +const categories = require('../categories'); +const privileges = require('../privileges'); +const meta = require('../meta'); +const utils = require('../utils'); +const plugins = require('../plugins'); +module.exports = function (Topics) { + Topics.getTotalUnread = async function (uid, filter) { + filter = filter || ''; + const counts = await Topics.getUnreadTids({ + cid: 0, + uid: uid, + count: true + }); + return counts && counts[filter]; + }; + Topics.getUnreadTopics = async function (params) { + const unreadTopics = { + showSelect: true, + nextStart: 0, + topics: [] + }; + let tids = await Topics.getUnreadTids(params); + unreadTopics.topicCount = tids.length; + if (!tids.length) { + return unreadTopics; + } + tids = tids.slice(params.start, params.stop !== -1 ? params.stop + 1 : undefined); + const topicData = await Topics.getTopicsByTids(tids, params.uid); + if (!topicData.length) { + return unreadTopics; + } + Topics.calculateTopicIndices(topicData, params.start); + unreadTopics.topics = topicData; + unreadTopics.nextStart = params.stop + 1; + return unreadTopics; + }; + Topics.unreadCutoff = async function (uid) { + const cutoff = Date.now() - meta.config.unreadCutoff * 86400000; + const data = await plugins.hooks.fire('filter:topics.unreadCutoff', { + uid: uid, + cutoff: cutoff + }); + return parseInt(data.cutoff, 10); + }; + Topics.getUnreadTids = async function (params) { + const results = await Topics.getUnreadData(params); + return params.count ? results.counts : results.tids; + }; + Topics.getUnreadData = async function (params) { + const uid = parseInt(params.uid, 10); + params.filter = params.filter || ''; + if (params.cid && !Array.isArray(params.cid)) { + params.cid = [params.cid]; + } + if (params.tag && !Array.isArray(params.tag)) { + params.tag = [params.tag]; + } + const data = await getTids(params); + if (uid <= 0) { + return data; + } + const result = await plugins.hooks.fire('filter:topics.getUnreadTids', { + uid: uid, + tids: data.tids, + counts: data.counts, + tidsByFilter: data.tidsByFilter, + unreadCids: data.unreadCids, + cid: params.cid, + filter: params.filter, + query: params.query || {} + }); + return result; + }; + async function getTids(params) { + const counts = { + '': 0, + new: 0, + watched: 0, + unreplied: 0 + }; + const tidsByFilter = { + '': [], + new: [], + watched: [], + unreplied: [] + }; + const unreadCids = []; + if (params.uid <= 0) { + return { + counts, + tids: [], + tidsByFilter, + unreadCids + }; + } + params.cutoff = await Topics.unreadCutoff(params.uid); + const [followedTids, ignoredTids, categoryTids, userScores, tids_unread] = await Promise.all([getFollowedTids(params), user.getIgnoredTids(params.uid, 0, -1), getCategoryTids(params), db.getSortedSetRevRangeByScoreWithScores(`uid:${params.uid}:tids_read`, 0, -1, '+inf', params.cutoff), db.getSortedSetRevRangeWithScores(`uid:${params.uid}:tids_unread`, 0, -1)]); + const userReadTimes = _.mapValues(_.keyBy(userScores, 'value'), 'score'); + const isTopicsFollowed = {}; + followedTids.forEach(t => { + isTopicsFollowed[t.value] = true; + }); + const unreadFollowed = await db.isSortedSetMembers(`uid:${params.uid}:followed_tids`, tids_unread.map(t => t.value)); + tids_unread.forEach((t, i) => { + isTopicsFollowed[t.value] = unreadFollowed[i]; + }); + const unreadTopics = _.unionWith(categoryTids, followedTids, (a, b) => a.value === b.value).filter(t => !ignoredTids.includes(t.value) && (!userReadTimes[t.value] || t.score > userReadTimes[t.value])).concat(tids_unread.filter(t => !ignoredTids.includes(t.value))).sort((a, b) => b.score - a.score); + let tids = _.uniq(unreadTopics.map(topic => topic.value)).slice(0, 200); + if (!tids.length) { + return { + counts, + tids, + tidsByFilter, + unreadCids + }; + } + const blockedUids = await user.blocks.list(params.uid); + tids = await filterTidsThatHaveBlockedPosts({ + uid: params.uid, + tids: tids, + blockedUids: blockedUids, + recentTids: categoryTids + }); + tids = await privileges.topics.filterTids('topics:read', tids, params.uid); + const topicData = (await Topics.getTopicsFields(tids, ['tid', 'cid', 'uid', 'postcount', 'deleted', 'scheduled', 'tags'])).filter(t => t.scheduled || !t.deleted); + const topicCids = _.uniq(topicData.map(topic => topic.cid)).filter(Boolean); + const categoryWatchState = await categories.getWatchState(topicCids, params.uid); + const userCidState = _.zipObject(topicCids, categoryWatchState); + const filterCids = params.cid && params.cid.map(cid => parseInt(cid, 10)); + const filterTags = params.tag && params.tag.map(tag => String(tag)); + topicData.forEach(topic => { + if (topic && topic.cid && (!filterCids || filterCids.includes(topic.cid)) && (!filterTags || filterTags.every(tag => topic.tags.find(topicTag => topicTag.value === tag))) && !blockedUids.includes(topic.uid)) { + if (isTopicsFollowed[topic.tid] || [categories.watchStates.watching, categories.watchStates.tracking].includes(userCidState[topic.cid])) { + tidsByFilter[''].push(topic.tid); + unreadCids.push(topic.cid); + } + if (isTopicsFollowed[topic.tid]) { + tidsByFilter.watched.push(topic.tid); + } + if (topic.postcount <= 1) { + tidsByFilter.unreplied.push(topic.tid); + } + if (!userReadTimes[topic.tid]) { + tidsByFilter.new.push(topic.tid); + } + } + }); + counts[''] = tidsByFilter[''].length; + counts.watched = tidsByFilter.watched.length; + counts.unreplied = tidsByFilter.unreplied.length; + counts.new = tidsByFilter.new.length; + return { + counts: counts, + tids: tidsByFilter[params.filter], + tidsByFilter: tidsByFilter, + unreadCids: _.uniq(unreadCids) + }; + } + async function getCategoryTids(params) { + if (plugins.hooks.hasListeners('filter:topics.unread.getCategoryTids')) { + const result = await plugins.hooks.fire('filter:topics.unread.getCategoryTids', { + params: params, + tids: [] + }); + return result.tids; + } + if (params.filter === 'watched') { + return []; + } + const cids = params.cid || (await getWatchedTrackedCids(params.uid)); + const keys = cids.map(cid => `cid:${cid}:tids:lastposttime`); + return await db.getSortedSetRevRangeByScoreWithScores(keys, 0, -1, '+inf', params.cutoff); + } + async function getWatchedTrackedCids(uid) { + if (!(parseInt(uid, 10) > 0)) { + return []; + } + const cids = await user.getCategoriesByStates(uid, [categories.watchStates.watching, categories.watchStates.tracking]); + const categoryData = await categories.getCategoriesFields(cids, ['disabled']); + return cids.filter((cid, index) => categoryData[index] && !categoryData[index].disabled); + } + async function getFollowedTids(params) { + const keys = params.cid ? params.cid.map(cid => `cid:${cid}:tids:lastposttime`) : 'topics:recent'; + const recentTopicData = await db.getSortedSetRevRangeByScoreWithScores(keys, 0, -1, '+inf', params.cutoff); + const isFollowed = await db.isSortedSetMembers(`uid:${params.uid}:followed_tids`, recentTopicData.map(t => t.tid)); + return recentTopicData.filter((t, i) => isFollowed[i]); + } + async function filterTidsThatHaveBlockedPosts(params) { + if (!params.blockedUids.length) { + return params.tids; + } + const topicScores = _.mapValues(_.keyBy(params.recentTids, 'value'), 'score'); + const results = await db.sortedSetScores(`uid:${params.uid}:tids_read`, params.tids); + const userScores = _.zipObject(params.tids, results); + return await async.filter(params.tids, async tid => await doesTidHaveUnblockedUnreadPosts(tid, { + blockedUids: params.blockedUids, + topicTimestamp: topicScores[tid], + userLastReadTimestamp: userScores[tid] + })); + } + async function doesTidHaveUnblockedUnreadPosts(tid, params) { + const { + userLastReadTimestamp + } = params; + if (!userLastReadTimestamp) { + return true; + } + let start = 0; + const count = 3; + let done = false; + let hasUnblockedUnread = params.topicTimestamp > userLastReadTimestamp; + if (!params.blockedUids.length) { + return hasUnblockedUnread; + } + while (!done) { + const pidsSinceLastVisit = await db.getSortedSetRangeByScore(`tid:${tid}:posts`, start, count, userLastReadTimestamp, '+inf'); + if (!pidsSinceLastVisit.length) { + return hasUnblockedUnread; + } + let postData = await posts.getPostsFields(pidsSinceLastVisit, ['pid', 'uid']); + postData = postData.filter(post => !params.blockedUids.includes(parseInt(post.uid, 10))); + done = postData.length > 0; + hasUnblockedUnread = postData.length > 0; + start += count; + } + return hasUnblockedUnread; + } + Topics.pushUnreadCount = async function (uid) { + if (!uid || parseInt(uid, 10) <= 0) { + return; + } + const results = await Topics.getUnreadTids({ + uid: uid, + count: true + }); + require('../socket.io').in(`uid_${uid}`).emit('event:unread.updateCount', { + unreadTopicCount: results[''], + unreadNewTopicCount: results.new, + unreadWatchedTopicCount: results.watched, + unreadUnrepliedTopicCount: results.unreplied + }); + }; + Topics.markAsUnreadForAll = async function (tid) { + const now = Date.now(); + const cid = await Topics.getTopicField(tid, 'cid'); + await Topics.updateRecent(tid, now); + await db.sortedSetAdd(`cid:${cid}:tids:lastposttime`, now, tid); + await Topics.setTopicField(tid, 'lastposttime', now); + }; + Topics.markAsRead = async function (tids, uid) { + if (!Array.isArray(tids) || !tids.length) { + return false; + } + tids = _.uniq(tids).filter(tid => tid && utils.isNumber(tid)); + if (!tids.length) { + return false; + } + const [topicScores, userScores] = await Promise.all([Topics.getTopicsFields(tids, ['tid', 'lastposttime', 'scheduled']), db.sortedSetScores(`uid:${uid}:tids_read`, tids)]); + const now = Date.now(); + const topics = topicScores.filter((t, i) => t.lastposttime && (!userScores[i] || userScores[i] < t.lastposttime || userScores[i] > now)); + tids = topics.map(t => t.tid); + if (!tids.length) { + return false; + } + const scores = topics.map(topic => topic.scheduled ? topic.lastposttime : now); + await Promise.all([db.sortedSetAdd(`uid:${uid}:tids_read`, scores, tids), db.sortedSetRemove(`uid:${uid}:tids_unread`, tids)]); + plugins.hooks.fire('action:topics.markAsRead', { + uid: uid, + tids: tids + }); + return true; + }; + Topics.markAllRead = async function (uid) { + const cutoff = await Topics.unreadCutoff(uid); + let tids = await db.getSortedSetRevRangeByScore('topics:recent', 0, -1, '+inf', cutoff); + tids = await privileges.topics.filterTids('topics:read', tids, uid); + Topics.markTopicNotificationsRead(tids, uid); + await Topics.markAsRead(tids, uid); + await db.delete(`uid:${uid}:tids_unread`); + }; + Topics.markTopicNotificationsRead = async function (tids, uid) { + if (!Array.isArray(tids) || !tids.length) { + return; + } + const nids = await user.notifications.getUnreadByField(uid, 'tid', tids); + await notifications.markReadMultiple(nids, uid); + user.notifications.pushCount(uid); + }; + Topics.markCategoryUnreadForAll = async function () { + console.warn('[deprecated] Topics.markCategoryUnreadForAll deprecated'); + }; + Topics.hasReadTopics = async function (tids, uid) { + if (!(parseInt(uid, 10) > 0)) { + return tids.map(() => false); + } + const [topicScores, userScores, tids_unread, blockedUids] = await Promise.all([db.sortedSetScores('topics:recent', tids), db.sortedSetScores(`uid:${uid}:tids_read`, tids), db.sortedSetScores(`uid:${uid}:tids_unread`, tids), user.blocks.list(uid)]); + const cutoff = await Topics.unreadCutoff(uid); + const result = tids.map((tid, index) => { + const read = !tids_unread[index] && (topicScores[index] < cutoff || !!(userScores[index] && userScores[index] >= topicScores[index])); + return { + tid: tid, + read: read, + index: index + }; + }); + return await async.map(result, async data => { + if (data.read) { + return true; + } + const hasUnblockedUnread = await doesTidHaveUnblockedUnreadPosts(data.tid, { + topicTimestamp: topicScores[data.index], + userLastReadTimestamp: userScores[data.index], + blockedUids: blockedUids + }); + if (!hasUnblockedUnread) { + data.read = true; + } + return data.read; + }); + }; + Topics.hasReadTopic = async function (tid, uid) { + const hasRead = await Topics.hasReadTopics([tid], uid); + return Array.isArray(hasRead) && hasRead.length ? hasRead[0] : false; + }; + Topics.markUnread = async function (tid, uid) { + const exists = await Topics.exists(tid); + if (!exists) { + throw new Error('[[error:no-topic]]'); + } + await Promise.all([db.sortedSetRemoveBulk([[`uid:${uid}:tids_read`, tid], [`tid:${tid}:bookmarks`, uid]]), db.sortedSetAdd(`uid:${uid}:tids_unread`, Date.now(), tid)]); + }; + Topics.filterNewTids = async function (tids, uid) { + if (parseInt(uid, 10) <= 0) { + return []; + } + const scores = await db.sortedSetScores(`uid:${uid}:tids_read`, tids); + return tids.filter((tid, index) => tid && !scores[index]); + }; + Topics.filterUnrepliedTids = async function (tids) { + const scores = await db.sortedSetScores('topics:posts', tids); + return tids.filter((tid, index) => tid && scores[index] !== null && scores[index] <= 1); + }; +}; \ No newline at end of file diff --git a/lib/topics/user.js b/lib/topics/user.js new file mode 100644 index 0000000000..7ca619ef4d --- /dev/null +++ b/lib/topics/user.js @@ -0,0 +1,16 @@ +'use strict'; + +const db = require('../database'); +module.exports = function (Topics) { + Topics.isOwner = async function (tid, uid) { + uid = parseInt(uid, 10); + if (uid <= 0) { + return false; + } + const author = await Topics.getTopicField(tid, 'uid'); + return author === uid; + }; + Topics.getUids = async function (tid) { + return await db.getSortedSetRevRangeByScore(`tid:${tid}:posters`, 0, -1, '+inf', 1); + }; +}; \ No newline at end of file diff --git a/lib/translator.js b/lib/translator.js new file mode 100644 index 0000000000..a984be24c8 --- /dev/null +++ b/lib/translator.js @@ -0,0 +1,12 @@ +'use strict'; + +const winston = require('winston'); +function warn(msg) { + if (global.env === 'development') { + winston.warn(msg); + } +} +module.exports = require('../public/src/modules/translator.common')(require('./utils'), (lang, namespace) => { + const languages = require('./languages'); + return languages.get(lang, namespace); +}, warn); \ No newline at end of file diff --git a/lib/upgrade.js b/lib/upgrade.js new file mode 100644 index 0000000000..d1a0fad30a --- /dev/null +++ b/lib/upgrade.js @@ -0,0 +1,152 @@ +'use strict'; + +const path = require('path'); +const util = require('util'); +const semver = require('semver'); +const readline = require('readline'); +const winston = require('winston'); +const chalk = require('chalk'); +const plugins = require('./plugins'); +const db = require('./database'); +const file = require('./file'); +const { + paths +} = require('./constants'); +const Upgrade = module.exports; +Upgrade.getAll = async function () { + let files = await file.walk(path.join(__dirname, './upgrades')); + files = files.filter(file => path.basename(file) !== 'TEMPLATE').sort((a, b) => { + const versionA = path.dirname(a).split(path.sep).pop(); + const versionB = path.dirname(b).split(path.sep).pop(); + const semverCompare = semver.compare(versionA, versionB); + if (semverCompare) { + return semverCompare; + } + const timestampA = require(a).timestamp; + const timestampB = require(b).timestamp; + return timestampA - timestampB; + }); + await Upgrade.appendPluginScripts(files); + const seen = {}; + const dupes = []; + files.forEach(file => { + if (seen[file]) { + dupes.push(file); + } else { + seen[file] = true; + } + }); + if (dupes.length) { + winston.error(`Found duplicate upgrade scripts\n${dupes}`); + throw new Error('[[error:duplicate-upgrade-scripts]]'); + } + return files; +}; +Upgrade.appendPluginScripts = async function (files) { + const activePlugins = await plugins.getActive(); + activePlugins.forEach(plugin => { + const configPath = path.join(paths.nodeModules, plugin, 'plugin.json'); + try { + const pluginConfig = require(configPath); + if (pluginConfig.hasOwnProperty('upgrades') && Array.isArray(pluginConfig.upgrades)) { + pluginConfig.upgrades.forEach(script => { + files.push(path.join(path.dirname(configPath), script)); + }); + } + } catch (e) { + if (e.code !== 'MODULE_NOT_FOUND') { + winston.error(e.stack); + } + } + }); + return files; +}; +Upgrade.check = async function () { + const files = await Upgrade.getAll(); + const executed = await db.getSortedSetRange('schemaLog', 0, -1); + const remainder = files.filter(name => !executed.includes(path.basename(name, '.js'))); + if (remainder.length > 0) { + throw new Error('schema-out-of-date'); + } +}; +Upgrade.run = async function () { + console.log('\nParsing upgrade scripts... '); + const [completed, available] = await Promise.all([db.getSortedSetRange('schemaLog', 0, -1), Upgrade.getAll()]); + let skipped = 0; + const queue = available.filter(cur => { + const upgradeRan = completed.includes(path.basename(cur, '.js')); + if (upgradeRan) { + skipped += 1; + } + return !upgradeRan; + }); + await Upgrade.process(queue, skipped); +}; +Upgrade.runParticular = async function (names) { + console.log('\nParsing upgrade scripts... '); + const files = await file.walk(path.join(__dirname, './upgrades')); + await Upgrade.appendPluginScripts(files); + const upgrades = files.filter(file => names.includes(path.basename(file, '.js'))); + await Upgrade.process(upgrades, 0); +}; +Upgrade.process = async function (files, skipCount) { + console.log(`${chalk.green('OK')} | ${chalk.cyan(`${files.length} script(s) found`)}${skipCount > 0 ? chalk.cyan(`, ${skipCount} skipped`) : ''}`); + const [schemaDate, schemaLogCount] = await Promise.all([db.get('schemaDate'), db.sortedSetCard('schemaLog')]); + for (const file of files) { + const scriptExport = require(file); + const date = new Date(scriptExport.timestamp); + const version = path.dirname(file).split('/').pop(); + const progress = { + current: 0, + counter: 0, + total: 0, + incr: Upgrade.incrementProgress, + script: scriptExport, + date: date + }; + process.stdout.write(`${chalk.white(' → ') + chalk.gray(`[${[date.getUTCFullYear(), date.getUTCMonth() + 1, date.getUTCDate()].join('/')}] `) + scriptExport.name}...`); + if (!schemaDate && !schemaLogCount || scriptExport.timestamp <= schemaDate && semver.lt(version, '1.5.0')) { + process.stdout.write(chalk.grey(' skipped\n')); + await db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js')); + continue; + } + if (scriptExport.method.constructor && scriptExport.method.constructor.name !== 'AsyncFunction') { + scriptExport.method = util.promisify(scriptExport.method); + } + const upgradeStart = Date.now(); + try { + await scriptExport.method.bind({ + progress: progress + })(); + } catch (err) { + console.error('Error occurred'); + throw err; + } + const upgradeDuration = ((Date.now() - upgradeStart) / 1000).toFixed(2); + process.stdout.write(chalk.green(` OK (${upgradeDuration} seconds)\n`)); + await db.sortedSetAdd('schemaLog', Date.now(), path.basename(file, '.js')); + } + console.log(chalk.green('Schema update complete!\n')); +}; +Upgrade.incrementProgress = function (value) { + if (this.current === 0) { + process.stdout.write('\n'); + } + this.current += value || 1; + this.counter += value || 1; + const step = this.total ? Math.floor(this.total / 100) : 100; + if (this.counter > step || this.current >= this.total) { + this.counter -= step; + let percentage = 0; + let filled = 0; + let unfilled = 15; + if (this.total) { + percentage = `${Math.floor(this.current / this.total * 100)}%`; + filled = Math.floor(this.current / this.total * 15); + unfilled = Math.max(0, 15 - filled); + } + readline.cursorTo(process.stdout, 0); + process.stdout.write(` [${filled ? new Array(filled).join('#') : ''}${new Array(unfilled).join(' ')}] (${this.current}/${this.total || '??'}) ${percentage} `); + } +}; +require('./promisify')(Upgrade); \ No newline at end of file diff --git a/lib/upgrades/1.0.0/chat_room_hashes.js b/lib/upgrades/1.0.0/chat_room_hashes.js new file mode 100644 index 0000000000..2e0df7350d --- /dev/null +++ b/lib/upgrades/1.0.0/chat_room_hashes.js @@ -0,0 +1,39 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +module.exports = { + name: 'Chat room hashes', + timestamp: Date.UTC(2015, 11, 23), + method: function (callback) { + db.getObjectField('global', 'nextChatRoomId', (err, nextChatRoomId) => { + if (err) { + return callback(err); + } + let currentChatRoomId = 1; + async.whilst(next => { + next(null, currentChatRoomId <= nextChatRoomId); + }, next => { + db.getSortedSetRange(`chat:room:${currentChatRoomId}:uids`, 0, 0, (err, uids) => { + if (err) { + return next(err); + } + if (!Array.isArray(uids) || !uids.length || !uids[0]) { + currentChatRoomId += 1; + return next(); + } + db.setObject(`chat:room:${currentChatRoomId}`, { + owner: uids[0], + roomId: currentChatRoomId + }, err => { + if (err) { + return next(err); + } + currentChatRoomId += 1; + next(); + }); + }); + }, callback); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.0.0/chat_upgrade.js b/lib/upgrades/1.0.0/chat_upgrade.js new file mode 100644 index 0000000000..28d49a9180 --- /dev/null +++ b/lib/upgrades/1.0.0/chat_upgrade.js @@ -0,0 +1,68 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Upgrading chats', + timestamp: Date.UTC(2015, 11, 15), + method: function (callback) { + db.getObjectFields('global', ['nextMid', 'nextChatRoomId'], (err, globalData) => { + if (err) { + return callback(err); + } + const rooms = {}; + let roomId = globalData.nextChatRoomId || 1; + let currentMid = 1; + async.whilst(next => { + next(null, currentMid <= globalData.nextMid); + }, next => { + db.getObject(`message:${currentMid}`, (err, message) => { + if (err || !message) { + winston.verbose('skipping chat message ', currentMid); + currentMid += 1; + return next(err); + } + const pairID = [parseInt(message.fromuid, 10), parseInt(message.touid, 10)].sort().join(':'); + const msgTime = parseInt(message.timestamp, 10); + function addMessageToUids(roomId, callback) { + async.parallel([function (next) { + db.sortedSetAdd(`uid:${message.fromuid}:chat:room:${roomId}:mids`, msgTime, currentMid, next); + }, function (next) { + db.sortedSetAdd(`uid:${message.touid}:chat:room:${roomId}:mids`, msgTime, currentMid, next); + }], callback); + } + if (rooms[pairID]) { + winston.verbose(`adding message ${currentMid} to existing roomID ${roomId}`); + addMessageToUids(rooms[pairID], err => { + if (err) { + return next(err); + } + currentMid += 1; + next(); + }); + } else { + winston.verbose(`adding message ${currentMid} to new roomID ${roomId}`); + async.parallel([function (next) { + db.sortedSetAdd(`uid:${message.fromuid}:chat:rooms`, msgTime, roomId, next); + }, function (next) { + db.sortedSetAdd(`uid:${message.touid}:chat:rooms`, msgTime, roomId, next); + }, function (next) { + db.sortedSetAdd(`chat:room:${roomId}:uids`, [msgTime, msgTime + 1], [message.fromuid, message.touid], next); + }, function (next) { + addMessageToUids(roomId, next); + }], err => { + if (err) { + return next(err); + } + rooms[pairID] = roomId; + roomId += 1; + currentMid += 1; + db.setObjectField('global', 'nextChatRoomId', roomId, next); + }); + } + }); + }, callback); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.0.0/global_moderators.js b/lib/upgrades/1.0.0/global_moderators.js new file mode 100644 index 0000000000..8208dd93b7 --- /dev/null +++ b/lib/upgrades/1.0.0/global_moderators.js @@ -0,0 +1,22 @@ +'use strict'; + +module.exports = { + name: 'Creating Global moderators group', + timestamp: Date.UTC(2016, 0, 23), + method: async function () { + const groups = require('../../groups'); + const exists = await groups.exists('Global Moderators'); + if (exists) { + return; + } + await groups.create({ + name: 'Global Moderators', + userTitle: 'Global Moderator', + description: 'Forum wide moderators', + hidden: 0, + private: 1, + disableJoinRequests: 1 + }); + await groups.show('Global Moderators'); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.0.0/social_post_sharing.js b/lib/upgrades/1.0.0/social_post_sharing.js new file mode 100644 index 0000000000..3edc4788fc --- /dev/null +++ b/lib/upgrades/1.0.0/social_post_sharing.js @@ -0,0 +1,16 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +module.exports = { + name: 'Social: Post Sharing', + timestamp: Date.UTC(2016, 1, 25), + method: function (callback) { + const social = require('../../social'); + async.parallel([function (next) { + social.setActivePostSharingNetworks(['facebook', 'google', 'twitter'], next); + }, function (next) { + db.deleteObjectField('config', 'disableSocialButtons', next); + }], callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.0.0/theme_to_active_plugins.js b/lib/upgrades/1.0.0/theme_to_active_plugins.js new file mode 100644 index 0000000000..f8653419f1 --- /dev/null +++ b/lib/upgrades/1.0.0/theme_to_active_plugins.js @@ -0,0 +1,11 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Adding theme to active plugins sorted set', + timestamp: Date.UTC(2015, 11, 23), + method: async function () { + const themeId = await db.getObjectField('config', 'theme:id'); + await db.sortedSetAdd('plugins:active', 0, themeId); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.0.0/user_best_posts.js b/lib/upgrades/1.0.0/user_best_posts.js new file mode 100644 index 0000000000..4cd38603e9 --- /dev/null +++ b/lib/upgrades/1.0.0/user_best_posts.js @@ -0,0 +1,32 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Creating user best post sorted sets', + timestamp: Date.UTC(2016, 0, 14), + method: function (callback) { + const batch = require('../../batch'); + const { + progress + } = this; + batch.processSortedSet('posts:pid', (ids, next) => { + async.eachSeries(ids, (id, next) => { + db.getObjectFields(`post:${id}`, ['pid', 'uid', 'votes'], (err, postData) => { + if (err) { + return next(err); + } + if (!postData || !parseInt(postData.votes, 10) || !parseInt(postData.uid, 10)) { + return next(); + } + winston.verbose(`processing pid: ${postData.pid} uid: ${postData.uid} votes: ${postData.votes}`); + db.sortedSetAdd(`uid:${postData.uid}:posts:votes`, postData.votes, postData.pid, next); + progress.incr(); + }); + }, next); + }, { + progress: progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.0.0/users_notvalidated.js b/lib/upgrades/1.0.0/users_notvalidated.js new file mode 100644 index 0000000000..fb9d59be1e --- /dev/null +++ b/lib/upgrades/1.0.0/users_notvalidated.js @@ -0,0 +1,27 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Creating users:notvalidated', + timestamp: Date.UTC(2016, 0, 20), + method: function (callback) { + const batch = require('../../batch'); + const now = Date.now(); + batch.processSortedSet('users:joindate', (ids, next) => { + async.eachSeries(ids, (id, next) => { + db.getObjectFields(`user:${id}`, ['uid', 'email:confirmed'], (err, userData) => { + if (err) { + return next(err); + } + if (!userData || !parseInt(userData.uid, 10) || parseInt(userData['email:confirmed'], 10) === 1) { + return next(); + } + winston.verbose(`processing uid: ${userData.uid} email:confirmed: ${userData['email:confirmed']}`); + db.sortedSetAdd('users:notvalidated', now, userData.uid, next); + }); + }, next); + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.1.0/assign_topic_read_privilege.js b/lib/upgrades/1.1.0/assign_topic_read_privilege.js new file mode 100644 index 0000000000..357b9f9258 --- /dev/null +++ b/lib/upgrades/1.1.0/assign_topic_read_privilege.js @@ -0,0 +1,32 @@ +'use strict'; + +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Giving topics:read privs to any group/user that was previously allowed to Find & Access Category', + timestamp: Date.UTC(2016, 4, 28), + method: async function () { + const groupsAPI = require('../../groups'); + const privilegesAPI = require('../../privileges'); + const cids = await db.getSortedSetRange('categories:cid', 0, -1); + for (const cid of cids) { + const { + groups, + users + } = await privilegesAPI.categories.list(cid); + for (const group of groups) { + if (group.privileges['groups:read']) { + await groupsAPI.join(`cid:${cid}:privileges:groups:topics:read`, group.name); + winston.verbose(`cid:${cid}:privileges:groups:topics:read granted to gid: ${group.name}`); + } + } + for (const user of users) { + if (user.privileges.read) { + await groupsAPI.join(`cid:${cid}:privileges:topics:read`, user.uid); + winston.verbose(`cid:${cid}:privileges:topics:read granted to uid: ${user.uid}`); + } + } + winston.verbose(`-- cid ${cid} upgraded`); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js b/lib/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js new file mode 100644 index 0000000000..cd14e3ac75 --- /dev/null +++ b/lib/upgrades/1.1.0/dismiss_flags_from_deleted_topics.js @@ -0,0 +1,32 @@ +'use strict'; + +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Dismiss flags from deleted topics', + timestamp: Date.UTC(2016, 3, 29), + method: async function () { + const posts = require('../../posts'); + const topics = require('../../topics'); + const pids = await db.getSortedSetRange('posts:flagged', 0, -1); + const postData = await posts.getPostsFields(pids, ['tid']); + const tids = postData.map(t => t.tid); + const topicData = await topics.getTopicsFields(tids, ['deleted']); + const toDismiss = topicData.map((t, idx) => parseInt(t.deleted, 10) === 1 ? pids[idx] : null).filter(Boolean); + winston.verbose(`[2016/04/29] ${toDismiss.length} dismissable flags found`); + await Promise.all(toDismiss.map(dismissFlag)); + } +}; +async function dismissFlag(pid) { + const postData = await db.getObjectFields(`post:${pid}`, ['pid', 'uid', 'flags']); + if (!postData.pid) { + return; + } + if (parseInt(postData.uid, 10) && parseInt(postData.flags, 10) > 0) { + await Promise.all([db.sortedSetIncrBy('users:flags', -postData.flags, postData.uid), db.incrObjectFieldBy(`user:${postData.uid}`, 'flags', -postData.flags)]); + } + const uids = await db.getSortedSetRange(`pid:${pid}:flag:uids`, 0, -1); + const nids = uids.map(uid => `post_flag:${pid}:uid:${uid}`); + await Promise.all([db.deleteAll(nids.map(nid => `notifications:${nid}`)), db.sortedSetRemove('notifications', nids), db.delete(`pid:${pid}:flag:uids`), db.sortedSetsRemove(['posts:flagged', 'posts:flags:count', `uid:${postData.uid}:flag:pids`], pid), db.deleteObjectField(`post:${pid}`, 'flags'), db.delete(`pid:${pid}:flag:uid:reason`), db.deleteObjectFields(`post:${pid}`, ['flag:state', 'flag:assignee', 'flag:notes', 'flag:history'])]); + await db.sortedSetsRemoveRangeByScore(['users:flags'], '-inf', 0); +} \ No newline at end of file diff --git a/lib/upgrades/1.1.0/group_title_update.js b/lib/upgrades/1.1.0/group_title_update.js new file mode 100644 index 0000000000..4e16e6554e --- /dev/null +++ b/lib/upgrades/1.1.0/group_title_update.js @@ -0,0 +1,27 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Group title from settings to user profile', + timestamp: Date.UTC(2016, 3, 14), + method: function (callback) { + const user = require('../../user'); + const batch = require('../../batch'); + let count = 0; + batch.processSortedSet('users:joindate', (uids, next) => { + winston.verbose(`upgraded ${count} users`); + user.getMultipleUserSettings(uids, (err, settings) => { + if (err) { + return next(err); + } + count += uids.length; + settings = settings.filter(setting => setting && setting.groupTitle); + async.each(settings, (setting, next) => { + db.setObjectField(`user:${setting.uid}`, 'groupTitle', setting.groupTitle, next); + }, next); + }); + }, {}, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.1.0/separate_upvote_downvote.js b/lib/upgrades/1.1.0/separate_upvote_downvote.js new file mode 100644 index 0000000000..b128f38edd --- /dev/null +++ b/lib/upgrades/1.1.0/separate_upvote_downvote.js @@ -0,0 +1,50 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Store upvotes/downvotes separately', + timestamp: Date.UTC(2016, 5, 13), + method: function (callback) { + const batch = require('../../batch'); + const posts = require('../../posts'); + let count = 0; + const { + progress + } = this; + batch.processSortedSet('posts:pid', (pids, next) => { + winston.verbose(`upgraded ${count} posts`); + count += pids.length; + async.each(pids, (pid, next) => { + async.parallel({ + upvotes: function (next) { + db.setCount(`pid:${pid}:upvote`, next); + }, + downvotes: function (next) { + db.setCount(`pid:${pid}:downvote`, next); + } + }, (err, results) => { + if (err) { + return next(err); + } + const data = {}; + if (parseInt(results.upvotes, 10) > 0) { + data.upvotes = results.upvotes; + } + if (parseInt(results.downvotes, 10) > 0) { + data.downvotes = results.downvotes; + } + if (Object.keys(data).length) { + posts.setPostFields(pid, data, next); + } else { + next(); + } + progress.incr(); + }, next); + }, next); + }, { + progress: progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.1.0/user_post_count_per_tid.js b/lib/upgrades/1.1.0/user_post_count_per_tid.js new file mode 100644 index 0000000000..834152dd0c --- /dev/null +++ b/lib/upgrades/1.1.0/user_post_count_per_tid.js @@ -0,0 +1,44 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Users post count per tid', + timestamp: Date.UTC(2016, 3, 19), + method: function (callback) { + const batch = require('../../batch'); + const topics = require('../../topics'); + let count = 0; + batch.processSortedSet('topics:tid', (tids, next) => { + winston.verbose(`upgraded ${count} topics`); + count += tids.length; + async.each(tids, (tid, next) => { + db.delete(`tid:${tid}:posters`, err => { + if (err) { + return next(err); + } + topics.getPids(tid, (err, pids) => { + if (err) { + return next(err); + } + if (!pids.length) { + return next(); + } + async.eachSeries(pids, (pid, next) => { + db.getObjectField(`post:${pid}`, 'uid', (err, uid) => { + if (err) { + return next(err); + } + if (!parseInt(uid, 10)) { + return next(); + } + db.sortedSetIncrBy(`tid:${tid}:posters`, 1, uid, next); + }); + }, next); + }); + }); + }, next); + }, {}, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.1.1/remove_negative_best_posts.js b/lib/upgrades/1.1.1/remove_negative_best_posts.js new file mode 100644 index 0000000000..58f50bdb6d --- /dev/null +++ b/lib/upgrades/1.1.1/remove_negative_best_posts.js @@ -0,0 +1,18 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Removing best posts with negative scores', + timestamp: Date.UTC(2016, 7, 5), + method: function (callback) { + const batch = require('../../batch'); + batch.processSortedSet('users:joindate', (ids, next) => { + async.each(ids, (id, next) => { + winston.verbose(`processing uid ${id}`); + db.sortedSetsRemoveRangeByScore([`uid:${id}:posts:votes`], '-inf', 0, next); + }, next); + }, {}, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.1.1/upload_privileges.js b/lib/upgrades/1.1.1/upload_privileges.js new file mode 100644 index 0000000000..f632aba51d --- /dev/null +++ b/lib/upgrades/1.1.1/upload_privileges.js @@ -0,0 +1,34 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +module.exports = { + name: 'Giving upload privileges', + timestamp: Date.UTC(2016, 6, 12), + method: function (callback) { + const privilegesAPI = require('../../privileges'); + const meta = require('../../meta'); + db.getSortedSetRange('categories:cid', 0, -1, (err, cids) => { + if (err) { + return callback(err); + } + async.eachSeries(cids, (cid, next) => { + privilegesAPI.categories.list(cid, (err, data) => { + if (err) { + return next(err); + } + async.eachSeries(data.groups, (group, next) => { + if (group.name === 'guests' && parseInt(meta.config.allowGuestUploads, 10) !== 1) { + return next(); + } + if (group.privileges['groups:read']) { + privilegesAPI.categories.give(['upload:post:image'], cid, group.name, next); + } else { + next(); + } + }, next); + }); + }, callback); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.0/hash_recent_ip_addresses.js b/lib/upgrades/1.10.0/hash_recent_ip_addresses.js new file mode 100644 index 0000000000..a2225b55e1 --- /dev/null +++ b/lib/upgrades/1.10.0/hash_recent_ip_addresses.js @@ -0,0 +1,34 @@ +'use strict'; + +const async = require('async'); +const crypto = require('crypto'); +const nconf = require('nconf'); +const batch = require('../../batch'); +const db = require('../../database'); +module.exports = { + name: 'Hash all IP addresses stored in Recent IPs zset', + timestamp: Date.UTC(2018, 5, 22), + method: function (callback) { + const { + progress + } = this; + const hashed = /[a-f0-9]{32}/; + let hash; + batch.processSortedSet('ip:recent', (ips, next) => { + async.each(ips, (set, next) => { + if (hashed.test(set.value)) { + progress.incr(); + return setImmediate(next); + } + hash = crypto.createHash('sha1').update(set.value + nconf.get('secret')).digest('hex'); + async.series([async.apply(db.sortedSetAdd, 'ip:recent', set.score, hash), async.apply(db.sortedSetRemove, 'ip:recent', set.value)], err => { + progress.incr(); + next(err); + }); + }, next); + }, { + withScores: 1, + progress: this.progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.0/post_history_privilege.js b/lib/upgrades/1.10.0/post_history_privilege.js new file mode 100644 index 0000000000..0e14b875ac --- /dev/null +++ b/lib/upgrades/1.10.0/post_history_privilege.js @@ -0,0 +1,19 @@ +'use strict'; + +const async = require('async'); +const privileges = require('../../privileges'); +const db = require('../../database'); +module.exports = { + name: 'Give post history viewing privilege to registered-users on all categories', + timestamp: Date.UTC(2018, 5, 7), + method: function (callback) { + db.getSortedSetRange('categories:cid', 0, -1, (err, cids) => { + if (err) { + return callback(err); + } + async.eachSeries(cids, (cid, next) => { + privileges.categories.give(['groups:posts:history'], cid, 'registered-users', next); + }, callback); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.0/search_privileges.js b/lib/upgrades/1.10.0/search_privileges.js new file mode 100644 index 0000000000..c1dcee7175 --- /dev/null +++ b/lib/upgrades/1.10.0/search_privileges.js @@ -0,0 +1,22 @@ +'use strict'; + +module.exports = { + name: 'Give global search privileges', + timestamp: Date.UTC(2018, 4, 28), + method: async function () { + const meta = require('../../meta'); + const privileges = require('../../privileges'); + const allowGuestSearching = parseInt(meta.config.allowGuestSearching, 10) === 1; + const allowGuestUserSearching = parseInt(meta.config.allowGuestUserSearching, 10) === 1; + await privileges.global.give(['groups:search:content', 'groups:search:users', 'groups:search:tags'], 'registered-users'); + const guestPrivs = []; + if (allowGuestSearching) { + guestPrivs.push('groups:search:content'); + } + if (allowGuestUserSearching) { + guestPrivs.push('groups:search:users'); + } + guestPrivs.push('groups:search:tags'); + await privileges.global.give(guestPrivs, 'guests'); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.0/view_deleted_privilege.js b/lib/upgrades/1.10.0/view_deleted_privilege.js new file mode 100644 index 0000000000..3abc522719 --- /dev/null +++ b/lib/upgrades/1.10.0/view_deleted_privilege.js @@ -0,0 +1,21 @@ +'use strict'; + +const groups = require('../../groups'); +const db = require('../../database'); +module.exports = { + name: 'Give deleted post viewing privilege to moderators on all categories', + timestamp: Date.UTC(2018, 5, 8), + method: async function () { + const { + progress + } = this; + const cids = await db.getSortedSetRange('categories:cid', 0, -1); + for (const cid of cids) { + const uids = await db.getSortedSetRange(`group:cid:${cid}:privileges:moderate:members`, 0, -1); + for (const uid of uids) { + await groups.join(`cid:${cid}:privileges:posts:view_deleted`, uid); + } + progress.incr(); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.2/event_filters.js b/lib/upgrades/1.10.2/event_filters.js new file mode 100644 index 0000000000..75a840d4b8 --- /dev/null +++ b/lib/upgrades/1.10.2/event_filters.js @@ -0,0 +1,32 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'add filters to events', + timestamp: Date.UTC(2018, 9, 4), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('events:time', async eids => { + for (const eid of eids) { + progress.incr(); + const eventData = await db.getObject(`event:${eid}`); + if (!eventData) { + await db.sortedSetRemove('events:time', eid); + return; + } + if (!eventData.type && eventData.privilege) { + eventData.type = 'privilege-change'; + await db.setObjectField(`event:${eid}`, 'type', 'privilege-change'); + await db.sortedSetAdd(`events:time:${eventData.type}`, eventData.timestamp, eid); + return; + } + await db.sortedSetAdd(`events:time:${eventData.type || ''}`, eventData.timestamp, eid); + } + }, { + progress: this.progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.2/fix_category_post_zsets.js b/lib/upgrades/1.10.2/fix_category_post_zsets.js new file mode 100644 index 0000000000..779fd6aec7 --- /dev/null +++ b/lib/upgrades/1.10.2/fix_category_post_zsets.js @@ -0,0 +1,30 @@ +'use strict'; + +const db = require('../../database'); +const posts = require('../../posts'); +const topics = require('../../topics'); +const batch = require('../../batch'); +module.exports = { + name: 'Fix category post zsets', + timestamp: Date.UTC(2018, 9, 10), + method: async function () { + const { + progress + } = this; + const cids = await db.getSortedSetRange('categories:cid', 0, -1); + const keys = cids.map(cid => `cid:${cid}:pids`); + await batch.processSortedSet('posts:pid', async postData => { + const pids = postData.map(p => p.value); + const topicData = await posts.getPostsFields(pids, ['tid']); + const categoryData = await topics.getTopicsFields(topicData.map(t => t.tid), ['cid']); + await db.sortedSetRemove(keys, pids); + const bulkAdd = postData.map((p, i) => [`cid:${categoryData[i].cid}:pids`, p.score, p.value]); + await db.sortedSetAddBulk(bulkAdd); + progress.incr(postData.length); + }, { + batch: 500, + progress: progress, + withScores: true + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.2/fix_category_topic_zsets.js b/lib/upgrades/1.10.2/fix_category_topic_zsets.js new file mode 100644 index 0000000000..0aa0b22663 --- /dev/null +++ b/lib/upgrades/1.10.2/fix_category_topic_zsets.js @@ -0,0 +1,27 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Fix category topic zsets', + timestamp: Date.UTC(2018, 9, 11), + method: async function () { + const { + progress + } = this; + const topics = require('../../topics'); + await batch.processSortedSet('topics:tid', async tids => { + for (const tid of tids) { + progress.incr(); + const topicData = await db.getObjectFields(`topic:${tid}`, ['cid', 'pinned', 'postcount']); + if (parseInt(topicData.pinned, 10) !== 1) { + topicData.postcount = parseInt(topicData.postcount, 10) || 0; + await db.sortedSetAdd(`cid:${topicData.cid}:tids:posts`, topicData.postcount, tid); + } + await topics.updateLastPostTimeFromLastPid(tid); + } + }, { + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.2/local_login_privileges.js b/lib/upgrades/1.10.2/local_login_privileges.js new file mode 100644 index 0000000000..ccf07604e0 --- /dev/null +++ b/lib/upgrades/1.10.2/local_login_privileges.js @@ -0,0 +1,16 @@ +'use strict'; + +module.exports = { + name: 'Give global local login privileges', + timestamp: Date.UTC(2018, 8, 28), + method: function (callback) { + const meta = require('../../meta'); + const privileges = require('../../privileges'); + const allowLocalLogin = parseInt(meta.config.allowLocalLogin, 10) !== 0; + if (allowLocalLogin) { + privileges.global.give(['groups:local:login'], 'registered-users', callback); + } else { + callback(); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.2/postgres_sessions.js b/lib/upgrades/1.10.2/postgres_sessions.js new file mode 100644 index 0000000000..7f4567b364 --- /dev/null +++ b/lib/upgrades/1.10.2/postgres_sessions.js @@ -0,0 +1,39 @@ +'use strict'; + +const nconf = require('nconf'); +const db = require('../../database'); +module.exports = { + name: 'Optimize PostgreSQL sessions', + timestamp: Date.UTC(2018, 9, 1), + method: function (callback) { + if (nconf.get('database') !== 'postgres' || nconf.get('redis')) { + return callback(); + } + db.pool.query(` +BEGIN TRANSACTION; + +CREATE TABLE IF NOT EXISTS "session" ( + "sid" CHAR(32) NOT NULL + COLLATE "C" + PRIMARY KEY, + "sess" JSONB NOT NULL, + "expire" TIMESTAMPTZ NOT NULL +) WITHOUT OIDS; + +CREATE INDEX IF NOT EXISTS "session_expire_idx" ON "session"("expire"); + +ALTER TABLE "session" + ALTER "sid" TYPE CHAR(32) COLLATE "C", + ALTER "sid" SET STORAGE PLAIN, + ALTER "sess" TYPE JSONB, + ALTER "expire" TYPE TIMESTAMPTZ, + CLUSTER ON "session_expire_idx"; + +CLUSTER "session"; +ANALYZE "session"; + +COMMIT;`, err => { + callback(err); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.10.2/upgrade_bans_to_hashes.js b/lib/upgrades/1.10.2/upgrade_bans_to_hashes.js new file mode 100644 index 0000000000..df143c5643 --- /dev/null +++ b/lib/upgrades/1.10.2/upgrade_bans_to_hashes.js @@ -0,0 +1,47 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Upgrade bans to hashes', + timestamp: Date.UTC(2018, 8, 24), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('users:joindate', async uids => { + for (const uid of uids) { + progress.incr(); + const [bans, reasons, userData] = await Promise.all([db.getSortedSetRevRangeWithScores(`uid:${uid}:bans`, 0, -1), db.getSortedSetRevRangeWithScores(`banned:${uid}:reasons`, 0, -1), db.getObjectFields(`user:${uid}`, ['banned', 'banned:expire', 'joindate', 'lastposttime', 'lastonline'])]); + if (!bans.length && parseInt(userData.banned, 10)) { + const banTimestamp = userData.lastonline || userData.lastposttime || userData.joindate || Date.now(); + const banKey = `uid:${uid}:ban:${banTimestamp}`; + await addBan(uid, banKey, { + uid: uid, + timestamp: banTimestamp + }); + } else if (bans.length) { + for (const ban of bans) { + const reasonData = reasons.find(reasonData => reasonData.score === ban.score); + const banKey = `uid:${uid}:ban:${ban.score}`; + const data = { + uid: uid, + timestamp: ban.score, + expire: parseInt(ban.value, 10) + }; + if (reasonData) { + data.reason = reasonData.value; + } + await addBan(uid, banKey, data); + } + } + } + }, { + progress: this.progress + }); + } +}; +async function addBan(uid, key, data) { + await db.setObject(key, data); + await db.sortedSetAdd(`uid:${uid}:bans:timestamp`, data.timestamp, key); +} \ No newline at end of file diff --git a/lib/upgrades/1.10.2/username_email_history.js b/lib/upgrades/1.10.2/username_email_history.js new file mode 100644 index 0000000000..3db5210501 --- /dev/null +++ b/lib/upgrades/1.10.2/username_email_history.js @@ -0,0 +1,31 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +const user = require('../../user'); +module.exports = { + name: 'Record first entry in username/email history', + timestamp: Date.UTC(2018, 7, 28), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('users:joindate', async uids => { + async function updateHistory(uid, set, fieldName) { + const count = await db.sortedSetCard(set); + if (count <= 0) { + const userData = await user.getUserFields(uid, [fieldName, 'joindate']); + if (userData && userData.joindate && userData[fieldName]) { + await db.sortedSetAdd(set, userData.joindate, [userData[fieldName], userData.joindate].join(':')); + } + } + } + await Promise.all(uids.map(async uid => { + await Promise.all([updateHistory(uid, `user:${uid}:usernames`, 'username'), updateHistory(uid, `user:${uid}:emails`, 'email')]); + progress.incr(); + })); + }, { + progress: this.progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.11.0/navigation_visibility_groups.js b/lib/upgrades/1.11.0/navigation_visibility_groups.js new file mode 100644 index 0000000000..f5e0b18cd7 --- /dev/null +++ b/lib/upgrades/1.11.0/navigation_visibility_groups.js @@ -0,0 +1,53 @@ +'use strict'; + +module.exports = { + name: 'Navigation item visibility groups', + timestamp: Date.UTC(2018, 10, 10), + method: async function () { + const data = await navigationAdminGet(); + data.forEach(navItem => { + if (navItem && navItem.properties) { + navItem.groups = []; + if (navItem.properties.adminOnly) { + navItem.groups.push('administrators'); + } else if (navItem.properties.globalMod) { + navItem.groups.push('Global Moderators'); + } + if (navItem.properties.loggedIn) { + navItem.groups.push('registered-users'); + } else if (navItem.properties.guestOnly) { + navItem.groups.push('guests'); + } + } + }); + await navigationAdminSave(data); + } +}; +async function navigationAdminGet() { + const db = require('../../database'); + const data = await db.getSortedSetRange('navigation:enabled', 0, -1); + return data.filter(Boolean).map(item => { + item = JSON.parse(item); + item.groups = item.groups || []; + if (item.groups && !Array.isArray(item.groups)) { + item.groups = [item.groups]; + } + return item; + }); +} +async function navigationAdminSave(data) { + const db = require('../../database'); + const translator = require('../../translator'); + const order = Object.keys(data); + const items = data.map((item, index) => { + Object.keys(item).forEach(key => { + if (item.hasOwnProperty(key) && typeof item[key] === 'string' && (key === 'title' || key === 'text')) { + item[key] = translator.escape(item[key]); + } + }); + item.order = order[index]; + return JSON.stringify(item); + }); + await db.delete('navigation:enabled'); + await db.sortedSetAdd('navigation:enabled', order, items); +} \ No newline at end of file diff --git a/lib/upgrades/1.11.0/resize_image_width.js b/lib/upgrades/1.11.0/resize_image_width.js new file mode 100644 index 0000000000..de03686cfd --- /dev/null +++ b/lib/upgrades/1.11.0/resize_image_width.js @@ -0,0 +1,13 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Rename maximumImageWidth to resizeImageWidth', + timestamp: Date.UTC(2018, 9, 24), + method: async function () { + const meta = require('../../meta'); + const value = await meta.configs.get('maximumImageWidth'); + await meta.configs.set('resizeImageWidth', value); + await db.deleteObjectField('config', 'maximumImageWidth'); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.11.0/widget_visibility_groups.js b/lib/upgrades/1.11.0/widget_visibility_groups.js new file mode 100644 index 0000000000..288a4008a7 --- /dev/null +++ b/lib/upgrades/1.11.0/widget_visibility_groups.js @@ -0,0 +1,32 @@ +'use strict'; + +module.exports = { + name: 'Widget visibility groups', + timestamp: Date.UTC(2018, 10, 10), + method: async function () { + const widgetAdmin = require('../../widgets/admin'); + const widgets = require('../../widgets'); + const areas = await widgetAdmin.getAreas(); + for (const area of areas) { + if (area.data.length) { + area.widgets = area.data; + area.widgets.forEach(widget => { + if (widget && widget.data) { + const groupsToShow = ['administrators', 'Global Moderators']; + if (widget.data['hide-guests'] !== 'on') { + groupsToShow.push('guests'); + } + if (widget.data['hide-registered'] !== 'on') { + groupsToShow.push('registered-users'); + } + widget.data.groups = groupsToShow; + if (groupsToShow.length === 4) { + widget.data.groups.length = 0; + } + } + }); + await widgets.setArea(area); + } + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.11.1/remove_ignored_cids_per_user.js b/lib/upgrades/1.11.1/remove_ignored_cids_per_user.js new file mode 100644 index 0000000000..fb71f05802 --- /dev/null +++ b/lib/upgrades/1.11.1/remove_ignored_cids_per_user.js @@ -0,0 +1,21 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Remove uid::ignored:cids', + timestamp: Date.UTC(2018, 11, 11), + method: function (callback) { + const { + progress + } = this; + batch.processSortedSet('users:joindate', (uids, next) => { + progress.incr(uids.length); + const keys = uids.map(uid => `uid:${uid}:ignored:cids`); + db.deleteAll(keys, next); + }, { + progress: this.progress, + batch: 500 + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.0/category_watch_state.js b/lib/upgrades/1.12.0/category_watch_state.js new file mode 100644 index 0000000000..444df64ae1 --- /dev/null +++ b/lib/upgrades/1.12.0/category_watch_state.js @@ -0,0 +1,31 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +const categories = require('../../categories'); +module.exports = { + name: 'Update category watch data', + timestamp: Date.UTC(2018, 11, 13), + method: async function () { + const { + progress + } = this; + const cids = await db.getSortedSetRange('categories:cid', 0, -1); + const keys = cids.map(cid => `cid:${cid}:ignorers`); + await batch.processSortedSet('users:joindate', async uids => { + progress.incr(uids.length); + for (const cid of cids) { + const isMembers = await db.isSortedSetMembers(`cid:${cid}:ignorers`, uids); + uids = uids.filter((uid, index) => isMembers[index]); + if (uids.length) { + const states = uids.map(() => categories.watchStates.ignoring); + await db.sortedSetAdd(`cid:${cid}:uid:watch:state`, states, uids); + } + } + }, { + progress: progress, + batch: 500 + }); + await db.deleteAll(keys); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.0/global_view_privileges.js b/lib/upgrades/1.12.0/global_view_privileges.js new file mode 100644 index 0000000000..3247c79676 --- /dev/null +++ b/lib/upgrades/1.12.0/global_view_privileges.js @@ -0,0 +1,21 @@ +'use strict'; + +const async = require('async'); +const privileges = require('../../privileges'); +module.exports = { + name: 'Global view privileges', + timestamp: Date.UTC(2019, 0, 5), + method: function (callback) { + const meta = require('../../meta'); + const tasks = [async.apply(privileges.global.give, ['groups:view:users', 'groups:view:tags', 'groups:view:groups'], 'registered-users')]; + if (parseInt(meta.config.privateUserInfo, 10) !== 1) { + tasks.push(async.apply(privileges.global.give, ['groups:view:users', 'groups:view:groups'], 'guests')); + tasks.push(async.apply(privileges.global.give, ['groups:view:users', 'groups:view:groups'], 'spiders')); + } + if (parseInt(meta.config.privateTagListing, 10) !== 1) { + tasks.push(async.apply(privileges.global.give, ['groups:view:tags'], 'guests')); + tasks.push(async.apply(privileges.global.give, ['groups:view:tags'], 'spiders')); + } + async.series(tasks, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.0/group_create_privilege.js b/lib/upgrades/1.12.0/group_create_privilege.js new file mode 100644 index 0000000000..81e6a13c23 --- /dev/null +++ b/lib/upgrades/1.12.0/group_create_privilege.js @@ -0,0 +1,15 @@ +'use strict'; + +const privileges = require('../../privileges'); +module.exports = { + name: 'Group create global privilege', + timestamp: Date.UTC(2019, 0, 4), + method: function (callback) { + const meta = require('../../meta'); + if (parseInt(meta.config.allowGroupCreation, 10) === 1) { + privileges.global.give(['groups:group:create'], 'registered-users', callback); + } else { + setImmediate(callback); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.1/clear_username_email_history.js b/lib/upgrades/1.12.1/clear_username_email_history.js new file mode 100644 index 0000000000..16e711f79a --- /dev/null +++ b/lib/upgrades/1.12.1/clear_username_email_history.js @@ -0,0 +1,44 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +const user = require('../../user'); +module.exports = { + name: 'Delete username email history for deleted users', + timestamp: Date.UTC(2019, 2, 25), + method: function (callback) { + const { + progress + } = this; + let currentUid = 1; + db.getObjectField('global', 'nextUid', (err, nextUid) => { + if (err) { + return callback(err); + } + progress.total = nextUid; + async.whilst(next => { + next(null, currentUid < nextUid); + }, next => { + progress.incr(); + user.exists(currentUid, (err, exists) => { + if (err) { + return next(err); + } + if (exists) { + currentUid += 1; + return next(); + } + db.deleteAll([`user:${currentUid}:usernames`, `user:${currentUid}:emails`], err => { + if (err) { + return next(err); + } + currentUid += 1; + next(); + }); + }); + }, err => { + callback(err); + }); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.1/moderation_notes_refactor.js b/lib/upgrades/1.12.1/moderation_notes_refactor.js new file mode 100644 index 0000000000..627f6f3bae --- /dev/null +++ b/lib/upgrades/1.12.1/moderation_notes_refactor.js @@ -0,0 +1,32 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Update moderation notes to hashes', + timestamp: Date.UTC(2019, 3, 5), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('users:joindate', async uids => { + await Promise.all(uids.map(async uid => { + progress.incr(); + const notes = await db.getSortedSetRevRange(`uid:${uid}:moderation:notes`, 0, -1); + for (const note of notes) { + const noteData = JSON.parse(note); + noteData.timestamp = noteData.timestamp || Date.now(); + await db.sortedSetRemove(`uid:${uid}:moderation:notes`, note); + await db.setObject(`uid:${uid}:moderation:note:${noteData.timestamp}`, { + uid: noteData.uid, + timestamp: noteData.timestamp, + note: noteData.note + }); + await db.sortedSetAdd(`uid:${uid}:moderation:notes`, noteData.timestamp, noteData.timestamp); + } + })); + }, { + progress: this.progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.1/post_upload_sizes.js b/lib/upgrades/1.12.1/post_upload_sizes.js new file mode 100644 index 0000000000..3cbd1e7507 --- /dev/null +++ b/lib/upgrades/1.12.1/post_upload_sizes.js @@ -0,0 +1,23 @@ +'use strict'; + +const batch = require('../../batch'); +const posts = require('../../posts'); +const db = require('../../database'); +module.exports = { + name: 'Calculate image sizes of all uploaded images', + timestamp: Date.UTC(2019, 2, 16), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('posts:pid', async pids => { + const keys = pids.map(p => `post:${p}:uploads`); + const uploads = await db.getSortedSetRange(keys, 0, -1); + await posts.uploads.saveSize(uploads); + progress.incr(pids.length); + }, { + batch: 100, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.3/disable_plugin_metrics.js b/lib/upgrades/1.12.3/disable_plugin_metrics.js new file mode 100644 index 0000000000..a6f4b0a310 --- /dev/null +++ b/lib/upgrades/1.12.3/disable_plugin_metrics.js @@ -0,0 +1,10 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Disable plugin metrics for existing installs', + timestamp: Date.UTC(2019, 4, 21), + method: async function (callback) { + db.setObjectField('config', 'submitPluginUsage', 0, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.3/give_mod_info_privilege.js b/lib/upgrades/1.12.3/give_mod_info_privilege.js new file mode 100644 index 0000000000..b8dac1ba5f --- /dev/null +++ b/lib/upgrades/1.12.3/give_mod_info_privilege.js @@ -0,0 +1,23 @@ +'use strict'; + +const db = require('../../database'); +const privileges = require('../../privileges'); +const groups = require('../../groups'); +module.exports = { + name: 'give mod info privilege', + timestamp: Date.UTC(2019, 9, 8), + method: async function () { + const cids = await db.getSortedSetRevRange('categories:cid', 0, -1); + for (const cid of cids) { + await givePrivsToModerators(cid, ''); + await givePrivsToModerators(cid, 'groups:'); + } + await privileges.global.give(['groups:view:users:info'], 'Global Moderators'); + async function givePrivsToModerators(cid, groupPrefix) { + const members = await db.getSortedSetRevRange(`group:cid:${cid}:privileges:${groupPrefix}moderate:members`, 0, -1); + for (const member of members) { + await groups.join(['cid:0:privileges:view:users:info'], member); + } + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.3/give_mod_privileges.js b/lib/upgrades/1.12.3/give_mod_privileges.js new file mode 100644 index 0000000000..447546f212 --- /dev/null +++ b/lib/upgrades/1.12.3/give_mod_privileges.js @@ -0,0 +1,28 @@ +'use strict'; + +const privileges = require('../../privileges'); +const groups = require('../../groups'); +const db = require('../../database'); +module.exports = { + name: 'Give mods explicit privileges', + timestamp: Date.UTC(2019, 4, 28), + method: async function () { + const defaultPrivileges = ['find', 'read', 'topics:read', 'topics:create', 'topics:reply', 'topics:tag', 'posts:edit', 'posts:history', 'posts:delete', 'posts:upvote', 'posts:downvote', 'topics:delete']; + const modPrivileges = defaultPrivileges.concat(['posts:view_deleted', 'purge']); + const globalModPrivs = ['groups:chat', 'groups:upload:post:image', 'groups:upload:post:file', 'groups:signature', 'groups:ban', 'groups:search:content', 'groups:search:users', 'groups:search:tags', 'groups:view:users', 'groups:view:tags', 'groups:view:groups', 'groups:local:login']; + const cids = await db.getSortedSetRevRange('categories:cid', 0, -1); + for (const cid of cids) { + await givePrivsToModerators(cid, ''); + await givePrivsToModerators(cid, 'groups:'); + await privileges.categories.give(modPrivileges.map(p => `groups:${p}`), cid, ['Global Moderators']); + } + await privileges.global.give(globalModPrivs, 'Global Moderators'); + async function givePrivsToModerators(cid, groupPrefix) { + const privGroups = modPrivileges.map(priv => `cid:${cid}:privileges:${groupPrefix}${priv}`); + const members = await db.getSortedSetRevRange(`group:cid:${cid}:privileges:${groupPrefix}moderate:members`, 0, -1); + for (const member of members) { + await groups.join(privGroups, member); + } + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.3/update_registration_type.js b/lib/upgrades/1.12.3/update_registration_type.js new file mode 100644 index 0000000000..de39d43820 --- /dev/null +++ b/lib/upgrades/1.12.3/update_registration_type.js @@ -0,0 +1,19 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Update registration type', + timestamp: Date.UTC(2019, 5, 4), + method: function (callback) { + const meta = require('../../meta'); + const registrationType = meta.config.registrationType || 'normal'; + if (registrationType === 'admin-approval' || registrationType === 'admin-approval-ip') { + db.setObject('config', { + registrationType: 'normal', + registrationApprovalType: registrationType + }, callback); + } else { + setImmediate(callback); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.12.3/user_pid_sets.js b/lib/upgrades/1.12.3/user_pid_sets.js new file mode 100644 index 0000000000..0841a92434 --- /dev/null +++ b/lib/upgrades/1.12.3/user_pid_sets.js @@ -0,0 +1,33 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +const posts = require('../../posts'); +const topics = require('../../topics'); +module.exports = { + name: 'Create zsets for user posts per category', + timestamp: Date.UTC(2019, 5, 23), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('posts:pid', async pids => { + progress.incr(pids.length); + const postData = await posts.getPostsFields(pids, ['pid', 'uid', 'tid', 'upvotes', 'downvotes', 'timestamp']); + const tids = postData.map(p => p.tid); + const topicData = await topics.getTopicsFields(tids, ['cid']); + const bulk = []; + postData.forEach((p, index) => { + if (p && p.uid && p.pid && p.tid && p.timestamp) { + bulk.push([`cid:${topicData[index].cid}:uid:${p.uid}:pids`, p.timestamp, p.pid]); + if (p.votes > 0) { + bulk.push([`cid:${topicData[index].cid}:uid:${p.uid}:pids:votes`, p.votes, p.pid]); + } + } + }); + await db.sortedSetAddBulk(bulk); + }, { + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.13.0/clean_flag_byCid.js b/lib/upgrades/1.13.0/clean_flag_byCid.js new file mode 100644 index 0000000000..4c5a8ce250 --- /dev/null +++ b/lib/upgrades/1.13.0/clean_flag_byCid.js @@ -0,0 +1,26 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Clean flag byCid zsets', + timestamp: Date.UTC(2019, 8, 24), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('flags:datetime', async flagIds => { + progress.incr(flagIds.length); + const flagData = await db.getObjects(flagIds.map(id => `flag:${id}`)); + const bulkRemove = []; + for (const flagObj of flagData) { + if (flagObj && flagObj.type === 'user' && flagObj.targetId && flagObj.flagId) { + bulkRemove.push([`flags:byCid:${flagObj.targetId}`, flagObj.flagId]); + } + } + await db.sortedSetRemoveBulk(bulkRemove); + }, { + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.13.0/clean_post_topic_hash.js b/lib/upgrades/1.13.0/clean_post_topic_hash.js new file mode 100644 index 0000000000..f544288b78 --- /dev/null +++ b/lib/upgrades/1.13.0/clean_post_topic_hash.js @@ -0,0 +1,81 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Clean up post hash data', + timestamp: Date.UTC(2019, 9, 7), + method: async function () { + const { + progress + } = this; + await cleanPost(progress); + await cleanTopic(progress); + } +}; +async function cleanPost(progress) { + await batch.processSortedSet('posts:pid', async pids => { + progress.incr(pids.length); + const postData = await db.getObjects(pids.map(pid => `post:${pid}`)); + await Promise.all(postData.map(async post => { + if (!post) { + return; + } + const fieldsToDelete = []; + if (post.hasOwnProperty('editor') && post.editor === '') { + fieldsToDelete.push('editor'); + } + if (post.hasOwnProperty('deleted') && parseInt(post.deleted, 10) === 0) { + fieldsToDelete.push('deleted'); + } + if (post.hasOwnProperty('edited') && parseInt(post.edited, 10) === 0) { + fieldsToDelete.push('edited'); + } + const legacyFields = ['show_banned', 'fav_star_class', 'relativeEditTime', 'post_rep', 'relativeTime', 'fav_button_class', 'edited-class']; + legacyFields.forEach(field => { + if (post.hasOwnProperty(field)) { + fieldsToDelete.push(field); + } + }); + if (fieldsToDelete.length) { + await db.deleteObjectFields(`post:${post.pid}`, fieldsToDelete); + } + })); + }, { + batch: 500, + progress: progress + }); +} +async function cleanTopic(progress) { + await batch.processSortedSet('topics:tid', async tids => { + progress.incr(tids.length); + const topicData = await db.getObjects(tids.map(tid => `topic:${tid}`)); + await Promise.all(topicData.map(async topic => { + if (!topic) { + return; + } + const fieldsToDelete = []; + if (topic.hasOwnProperty('deleted') && parseInt(topic.deleted, 10) === 0) { + fieldsToDelete.push('deleted'); + } + if (topic.hasOwnProperty('pinned') && parseInt(topic.pinned, 10) === 0) { + fieldsToDelete.push('pinned'); + } + if (topic.hasOwnProperty('locked') && parseInt(topic.locked, 10) === 0) { + fieldsToDelete.push('locked'); + } + const legacyFields = ['category_name', 'category_slug']; + legacyFields.forEach(field => { + if (topic.hasOwnProperty(field)) { + fieldsToDelete.push(field); + } + }); + if (fieldsToDelete.length) { + await db.deleteObjectFields(`topic:${topic.tid}`, fieldsToDelete); + } + })); + }, { + batch: 500, + progress: progress + }); +} \ No newline at end of file diff --git a/lib/upgrades/1.13.0/cleanup_old_notifications.js b/lib/upgrades/1.13.0/cleanup_old_notifications.js new file mode 100644 index 0000000000..91b37110d0 --- /dev/null +++ b/lib/upgrades/1.13.0/cleanup_old_notifications.js @@ -0,0 +1,49 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +const user = require('../../user'); +module.exports = { + name: 'Clean up old notifications and hash data', + timestamp: Date.UTC(2019, 9, 7), + method: async function () { + const { + progress + } = this; + const week = 604800000; + const cutoffTime = Date.now() - week; + await batch.processSortedSet('users:joindate', async uids => { + progress.incr(uids.length); + await Promise.all([db.sortedSetsRemoveRangeByScore(uids.map(uid => `uid:${uid}:notifications:unread`), '-inf', cutoffTime), db.sortedSetsRemoveRangeByScore(uids.map(uid => `uid:${uid}:notifications:read`), '-inf', cutoffTime)]); + const userData = await user.getUsersData(uids); + await Promise.all(userData.map(async user => { + if (!user) { + return; + } + const fields = []; + ['picture', 'fullname', 'location', 'birthday', 'website', 'signature', 'uploadedpicture'].forEach(field => { + if (user[field] === '') { + fields.push(field); + } + }); + ['profileviews', 'reputation', 'postcount', 'topiccount', 'lastposttime', 'banned', 'followerCount', 'followingCount'].forEach(field => { + if (user[field] === 0) { + fields.push(field); + } + }); + if (user['icon:text']) { + fields.push('icon:text'); + } + if (user['icon:bgColor']) { + fields.push('icon:bgColor'); + } + if (fields.length) { + await db.deleteObjectFields(`user:${user.uid}`, fields); + } + })); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.13.3/fix_users_sorted_sets.js b/lib/upgrades/1.13.3/fix_users_sorted_sets.js new file mode 100644 index 0000000000..60adaf1a8a --- /dev/null +++ b/lib/upgrades/1.13.3/fix_users_sorted_sets.js @@ -0,0 +1,44 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Fix user sorted sets', + timestamp: Date.UTC(2020, 4, 2), + method: async function () { + const { + progress + } = this; + const nextUid = await db.getObjectField('global', 'nextUid'); + const allUids = []; + for (let i = 1; i <= nextUid; i++) { + allUids.push(i); + } + progress.total = nextUid; + let totalUserCount = 0; + await db.delete('user:null'); + await db.sortedSetsRemove(['users:joindate', 'users:reputation', 'users:postcount', 'users:flags'], 'null'); + await batch.processArray(allUids, async uids => { + progress.incr(uids.length); + const userData = await db.getObjects(uids.map(id => `user:${id}`)); + await Promise.all(userData.map(async (userData, index) => { + if (!userData || !userData.uid) { + await db.sortedSetsRemove(['users:joindate', 'users:reputation', 'users:postcount', 'users:flags'], uids[index]); + if (userData && !userData.uid) { + await db.delete(`user:${uids[index]}`); + } + return; + } + totalUserCount += 1; + await db.sortedSetAddBulk([['users:joindate', userData.joindate || Date.now(), uids[index]], ['users:reputation', userData.reputation || 0, uids[index]], ['users:postcount', userData.postcount || 0, uids[index]]]); + if (userData.hasOwnProperty('flags') && parseInt(userData.flags, 10) > 0) { + await db.sortedSetAdd('users:flags', userData.flags, uids[index]); + } + })); + }, { + progress: progress, + batch: 500 + }); + await db.setObjectField('global', 'userCount', totalUserCount); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.13.4/remove_allowFileUploads_priv.js b/lib/upgrades/1.13.4/remove_allowFileUploads_priv.js new file mode 100644 index 0000000000..c7e26f73c1 --- /dev/null +++ b/lib/upgrades/1.13.4/remove_allowFileUploads_priv.js @@ -0,0 +1,17 @@ +'use strict'; + +const db = require('../../database'); +const privileges = require('../../privileges'); +module.exports = { + name: 'Removing file upload privilege if file uploads were disabled (`allowFileUploads`)', + timestamp: Date.UTC(2020, 4, 21), + method: async () => { + const allowFileUploads = parseInt(await db.getObjectField('config', 'allowFileUploads'), 10); + if (allowFileUploads === 1) { + await db.deleteObjectField('config', 'allowFileUploads'); + return; + } + await privileges.categories.rescind(['groups:upload:post:file'], 0, ['guests', 'registered-users', 'Global Moderators']); + await db.deleteObjectField('config', 'allowFileUploads'); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.14.0/fix_category_image_field.js b/lib/upgrades/1.14.0/fix_category_image_field.js new file mode 100644 index 0000000000..fa17f0533a --- /dev/null +++ b/lib/upgrades/1.14.0/fix_category_image_field.js @@ -0,0 +1,24 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Remove duplicate image field for categories', + timestamp: Date.UTC(2020, 5, 9), + method: async () => { + const batch = require('../../batch'); + await batch.processSortedSet('categories:cid', async cids => { + let categoryData = await db.getObjects(cids.map(c => `category:${c}`)); + categoryData = categoryData.filter(c => c && (c.image || c.backgroundImage)); + if (categoryData.length) { + await Promise.all(categoryData.map(async data => { + if (data.image && !data.backgroundImage) { + await db.setObjectField(`category:${data.cid}`, 'backgroundImage', data.image); + } + await db.deleteObjectField(`category:${data.cid}`, 'image', data.image); + })); + } + }, { + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.14.0/unescape_navigation_titles.js b/lib/upgrades/1.14.0/unescape_navigation_titles.js new file mode 100644 index 0000000000..fba880dacd --- /dev/null +++ b/lib/upgrades/1.14.0/unescape_navigation_titles.js @@ -0,0 +1,31 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Unescape navigation titles', + timestamp: Date.UTC(2020, 5, 26), + method: async function () { + const data = await db.getSortedSetRangeWithScores('navigation:enabled', 0, -1); + const translator = require('../../translator'); + const order = []; + const items = []; + data.forEach(item => { + const navItem = JSON.parse(item.value); + if (navItem.hasOwnProperty('title')) { + navItem.title = translator.unescape(navItem.title); + navItem.title = navItem.title.replace(/\/g, ''); + } + if (navItem.hasOwnProperty('text')) { + navItem.text = translator.unescape(navItem.text); + navItem.text = navItem.text.replace(/\/g, ''); + } + if (navItem.hasOwnProperty('route')) { + navItem.route = navItem.route.replace('/', '/'); + } + order.push(item.score); + items.push(JSON.stringify(navItem)); + }); + await db.delete('navigation:enabled'); + await db.sortedSetAdd('navigation:enabled', order, items); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.14.1/readd_deleted_recent_topics.js b/lib/upgrades/1.14.1/readd_deleted_recent_topics.js new file mode 100644 index 0000000000..acf8846946 --- /dev/null +++ b/lib/upgrades/1.14.1/readd_deleted_recent_topics.js @@ -0,0 +1,32 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Re-add deleted topics to topics:recent', + timestamp: Date.UTC(2018, 9, 11), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('topics:tid', async tids => { + progress.incr(tids.length); + const topicData = await db.getObjectsFields(tids.map(tid => `topic:${tid}`), ['tid', 'lastposttime', 'viewcount', 'postcount', 'upvotes', 'downvotes']); + if (!topicData.tid) { + return; + } + topicData.forEach(t => { + if (t.hasOwnProperty('upvotes') && t.hasOwnProperty('downvotes')) { + t.votes = parseInt(t.upvotes, 10) - parseInt(t.downvotes, 10); + } + }); + await db.sortedSetAdd('topics:recent', topicData.map(t => t.lastposttime || 0), topicData.map(t => t.tid)); + await db.sortedSetAdd('topics:views', topicData.map(t => t.viewcount || 0), topicData.map(t => t.tid)); + await db.sortedSetAdd('topics:posts', topicData.map(t => t.postcount || 0), topicData.map(t => t.tid)); + await db.sortedSetAdd('topics:votes', topicData.map(t => t.votes || 0), topicData.map(t => t.tid)); + }, { + progress: progress, + batchSize: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/add_target_uid_to_flags.js b/lib/upgrades/1.15.0/add_target_uid_to_flags.js new file mode 100644 index 0000000000..f3dbb8fb6b --- /dev/null +++ b/lib/upgrades/1.15.0/add_target_uid_to_flags.js @@ -0,0 +1,38 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +const posts = require('../../posts'); +module.exports = { + name: 'Add target uid to flag objects', + timestamp: Date.UTC(2020, 7, 22), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('flags:datetime', async flagIds => { + progress.incr(flagIds.length); + const flagData = await db.getObjects(flagIds.map(id => `flag:${id}`)); + for (const flagObj of flagData) { + if (flagObj) { + const { + targetId + } = flagObj; + if (targetId) { + if (flagObj.type === 'post') { + const targetUid = await posts.getPostField(targetId, 'uid'); + if (targetUid) { + await db.setObjectField(`flag:${flagObj.flagId}`, 'targetUid', targetUid); + } + } else if (flagObj.type === 'user') { + await db.setObjectField(`flag:${flagObj.flagId}`, 'targetUid', targetId); + } + } + } + } + }, { + progress: progress, + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/consolidate_flags.js b/lib/upgrades/1.15.0/consolidate_flags.js new file mode 100644 index 0000000000..3b30ea57e8 --- /dev/null +++ b/lib/upgrades/1.15.0/consolidate_flags.js @@ -0,0 +1,38 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +const posts = require('../../posts'); +const user = require('../../user'); +module.exports = { + name: 'Consolidate multiple flags reports, going forward', + timestamp: Date.UTC(2020, 6, 16), + method: async function () { + const { + progress + } = this; + let flags = await db.getSortedSetRange('flags:datetime', 0, -1); + flags = flags.map(flagId => `flag:${flagId}`); + flags = await db.getObjectsFields(flags, ['flagId', 'type', 'targetId', 'uid', 'description', 'datetime']); + progress.total = flags.length; + await batch.processArray(flags, async subset => { + progress.incr(subset.length); + await Promise.all(subset.map(async flagObj => { + const methods = []; + switch (flagObj.type) { + case 'post': + methods.push(posts.setPostField.bind(posts, flagObj.targetId, 'flagId', flagObj.flagId)); + break; + case 'user': + methods.push(user.setUserField.bind(user, flagObj.targetId, 'flagId', flagObj.flagId)); + break; + } + methods.push(db.sortedSetAdd.bind(db, `flag:${flagObj.flagId}:reports`, flagObj.datetime, String(flagObj.description).slice(0, 250)), db.sortedSetAdd.bind(db, `flag:${flagObj.flagId}:reporters`, flagObj.datetime, flagObj.uid)); + await Promise.all(methods.map(async method => method())); + })); + }, { + progress: progress, + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/disable_sounds_plugin.js b/lib/upgrades/1.15.0/disable_sounds_plugin.js new file mode 100644 index 0000000000..1e472481ae --- /dev/null +++ b/lib/upgrades/1.15.0/disable_sounds_plugin.js @@ -0,0 +1,10 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Disable nodebb-plugin-soundpack-default', + timestamp: Date.UTC(2020, 8, 6), + method: async function () { + await db.sortedSetRemove('plugins:active', 'nodebb-plugin-soundpack-default'); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/fix_category_colors.js b/lib/upgrades/1.15.0/fix_category_colors.js new file mode 100644 index 0000000000..b71238846a --- /dev/null +++ b/lib/upgrades/1.15.0/fix_category_colors.js @@ -0,0 +1,22 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Fix category colors that are 3 digit hex colors', + timestamp: Date.UTC(2020, 9, 11), + method: async () => { + const batch = require('../../batch'); + await batch.processSortedSet('categories:cid', async cids => { + let categoryData = await db.getObjects(cids.map(c => `category:${c}`)); + categoryData = categoryData.filter(c => c && (c.color === '#fff' || c.color === '#333' || String(c.color).length !== 7)); + if (categoryData.length) { + await Promise.all(categoryData.map(async data => { + const color = `#${new Array(6).fill(data.color && data.color[1] || 'f').join('')}`; + await db.setObjectField(`category:${data.cid}`, 'color', color); + })); + } + }, { + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/fullname_search_set.js b/lib/upgrades/1.15.0/fullname_search_set.js new file mode 100644 index 0000000000..51dd090f33 --- /dev/null +++ b/lib/upgrades/1.15.0/fullname_search_set.js @@ -0,0 +1,23 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +const user = require('../../user'); +module.exports = { + name: 'Create fullname search set', + timestamp: Date.UTC(2020, 8, 11), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('users:joindate', async uids => { + progress.incr(uids.length); + const userData = await user.getUsersFields(uids, ['uid', 'fullname']); + const bulkAdd = userData.filter(u => u.uid && u.fullname).map(u => ['fullname:sorted', 0, `${String(u.fullname).slice(0, 255).toLowerCase()}:${u.uid}`]); + await db.sortedSetAddBulk(bulkAdd); + }, { + batch: 500, + progress: this.progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/remove_allow_from_uri.js b/lib/upgrades/1.15.0/remove_allow_from_uri.js new file mode 100644 index 0000000000..dbe56d8ae3 --- /dev/null +++ b/lib/upgrades/1.15.0/remove_allow_from_uri.js @@ -0,0 +1,14 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Remove allow from uri setting', + timestamp: Date.UTC(2020, 8, 6), + method: async function () { + const meta = require('../../meta'); + if (meta.config['allow-from-uri']) { + await db.setObjectField('config', 'csp-frame-ancestors', meta.config['allow-from-uri']); + } + await db.deleteObjectField('config', 'allow-from-uri'); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/remove_flag_reporters_zset.js b/lib/upgrades/1.15.0/remove_flag_reporters_zset.js new file mode 100644 index 0000000000..ab6f6e5ab0 --- /dev/null +++ b/lib/upgrades/1.15.0/remove_flag_reporters_zset.js @@ -0,0 +1,28 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Remove flag reporters sorted set', + timestamp: Date.UTC(2020, 6, 31), + method: async function () { + const { + progress + } = this; + progress.total = await db.sortedSetCard('flags:datetime'); + await batch.processSortedSet('flags:datetime', async flagIds => { + await Promise.all(flagIds.map(async flagId => { + const [reports, reporterUids] = await Promise.all([db.getSortedSetRevRangeWithScores(`flag:${flagId}:reports`, 0, -1), db.getSortedSetRevRange(`flag:${flagId}:reporters`, 0, -1)]); + const values = reports.reduce((memo, cur, idx) => { + memo.push([`flag:${flagId}:reports`, cur.score, [reporterUids[idx] || 0, cur.value].join(';')]); + return memo; + }, []); + await db.delete(`flag:${flagId}:reports`); + await db.sortedSetAddBulk(values); + })); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/topic_poster_count.js b/lib/upgrades/1.15.0/topic_poster_count.js new file mode 100644 index 0000000000..0a8554e3c2 --- /dev/null +++ b/lib/upgrades/1.15.0/topic_poster_count.js @@ -0,0 +1,31 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Store poster count in topic hash', + timestamp: Date.UTC(2020, 9, 24), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('topics:tid', async tids => { + progress.incr(tids.length); + const keys = tids.map(tid => `tid:${tid}:posters`); + await db.sortedSetsRemoveRangeByScore(keys, '-inf', 0); + const counts = await db.sortedSetsCard(keys); + const bulkSet = []; + for (let i = 0; i < tids.length; i++) { + if (counts[i] > 0) { + bulkSet.push([`topic:${tids[i]}`, { + postercount: counts[i] + }]); + } + } + await db.setObjectBulk(bulkSet); + }, { + progress: progress, + batchSize: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/track_flags_by_target.js b/lib/upgrades/1.15.0/track_flags_by_target.js new file mode 100644 index 0000000000..1ae759d47b --- /dev/null +++ b/lib/upgrades/1.15.0/track_flags_by_target.js @@ -0,0 +1,14 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'New sorted set for tracking flags by target', + timestamp: Date.UTC(2020, 6, 15), + method: async () => { + const flags = await db.getSortedSetRange('flags:hash', 0, -1); + await Promise.all(flags.map(async flag => { + flag = flag.split(':').slice(0, 2); + await db.sortedSetIncrBy('flags:byTarget', 1, flag.join(':')); + })); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.15.0/verified_users_group.js b/lib/upgrades/1.15.0/verified_users_group.js new file mode 100644 index 0000000000..be303cb96d --- /dev/null +++ b/lib/upgrades/1.15.0/verified_users_group.js @@ -0,0 +1,85 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +const user = require('../../user'); +const groups = require('../../groups'); +const meta = require('../../meta'); +const privileges = require('../../privileges'); +const now = Date.now(); +module.exports = { + name: 'Create verified/unverified user groups', + timestamp: Date.UTC(2020, 9, 13), + method: async function () { + const { + progress + } = this; + const maxGroupLength = meta.config.maximumGroupNameLength; + meta.config.maximumGroupNameLength = 30; + const timestamp = await db.getObjectField('group:administrators', 'timestamp'); + const verifiedExists = await groups.exists('verified-users'); + if (!verifiedExists) { + await groups.create({ + name: 'verified-users', + hidden: 1, + private: 1, + system: 1, + disableLeave: 1, + disableJoinRequests: 1, + timestamp: timestamp + 1 + }); + } + const unverifiedExists = await groups.exists('unverified-users'); + if (!unverifiedExists) { + await groups.create({ + name: 'unverified-users', + hidden: 1, + private: 1, + system: 1, + disableLeave: 1, + disableJoinRequests: 1, + timestamp: timestamp + 1 + }); + } + meta.config.maximumGroupNameLength = maxGroupLength; + await batch.processSortedSet('users:joindate', async uids => { + progress.incr(uids.length); + const userData = await user.getUsersFields(uids, ['uid', 'email:confirmed']); + const verified = userData.filter(u => parseInt(u['email:confirmed'], 10) === 1); + const unverified = userData.filter(u => parseInt(u['email:confirmed'], 10) !== 1); + await db.sortedSetAdd('group:verified-users:members', verified.map(() => now), verified.map(u => u.uid)); + await db.sortedSetAdd('group:unverified-users:members', unverified.map(() => now), unverified.map(u => u.uid)); + }, { + batch: 500, + progress: this.progress + }); + await db.delete('users:notvalidated'); + await updatePrivilges(); + const verifiedCount = await db.sortedSetCard('group:verified-users:members'); + const unverifiedCount = await db.sortedSetCard('group:unverified-users:members'); + await db.setObjectField('group:verified-users', 'memberCount', verifiedCount); + await db.setObjectField('group:unverified-users', 'memberCount', unverifiedCount); + } +}; +async function updatePrivilges() { + if (meta.config.requireEmailConfirmation) { + const cids = await db.getSortedSetRevRange('categories:cid', 0, -1); + const canChat = await privileges.global.canGroup('chat', 'registered-users'); + if (canChat) { + await privileges.global.give(['groups:chat'], 'verified-users'); + await privileges.global.rescind(['groups:chat'], 'registered-users'); + } + for (const cid of cids) { + const data = await privileges.categories.list(cid); + const registeredUsersPrivs = data.groups.find(d => d.name === 'registered-users').privileges; + if (registeredUsersPrivs['groups:topics:create']) { + await privileges.categories.give(['groups:topics:create'], cid, 'verified-users'); + await privileges.categories.rescind(['groups:topics:create'], cid, 'registered-users'); + } + if (registeredUsersPrivs['groups:topics:reply']) { + await privileges.categories.give(['groups:topics:reply'], cid, 'verified-users'); + await privileges.categories.rescind(['groups:topics:reply'], cid, 'registered-users'); + } + } + } +} \ No newline at end of file diff --git a/lib/upgrades/1.15.4/clear_purged_replies.js b/lib/upgrades/1.15.4/clear_purged_replies.js new file mode 100644 index 0000000000..5c20aa187d --- /dev/null +++ b/lib/upgrades/1.15.4/clear_purged_replies.js @@ -0,0 +1,31 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Clear purged replies and toPid', + timestamp: Date.UTC(2020, 10, 26), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('posts:pid', async pids => { + progress.incr(pids.length); + let postData = await db.getObjects(pids.map(pid => `post:${pid}`)); + postData = postData.filter(p => p && parseInt(p.toPid, 10)); + if (!postData.length) { + return; + } + const toPids = postData.map(p => p.toPid); + const exists = await db.exists(toPids.map(pid => `post:${pid}`)); + const pidsToDelete = postData.filter((p, index) => !exists[index]).map(p => p.pid); + await db.deleteObjectFields(pidsToDelete.map(pid => `post:${pid}`), ['toPid']); + const repliesToDelete = _.uniq(toPids.filter((pid, index) => !exists[index])); + await db.deleteAll(repliesToDelete.map(pid => `pid:${pid}:replies`)); + }, { + progress: progress, + batchSize: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.16.0/category_tags.js b/lib/upgrades/1.16.0/category_tags.js new file mode 100644 index 0000000000..dce08e7743 --- /dev/null +++ b/lib/upgrades/1.16.0/category_tags.js @@ -0,0 +1,38 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +const batch = require('../../batch'); +const topics = require('../../topics'); +module.exports = { + name: 'Create category tags sorted sets', + timestamp: Date.UTC(2020, 10, 23), + method: async function () { + const { + progress + } = this; + async function getTopicsTags(tids) { + return await db.getSetsMembers(tids.map(tid => `topic:${tid}:tags`)); + } + await batch.processSortedSet('topics:tid', async tids => { + const [topicData, tags] = await Promise.all([topics.getTopicsFields(tids, ['tid', 'cid', 'timestamp']), getTopicsTags(tids)]); + const topicsWithTags = topicData.map((t, i) => { + t.tags = tags[i]; + return t; + }).filter(t => t && t.tags.length); + await async.eachSeries(topicsWithTags, async topicObj => { + const { + cid, + tags + } = topicObj; + await db.sortedSetsAdd(tags.map(tag => `cid:${cid}:tag:${tag}:topics`), topicObj.timestamp, topicObj.tid); + const counts = await db.sortedSetsCard(tags.map(tag => `cid:${cid}:tag:${tag}:topics`)); + await db.sortedSetAdd(`cid:${cid}:tags`, counts, tags); + }); + progress.incr(tids.length); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.16.0/migrate_thumbs.js b/lib/upgrades/1.16.0/migrate_thumbs.js new file mode 100644 index 0000000000..209d94e9ac --- /dev/null +++ b/lib/upgrades/1.16.0/migrate_thumbs.js @@ -0,0 +1,40 @@ +'use strict'; + +const nconf = require('nconf'); +const db = require('../../database'); +const meta = require('../../meta'); +const topics = require('../../topics'); +const batch = require('../../batch'); +module.exports = { + name: 'Migrate existing topic thumbnails to new format', + timestamp: Date.UTC(2020, 11, 11), + method: async function () { + const { + progress + } = this; + const current = await meta.configs.get('topicThumbSize'); + if (parseInt(current, 10) === 120) { + await meta.configs.set('topicThumbSize', 512); + } + await batch.processSortedSet('topics:tid', async tids => { + const keys = tids.map(tid => `topic:${tid}`); + const topicThumbs = (await db.getObjectsFields(keys, ['thumb'])).map(obj => obj.thumb ? obj.thumb.replace(nconf.get('upload_url'), '') : null); + await Promise.all(tids.map(async (tid, idx) => { + const path = topicThumbs[idx]; + if (path) { + if (path.length < 255 && !path.startsWith('data:')) { + await topics.thumbs.associate({ + id: tid, + path + }); + } + await db.deleteObjectField(keys[idx], 'thumb'); + } + progress.incr(); + })); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.17.0/banned_users_group.js b/lib/upgrades/1.17.0/banned_users_group.js new file mode 100644 index 0000000000..7415dfcc49 --- /dev/null +++ b/lib/upgrades/1.17.0/banned_users_group.js @@ -0,0 +1,46 @@ +'use strict'; + +const batch = require('../../batch'); +const db = require('../../database'); +const groups = require('../../groups'); +const now = Date.now(); +module.exports = { + name: 'Move banned users to banned-users group', + timestamp: Date.UTC(2020, 11, 13), + method: async function () { + const { + progress + } = this; + const timestamp = await db.getObjectField('group:administrators', 'timestamp'); + const bannedExists = await groups.exists('banned-users'); + if (!bannedExists) { + await groups.create({ + name: 'banned-users', + hidden: 1, + private: 1, + system: 1, + disableLeave: 1, + disableJoinRequests: 1, + timestamp: timestamp + 1 + }); + } + await batch.processSortedSet('users:banned', async uids => { + progress.incr(uids.length); + await db.sortedSetAdd('group:banned-users:members', uids.map(() => now), uids); + await db.sortedSetRemove(['group:registered-users:members', 'group:verified-users:members', 'group:unverified-users:members', 'group:Global Moderators:members'], uids); + }, { + batch: 500, + progress: this.progress + }); + const bannedCount = await db.sortedSetCard('group:banned-users:members'); + const registeredCount = await db.sortedSetCard('group:registered-users:members'); + const verifiedCount = await db.sortedSetCard('group:verified-users:members'); + const unverifiedCount = await db.sortedSetCard('group:unverified-users:members'); + const globalModCount = await db.sortedSetCard('group:Global Moderators:members'); + await db.setObjectField('group:banned-users', 'memberCount', bannedCount); + await db.setObjectField('group:registered-users', 'memberCount', registeredCount); + await db.setObjectField('group:verified-users', 'memberCount', verifiedCount); + await db.setObjectField('group:unverified-users', 'memberCount', unverifiedCount); + await db.setObjectField('group:Global Moderators', 'memberCount', globalModCount); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.17.0/category_name_zset.js b/lib/upgrades/1.17.0/category_name_zset.js new file mode 100644 index 0000000000..25cf1a6ba6 --- /dev/null +++ b/lib/upgrades/1.17.0/category_name_zset.js @@ -0,0 +1,24 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Create category name sorted set', + timestamp: Date.UTC(2021, 0, 27), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('categories:cid', async cids => { + const keys = cids.map(cid => `category:${cid}`); + let categoryData = await db.getObjectsFields(keys, ['cid', 'name']); + categoryData = categoryData.filter(c => c.cid && c.name); + const bulkAdd = categoryData.map(cat => ['categories:name', 0, `${String(cat.name).slice(0, 200).toLowerCase()}:${cat.cid}`]); + await db.sortedSetAddBulk(bulkAdd); + progress.incr(cids.length); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.17.0/default_favicon.js b/lib/upgrades/1.17.0/default_favicon.js new file mode 100644 index 0000000000..3d70f4a5a2 --- /dev/null +++ b/lib/upgrades/1.17.0/default_favicon.js @@ -0,0 +1,19 @@ +'use strict'; + +const nconf = require('nconf'); +const path = require('path'); +const fs = require('fs'); +const file = require('../../file'); +module.exports = { + name: 'Store default favicon if it does not exist', + timestamp: Date.UTC(2021, 2, 9), + method: async function () { + const pathToIco = path.join(nconf.get('upload_path'), 'system', 'favicon.ico'); + const defaultIco = path.join(nconf.get('base_dir'), 'public', 'favicon.ico'); + const targetExists = await file.exists(pathToIco); + const defaultExists = await file.exists(defaultIco); + if (defaultExists && !targetExists) { + await fs.promises.copyFile(defaultIco, pathToIco); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.17.0/schedule_privilege_for_existing_categories.js b/lib/upgrades/1.17.0/schedule_privilege_for_existing_categories.js new file mode 100644 index 0000000000..5cc16e38f2 --- /dev/null +++ b/lib/upgrades/1.17.0/schedule_privilege_for_existing_categories.js @@ -0,0 +1,15 @@ +'use strict'; + +const db = require('../../database'); +const privileges = require('../../privileges'); +module.exports = { + name: 'Add "schedule" to default privileges of admins and gmods for existing categories', + timestamp: Date.UTC(2021, 2, 11), + method: async () => { + const privilegeToGive = ['groups:topics:schedule']; + const cids = await db.getSortedSetRevRange('categories:cid', 0, -1); + for (const cid of cids) { + await privileges.categories.give(privilegeToGive, cid, ['administrators', 'Global Moderators']); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.17.0/subcategories_per_page.js b/lib/upgrades/1.17.0/subcategories_per_page.js new file mode 100644 index 0000000000..b9d003ac8f --- /dev/null +++ b/lib/upgrades/1.17.0/subcategories_per_page.js @@ -0,0 +1,23 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Create subCategoriesPerPage property for categories', + timestamp: Date.UTC(2021, 0, 31), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('categories:cid', async cids => { + const keys = cids.map(cid => `category:${cid}`); + await db.setObject(keys, { + subCategoriesPerPage: 10 + }); + progress.incr(cids.length); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.17.0/topic_thumb_count.js b/lib/upgrades/1.17.0/topic_thumb_count.js new file mode 100644 index 0000000000..83da305dd9 --- /dev/null +++ b/lib/upgrades/1.17.0/topic_thumb_count.js @@ -0,0 +1,27 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Store number of thumbs a topic has in the topic object', + timestamp: Date.UTC(2021, 1, 7), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('topics:tid', async tids => { + const keys = tids.map(tid => `topic:${tid}:thumbs`); + const counts = await db.sortedSetsCard(keys); + const tidToCount = _.zipObject(tids, counts); + const tidsWithThumbs = tids.filter((t, i) => counts[i] > 0); + await db.setObjectBulk(tidsWithThumbs.map(tid => [`topic:${tid}`, { + numThumbs: tidToCount[tid] + }])); + progress.incr(tids.length); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.18.0/enable_include_unverified_emails.js b/lib/upgrades/1.18.0/enable_include_unverified_emails.js new file mode 100644 index 0000000000..c496e44687 --- /dev/null +++ b/lib/upgrades/1.18.0/enable_include_unverified_emails.js @@ -0,0 +1,10 @@ +'use strict'; + +const meta = require('../../meta'); +module.exports = { + name: 'Enable setting to include unverified emails for all mailings', + timestamp: Date.UTC(2021, 5, 18), + method: async () => { + await meta.configs.set('includeUnverifiedEmails', 1); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.18.0/topic_tags_refactor.js b/lib/upgrades/1.18.0/topic_tags_refactor.js new file mode 100644 index 0000000000..884e06aa11 --- /dev/null +++ b/lib/upgrades/1.18.0/topic_tags_refactor.js @@ -0,0 +1,34 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Store tags in topic hash', + timestamp: Date.UTC(2021, 8, 9), + method: async function () { + const { + progress + } = this; + async function getTopicsTags(tids) { + return await db.getSetsMembers(tids.map(tid => `topic:${tid}:tags`)); + } + await batch.processSortedSet('topics:tid', async tids => { + const tags = await getTopicsTags(tids); + const topicsWithTags = tids.map((tid, i) => { + const topic = { + tid: tid + }; + topic.tags = tags[i]; + return topic; + }).filter(t => t && t.tags.length); + await db.setObjectBulk(topicsWithTags.map(t => [`topic:${t.tid}`, { + tags: t.tags.join(',') + }])); + await db.deleteAll(tids.map(tid => `topic:${tid}:tags`)); + progress.incr(tids.length); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.18.4/category_topics_views.js b/lib/upgrades/1.18.4/category_topics_views.js new file mode 100644 index 0000000000..4a4027d4e9 --- /dev/null +++ b/lib/upgrades/1.18.4/category_topics_views.js @@ -0,0 +1,23 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +const topics = require('../../topics'); +module.exports = { + name: 'Category topics sorted sets by views', + timestamp: Date.UTC(2021, 8, 28), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('topics:tid', async tids => { + let topicData = await topics.getTopicsData(tids); + topicData = topicData.filter(t => t && t.cid); + await db.sortedSetAddBulk(topicData.map(t => [`cid:${t.cid}:tids:views`, t.viewcount || 0, t.tid])); + progress.incr(tids.length); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.19.0/navigation-enabled-hashes.js b/lib/upgrades/1.19.0/navigation-enabled-hashes.js new file mode 100644 index 0000000000..f13e4466c1 --- /dev/null +++ b/lib/upgrades/1.19.0/navigation-enabled-hashes.js @@ -0,0 +1,29 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Upgrade navigation items to hashes', + timestamp: Date.UTC(2021, 11, 13), + method: async function () { + const data = await db.getSortedSetRangeWithScores('navigation:enabled', 0, -1); + const order = []; + const bulkSet = []; + data.forEach(item => { + const navItem = JSON.parse(item.value); + if (navItem.hasOwnProperty('properties') && navItem.properties) { + if (navItem.properties.hasOwnProperty('targetBlank')) { + navItem.targetBlank = navItem.properties.targetBlank; + } + delete navItem.properties; + } + if (navItem.hasOwnProperty('groups') && (Array.isArray(navItem.groups) || typeof navItem.groups === 'string')) { + navItem.groups = JSON.stringify(navItem.groups); + } + bulkSet.push([`navigation:enabled:${item.score}`, navItem]); + order.push(item.score); + }); + await db.setObjectBulk(bulkSet); + await db.delete('navigation:enabled'); + await db.sortedSetAdd('navigation:enabled', order, order); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.19.0/reenable-username-login.js b/lib/upgrades/1.19.0/reenable-username-login.js new file mode 100644 index 0000000000..796a647410 --- /dev/null +++ b/lib/upgrades/1.19.0/reenable-username-login.js @@ -0,0 +1,13 @@ +'use strict'; + +const meta = require('../../meta'); +module.exports = { + name: 'Re-enable username login', + timestamp: Date.UTC(2021, 10, 23), + method: async () => { + const setting = await meta.config.allowLoginWith; + if (setting === 'email') { + await meta.configs.set('allowLoginWith', 'username-email'); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.19.2/remove_leftover_thumbs_after_topic_purge.js b/lib/upgrades/1.19.2/remove_leftover_thumbs_after_topic_purge.js new file mode 100644 index 0000000000..d5f420e7f8 --- /dev/null +++ b/lib/upgrades/1.19.2/remove_leftover_thumbs_after_topic_purge.js @@ -0,0 +1,42 @@ +'use strict'; + +const path = require('path'); +const fs = require('fs').promises; +const nconf = require('nconf'); +const db = require('../../database'); +const batch = require('../../batch'); +const file = require('../../file'); +module.exports = { + name: 'Clean up leftover topic thumb sorted sets and files for since-purged topics', + timestamp: Date.UTC(2022, 1, 7), + method: async function () { + const { + progress + } = this; + const nextTid = await db.getObjectField('global', 'nextTid'); + const tids = []; + for (let x = 1; x < nextTid; x++) { + tids.push(x); + } + const purgedTids = (await db.isSortedSetMembers('topics:tid', tids)).map((exists, idx) => exists ? false : tids[idx]).filter(Boolean); + const affectedTids = (await db.exists(purgedTids.map(tid => `topic:${tid}:thumbs`))).map((exists, idx) => exists ? purgedTids[idx] : false).filter(Boolean); + progress.total = affectedTids.length; + await batch.processArray(affectedTids, async tids => { + await Promise.all(tids.map(async tid => { + const relativePaths = await db.getSortedSetMembers(`topic:${tid}:thumbs`); + const absolutePaths = relativePaths.map(relativePath => path.join(nconf.get('upload_path'), relativePath)); + await Promise.all(absolutePaths.map(async absolutePath => { + const exists = await file.exists(absolutePath); + if (exists) { + await fs.unlink(absolutePath); + } + })); + await db.delete(`topic:${tid}:thumbs`); + progress.incr(); + })); + }, { + progress, + batch: 100 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.19.2/store_downvoted_posts_in_zset.js b/lib/upgrades/1.19.2/store_downvoted_posts_in_zset.js new file mode 100644 index 0000000000..ff00d6a6cb --- /dev/null +++ b/lib/upgrades/1.19.2/store_downvoted_posts_in_zset.js @@ -0,0 +1,30 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Store downvoted posts in user votes sorted set', + timestamp: Date.UTC(2022, 1, 4), + method: async function () { + const batch = require('../../batch'); + const posts = require('../../posts'); + const { + progress + } = this; + await batch.processSortedSet('posts:pid', async pids => { + const postData = await posts.getPostsFields(pids, ['pid', 'uid', 'upvotes', 'downvotes']); + const cids = await posts.getCidsByPids(pids); + const bulkAdd = []; + postData.forEach((post, index) => { + if (post.votes > 0 || post.votes < 0) { + const cid = cids[index]; + bulkAdd.push([`cid:${cid}:uid:${post.uid}:pids:votes`, post.votes, post.pid]); + } + }); + await db.sortedSetAddBulk(bulkAdd); + progress.incr(postData.length); + }, { + progress, + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.19.3/fix_user_uploads_zset.js b/lib/upgrades/1.19.3/fix_user_uploads_zset.js new file mode 100644 index 0000000000..db24514a07 --- /dev/null +++ b/lib/upgrades/1.19.3/fix_user_uploads_zset.js @@ -0,0 +1,34 @@ +'use strict'; + +const crypto = require('crypto'); +const db = require('../../database'); +const batch = require('../../batch'); +const md5 = filename => crypto.createHash('md5').update(filename).digest('hex'); +module.exports = { + name: 'Fix paths in user uploads sorted sets', + timestamp: Date.UTC(2022, 1, 10), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('users:joindate', async uids => { + progress.incr(uids.length); + await Promise.all(uids.map(async uid => { + const key = `uid:${uid}:uploads`; + let uploads = await db.getSortedSetRangeWithScores(key, 0, -1); + if (uploads.length) { + uploads = uploads.filter(upload => upload.value.startsWith('/files/')); + await db.sortedSetRemove(key, uploads.map(upload => upload.value)); + await db.sortedSetAdd(key, uploads.map(upload => upload.score), uploads.map(upload => upload.value.slice(1))); + uploads = await db.getSortedSetMembers(key); + await db.setObjectBulk(uploads.map(relativePath => [`upload:${md5(relativePath)}`, { + uid: uid + }])); + } + })); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.19.3/rename_post_upload_hashes.js b/lib/upgrades/1.19.3/rename_post_upload_hashes.js new file mode 100644 index 0000000000..830730132b --- /dev/null +++ b/lib/upgrades/1.19.3/rename_post_upload_hashes.js @@ -0,0 +1,42 @@ +'use strict'; + +const crypto = require('crypto'); +const db = require('../../database'); +const batch = require('../../batch'); +const md5 = filename => crypto.createHash('md5').update(filename).digest('hex'); +module.exports = { + name: 'Rename object and sorted sets used in post uploads', + timestamp: Date.UTC(2022, 1, 10), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('posts:pid', async pids => { + let keys = pids.map(pid => `post:${pid}:uploads`); + const exists = await db.exists(keys); + keys = keys.filter((key, idx) => exists[idx]); + progress.incr(pids.length); + for (const key of keys) { + let uploads = await db.getSortedSetRangeWithScores(key, 0, -1); + uploads = uploads.filter(upload => upload && upload.value && !upload.value.startsWith('files/')); + await db.sortedSetRemove(key, uploads.map(upload => upload.value)); + await db.sortedSetAdd(key, uploads.map(upload => upload.score), uploads.map(upload => `files/${upload.value}`)); + const hashes = uploads.map(upload => md5(upload.value)); + const newHashes = uploads.map(upload => md5(`files/${upload.value}`)); + const oldData = await db.getObjects(hashes.map(hash => `upload:${hash}`)); + const bulkSet = []; + oldData.forEach((data, idx) => { + if (data) { + bulkSet.push([`upload:${newHashes[idx]}`, data]); + } + }); + await db.setObjectBulk(bulkSet); + await db.deleteAll(hashes.map(hash => `upload:${hash}`)); + await Promise.all(hashes.map((hash, idx) => db.rename(`upload:${hash}:pids`, `upload:${newHashes[idx]}:pids`))); + } + }, { + batch: 100, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.2.0/category_recent_tids.js b/lib/upgrades/1.2.0/category_recent_tids.js new file mode 100644 index 0000000000..7483ba7860 --- /dev/null +++ b/lib/upgrades/1.2.0/category_recent_tids.js @@ -0,0 +1,28 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +module.exports = { + name: 'Category recent tids', + timestamp: Date.UTC(2016, 8, 22), + method: function (callback) { + db.getSortedSetRange('categories:cid', 0, -1, (err, cids) => { + if (err) { + return callback(err); + } + async.eachSeries(cids, (cid, next) => { + db.getSortedSetRevRange(`cid:${cid}:pids`, 0, 0, (err, pid) => { + if (err || !pid) { + return next(err); + } + db.getObjectFields(`post:${pid}`, ['tid', 'timestamp'], (err, postData) => { + if (err || !postData || !postData.tid) { + return next(err); + } + db.sortedSetAdd(`cid:${cid}:recent_tids`, postData.timestamp, postData.tid, next); + }); + }); + }, callback); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.2.0/edit_delete_deletetopic_privileges.js b/lib/upgrades/1.2.0/edit_delete_deletetopic_privileges.js new file mode 100644 index 0000000000..58fef05de6 --- /dev/null +++ b/lib/upgrades/1.2.0/edit_delete_deletetopic_privileges.js @@ -0,0 +1,41 @@ +'use strict'; + +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Granting edit/delete/delete topic on existing categories', + timestamp: Date.UTC(2016, 7, 7), + method: async function () { + const groupsAPI = require('../../groups'); + const privilegesAPI = require('../../privileges'); + const cids = await db.getSortedSetRange('categories:cid', 0, -1); + for (const cid of cids) { + const data = await privilegesAPI.categories.list(cid); + const { + groups, + users + } = data; + for (const group of groups) { + if (group.privileges['groups:topics:reply']) { + await Promise.all([groupsAPI.join(`cid:${cid}:privileges:groups:posts:edit`, group.name), groupsAPI.join(`cid:${cid}:privileges:groups:posts:delete`, group.name)]); + winston.verbose(`cid:${cid}:privileges:groups:posts:edit, cid:${cid}:privileges:groups:posts:delete granted to gid: ${group.name}`); + } + if (group.privileges['groups:topics:create']) { + await groupsAPI.join(`cid:${cid}:privileges:groups:topics:delete`, group.name); + winston.verbose(`cid:${cid}:privileges:groups:topics:delete granted to gid: ${group.name}`); + } + } + for (const user of users) { + if (user.privileges['topics:reply']) { + await Promise.all([groupsAPI.join(`cid:${cid}:privileges:posts:edit`, user.uid), groupsAPI.join(`cid:${cid}:privileges:posts:delete`, user.uid)]); + winston.verbose(`cid:${cid}:privileges:posts:edit, cid:${cid}:privileges:posts:delete granted to uid: ${user.uid}`); + } + if (user.privileges['topics:create']) { + await groupsAPI.join(`cid:${cid}:privileges:topics:delete`, user.uid); + winston.verbose(`cid:${cid}:privileges:topics:delete granted to uid: ${user.uid}`); + } + } + winston.verbose(`-- cid ${cid} upgraded`); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.3.0/favourites_to_bookmarks.js b/lib/upgrades/1.3.0/favourites_to_bookmarks.js new file mode 100644 index 0000000000..4925a49d57 --- /dev/null +++ b/lib/upgrades/1.3.0/favourites_to_bookmarks.js @@ -0,0 +1,37 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Favourites to Bookmarks', + timestamp: Date.UTC(2016, 9, 8), + method: async function () { + const { + progress + } = this; + const batch = require('../../batch'); + async function upgradePosts() { + await batch.processSortedSet('posts:pid', async ids => { + await Promise.all(ids.map(async id => { + progress.incr(); + await db.rename(`pid:${id}:users_favourited`, `pid:${id}:users_bookmarked`); + const reputation = await db.getObjectField(`post:${id}`, 'reputation'); + if (parseInt(reputation, 10)) { + await db.setObjectField(`post:${id}`, 'bookmarks', reputation); + } + await db.deleteObjectField(`post:${id}`, 'reputation'); + })); + }, { + progress: progress + }); + } + async function upgradeUsers() { + await batch.processSortedSet('users:joindate', async ids => { + await Promise.all(ids.map(async id => { + await db.rename(`uid:${id}:favourites`, `uid:${id}:bookmarks`); + })); + }, {}); + } + await upgradePosts(); + await upgradeUsers(); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.3.0/sorted_sets_for_post_replies.js b/lib/upgrades/1.3.0/sorted_sets_for_post_replies.js new file mode 100644 index 0000000000..48770f2a69 --- /dev/null +++ b/lib/upgrades/1.3.0/sorted_sets_for_post_replies.js @@ -0,0 +1,33 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Sorted sets for post replies', + timestamp: Date.UTC(2016, 9, 14), + method: function (callback) { + const posts = require('../../posts'); + const batch = require('../../batch'); + const { + progress + } = this; + batch.processSortedSet('posts:pid', (ids, next) => { + posts.getPostsFields(ids, ['pid', 'toPid', 'timestamp'], (err, data) => { + if (err) { + return next(err); + } + progress.incr(); + async.eachSeries(data, (postData, next) => { + if (!parseInt(postData.toPid, 10)) { + return next(null); + } + winston.verbose(`processing pid: ${postData.pid} toPid: ${postData.toPid}`); + async.parallel([async.apply(db.sortedSetAdd, `pid:${postData.toPid}:replies`, postData.timestamp, postData.pid), async.apply(db.incrObjectField, `post:${postData.toPid}`, 'replies')], next); + }, next); + }); + }, { + progress: progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.4.0/global_and_user_language_keys.js b/lib/upgrades/1.4.0/global_and_user_language_keys.js new file mode 100644 index 0000000000..9fba956900 --- /dev/null +++ b/lib/upgrades/1.4.0/global_and_user_language_keys.js @@ -0,0 +1,36 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Update global and user language keys', + timestamp: Date.UTC(2016, 10, 22), + method: async function () { + const { + progress + } = this; + const user = require('../../user'); + const meta = require('../../meta'); + const batch = require('../../batch'); + const defaultLang = await meta.configs.get('defaultLang'); + if (defaultLang) { + const newLanguage = defaultLang.replace('_', '-').replace('@', '-x-'); + if (newLanguage !== defaultLang) { + await meta.configs.set('defaultLang', newLanguage); + } + } + await batch.processSortedSet('users:joindate', async ids => { + await Promise.all(ids.map(async uid => { + progress.incr(); + const language = await db.getObjectField(`user:${uid}:settings`, 'userLang'); + if (language) { + const newLanguage = language.replace('_', '-').replace('@', '-x-'); + if (newLanguage !== language) { + await user.setSetting(uid, 'userLang', newLanguage); + } + } + })); + }, { + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.4.0/sorted_set_for_pinned_topics.js b/lib/upgrades/1.4.0/sorted_set_for_pinned_topics.js new file mode 100644 index 0000000000..449d7404a2 --- /dev/null +++ b/lib/upgrades/1.4.0/sorted_set_for_pinned_topics.js @@ -0,0 +1,25 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const db = require('../../database'); +module.exports = { + name: 'Sorted set for pinned topics', + timestamp: Date.UTC(2016, 10, 25), + method: function (callback) { + const topics = require('../../topics'); + const batch = require('../../batch'); + batch.processSortedSet('topics:tid', (ids, next) => { + topics.getTopicsFields(ids, ['tid', 'cid', 'pinned', 'lastposttime'], (err, data) => { + if (err) { + return next(err); + } + data = data.filter(topicData => parseInt(topicData.pinned, 10) === 1); + async.eachSeries(data, (topicData, next) => { + winston.verbose(`processing tid: ${topicData.tid}`); + async.parallel([async.apply(db.sortedSetAdd, `cid:${topicData.cid}:tids:pinned`, Date.now(), topicData.tid), async.apply(db.sortedSetRemove, `cid:${topicData.cid}:tids`, topicData.tid), async.apply(db.sortedSetRemove, `cid:${topicData.cid}:tids:posts`, topicData.tid)], next); + }, next); + }); + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.4.4/config_urls_update.js b/lib/upgrades/1.4.4/config_urls_update.js new file mode 100644 index 0000000000..a95cf4c937 --- /dev/null +++ b/lib/upgrades/1.4.4/config_urls_update.js @@ -0,0 +1,21 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Upgrading config urls to use assets route', + timestamp: Date.UTC(2017, 1, 28), + method: async function () { + const config = await db.getObject('config'); + if (config) { + const keys = ['brand:favicon', 'brand:touchicon', 'og:image', 'brand:logo:url', 'defaultAvatar', 'profile:defaultCovers']; + keys.forEach(key => { + const oldValue = config[key]; + if (!oldValue || typeof oldValue !== 'string') { + return; + } + config[key] = oldValue.replace(/(?:\/assets)?\/(images|uploads)\//g, '/assets/$1/'); + }); + await db.setObject('config', config); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.4.4/sound_settings.js b/lib/upgrades/1.4.4/sound_settings.js new file mode 100644 index 0000000000..30e94b73ff --- /dev/null +++ b/lib/upgrades/1.4.4/sound_settings.js @@ -0,0 +1,53 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +module.exports = { + name: 'Update global and user sound settings', + timestamp: Date.UTC(2017, 1, 25), + method: function (callback) { + const meta = require('../../meta'); + const batch = require('../../batch'); + const map = { + 'notification.mp3': 'Default | Deedle-dum', + 'waterdrop-high.mp3': 'Default | Water drop (high)', + 'waterdrop-low.mp3': 'Default | Water drop (low)' + }; + async.parallel([function (cb) { + const keys = ['chat-incoming', 'chat-outgoing', 'notification']; + db.getObject('settings:sounds', (err, settings) => { + if (err || !settings) { + return cb(err); + } + keys.forEach(key => { + if (settings[key] && !settings[key].includes(' | ')) { + settings[key] = map[settings[key]] || ''; + } + }); + meta.configs.setMultiple(settings, cb); + }); + }, function (cb) { + const keys = ['notificationSound', 'incomingChatSound', 'outgoingChatSound']; + batch.processSortedSet('users:joindate', (ids, next) => { + async.each(ids, (uid, next) => { + db.getObject(`user:${uid}:settings`, (err, settings) => { + if (err || !settings) { + return next(err); + } + const newSettings = {}; + keys.forEach(key => { + if (settings[key] && !settings[key].includes(' | ')) { + newSettings[key] = map[settings[key]] || ''; + } + }); + if (Object.keys(newSettings).length) { + db.setObject(`user:${uid}:settings`, newSettings, next); + } else { + setImmediate(next); + } + }); + }, next); + }, cb); + }], callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.4.6/delete_sessions.js b/lib/upgrades/1.4.6/delete_sessions.js new file mode 100644 index 0000000000..1e94f58949 --- /dev/null +++ b/lib/upgrades/1.4.6/delete_sessions.js @@ -0,0 +1,43 @@ +'use strict'; + +const nconf = require('nconf'); +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Delete accidentally long-lived sessions', + timestamp: Date.UTC(2017, 3, 16), + method: async function () { + let configJSON; + try { + configJSON = require('../../../config.json') || { + [process.env.database]: true + }; + } catch (err) { + configJSON = { + [process.env.database]: true + }; + } + const isRedisSessionStore = configJSON.hasOwnProperty('redis'); + const { + progress + } = this; + if (isRedisSessionStore) { + const connection = require('../../database/redis/connection'); + const client = await connection.connect(nconf.get('redis')); + const sessionKeys = await client.keys('sess:*'); + progress.total = sessionKeys.length; + await batch.processArray(sessionKeys, async keys => { + const multi = client.multi(); + keys.forEach(key => { + progress.incr(); + multi.del(key); + }); + await multi.exec(); + }, { + batch: 1000 + }); + } else if (db.client && db.client.collection) { + await db.client.collection('sessions').deleteMany({}, {}); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.5.0/allowed_file_extensions.js b/lib/upgrades/1.5.0/allowed_file_extensions.js new file mode 100644 index 0000000000..17a41438fa --- /dev/null +++ b/lib/upgrades/1.5.0/allowed_file_extensions.js @@ -0,0 +1,15 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Set default allowed file extensions', + timestamp: Date.UTC(2017, 3, 14), + method: function (callback) { + db.getObjectField('config', 'allowedFileExtensions', (err, value) => { + if (err || value) { + return callback(err); + } + db.setObjectField('config', 'allowedFileExtensions', 'png,jpg,bmp', callback); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.5.0/flags_refactor.js b/lib/upgrades/1.5.0/flags_refactor.js new file mode 100644 index 0000000000..b7115db464 --- /dev/null +++ b/lib/upgrades/1.5.0/flags_refactor.js @@ -0,0 +1,48 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Migrating flags to new schema', + timestamp: Date.UTC(2016, 11, 7), + method: async function () { + const batch = require('../../batch'); + const posts = require('../../posts'); + const flags = require('../../flags'); + const { + progress + } = this; + await batch.processSortedSet('posts:pid', async ids => { + let postData = await posts.getPostsByPids(ids, 1); + postData = postData.filter(post => post.hasOwnProperty('flags')); + await Promise.all(postData.map(async post => { + progress.incr(); + const [uids, reasons] = await Promise.all([db.getSortedSetRangeWithScores(`pid:${post.pid}:flag:uids`, 0, -1), db.getSortedSetRange(`pid:${post.pid}:flag:uid:reason`, 0, -1)]); + if (uids.length && reasons.length) { + const datetime = uids[0].score; + const reason = reasons[0].split(':')[1]; + try { + const flagObj = await flags.create('post', post.pid, uids[0].value, reason, datetime); + if (post['flag:state'] || post['flag:assignee']) { + await flags.update(flagObj.flagId, 1, { + state: post['flag:state'], + assignee: post['flag:assignee'], + datetime: datetime + }); + } + if (post.hasOwnProperty('flag:notes') && post['flag:notes'].length) { + let history = JSON.parse(post['flag:history']); + history = history.filter(event => event.type === 'notes')[0]; + await flags.appendNote(flagObj.flagId, history.uid, post['flag:notes'], history.timestamp); + } + } catch (err) { + if (err.message !== '[[error:post-already-flagged]]') { + throw err; + } + } + } + })); + }, { + progress: this.progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.5.0/moderation_history_refactor.js b/lib/upgrades/1.5.0/moderation_history_refactor.js new file mode 100644 index 0000000000..c8d73a941c --- /dev/null +++ b/lib/upgrades/1.5.0/moderation_history_refactor.js @@ -0,0 +1,33 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Update moderation notes to zset', + timestamp: Date.UTC(2017, 2, 22), + method: function (callback) { + const { + progress + } = this; + batch.processSortedSet('users:joindate', (ids, next) => { + async.each(ids, (uid, next) => { + db.getObjectField(`user:${uid}`, 'moderationNote', (err, moderationNote) => { + if (err || !moderationNote) { + progress.incr(); + return next(err); + } + const note = { + uid: 1, + note: moderationNote, + timestamp: Date.now() + }; + progress.incr(); + db.sortedSetAdd(`uid:${uid}:moderation:notes`, note.timestamp, JSON.stringify(note), next); + }); + }, next); + }, { + progress: this.progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.5.0/post_votes_zset.js b/lib/upgrades/1.5.0/post_votes_zset.js new file mode 100644 index 0000000000..620bb275c0 --- /dev/null +++ b/lib/upgrades/1.5.0/post_votes_zset.js @@ -0,0 +1,27 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +module.exports = { + name: 'New sorted set posts:votes', + timestamp: Date.UTC(2017, 1, 27), + method: function (callback) { + const { + progress + } = this; + require('../../batch').processSortedSet('posts:pid', (pids, next) => { + async.each(pids, (pid, next) => { + db.getObjectFields(`post:${pid}`, ['upvotes', 'downvotes'], (err, postData) => { + if (err || !postData) { + return next(err); + } + progress.incr(); + const votes = parseInt(postData.upvotes || 0, 10) - parseInt(postData.downvotes || 0, 10); + db.sortedSetAdd('posts:votes', votes, pid, next); + }); + }, next); + }, { + progress: this.progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.5.0/remove_relative_uploaded_profile_cover.js b/lib/upgrades/1.5.0/remove_relative_uploaded_profile_cover.js new file mode 100644 index 0000000000..709a614db1 --- /dev/null +++ b/lib/upgrades/1.5.0/remove_relative_uploaded_profile_cover.js @@ -0,0 +1,25 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Remove relative_path from uploaded profile cover urls', + timestamp: Date.UTC(2017, 3, 26), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('users:joindate', async ids => { + await Promise.all(ids.map(async uid => { + const url = await db.getObjectField(`user:${uid}`, 'cover:url'); + progress.incr(); + if (url) { + const newUrl = url.replace(/^.*?\/uploads\//, '/assets/uploads/'); + await db.setObjectField(`user:${uid}`, 'cover:url', newUrl); + } + })); + }, { + progress: this.progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.5.1/rename_mods_group.js b/lib/upgrades/1.5.1/rename_mods_group.js new file mode 100644 index 0000000000..7f01d31cc1 --- /dev/null +++ b/lib/upgrades/1.5.1/rename_mods_group.js @@ -0,0 +1,32 @@ +'use strict'; + +const async = require('async'); +const winston = require('winston'); +const batch = require('../../batch'); +const groups = require('../../groups'); +module.exports = { + name: 'rename user mod privileges group', + timestamp: Date.UTC(2017, 4, 26), + method: function (callback) { + const { + progress + } = this; + batch.processSortedSet('categories:cid', (cids, next) => { + async.eachSeries(cids, (cid, next) => { + const groupName = `cid:${cid}:privileges:mods`; + const newName = `cid:${cid}:privileges:moderate`; + groups.exists(groupName, (err, exists) => { + if (err || !exists) { + progress.incr(); + return next(err); + } + winston.verbose(`renaming ${groupName} to ${newName}`); + progress.incr(); + groups.renameGroup(groupName, newName, next); + }); + }, next); + }, { + progress: progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.5.2/rss_token_wipe.js b/lib/upgrades/1.5.2/rss_token_wipe.js new file mode 100644 index 0000000000..3629e8e065 --- /dev/null +++ b/lib/upgrades/1.5.2/rss_token_wipe.js @@ -0,0 +1,22 @@ +'use strict'; + +const async = require('async'); +const batch = require('../../batch'); +const db = require('../../database'); +module.exports = { + name: 'Wipe all existing RSS tokens', + timestamp: Date.UTC(2017, 6, 5), + method: function (callback) { + const { + progress + } = this; + batch.processSortedSet('users:joindate', (uids, next) => { + async.eachLimit(uids, 500, (uid, next) => { + progress.incr(); + db.deleteObjectField(`user:${uid}`, 'rss_token', next); + }, next); + }, { + progress: progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.5.2/tags_privilege.js b/lib/upgrades/1.5.2/tags_privilege.js new file mode 100644 index 0000000000..846c668bff --- /dev/null +++ b/lib/upgrades/1.5.2/tags_privilege.js @@ -0,0 +1,22 @@ +'use strict'; + +const async = require('async'); +const batch = require('../../batch'); +module.exports = { + name: 'Give tag privilege to registered-users on all categories', + timestamp: Date.UTC(2017, 5, 16), + method: function (callback) { + const { + progress + } = this; + const privileges = require('../../privileges'); + batch.processSortedSet('categories:cid', (cids, next) => { + async.eachSeries(cids, (cid, next) => { + progress.incr(); + privileges.categories.give(['groups:topics:tag'], cid, 'registered-users', next); + }, next); + }, { + progress: progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.6.0/clear-stale-digest-template.js b/lib/upgrades/1.6.0/clear-stale-digest-template.js new file mode 100644 index 0000000000..8c80e1d8f9 --- /dev/null +++ b/lib/upgrades/1.6.0/clear-stale-digest-template.js @@ -0,0 +1,16 @@ +'use strict'; + +const crypto = require('crypto'); +const meta = require('../../meta'); +module.exports = { + name: 'Clearing stale digest templates that were accidentally saved as custom', + timestamp: Date.UTC(2017, 8, 6), + method: async function () { + const matches = ['112e541b40023d6530dd44df4b0d9c5d', '110b8805f70395b0282fd10555059e9f', '9538e7249edb369b2a25b03f2bd3282b']; + const fieldset = await meta.configs.getFields(['email:custom:digest']); + const hash = fieldset['email:custom:digest'] ? crypto.createHash('md5').update(fieldset['email:custom:digest']).digest('hex') : null; + if (matches.includes(hash)) { + await meta.configs.remove('email:custom:digest'); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.6.0/generate-email-logo.js b/lib/upgrades/1.6.0/generate-email-logo.js new file mode 100644 index 0000000000..e96bd5daf7 --- /dev/null +++ b/lib/upgrades/1.6.0/generate-email-logo.js @@ -0,0 +1,42 @@ +'use strict'; + +const async = require('async'); +const path = require('path'); +const nconf = require('nconf'); +const fs = require('fs'); +const meta = require('../../meta'); +const image = require('../../image'); +module.exports = { + name: 'Generate email logo for use in email header', + timestamp: Date.UTC(2017, 6, 17), + method: function (callback) { + let skip = false; + async.series([function (next) { + const uploadPath = path.join(nconf.get('upload_path'), 'system', 'site-logo-x50.png'); + const sourcePath = meta.config['brand:logo'] ? path.join(nconf.get('upload_path'), 'system', path.basename(meta.config['brand:logo'])) : null; + if (!sourcePath) { + skip = true; + return setImmediate(next); + } + fs.access(sourcePath, err => { + if (err || path.extname(sourcePath) === '.svg') { + skip = true; + return setImmediate(next); + } + image.resizeImage({ + path: sourcePath, + target: uploadPath, + height: 50 + }, next); + }); + }, function (next) { + if (skip) { + return setImmediate(next); + } + meta.configs.setMultiple({ + 'brand:logo': path.join('/assets/uploads/system', path.basename(meta.config['brand:logo'])), + 'brand:emailLogo': '/assets/uploads/system/site-logo-x50.png' + }, next); + }], callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.6.0/ipblacklist-fix.js b/lib/upgrades/1.6.0/ipblacklist-fix.js new file mode 100644 index 0000000000..673f15fa5e --- /dev/null +++ b/lib/upgrades/1.6.0/ipblacklist-fix.js @@ -0,0 +1,14 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Changing ip blacklist storage to object', + timestamp: Date.UTC(2017, 8, 7), + method: async function () { + const rules = await db.get('ip-blacklist-rules'); + await db.delete('ip-blacklist-rules'); + await db.setObject('ip-blacklist-rules', { + rules: rules + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.6.0/robots-config-change.js b/lib/upgrades/1.6.0/robots-config-change.js new file mode 100644 index 0000000000..778ea9be7f --- /dev/null +++ b/lib/upgrades/1.6.0/robots-config-change.js @@ -0,0 +1,19 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Fix incorrect robots.txt schema', + timestamp: Date.UTC(2017, 6, 10), + method: async function () { + const config = await db.getObject('config'); + if (config) { + if (config.robots && config.robots.txt) { + await db.setObjectField('config', 'robots:txt', config.robots.txt); + } else if (typeof config['robots.txt'] === 'string' && config['robots.txt']) { + await db.setObjectField('config', 'robots:txt', config['robots.txt']); + } + await db.deleteObjectField('config', 'robots'); + await db.deleteObjectField('config', 'robots.txt'); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.6.2/topics_lastposttime_zset.js b/lib/upgrades/1.6.2/topics_lastposttime_zset.js new file mode 100644 index 0000000000..b5f8c66556 --- /dev/null +++ b/lib/upgrades/1.6.2/topics_lastposttime_zset.js @@ -0,0 +1,27 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +module.exports = { + name: 'New sorted set cid::tids:lastposttime', + timestamp: Date.UTC(2017, 9, 30), + method: function (callback) { + const { + progress + } = this; + require('../../batch').processSortedSet('topics:tid', (tids, next) => { + async.eachSeries(tids, (tid, next) => { + db.getObjectFields(`topic:${tid}`, ['cid', 'timestamp', 'lastposttime'], (err, topicData) => { + if (err || !topicData) { + return next(err); + } + progress.incr(); + const timestamp = topicData.lastposttime || topicData.timestamp || Date.now(); + db.sortedSetAdd(`cid:${topicData.cid}:tids:lastposttime`, timestamp, tid, next); + }, next); + }, next); + }, { + progress: this.progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.0/generate-custom-html.js b/lib/upgrades/1.7.0/generate-custom-html.js new file mode 100644 index 0000000000..450d5a8789 --- /dev/null +++ b/lib/upgrades/1.7.0/generate-custom-html.js @@ -0,0 +1,30 @@ +'use strict'; + +const db = require('../../database'); +const meta = require('../../meta'); +module.exports = { + name: 'Generate customHTML block from old customJS setting', + timestamp: Date.UTC(2017, 9, 12), + method: function (callback) { + db.getObjectField('config', 'customJS', (err, newHTML) => { + if (err) { + return callback(err); + } + let newJS = []; + const scriptMatch = /^([\s\S]+?)<\/script>/m; + let match = scriptMatch.exec(newHTML); + while (match) { + if (match[1]) { + newJS.push(match[1].trim()); + newHTML = ((match.index > 0 ? newHTML.slice(0, match.index) : '') + newHTML.slice(match.index + match[0].length)).trim(); + } + match = scriptMatch.exec(newHTML); + } + newJS = newJS.join('\n\n'); + meta.configs.setMultiple({ + customHTML: newHTML, + customJS: newJS + }, callback); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.1/notification-settings.js b/lib/upgrades/1.7.1/notification-settings.js new file mode 100644 index 0000000000..275e90b77f --- /dev/null +++ b/lib/upgrades/1.7.1/notification-settings.js @@ -0,0 +1,31 @@ +'use strict'; + +const batch = require('../../batch'); +const db = require('../../database'); +module.exports = { + name: 'Convert old notification digest settings', + timestamp: Date.UTC(2017, 10, 15), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('users:joindate', async uids => { + await Promise.all(uids.map(async uid => { + progress.incr(); + const userSettings = await db.getObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']); + if (userSettings) { + if (parseInt(userSettings.sendChatNotifications, 10) === 1) { + await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-chat', 'notificationemail'); + } + if (parseInt(userSettings.sendPostNotifications, 10) === 1) { + await db.setObjectField(`user:${uid}:settings`, 'notificationType_new-reply', 'notificationemail'); + } + } + await db.deleteObjectFields(`user:${uid}:settings`, ['sendChatNotifications', 'sendPostNotifications']); + })); + }, { + progress: progress, + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.3/key_value_schema_change.js b/lib/upgrades/1.7.3/key_value_schema_change.js new file mode 100644 index 0000000000..01ee6d53d3 --- /dev/null +++ b/lib/upgrades/1.7.3/key_value_schema_change.js @@ -0,0 +1,63 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Change the schema of simple keys so they don\'t use value field (mongodb only)', + timestamp: Date.UTC(2017, 11, 18), + method: async function () { + let configJSON; + try { + configJSON = require('../../../config.json') || { + [process.env.database]: true, + database: process.env.database + }; + } catch (err) { + configJSON = { + [process.env.database]: true, + database: process.env.database + }; + } + const isMongo = configJSON.hasOwnProperty('mongo') && configJSON.database === 'mongo'; + const { + progress + } = this; + if (!isMongo) { + return; + } + const { + client + } = db; + const query = { + _key: { + $exists: true + }, + value: { + $exists: true + }, + score: { + $exists: false + } + }; + progress.total = await client.collection('objects').countDocuments(query); + const cursor = await client.collection('objects').find(query).batchSize(1000); + let done = false; + while (!done) { + const item = await cursor.next(); + progress.incr(); + if (item === null) { + done = true; + } else { + delete item.expireAt; + if (Object.keys(item).length === 3 && item.hasOwnProperty('_key') && item.hasOwnProperty('value')) { + await client.collection('objects').updateOne({ + _key: item._key + }, { + $rename: { + value: 'data' + } + }); + } + } + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.3/topic_votes.js b/lib/upgrades/1.7.3/topic_votes.js new file mode 100644 index 0000000000..4de532744d --- /dev/null +++ b/lib/upgrades/1.7.3/topic_votes.js @@ -0,0 +1,38 @@ +'use strict'; + +const batch = require('../../batch'); +const db = require('../../database'); +module.exports = { + name: 'Add votes to topics', + timestamp: Date.UTC(2017, 11, 8), + method: async function () { + const { + progress + } = this; + batch.processSortedSet('topics:tid', async tids => { + await Promise.all(tids.map(async tid => { + progress.incr(); + const topicData = await db.getObjectFields(`topic:${tid}`, ['mainPid', 'cid', 'pinned']); + if (topicData.mainPid && topicData.cid) { + const postData = await db.getObject(`post:${topicData.mainPid}`); + if (postData) { + const upvotes = parseInt(postData.upvotes, 10) || 0; + const downvotes = parseInt(postData.downvotes, 10) || 0; + const data = { + upvotes: upvotes, + downvotes: downvotes + }; + const votes = upvotes - downvotes; + await Promise.all([db.setObject(`topic:${tid}`, data), db.sortedSetAdd('topics:votes', votes, tid)]); + if (parseInt(topicData.pinned, 10) !== 1) { + await db.sortedSetAdd(`cid:${topicData.cid}:tids:votes`, votes, tid); + } + } + } + })); + }, { + progress: progress, + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.4/chat_privilege.js b/lib/upgrades/1.7.4/chat_privilege.js new file mode 100644 index 0000000000..642065e396 --- /dev/null +++ b/lib/upgrades/1.7.4/chat_privilege.js @@ -0,0 +1,10 @@ +'use strict'; + +const groups = require('../../groups'); +module.exports = { + name: 'Give chat privilege to registered-users', + timestamp: Date.UTC(2017, 11, 18), + method: function (callback) { + groups.join('cid:0:privileges:groups:chat', 'registered-users', callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.4/fix_moved_topics_byvotes.js b/lib/upgrades/1.7.4/fix_moved_topics_byvotes.js new file mode 100644 index 0000000000..a059c954a7 --- /dev/null +++ b/lib/upgrades/1.7.4/fix_moved_topics_byvotes.js @@ -0,0 +1,31 @@ +'use strict'; + +const batch = require('../../batch'); +const db = require('../../database'); +module.exports = { + name: 'Fix sort by votes for moved topics', + timestamp: Date.UTC(2018, 0, 8), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('topics:tid', async tids => { + await Promise.all(tids.map(async tid => { + progress.incr(); + const topicData = await db.getObjectFields(`topic:${tid}`, ['cid', 'oldCid', 'upvotes', 'downvotes', 'pinned']); + if (topicData.cid && topicData.oldCid) { + const upvotes = parseInt(topicData.upvotes, 10) || 0; + const downvotes = parseInt(topicData.downvotes, 10) || 0; + const votes = upvotes - downvotes; + await db.sortedSetRemove(`cid:${topicData.oldCid}:tids:votes`, tid); + if (parseInt(topicData.pinned, 10) !== 1) { + await db.sortedSetAdd(`cid:${topicData.cid}:tids:votes`, votes, tid); + } + } + })); + }, { + progress: progress, + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.4/fix_user_topics_per_category.js b/lib/upgrades/1.7.4/fix_user_topics_per_category.js new file mode 100644 index 0000000000..50b2733507 --- /dev/null +++ b/lib/upgrades/1.7.4/fix_user_topics_per_category.js @@ -0,0 +1,29 @@ +'use strict'; + +const batch = require('../../batch'); +const db = require('../../database'); +module.exports = { + name: 'Fix topics in categories per user if they were moved', + timestamp: Date.UTC(2018, 0, 22), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('topics:tid', async tids => { + await Promise.all(tids.map(async tid => { + progress.incr(); + const topicData = await db.getObjectFields(`topic:${tid}`, ['cid', 'tid', 'uid', 'oldCid', 'timestamp']); + if (topicData.cid && topicData.oldCid) { + const isMember = await db.isSortedSetMember(`cid:${topicData.oldCid}:uid:${topicData.uid}:tids`, topicData.tid); + if (isMember) { + await db.sortedSetRemove(`cid:${topicData.oldCid}:uid:${topicData.uid}:tids`, tid); + await db.sortedSetAdd(`cid:${topicData.cid}:uid:${topicData.uid}:tids`, topicData.timestamp, tid); + } + } + })); + }, { + progress: progress, + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.4/global_upload_privilege.js b/lib/upgrades/1.7.4/global_upload_privilege.js new file mode 100644 index 0000000000..1a8fa0a4bb --- /dev/null +++ b/lib/upgrades/1.7.4/global_upload_privilege.js @@ -0,0 +1,39 @@ +'use strict'; + +const async = require('async'); +const groups = require('../../groups'); +const privileges = require('../../privileges'); +const db = require('../../database'); +module.exports = { + name: 'Give upload privilege to registered-users globally if it is given on a category', + timestamp: Date.UTC(2018, 0, 3), + method: function (callback) { + db.getSortedSetRange('categories:cid', 0, -1, (err, cids) => { + if (err) { + return callback(err); + } + async.eachSeries(cids, (cid, next) => { + getGroupPrivileges(cid, (err, groupPrivileges) => { + if (err) { + return next(err); + } + const privs = []; + if (groupPrivileges['groups:upload:post:image']) { + privs.push('groups:upload:post:image'); + } + if (groupPrivileges['groups:upload:post:file']) { + privs.push('groups:upload:post:file'); + } + privileges.global.give(privs, 'registered-users', next); + }); + }, callback); + }); + } +}; +function getGroupPrivileges(cid, callback) { + const tasks = {}; + ['groups:upload:post:image', 'groups:upload:post:file'].forEach(privilege => { + tasks[privilege] = async.apply(groups.isMember, 'registered-users', `cid:${cid}:privileges:${privilege}`); + }); + async.parallel(tasks, callback); +} \ No newline at end of file diff --git a/lib/upgrades/1.7.4/rename_min_reputation_settings.js b/lib/upgrades/1.7.4/rename_min_reputation_settings.js new file mode 100644 index 0000000000..0e574315ee --- /dev/null +++ b/lib/upgrades/1.7.4/rename_min_reputation_settings.js @@ -0,0 +1,23 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Rename privileges:downvote and privileges:flag to min:rep:downvote, min:rep:flag respectively', + timestamp: Date.UTC(2018, 0, 12), + method: function (callback) { + db.getObjectFields('config', ['privileges:downvote', 'privileges:flag'], (err, config) => { + if (err) { + return callback(err); + } + db.setObject('config', { + 'min:rep:downvote': parseInt(config['privileges:downvote'], 10) || 0, + 'min:rep:flag': parseInt(config['privileges:downvote'], 10) || 0 + }, err => { + if (err) { + return callback(err); + } + db.deleteObjectFields('config', ['privileges:downvote', 'privileges:flag'], callback); + }); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.4/vote_privilege.js b/lib/upgrades/1.7.4/vote_privilege.js new file mode 100644 index 0000000000..f7c209ce13 --- /dev/null +++ b/lib/upgrades/1.7.4/vote_privilege.js @@ -0,0 +1,19 @@ +'use strict'; + +const async = require('async'); +const privileges = require('../../privileges'); +const db = require('../../database'); +module.exports = { + name: 'Give vote privilege to registered-users on all categories', + timestamp: Date.UTC(2018, 0, 9), + method: function (callback) { + db.getSortedSetRange('categories:cid', 0, -1, (err, cids) => { + if (err) { + return callback(err); + } + async.eachSeries(cids, (cid, next) => { + privileges.categories.give(['groups:posts:upvote', 'groups:posts:downvote'], cid, 'registered-users', next); + }, callback); + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.6/flatten_navigation_data.js b/lib/upgrades/1.7.6/flatten_navigation_data.js new file mode 100644 index 0000000000..13d23a08ec --- /dev/null +++ b/lib/upgrades/1.7.6/flatten_navigation_data.js @@ -0,0 +1,23 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Flatten navigation data', + timestamp: Date.UTC(2018, 1, 17), + method: async function () { + const data = await db.getSortedSetRangeWithScores('navigation:enabled', 0, -1); + const order = []; + const items = []; + data.forEach(item => { + let navItem = JSON.parse(item.value); + const keys = Object.keys(navItem); + if (keys.length && parseInt(keys[0], 10) >= 0) { + navItem = navItem[keys[0]]; + } + order.push(item.score); + items.push(JSON.stringify(navItem)); + }); + await db.delete('navigation:enabled'); + await db.sortedSetAdd('navigation:enabled', order, items); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.6/notification_types.js b/lib/upgrades/1.7.6/notification_types.js new file mode 100644 index 0000000000..d075171d05 --- /dev/null +++ b/lib/upgrades/1.7.6/notification_types.js @@ -0,0 +1,20 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Add default settings for notification delivery types', + timestamp: Date.UTC(2018, 1, 14), + method: async function () { + const config = await db.getObject('config'); + const postNotifications = parseInt(config.sendPostNotifications, 10) === 1 ? 'notification' : 'none'; + const chatNotifications = parseInt(config.sendChatNotifications, 10) === 1 ? 'notification' : 'none'; + await db.setObject('config', { + notificationType_upvote: config.notificationType_upvote || 'notification', + 'notificationType_new-topic': config['notificationType_new-topic'] || 'notification', + 'notificationType_new-reply': config['notificationType_new-reply'] || postNotifications, + notificationType_follow: config.notificationType_follow || 'notification', + 'notificationType_new-chat': config['notificationType_new-chat'] || chatNotifications, + 'notificationType_group-invite': config['notificationType_group-invite'] || 'notification' + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.7.6/update_min_pass_strength.js b/lib/upgrades/1.7.6/update_min_pass_strength.js new file mode 100644 index 0000000000..84434fdc43 --- /dev/null +++ b/lib/upgrades/1.7.6/update_min_pass_strength.js @@ -0,0 +1,13 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Revising minimum password strength to 1 (from 0)', + timestamp: Date.UTC(2018, 1, 21), + method: async function () { + const strength = await db.getObjectField('config', 'minimumPasswordStrength'); + if (!strength) { + await db.setObjectField('config', 'minimumPasswordStrength', 1); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.8.0/give_signature_privileges.js b/lib/upgrades/1.8.0/give_signature_privileges.js new file mode 100644 index 0000000000..e626e3dedf --- /dev/null +++ b/lib/upgrades/1.8.0/give_signature_privileges.js @@ -0,0 +1,10 @@ +'use strict'; + +const privileges = require('../../privileges'); +module.exports = { + name: 'Give registered users signature privilege', + timestamp: Date.UTC(2018, 1, 28), + method: function (callback) { + privileges.global.give(['groups:signature'], 'registered-users', callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.8.0/give_spiders_privileges.js b/lib/upgrades/1.8.0/give_spiders_privileges.js new file mode 100644 index 0000000000..aa70074622 --- /dev/null +++ b/lib/upgrades/1.8.0/give_spiders_privileges.js @@ -0,0 +1,42 @@ +'use strict'; + +const async = require('async'); +const groups = require('../../groups'); +const privileges = require('../../privileges'); +const db = require('../../database'); +module.exports = { + name: 'Give category access privileges to spiders system group', + timestamp: Date.UTC(2018, 0, 31), + method: function (callback) { + db.getSortedSetRange('categories:cid', 0, -1, (err, cids) => { + if (err) { + return callback(err); + } + async.eachSeries(cids, (cid, next) => { + getGroupPrivileges(cid, (err, groupPrivileges) => { + if (err) { + return next(err); + } + const privs = []; + if (groupPrivileges['groups:find']) { + privs.push('groups:find'); + } + if (groupPrivileges['groups:read']) { + privs.push('groups:read'); + } + if (groupPrivileges['groups:topics:read']) { + privs.push('groups:topics:read'); + } + privileges.categories.give(privs, cid, 'spiders', next); + }); + }, callback); + }); + } +}; +function getGroupPrivileges(cid, callback) { + const tasks = {}; + ['groups:find', 'groups:read', 'groups:topics:read'].forEach(privilege => { + tasks[privilege] = async.apply(groups.isMember, 'guests', `cid:${cid}:privileges:${privilege}`); + }); + async.parallel(tasks, callback); +} \ No newline at end of file diff --git a/lib/upgrades/1.8.1/diffs_zset_to_listhash.js b/lib/upgrades/1.8.1/diffs_zset_to_listhash.js new file mode 100644 index 0000000000..e5c769680b --- /dev/null +++ b/lib/upgrades/1.8.1/diffs_zset_to_listhash.js @@ -0,0 +1,46 @@ +'use strict'; + +const async = require('async'); +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Reformatting post diffs to be stored in lists and hash instead of single zset', + timestamp: Date.UTC(2018, 2, 15), + method: function (callback) { + const { + progress + } = this; + batch.processSortedSet('posts:pid', (pids, next) => { + async.each(pids, (pid, next) => { + db.getSortedSetRangeWithScores(`post:${pid}:diffs`, 0, -1, (err, diffs) => { + if (err) { + return next(err); + } + if (!diffs || !diffs.length) { + progress.incr(); + return next(); + } + async.each(diffs, (diff, next) => { + async.series([async.apply(db.delete.bind(db), `post:${pid}:diffs`), async.apply(db.listPrepend.bind(db), `post:${pid}:diffs`, diff.score), async.apply(db.setObject.bind(db), `diff:${pid}.${diff.score}`, { + pid: pid, + patch: diff.value + })], next); + }, err => { + if (err) { + return next(err); + } + progress.incr(); + return next(); + }); + }); + }, err => { + if (err) { + progress.incr(); + } + return next(); + }); + }, { + progress: progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/1.9.0/refresh_post_upload_associations.js b/lib/upgrades/1.9.0/refresh_post_upload_associations.js new file mode 100644 index 0000000000..7d034b8355 --- /dev/null +++ b/lib/upgrades/1.9.0/refresh_post_upload_associations.js @@ -0,0 +1,21 @@ +'use strict'; + +const async = require('async'); +const posts = require('../../posts'); +module.exports = { + name: 'Refresh post-upload associations', + timestamp: Date.UTC(2018, 3, 16), + method: function (callback) { + const { + progress + } = this; + require('../../batch').processSortedSet('posts:pid', (pids, next) => { + async.each(pids, (pid, next) => { + posts.uploads.sync(pid, next); + progress.incr(); + }, next); + }, { + progress: this.progress + }, callback); + } +}; \ No newline at end of file diff --git a/lib/upgrades/2.8.7/fix-email-sorted-sets.js b/lib/upgrades/2.8.7/fix-email-sorted-sets.js new file mode 100644 index 0000000000..aba8aeb564 --- /dev/null +++ b/lib/upgrades/2.8.7/fix-email-sorted-sets.js @@ -0,0 +1,43 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Fix user email sorted sets', + timestamp: Date.UTC(2023, 1, 4), + method: async function () { + const { + progress + } = this; + const bulkRemove = []; + await batch.processSortedSet('email:uid', async data => { + progress.incr(data.length); + const usersData = await db.getObjects(data.map(d => `user:${d.score}`)); + data.forEach((emailData, index) => { + const { + score: uid, + value: email + } = emailData; + const userData = usersData[index]; + if (!userData || !userData.email) { + bulkRemove.push(['email:uid', email]); + bulkRemove.push(['email:sorted', `${email.toLowerCase()}:${uid}`]); + return; + } + if (userData.email && userData.email.toLowerCase() !== email.toLowerCase()) { + bulkRemove.push(['email:uid', email]); + bulkRemove.push(['email:sorted', `${email.toLowerCase()}:${uid}`]); + } + }); + }, { + batch: 500, + withScores: true, + progress: progress + }); + await batch.processArray(bulkRemove, async bulk => { + await db.sortedSetRemoveBulk(bulk); + }, { + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.0.0/reset_bootswatch_skin.js b/lib/upgrades/3.0.0/reset_bootswatch_skin.js new file mode 100644 index 0000000000..92ce15e0b5 --- /dev/null +++ b/lib/upgrades/3.0.0/reset_bootswatch_skin.js @@ -0,0 +1,15 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Reset bootswatch skin', + timestamp: Date.UTC(2023, 3, 24), + method: async function () { + const config = await db.getObject('config'); + const currentSkin = config.bootswatchSkin || ''; + const css = require('../../meta/css'); + if (currentSkin && !css.supportedSkins.includes(currentSkin)) { + await db.setObjectField('config', 'bootswatchSkin', ''); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.1.0/reset_user_bootswatch_skin.js b/lib/upgrades/3.1.0/reset_user_bootswatch_skin.js new file mode 100644 index 0000000000..80d458107f --- /dev/null +++ b/lib/upgrades/3.1.0/reset_user_bootswatch_skin.js @@ -0,0 +1,20 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Reset old bootswatch skin for all users', + timestamp: Date.UTC(2023, 4, 1), + method: async function () { + const batch = require('../../batch'); + const css = require('../../meta/css'); + batch.processSortedSet('users:joindate', async uids => { + let settings = await db.getObjects(uids.map(uid => `user:${uid}:settings`)); + settings = settings.filter(s => s && s.bootswatchSkin && !css.supportedSkins.includes(s.bootswatchSkin)); + await db.setObjectBulk(settings.map(s => [`user:${s.uid}`, { + bootswatchSkin: '' + }])); + }, { + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.2.0/fix_username_zsets.js b/lib/upgrades/3.2.0/fix_username_zsets.js new file mode 100644 index 0000000000..488a30bbe4 --- /dev/null +++ b/lib/upgrades/3.2.0/fix_username_zsets.js @@ -0,0 +1,29 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Fix username zsets', + timestamp: Date.UTC(2023, 5, 5), + method: async function () { + const { + progress + } = this; + await db.deleteAll(['username:uid', 'username:sorted']); + await batch.processSortedSet('users:joindate', async uids => { + progress.incr(uids.length); + const usersData = await db.getObjects(uids.map(uid => `user:${uid}`)); + const bulkAdd = []; + usersData.forEach(userData => { + if (userData && userData.username) { + bulkAdd.push(['username:uid', userData.uid, userData.username]); + bulkAdd.push(['username:sorted', 0, `${String(userData.username).toLowerCase()}:${userData.uid}`]); + } + }); + await db.sortedSetAddBulk(bulkAdd); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.2.0/migrate_api_tokens.js b/lib/upgrades/3.2.0/migrate_api_tokens.js new file mode 100644 index 0000000000..762f1d7803 --- /dev/null +++ b/lib/upgrades/3.2.0/migrate_api_tokens.js @@ -0,0 +1,41 @@ +'use strict'; + +const assert = require('assert'); +const winston = require('winston'); +const db = require('../../database'); +const meta = require('../../meta'); +const api = require('../../api'); +module.exports = { + name: 'Migrate tokens away from sorted-list implementation', + timestamp: Date.UTC(2023, 4, 2), + method: async () => { + const { + tokens = [] + } = await meta.settings.get('core.api'); + await Promise.all(tokens.map(async tokenObj => { + const { + token, + uid, + description + } = tokenObj; + await api.utils.tokens.add({ + token, + uid, + description + }); + })); + const oldCount = await db.sortedSetCard('settings:core.api:sorted-list:tokens'); + const newCount = await db.sortedSetCard('tokens:createtime'); + try { + if (oldCount > 0) { + assert.strictEqual(oldCount, newCount); + } + await meta.settings.set('core.api', { + tokens: [] + }); + await db.delete('settings:core.api:sorted-lists'); + } catch (e) { + winston.warn('Old token count does not match migrated tokens count, leaving old tokens behind.'); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.2.0/migrate_post_sharing.js b/lib/upgrades/3.2.0/migrate_post_sharing.js new file mode 100644 index 0000000000..3b66b86e14 --- /dev/null +++ b/lib/upgrades/3.2.0/migrate_post_sharing.js @@ -0,0 +1,18 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Migrate post sharing values to config', + timestamp: Date.UTC(2023, 4, 23), + method: async () => { + const activated = await db.getSetMembers('social:posts.activated'); + if (activated.length) { + const data = {}; + activated.forEach(id => { + data[`post-sharing-${id}`] = 1; + }); + await db.setObject('config', data); + await db.delete('social:posts.activated'); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.3.0/chat_message_mids.js b/lib/upgrades/3.3.0/chat_message_mids.js new file mode 100644 index 0000000000..461705c667 --- /dev/null +++ b/lib/upgrades/3.3.0/chat_message_mids.js @@ -0,0 +1,36 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Set mid on message objects and create messages:mid', + timestamp: Date.UTC(2023, 6, 27), + method: async function () { + const { + progress + } = this; + const allRoomIds = await db.getSortedSetRange(`chat:rooms`, 0, -1); + progress.total = allRoomIds.length; + for (const roomId of allRoomIds) { + await batch.processSortedSet(`chat:room:${roomId}:mids`, async mids => { + let messageData = await db.getObjects(mids.map(mid => `message:${mid}`)); + messageData.forEach((m, idx) => { + if (m) { + m.mid = parseInt(mids[idx], 10); + } + }); + messageData = messageData.filter(Boolean); + const bulkSet = messageData.map(msg => [`message:${msg.mid}`, { + mid: msg.mid + }]); + await db.setObjectBulk(bulkSet); + await db.sortedSetAdd('messages:mid', messageData.map(msg => msg.timestamp), messageData.map(msg => msg.mid)); + }, { + batch: 500 + }); + progress.incr(1); + } + const count = await db.sortedSetCard(`messages:mid`); + await db.setObjectField('global', 'messageCount', count); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.3.0/chat_room_online_zset.js b/lib/upgrades/3.3.0/chat_room_online_zset.js new file mode 100644 index 0000000000..fb4f7e4919 --- /dev/null +++ b/lib/upgrades/3.3.0/chat_room_online_zset.js @@ -0,0 +1,28 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Create chat:room:uids:online zset', + timestamp: Date.UTC(2023, 6, 14), + method: async function () { + const { + progress + } = this; + progress.total = await db.sortedSetCard('chat:rooms'); + await batch.processSortedSet('chat:rooms', async roomIds => { + progress.incr(roomIds.length); + const arrayOfUids = await db.getSortedSetsMembersWithScores(roomIds.map(roomId => `chat:room:${roomId}:uids`)); + const bulkAdd = []; + arrayOfUids.forEach((uids, idx) => { + const roomId = roomIds[idx]; + uids.forEach(uid => { + bulkAdd.push([`chat:room:${roomId}:uids:online`, uid.score, uid.value]); + }); + }); + await db.sortedSetAddBulk(bulkAdd); + }, { + batch: 100 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.3.0/chat_room_owners.js b/lib/upgrades/3.3.0/chat_room_owners.js new file mode 100644 index 0000000000..ade92f6b24 --- /dev/null +++ b/lib/upgrades/3.3.0/chat_room_owners.js @@ -0,0 +1,31 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Create chat:room::owners zset', + timestamp: Date.UTC(2023, 6, 17), + method: async function () { + const { + progress + } = this; + progress.total = await db.sortedSetCard('chat:rooms'); + const users = await db.getSortedSetRangeWithScores(`users:joindate`, 0, 0); + const timestamp = users.length ? users[0].score : Date.now(); + await batch.processSortedSet('chat:rooms', async roomIds => { + progress.incr(roomIds.length); + const roomData = await db.getObjects(roomIds.map(id => `chat:room:${id}`)); + const arrayOfUids = await Promise.all(roomIds.map(roomId => db.getSortedSetRangeWithScores(`chat:room:${roomId}:uids`, 0, 0))); + const bulkAdd = []; + roomData.forEach((room, idx) => { + if (room && room.roomId && room.owner) { + room.timestamp = room.timestamp || (arrayOfUids[idx].length ? arrayOfUids[idx][0].score || timestamp : timestamp); + bulkAdd.push([`chat:room:${room.roomId}:owners`, room.timestamp, room.owner]); + } + }); + await db.sortedSetAddBulk(bulkAdd); + }, { + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.3.0/chat_room_refactor.js b/lib/upgrades/3.3.0/chat_room_refactor.js new file mode 100644 index 0000000000..266b1d306c --- /dev/null +++ b/lib/upgrades/3.3.0/chat_room_refactor.js @@ -0,0 +1,77 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Update chat messages to add roomId field', + timestamp: Date.UTC(2023, 6, 2), + method: async function () { + const { + progress + } = this; + const nextChatRoomId = await db.getObjectField('global', 'nextChatRoomId'); + const allRoomIds = []; + for (let i = 1; i <= nextChatRoomId; i++) { + allRoomIds.push(i); + } + progress.total = 0; + await batch.processArray(allRoomIds, async roomIds => { + const arrayOfRoomData = await db.getObjects(roomIds.map(roomId => `chat:room:${roomId}`)); + await Promise.all(roomIds.map(async (roomId, idx) => { + const roomData = arrayOfRoomData[idx]; + if (roomData) { + const userCount = await db.sortedSetCard(`chat:room:${roomId}:uids`); + progress.total += userCount; + } + })); + }, { + batch: 500 + }); + await batch.processArray(allRoomIds, async roomIds => { + const arrayOfRoomData = await db.getObjects(roomIds.map(roomId => `chat:room:${roomId}`)); + for (const roomData of arrayOfRoomData) { + if (roomData) { + const midsSeen = {}; + const { + roomId + } = roomData; + const uids = await db.getSortedSetRange(`chat:room:${roomId}:uids`, 0, -1); + for (const uid of uids) { + await batch.processSortedSet(`uid:${uid}:chat:room:${roomId}:mids`, async userMessageData => { + const uniqMessages = userMessageData.filter(m => !midsSeen.hasOwnProperty(m.value)); + const uniqMids = uniqMessages.map(m => m.value); + if (!uniqMids.length) { + return; + } + let messageData = await db.getObjects(uniqMids.map(mid => `message:${mid}`)); + messageData.forEach((m, idx) => { + if (m && uniqMessages[idx]) { + m.mid = parseInt(uniqMids[idx], 10); + m.timestamp = m.timestamp || uniqMessages[idx].score || 0; + } + }); + messageData = messageData.filter(Boolean); + const bulkSet = messageData.map(msg => [`message:${msg.mid}`, { + roomId: roomId, + timestamp: msg.timestamp + }]); + await db.setObjectBulk(bulkSet); + await db.sortedSetAdd(`chat:room:${roomId}:mids`, messageData.map(m => m.timestamp), messageData.map(m => m.mid)); + uniqMids.forEach(mid => { + midsSeen[mid] = 1; + }); + }, { + batch: 500, + withScores: true + }); + await db.deleteAll(`uid:${uid}:chat:room:${roomId}:mids`); + progress.incr(1); + } + await db.setObjectField(`chat:room:${roomId}`, 'userCount', uids.length); + } + } + }, { + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.3.0/save_rooms_zset.js b/lib/upgrades/3.3.0/save_rooms_zset.js new file mode 100644 index 0000000000..1769edfa80 --- /dev/null +++ b/lib/upgrades/3.3.0/save_rooms_zset.js @@ -0,0 +1,35 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Store list of chat rooms', + timestamp: Date.UTC(2023, 6, 3), + method: async function () { + const { + progress + } = this; + const lastRoomId = await db.getObjectField('global', 'nextChatRoomId'); + const allRoomIds = []; + for (let x = 1; x <= lastRoomId; x++) { + allRoomIds.push(x); + } + const users = await db.getSortedSetRangeWithScores(`users:joindate`, 0, 0); + const timestamp = users.length ? users[0].score : Date.now(); + progress.total = allRoomIds.length; + await batch.processArray(allRoomIds, async roomIds => { + progress.incr(roomIds.length); + const keys = roomIds.map(id => `chat:room:${id}`); + const exists = await db.exists(keys); + roomIds = roomIds.filter((_, idx) => exists[idx]); + const arrayOfUids = await Promise.all(roomIds.map(roomId => db.getSortedSetRangeWithScores(`chat:room:${roomId}:uids`, 0, 0))); + const timestamps = roomIds.map((id, idx) => arrayOfUids[idx].length ? arrayOfUids[idx][0].score || timestamp : timestamp); + await db.sortedSetAdd('chat:rooms', timestamps, roomIds); + await db.setObjectBulk(roomIds.map((id, idx) => [`chat:room:${id}`, { + timestamp: timestamps[idx] + }])); + }, { + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.5.0/notification_translations.js b/lib/upgrades/3.5.0/notification_translations.js new file mode 100644 index 0000000000..55d225d647 --- /dev/null +++ b/lib/upgrades/3.5.0/notification_translations.js @@ -0,0 +1,28 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Update translation keys in notification bodyShort', + timestamp: Date.UTC(2023, 9, 5), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet(`notifications`, async nids => { + const notifData = await db.getObjects(nids.map(nid => `notifications:${nid}`)); + notifData.forEach(n => { + if (n && n.bodyShort) { + n.bodyShort = n.bodyShort.replace(/_/g, '-'); + } + }); + const bulkSet = notifData.map(n => [`notifications:${n.nid}`, { + bodyShort: n.bodyShort + }]); + await db.setObjectBulk(bulkSet); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.6.0/category_tracking.js b/lib/upgrades/3.6.0/category_tracking.js new file mode 100644 index 0000000000..fd37f9bd5f --- /dev/null +++ b/lib/upgrades/3.6.0/category_tracking.js @@ -0,0 +1,29 @@ +'use strict'; + +const db = require('../../database'); +const user = require('../../user'); +const batch = require('../../batch'); +module.exports = { + name: 'Add tracking category state', + timestamp: Date.UTC(2023, 10, 3), + method: async function () { + const { + progress + } = this; + const current = await db.getObjectField('config', 'categoryWatchState'); + if (current === 'watching') { + await db.setObjectField('config', 'categoryWatchState', 'tracking'); + } + await batch.processSortedSet(`users:joindate`, async uids => { + const userSettings = await user.getMultipleUserSettings(uids); + const change = userSettings.filter(s => s && s.categoryWatchState === 'watching'); + await db.setObjectBulk(change.map(s => [`user:${s.uid}:settings`, { + categoryWatchState: 'tracking' + }])); + progress.incr(uids.length); + }, { + batch: 500, + progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.6.0/chat_message_counts.js b/lib/upgrades/3.6.0/chat_message_counts.js new file mode 100644 index 0000000000..36bb2150a7 --- /dev/null +++ b/lib/upgrades/3.6.0/chat_message_counts.js @@ -0,0 +1,21 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Set messageCount on chat rooms', + timestamp: Date.UTC(2023, 6, 27), + method: async function () { + const { + progress + } = this; + const allRoomIds = await db.getSortedSetRange(`chat:rooms`, 0, -1); + progress.total = allRoomIds.length; + for (const roomId of allRoomIds) { + const count = await db.sortedSetCard(`chat:room:${roomId}:mids`); + await db.setObject(`chat:room:${roomId}`, { + messageCount: count + }); + progress.incr(1); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.6.0/rename_newbie_config.js b/lib/upgrades/3.6.0/rename_newbie_config.js new file mode 100644 index 0000000000..a3785060a8 --- /dev/null +++ b/lib/upgrades/3.6.0/rename_newbie_config.js @@ -0,0 +1,12 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Rename newbiePostDelayThreshold to newbieReputationThreshold', + timestamp: Date.UTC(2023, 10, 7), + method: async function () { + const current = await db.getObjectField('config', 'newbiePostDelayThreshold'); + await db.setObjectField('config', 'newbieReputationThreshold', current); + await db.deleteObjectField('config', 'newbiePostDelayThreshold'); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.6.0/rewards_zsets.js b/lib/upgrades/3.6.0/rewards_zsets.js new file mode 100644 index 0000000000..fcf739bfb5 --- /dev/null +++ b/lib/upgrades/3.6.0/rewards_zsets.js @@ -0,0 +1,15 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Convert rewards:list to a sorted set', + timestamp: Date.UTC(2023, 10, 10), + method: async function () { + const rewards = await db.getSetMembers('rewards:list'); + if (rewards.length) { + rewards.sort((a, b) => a - b); + await db.delete('rewards:list'); + await db.sortedSetAdd('rewards:list', rewards.map((id, index) => index), rewards.map(id => id)); + } + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.7.0/category-read-by-uid.js b/lib/upgrades/3.7.0/category-read-by-uid.js new file mode 100644 index 0000000000..c65d839837 --- /dev/null +++ b/lib/upgrades/3.7.0/category-read-by-uid.js @@ -0,0 +1,25 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Remove cid::read_by_uid sets', + timestamp: Date.UTC(2024, 0, 29), + method: async function () { + const { + progress + } = this; + const nextCid = await db.getObjectField('global', 'nextCid'); + const allCids = []; + for (let i = 1; i <= nextCid; i++) { + allCids.push(i); + } + await batch.processArray(allCids, async cids => { + await db.deleteAll(cids.map(cid => `cid:${cid}:read_by_uid`)); + progress.incr(cids.length); + }, { + batch: 500, + progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.7.0/category-tid-created-zset.js b/lib/upgrades/3.7.0/category-tid-created-zset.js new file mode 100644 index 0000000000..9b59309f90 --- /dev/null +++ b/lib/upgrades/3.7.0/category-tid-created-zset.js @@ -0,0 +1,24 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'New sorted set cid::tids:create', + timestamp: Date.UTC(2024, 2, 4), + method: async function () { + const { + progress + } = this; + const batch = require('../../batch'); + await batch.processSortedSet('topics:tid', async tids => { + let topicData = await db.getObjectsFields(tids.map(tid => `topic:${tid}`), ['tid', 'cid', 'timestamp']); + topicData = topicData.filter(Boolean); + topicData.forEach(t => { + t.timestamp = t.timestamp || Date.now(); + }); + await db.sortedSetAddBulk(topicData.map(t => [`cid:${t.cid}:tids:create`, t.timestamp, t.tid])); + progress.incr(tids.length); + }, { + progress: this.progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.7.0/change-category-sort-settings.js b/lib/upgrades/3.7.0/change-category-sort-settings.js new file mode 100644 index 0000000000..022c9147bb --- /dev/null +++ b/lib/upgrades/3.7.0/change-category-sort-settings.js @@ -0,0 +1,33 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Change category sort settings', + timestamp: Date.UTC(2024, 2, 4), + method: async function () { + const { + progress + } = this; + const currentSort = await db.getObjectField('config', 'categoryTopicSort'); + if (currentSort === 'oldest_to_newest' || currentSort === 'newest_to_oldest') { + await db.setObjectField('config', 'categoryTopicSort', 'recently_replied'); + } + await batch.processSortedSet('users:joindate', async uids => { + progress.incr(uids.length); + const usersSettings = await db.getObjects(uids.map(uid => `user:${uid}:settings`)); + const bulkSet = []; + usersSettings.forEach((userSetting, i) => { + if (userSetting && (userSetting.categoryTopicSort === 'newest_to_oldest' || userSetting.categoryTopicSort === 'oldest_to_newest')) { + bulkSet.push([`user:${uids[i]}:settings`, { + categoryTopicSort: 'recently_replied' + }]); + } + }); + await db.setObjectBulk(bulkSet); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.8.0/events-uid-filter.js b/lib/upgrades/3.8.0/events-uid-filter.js new file mode 100644 index 0000000000..722d6944ba --- /dev/null +++ b/lib/upgrades/3.8.0/events-uid-filter.js @@ -0,0 +1,27 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Add user filter to acp events', + timestamp: Date.UTC(2024, 3, 1), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet(`events:time`, async eids => { + const eventData = await db.getObjects(eids.map(eid => `event:${eid}`)); + const bulkAdd = []; + eventData.forEach(event => { + if (event && event.hasOwnProperty('uid') && event.uid && event.eid) { + bulkAdd.push([`events:time:uid:${event.uid}`, event.timestamp || Date.now(), event.eid]); + } + }); + await db.sortedSetAddBulk(bulkAdd); + progress.incr(eids.length); + }, { + batch: 500, + progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.8.0/remove-privilege-slugs.js b/lib/upgrades/3.8.0/remove-privilege-slugs.js new file mode 100644 index 0000000000..4a967910ee --- /dev/null +++ b/lib/upgrades/3.8.0/remove-privilege-slugs.js @@ -0,0 +1,28 @@ +'use strict'; + +const db = require('../../database'); +const groups = require('../../groups'); +const batch = require('../../batch'); +module.exports = { + name: 'Remove privilege groups from groupslug:groupname object', + timestamp: Date.UTC(2024, 3, 8), + method: async function () { + const { + progress + } = this; + const slugsToNames = await db.getObject(`groupslug:groupname`); + const privilegeGroups = []; + for (const [slug, name] of Object.entries(slugsToNames)) { + if (groups.isPrivilegeGroup(name)) { + privilegeGroups.push(slug); + } + } + progress.total = privilegeGroups.length; + await batch.processArray(privilegeGroups, async slugs => { + progress.incr(slugs.length); + await db.deleteObjectFields(`groupslug:groupname`, slugs); + }, { + batch: 500 + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.8.0/user-upload-folders.js b/lib/upgrades/3.8.0/user-upload-folders.js new file mode 100644 index 0000000000..d1174b1003 --- /dev/null +++ b/lib/upgrades/3.8.0/user-upload-folders.js @@ -0,0 +1,61 @@ +'use strict'; + +const fs = require('fs'); +const nconf = require('nconf'); +const path = require('path'); +const { + mkdirp +} = require('mkdirp'); +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Create user upload folders', + timestamp: Date.UTC(2024, 2, 28), + method: async function () { + const { + progress + } = this; + const folder = path.join(nconf.get('upload_path'), 'profile'); + const userPicRegex = /^\d+-profile/; + const files = (await fs.promises.readdir(folder, { + withFileTypes: true + })).filter(item => !item.isDirectory() && String(item.name).match(userPicRegex)).map(item => item.name); + progress.total = files.length; + await batch.processArray(files, async files => { + progress.incr(files.length); + await Promise.all(files.map(async file => { + const uid = file.split('-')[0]; + if (parseInt(uid, 10) > 0) { + await mkdirp(path.join(folder, `uid-${uid}`)); + await fs.promises.rename(path.join(folder, file), path.join(folder, `uid-${uid}`, file)); + } + })); + }, { + batch: 500 + }); + await batch.processSortedSet('users:joindate', async uids => { + progress.incr(uids.length); + const usersData = await db.getObjects(uids.map(uid => `user:${uid}`)); + const bulkSet = []; + usersData.forEach(userData => { + const setObj = {}; + if (userData && userData.picture && userData.picture.includes(`/uploads/profile/${userData.uid}-`) && !userData.picture.includes(`/uploads/profile/uid-${userData.uid}/${userData.uid}-`)) { + setObj.picture = userData.picture.replace(`/uploads/profile/${userData.uid}-`, `/uploads/profile/uid-${userData.uid}/${userData.uid}-`); + } + if (userData && userData.uploadedpicture && userData.uploadedpicture.includes(`/uploads/profile/${userData.uid}-`) && !userData.uploadedpicture.includes(`/uploads/profile/uid-${userData.uid}/${userData.uid}-`)) { + setObj.uploadedpicture = userData.uploadedpicture.replace(`/uploads/profile/${userData.uid}-`, `/uploads/profile/uid-${userData.uid}/${userData.uid}-`); + } + if (userData && userData['cover:url'] && userData['cover:url'].includes(`/uploads/profile/${userData.uid}-`) && !userData['cover:url'].includes(`/uploads/profile/uid-${userData.uid}/${userData.uid}-`)) { + setObj['cover:url'] = userData['cover:url'].replace(`/uploads/profile/${userData.uid}-`, `/uploads/profile/uid-${userData.uid}/${userData.uid}-`); + } + if (Object.keys(setObj).length) { + bulkSet.push([`user:${userData.uid}`, setObj]); + } + }); + await db.setObjectBulk(bulkSet); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.8.2/vote-visibility-config.js b/lib/upgrades/3.8.2/vote-visibility-config.js new file mode 100644 index 0000000000..e3cdc4c69f --- /dev/null +++ b/lib/upgrades/3.8.2/vote-visibility-config.js @@ -0,0 +1,13 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Add vote visibility config field', + timestamp: Date.UTC(2024, 4, 24), + method: async function () { + const current = await db.getObjectField('config', 'votesArePublic'); + const isPublic = parseInt(current, 10) === 1; + await db.setObjectField('config', 'voteVisibility', isPublic ? 'all' : 'privileged'); + await db.deleteObjectField('config', 'votesArePublic'); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.8.3/remove-session-uuid.js b/lib/upgrades/3.8.3/remove-session-uuid.js new file mode 100644 index 0000000000..603c1f9462 --- /dev/null +++ b/lib/upgrades/3.8.3/remove-session-uuid.js @@ -0,0 +1,20 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Remove uid::sessionUUID:sessionId object', + timestamp: Date.UTC(2024, 5, 26), + method: async function () { + const { + progress + } = this; + await batch.processSortedSet('users:joindate', async uids => { + progress.incr(uids.length); + await db.deleteAll(uids.map(uid => `uid:${uid}:sessionUUID:sessionId`)); + }, { + batch: 500, + progress: progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.8.3/topic-event-ids.js b/lib/upgrades/3.8.3/topic-event-ids.js new file mode 100644 index 0000000000..dca5a35950 --- /dev/null +++ b/lib/upgrades/3.8.3/topic-event-ids.js @@ -0,0 +1,36 @@ +'use strict'; + +const db = require('../../database'); +const batch = require('../../batch'); +module.exports = { + name: 'Add id field to all topic events', + timestamp: Date.UTC(2024, 5, 24), + method: async function () { + const { + progress + } = this; + let nextId = await db.getObjectField('global', 'nextTopicEventId'); + nextId = parseInt(nextId, 10) || 0; + const ids = []; + for (let i = 1; i < nextId; i++) { + ids.push(i); + } + await batch.processArray(ids, async eids => { + const eventData = await db.getObjects(eids.map(eid => `topicEvent:${eid}`)); + const bulkSet = []; + eventData.forEach((event, idx) => { + if (event && event.type) { + const id = eids[idx]; + bulkSet.push([`topicEvent:${id}`, { + id: id + }]); + } + }); + await db.setObjectBulk(bulkSet); + progress.incr(eids.length); + }, { + batch: 500, + progress + }); + } +}; \ No newline at end of file diff --git a/lib/upgrades/3.8.4/downvote-visibility-config.js b/lib/upgrades/3.8.4/downvote-visibility-config.js new file mode 100644 index 0000000000..6d85e7108d --- /dev/null +++ b/lib/upgrades/3.8.4/downvote-visibility-config.js @@ -0,0 +1,17 @@ +'use strict'; + +const db = require('../../database'); +module.exports = { + name: 'Add downvote visibility config field', + timestamp: Date.UTC(2024, 6, 17), + method: async function () { + const current = await db.getObjectField('config', 'voteVisibility'); + if (current) { + await db.setObject('config', { + upvoteVisibility: current, + downvoteVisibility: current + }); + await db.deleteObjectField('config', 'voteVisibility'); + } + } +}; \ No newline at end of file diff --git a/lib/user/admin.js b/lib/user/admin.js new file mode 100644 index 0000000000..3750e0b12e --- /dev/null +++ b/lib/user/admin.js @@ -0,0 +1,83 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const winston = require('winston'); +const validator = require('validator'); +const json2csvAsync = require('json2csv').parseAsync; +const { + baseDir +} = require('../constants').paths; +const db = require('../database'); +const plugins = require('../plugins'); +const batch = require('../batch'); +module.exports = function (User) { + User.logIP = async function (uid, ip) { + if (!(parseInt(uid, 10) > 0)) { + return; + } + const now = Date.now(); + const bulk = [[`uid:${uid}:ip`, now, ip || 'Unknown']]; + if (ip) { + bulk.push([`ip:${ip}:uid`, now, uid]); + } + await db.sortedSetAddBulk(bulk); + }; + User.getIPs = async function (uid, stop) { + const ips = await db.getSortedSetRevRange(`uid:${uid}:ip`, 0, stop); + return ips.map(ip => validator.escape(String(ip))); + }; + User.getUsersCSV = async function () { + winston.verbose('[user/getUsersCSV] Compiling User CSV data'); + const data = await plugins.hooks.fire('filter:user.csvFields', { + fields: ['uid', 'email', 'username'] + }); + let csvContent = `${data.fields.join(',')}\n`; + await batch.processSortedSet('users:joindate', async uids => { + const usersData = await User.getUsersFields(uids, data.fields); + csvContent += usersData.reduce((memo, user) => { + memo += `${data.fields.map(field => user[field]).join(',')}\n`; + return memo; + }, ''); + }, {}); + return csvContent; + }; + User.exportUsersCSV = async function (fieldsToExport = ['email', 'username', 'uid', 'ip']) { + winston.verbose('[user/exportUsersCSV] Exporting User CSV data'); + const { + fields, + showIps + } = await plugins.hooks.fire('filter:user.csvFields', { + fields: fieldsToExport, + showIps: fieldsToExport.includes('ip') + }); + if (!showIps && fields.includes('ip')) { + fields.splice(fields.indexOf('ip'), 1); + } + const fd = await fs.promises.open(path.join(baseDir, 'build/export', 'users.csv'), 'w'); + fs.promises.appendFile(fd, `${fields.map(f => `"${f}"`).join(',')}\n`); + await batch.processSortedSet('group:administrators:members', async uids => { + const userFieldsToLoad = fields.filter(field => field !== 'ip' && field !== 'password'); + const usersData = await User.getUsersFields(uids, userFieldsToLoad); + let userIps = []; + if (showIps) { + userIps = await db.getSortedSetsMembers(uids.map(uid => `uid:${uid}:ip`)); + } + usersData.forEach((user, index) => { + if (Array.isArray(userIps[index])) { + user.ip = userIps[index].join(','); + } + }); + const opts = { + fields, + header: false + }; + const csv = await json2csvAsync(usersData, opts); + await fs.promises.appendFile(fd, csv); + }, { + batch: 5000, + interval: 250 + }); + await fd.close(); + }; +}; \ No newline at end of file diff --git a/lib/user/approval.js b/lib/user/approval.js new file mode 100644 index 0000000000..80be98245b --- /dev/null +++ b/lib/user/approval.js @@ -0,0 +1,147 @@ +'use strict'; + +const validator = require('validator'); +const winston = require('winston'); +const cronJob = require('cron').CronJob; +const db = require('../database'); +const meta = require('../meta'); +const emailer = require('../emailer'); +const notifications = require('../notifications'); +const groups = require('../groups'); +const utils = require('../utils'); +const slugify = require('../slugify'); +const plugins = require('../plugins'); +module.exports = function (User) { + new cronJob('0 * * * *', () => { + User.autoApprove(); + }, null, true); + User.addToApprovalQueue = async function (userData) { + userData.username = userData.username.trim(); + userData.userslug = slugify(userData.username); + await canQueue(userData); + const hashedPassword = await User.hashPassword(userData.password); + const data = { + username: userData.username, + email: userData.email, + ip: userData.ip, + hashedPassword: hashedPassword + }; + const results = await plugins.hooks.fire('filter:user.addToApprovalQueue', { + data: data, + userData: userData + }); + await db.setObject(`registration:queue:name:${userData.username}`, results.data); + await db.sortedSetAdd('registration:queue', Date.now(), userData.username); + await sendNotificationToAdmins(userData.username); + }; + async function canQueue(userData) { + await User.isDataValid(userData); + const usernames = await db.getSortedSetRange('registration:queue', 0, -1); + if (usernames.includes(userData.username)) { + throw new Error('[[error:username-taken]]'); + } + const keys = usernames.filter(Boolean).map(username => `registration:queue:name:${username}`); + const data = await db.getObjectsFields(keys, ['email']); + const emails = data.map(data => data && data.email).filter(Boolean); + if (userData.email && emails.includes(userData.email)) { + throw new Error('[[error:email-taken]]'); + } + } + async function sendNotificationToAdmins(username) { + const notifObj = await notifications.create({ + type: 'new-register', + bodyShort: `[[notifications:new-register, ${username}]]`, + nid: `new-register:${username}`, + path: '/admin/manage/registration', + mergeId: 'new-register' + }); + await notifications.pushGroup(notifObj, 'administrators'); + } + User.acceptRegistration = async function (username) { + const userData = await db.getObject(`registration:queue:name:${username}`); + if (!userData) { + throw new Error('[[error:invalid-data]]'); + } + const creation_time = await db.sortedSetScore('registration:queue', username); + const uid = await User.create(userData); + await User.setUserFields(uid, { + password: userData.hashedPassword, + 'password:shaWrapped': 1 + }); + await removeFromQueue(username); + await markNotificationRead(username); + await plugins.hooks.fire('filter:register.complete', { + uid: uid + }); + await emailer.send('registration_accepted', uid, { + username: username, + subject: `[[email:welcome-to, ${meta.config.title || meta.config.browserTitle || 'NodeBB'}]]`, + template: 'registration_accepted', + uid: uid + }).catch(err => winston.error(`[emailer.send] ${err.stack}`)); + const total = await db.incrObjectFieldBy('registration:queue:approval:times', 'totalTime', Math.floor((Date.now() - creation_time) / 60000)); + const counter = await db.incrObjectField('registration:queue:approval:times', 'counter'); + await db.setObjectField('registration:queue:approval:times', 'average', total / counter); + return uid; + }; + async function markNotificationRead(username) { + const nid = `new-register:${username}`; + const uids = await groups.getMembers('administrators', 0, -1); + const promises = uids.map(uid => notifications.markRead(nid, uid)); + await Promise.all(promises); + } + User.rejectRegistration = async function (username) { + await removeFromQueue(username); + await markNotificationRead(username); + }; + async function removeFromQueue(username) { + await Promise.all([db.sortedSetRemove('registration:queue', username), db.delete(`registration:queue:name:${username}`)]); + } + User.shouldQueueUser = async function (ip) { + const { + registrationApprovalType + } = meta.config; + if (registrationApprovalType === 'admin-approval') { + return true; + } else if (registrationApprovalType === 'admin-approval-ip') { + const count = await db.sortedSetCard(`ip:${ip}:uid`); + return !!count; + } + return false; + }; + User.getRegistrationQueue = async function (start, stop) { + const data = await db.getSortedSetRevRangeWithScores('registration:queue', start, stop); + const keys = data.filter(Boolean).map(user => `registration:queue:name:${user.value}`); + let users = await db.getObjects(keys); + users = users.filter(Boolean).map((user, index) => { + user.timestampISO = utils.toISOString(data[index].score); + user.email = validator.escape(String(user.email)); + user.usernameEscaped = validator.escape(String(user.username)); + delete user.hashedPassword; + return user; + }); + await Promise.all(users.map(async user => { + user.ip = user.ip.replace('::ffff:', ''); + await getIPMatchedUsers(user); + user.customActions = user.customActions || []; + })); + const results = await plugins.hooks.fire('filter:user.getRegistrationQueue', { + users: users + }); + return results.users; + }; + async function getIPMatchedUsers(user) { + const uids = await User.getUidsFromSet(`ip:${user.ip}:uid`, 0, -1); + user.ipMatch = await User.getUsersFields(uids, ['uid', 'username', 'picture']); + } + User.autoApprove = async function () { + if (meta.config.autoApproveTime <= 0) { + return; + } + const users = await db.getSortedSetRevRangeWithScores('registration:queue', 0, -1); + const now = Date.now(); + for (const user of users.filter(user => now - user.score >= meta.config.autoApproveTime * 3600000)) { + await User.acceptRegistration(user.value); + } + }; +}; \ No newline at end of file diff --git a/lib/user/auth.js b/lib/user/auth.js new file mode 100644 index 0000000000..62f12d155c --- /dev/null +++ b/lib/user/auth.js @@ -0,0 +1,125 @@ +'use strict'; + +const validator = require('validator'); +const _ = require('lodash'); +const db = require('../database'); +const meta = require('../meta'); +const events = require('../events'); +const batch = require('../batch'); +const utils = require('../utils'); +module.exports = function (User) { + User.auth = {}; + User.auth.logAttempt = async function (uid, ip) { + if (!(parseInt(uid, 10) > 0)) { + return; + } + const exists = await db.exists(`lockout:${uid}`); + if (exists) { + throw new Error('[[error:account-locked]]'); + } + const attempts = await db.increment(`loginAttempts:${uid}`); + if (attempts <= meta.config.loginAttempts) { + return await db.pexpire(`loginAttempts:${uid}`, 1000 * 60 * 60); + } + await db.set(`lockout:${uid}`, ''); + const duration = 1000 * 60 * meta.config.lockoutDuration; + await db.delete(`loginAttempts:${uid}`); + await db.pexpire(`lockout:${uid}`, duration); + await events.log({ + type: 'account-locked', + uid: uid, + ip: ip + }); + throw new Error('[[error:account-locked]]'); + }; + User.auth.getFeedToken = async function (uid) { + if (!(parseInt(uid, 10) > 0)) { + return; + } + const _token = await db.getObjectField(`user:${uid}`, 'rss_token'); + const token = _token || utils.generateUUID(); + if (!_token) { + await User.setUserField(uid, 'rss_token', token); + } + return token; + }; + User.auth.clearLoginAttempts = async function (uid) { + await db.delete(`loginAttempts:${uid}`); + }; + User.auth.resetLockout = async function (uid) { + await db.deleteAll([`loginAttempts:${uid}`, `lockout:${uid}`]); + }; + User.auth.getSessions = async function (uid, curSessionId) { + await cleanExpiredSessions(uid); + const sids = await db.getSortedSetRevRange(`uid:${uid}:sessions`, 0, 19); + let sessions = await Promise.all(sids.map(sid => db.sessionStoreGet(sid))); + sessions = sessions.map((sessObj, idx) => { + if (sessObj && sessObj.meta) { + sessObj.meta.current = curSessionId === sids[idx]; + sessObj.meta.datetimeISO = new Date(sessObj.meta.datetime).toISOString(); + sessObj.meta.ip = validator.escape(String(sessObj.meta.ip)); + } + return sessObj && sessObj.meta; + }).filter(Boolean); + return sessions; + }; + async function cleanExpiredSessions(uid) { + const sids = await db.getSortedSetRange(`uid:${uid}:sessions`, 0, -1); + if (!sids.length) { + return []; + } + const expiredSids = []; + const activeSids = []; + await Promise.all(sids.map(async sid => { + const sessionObj = await db.sessionStoreGet(sid); + const expired = !sessionObj || !sessionObj.hasOwnProperty('passport') || !sessionObj.passport.hasOwnProperty('user') || parseInt(sessionObj.passport.user, 10) !== parseInt(uid, 10); + if (expired) { + expiredSids.push(sid); + } else { + activeSids.push(sid); + } + })); + await db.sortedSetRemove(`uid:${uid}:sessions`, expiredSids); + return activeSids; + } + User.auth.addSession = async function (uid, sessionId) { + if (!(parseInt(uid, 10) > 0)) { + return; + } + const activeSids = await cleanExpiredSessions(uid); + await db.sortedSetAdd(`uid:${uid}:sessions`, Date.now(), sessionId); + await revokeSessionsAboveThreshold(activeSids.push(sessionId), uid); + }; + async function revokeSessionsAboveThreshold(activeSids, uid) { + if (meta.config.maxUserSessions > 0 && activeSids.length > meta.config.maxUserSessions) { + const sessionsToRevoke = activeSids.slice(0, activeSids.length - meta.config.maxUserSessions); + await User.auth.revokeSession(sessionsToRevoke, uid); + } + } + User.auth.revokeSession = async function (sessionIds, uid) { + sessionIds = Array.isArray(sessionIds) ? sessionIds : [sessionIds]; + const destroySids = sids => Promise.all(sids.map(db.sessionStoreDestroy)); + await Promise.all([db.sortedSetRemove(`uid:${uid}:sessions`, sessionIds), destroySids(sessionIds)]); + }; + User.auth.revokeAllSessions = async function (uids, except) { + uids = Array.isArray(uids) ? uids : [uids]; + const sids = await db.getSortedSetsMembers(uids.map(uid => `uid:${uid}:sessions`)); + const promises = []; + uids.forEach((uid, index) => { + const ids = sids[index].filter(id => id !== except); + if (ids.length) { + promises.push(User.auth.revokeSession(ids, uid)); + } + }); + await Promise.all(promises); + }; + User.auth.deleteAllSessions = async function () { + await batch.processSortedSet('users:joindate', async uids => { + const sessionKeys = uids.map(uid => `uid:${uid}:sessions`); + const sids = _.flatten(await db.getSortedSetRange(sessionKeys, 0, -1)); + await Promise.all([db.deleteAll(sessionKeys), ...sids.map(sid => db.sessionStoreDestroy(sid))]); + }, { + batch: 1000 + }); + }; +}; \ No newline at end of file diff --git a/lib/user/bans.js b/lib/user/bans.js new file mode 100644 index 0000000000..695b02c732 --- /dev/null +++ b/lib/user/bans.js @@ -0,0 +1,128 @@ +'use strict'; + +const winston = require('winston'); +const meta = require('../meta'); +const emailer = require('../emailer'); +const db = require('../database'); +const groups = require('../groups'); +const privileges = require('../privileges'); +module.exports = function (User) { + User.bans = {}; + User.bans.ban = async function (uid, until, reason) { + until = until || 0; + reason = reason || ''; + const now = Date.now(); + until = parseInt(until, 10); + if (isNaN(until)) { + throw new Error('[[error:ban-expiry-missing]]'); + } + const banKey = `uid:${uid}:ban:${now}`; + const banData = { + type: 'ban', + uid: uid, + timestamp: now, + expire: until > now ? until : 0 + }; + if (reason) { + banData.reason = reason; + } + const systemGroups = groups.systemGroups.filter(group => group !== groups.BANNED_USERS); + await groups.leave(systemGroups, uid); + await groups.join(groups.BANNED_USERS, uid); + await db.sortedSetAdd('users:banned', now, uid); + await db.sortedSetAdd(`uid:${uid}:bans:timestamp`, now, banKey); + await db.setObject(banKey, banData); + await User.setUserFields(uid, { + banned: 1, + 'banned:expire': banData.expire + }); + if (until > now) { + await db.sortedSetAdd('users:banned:expire', until, uid); + } else { + await db.sortedSetRemove('users:banned:expire', uid); + } + const username = await User.getUserField(uid, 'username'); + const siteTitle = meta.config.title || 'NodeBB'; + const data = { + subject: `[[email:banned.subject, ${siteTitle}]]`, + username: username, + until: until ? new Date(until).toUTCString().replace(/,/g, '\\,') : false, + reason: reason + }; + await emailer.send('banned', uid, data).catch(err => winston.error(`[emailer.send] ${err.stack}`)); + return banData; + }; + User.bans.unban = async function (uids, reason = '') { + const isArray = Array.isArray(uids); + uids = isArray ? uids : [uids]; + const userData = await User.getUsersFields(uids, ['email:confirmed']); + await db.setObject(uids.map(uid => `user:${uid}`), { + banned: 0, + 'banned:expire': 0 + }); + const now = Date.now(); + const unbanDataArray = []; + for (const user of userData) { + const systemGroupsToJoin = ['registered-users', parseInt(user['email:confirmed'], 10) === 1 ? 'verified-users' : 'unverified-users']; + const unbanKey = `uid:${user.uid}:unban:${now}`; + const unbanData = { + type: 'unban', + uid: user.uid, + reason, + timestamp: now + }; + await Promise.all([db.sortedSetAdd(`uid:${user.uid}:unbans:timestamp`, now, unbanKey), db.setObject(unbanKey, unbanData), groups.leave(groups.BANNED_USERS, user.uid), groups.join(systemGroupsToJoin, user.uid)]); + unbanDataArray.push(unbanData); + } + await db.sortedSetRemove(['users:banned', 'users:banned:expire'], uids); + return isArray ? unbanDataArray : unbanDataArray[0]; + }; + User.bans.isBanned = async function (uids) { + const isArray = Array.isArray(uids); + uids = isArray ? uids : [uids]; + const result = await User.bans.unbanIfExpired(uids); + return isArray ? result.map(r => r.banned) : result[0].banned; + }; + User.bans.canLoginIfBanned = async function (uid) { + let canLogin = true; + const { + banned + } = (await User.bans.unbanIfExpired([uid]))[0]; + if (banned) { + canLogin = await privileges.global.canGroup('local:login', groups.BANNED_USERS); + } + if (banned && !canLogin) { + canLogin = await groups.isMember(uid, 'cid:0:privileges:local:login'); + } + return canLogin; + }; + User.bans.unbanIfExpired = async function (uids) { + const userData = await User.getUsersFields(uids, ['banned', 'banned:expire']); + return User.bans.calcExpiredFromUserData(userData); + }; + User.bans.calcExpiredFromUserData = function (userData) { + const isArray = Array.isArray(userData); + userData = isArray ? userData : [userData]; + userData = userData.map(userData => ({ + banned: !!(userData && userData.banned), + 'banned:expire': userData && userData['banned:expire'], + banExpired: userData && userData['banned:expire'] <= Date.now() && userData['banned:expire'] !== 0 + })); + return isArray ? userData : userData[0]; + }; + User.bans.filterBanned = async function (uids) { + const isBanned = await User.bans.isBanned(uids); + return uids.filter((uid, index) => !isBanned[index]); + }; + User.bans.getReason = async function (uid) { + if (parseInt(uid, 10) <= 0) { + return ''; + } + const keys = await db.getSortedSetRevRange(`uid:${uid}:bans:timestamp`, 0, 0); + if (!keys.length) { + return ''; + } + const banObj = await db.getObject(keys[0]); + return banObj && banObj.reason ? banObj.reason : ''; + }; +}; \ No newline at end of file diff --git a/lib/user/blocks.js b/lib/user/blocks.js new file mode 100644 index 0000000000..7febe6162a --- /dev/null +++ b/lib/user/blocks.js @@ -0,0 +1,102 @@ +'use strict'; + +const db = require('../database'); +const plugins = require('../plugins'); +const cacheCreate = require('../cache/lru'); +module.exports = function (User) { + User.blocks = { + _cache: cacheCreate({ + name: 'user:blocks', + max: 100, + ttl: 0 + }) + }; + User.blocks.is = async function (targetUid, uids) { + const isArray = Array.isArray(uids); + uids = isArray ? uids : [uids]; + const blocks = await User.blocks.list(uids); + const isBlocked = uids.map((uid, index) => blocks[index] && blocks[index].includes(parseInt(targetUid, 10))); + return isArray ? isBlocked : isBlocked[0]; + }; + User.blocks.can = async function (callerUid, blockerUid, blockeeUid, type) { + if (blockerUid === 0 || blockeeUid === 0) { + throw new Error('[[error:cannot-block-guest]]'); + } else if (blockerUid === blockeeUid) { + throw new Error('[[error:cannot-block-self]]'); + } + const [isCallerAdminOrMod, isBlockeeAdminOrMod] = await Promise.all([User.isAdminOrGlobalMod(callerUid), User.isAdminOrGlobalMod(blockeeUid)]); + if (isBlockeeAdminOrMod && type === 'block') { + throw new Error('[[error:cannot-block-privileged]]'); + } + if (parseInt(callerUid, 10) !== parseInt(blockerUid, 10) && !isCallerAdminOrMod) { + throw new Error('[[error:no-privileges]]'); + } + }; + User.blocks.list = async function (uids) { + const isArray = Array.isArray(uids); + uids = (isArray ? uids : [uids]).map(uid => parseInt(uid, 10)); + const cachedData = {}; + const unCachedUids = User.blocks._cache.getUnCachedKeys(uids, cachedData); + if (unCachedUids.length) { + const unCachedData = await db.getSortedSetsMembers(unCachedUids.map(uid => `uid:${uid}:blocked_uids`)); + unCachedUids.forEach((uid, index) => { + cachedData[uid] = (unCachedData[index] || []).map(uid => parseInt(uid, 10)); + User.blocks._cache.set(uid, cachedData[uid]); + }); + } + const result = uids.map(uid => cachedData[uid] || []); + return isArray ? result.slice() : result[0]; + }; + User.blocks.add = async function (targetUid, uid) { + await User.blocks.applyChecks('block', targetUid, uid); + await db.sortedSetAdd(`uid:${uid}:blocked_uids`, Date.now(), targetUid); + await User.incrementUserFieldBy(uid, 'blocksCount', 1); + User.blocks._cache.del(parseInt(uid, 10)); + plugins.hooks.fire('action:user.blocks.add', { + uid: uid, + targetUid: targetUid + }); + }; + User.blocks.remove = async function (targetUid, uid) { + await User.blocks.applyChecks('unblock', targetUid, uid); + await db.sortedSetRemove(`uid:${uid}:blocked_uids`, targetUid); + await User.decrementUserFieldBy(uid, 'blocksCount', 1); + User.blocks._cache.del(parseInt(uid, 10)); + plugins.hooks.fire('action:user.blocks.remove', { + uid: uid, + targetUid: targetUid + }); + }; + User.blocks.applyChecks = async function (type, targetUid, uid) { + await User.blocks.can(uid, uid, targetUid); + const isBlock = type === 'block'; + const is = await User.blocks.is(targetUid, uid); + if (is === isBlock) { + throw new Error(`[[error:already-${isBlock ? 'blocked' : 'unblocked'}]]`); + } + }; + User.blocks.filterUids = async function (targetUid, uids) { + const isBlocked = await User.blocks.is(targetUid, uids); + return uids.filter((uid, index) => !isBlocked[index]); + }; + User.blocks.filter = async function (uid, property, set) { + if (Array.isArray(property) && typeof set === 'undefined') { + set = property; + property = 'uid'; + } + if (!Array.isArray(set) || !set.length) { + return set; + } + const isPlain = typeof set[0] !== 'object'; + const blocked_uids = await User.blocks.list(uid); + const blockedSet = new Set(blocked_uids); + set = set.filter(item => !blockedSet.has(parseInt(isPlain ? item : item && item[property], 10))); + const data = await plugins.hooks.fire('filter:user.blocks.filter', { + set: set, + property: property, + uid: uid, + blockedSet: blockedSet + }); + return data.set; + }; +}; \ No newline at end of file diff --git a/lib/user/categories.js b/lib/user/categories.js new file mode 100644 index 0000000000..b7ee7675c4 --- /dev/null +++ b/lib/user/categories.js @@ -0,0 +1,69 @@ +'use strict'; + +const _ = require('lodash'); +const db = require('../database'); +const categories = require('../categories'); +const plugins = require('../plugins'); +module.exports = function (User) { + User.setCategoryWatchState = async function (uid, cids, state) { + if (!(parseInt(uid, 10) > 0)) { + return; + } + const isStateValid = Object.values(categories.watchStates).includes(parseInt(state, 10)); + if (!isStateValid) { + throw new Error('[[error:invalid-watch-state]]'); + } + cids = Array.isArray(cids) ? cids : [cids]; + const exists = await categories.exists(cids); + if (exists.includes(false)) { + throw new Error('[[error:no-category]]'); + } + await db.sortedSetsAdd(cids.map(cid => `cid:${cid}:uid:watch:state`), state, uid); + }; + User.getCategoryWatchState = async function (uid) { + if (!(parseInt(uid, 10) > 0)) { + return {}; + } + const cids = await categories.getAllCidsFromSet('categories:cid'); + const states = await categories.getWatchState(cids, uid); + return _.zipObject(cids, states); + }; + User.getIgnoredCategories = async function (uid) { + if (!(parseInt(uid, 10) > 0)) { + return []; + } + const cids = await User.getCategoriesByStates(uid, [categories.watchStates.ignoring]); + const result = await plugins.hooks.fire('filter:user.getIgnoredCategories', { + uid: uid, + cids: cids + }); + return result.cids; + }; + User.getWatchedCategories = async function (uid) { + if (!(parseInt(uid, 10) > 0)) { + return []; + } + let cids = await User.getCategoriesByStates(uid, [categories.watchStates.watching]); + const categoryData = await categories.getCategoriesFields(cids, ['disabled']); + cids = cids.filter((cid, index) => categoryData[index] && !categoryData[index].disabled); + const result = await plugins.hooks.fire('filter:user.getWatchedCategories', { + uid: uid, + cids: cids + }); + return result.cids; + }; + User.getCategoriesByStates = async function (uid, states) { + const cids = await categories.getAllCidsFromSet('categories:cid'); + if (!(parseInt(uid, 10) > 0)) { + return cids; + } + const userState = await categories.getWatchState(cids, uid); + return cids.filter((cid, index) => states.includes(userState[index])); + }; + User.ignoreCategory = async function (uid, cid) { + await User.setCategoryWatchState(uid, cid, categories.watchStates.ignoring); + }; + User.watchCategory = async function (uid, cid) { + await User.setCategoryWatchState(uid, cid, categories.watchStates.watching); + }; +}; \ No newline at end of file diff --git a/lib/user/create.js b/lib/user/create.js new file mode 100644 index 0000000000..a04be1c7c3 --- /dev/null +++ b/lib/user/create.js @@ -0,0 +1,153 @@ +'use strict'; + +const zxcvbn = require('zxcvbn'); +const winston = require('winston'); +const db = require('../database'); +const utils = require('../utils'); +const slugify = require('../slugify'); +const plugins = require('../plugins'); +const groups = require('../groups'); +const meta = require('../meta'); +const analytics = require('../analytics'); +module.exports = function (User) { + User.create = async function (data) { + data.username = data.username.trim(); + data.userslug = slugify(data.username); + if (data.email !== undefined) { + data.email = String(data.email).trim(); + } + await User.isDataValid(data); + await lock(data.username, '[[error:username-taken]]'); + if (data.email && data.email !== data.username) { + await lock(data.email, '[[error:email-taken]]'); + } + try { + return await create(data); + } finally { + await db.deleteObjectFields('locks', [data.username, data.email]); + } + }; + async function lock(value, error) { + const count = await db.incrObjectField('locks', value); + if (count > 1) { + throw new Error(error); + } + } + async function create(data) { + const timestamp = data.timestamp || Date.now(); + let userData = { + username: data.username, + userslug: data.userslug, + joindate: timestamp, + lastonline: timestamp, + status: 'online' + }; + ['picture', 'fullname', 'location', 'birthday'].forEach(field => { + if (data[field]) { + userData[field] = data[field]; + } + }); + if (data.gdpr_consent === true) { + userData.gdpr_consent = 1; + } + if (data.acceptTos === true) { + userData.acceptTos = 1; + } + const renamedUsername = await User.uniqueUsername(userData); + const userNameChanged = !!renamedUsername; + if (userNameChanged) { + userData.username = renamedUsername; + userData.userslug = slugify(renamedUsername); + } + const results = await plugins.hooks.fire('filter:user.create', { + user: userData, + data: data + }); + userData = results.user; + const uid = await db.incrObjectField('global', 'nextUid'); + const isFirstUser = uid === 1; + userData.uid = uid; + await db.setObject(`user:${uid}`, userData); + const bulkAdd = [['username:uid', userData.uid, userData.username], [`user:${userData.uid}:usernames`, timestamp, `${userData.username}:${timestamp}`], ['username:sorted', 0, `${userData.username.toLowerCase()}:${userData.uid}`], ['userslug:uid', userData.uid, userData.userslug], ['users:joindate', timestamp, userData.uid], ['users:online', timestamp, userData.uid], ['users:postcount', 0, userData.uid], ['users:reputation', 0, userData.uid]]; + if (userData.fullname) { + bulkAdd.push(['fullname:sorted', 0, `${userData.fullname.toLowerCase()}:${userData.uid}`]); + } + await Promise.all([db.incrObjectField('global', 'userCount'), analytics.increment('registrations'), db.sortedSetAddBulk(bulkAdd), groups.join(['registered-users', 'unverified-users'], userData.uid), User.notifications.sendWelcomeNotification(userData.uid), storePassword(userData.uid, data.password), User.updateDigestSetting(userData.uid, meta.config.dailyDigestFreq)]); + if (data.email && isFirstUser) { + await User.setUserField(uid, 'email', data.email); + await User.email.confirmByUid(userData.uid); + } + if (data.email && userData.uid > 1) { + await User.email.sendValidationEmail(userData.uid, { + email: data.email, + template: 'welcome', + subject: `[[email:welcome-to, ${meta.config.title || meta.config.browserTitle || 'NodeBB'}]]` + }).catch(err => winston.error(`[user.create] Validation email failed to send\n[emailer.send] ${err.stack}`)); + } + if (userNameChanged) { + await User.notifications.sendNameChangeNotification(userData.uid, userData.username); + } + plugins.hooks.fire('action:user.create', { + user: userData, + data: data + }); + return userData.uid; + } + async function storePassword(uid, password) { + if (!password) { + return; + } + const hash = await User.hashPassword(password); + await Promise.all([User.setUserFields(uid, { + password: hash, + 'password:shaWrapped': 1 + }), User.reset.updateExpiry(uid)]); + } + User.isDataValid = async function (userData) { + if (userData.email && !utils.isEmailValid(userData.email)) { + throw new Error('[[error:invalid-email]]'); + } + if (!utils.isUserNameValid(userData.username) || !userData.userslug) { + throw new Error(`[[error:invalid-username, ${userData.username}]]`); + } + if (userData.password) { + User.isPasswordValid(userData.password); + } + if (userData.email) { + const available = await User.email.available(userData.email); + if (!available) { + throw new Error('[[error:email-taken]]'); + } + } + }; + User.isPasswordValid = function (password, minStrength) { + minStrength = minStrength || minStrength === 0 ? minStrength : meta.config.minimumPasswordStrength; + if (!password || !utils.isPasswordValid(password)) { + throw new Error('[[error:invalid-password]]'); + } + if (password.length < meta.config.minimumPasswordLength) { + throw new Error('[[reset_password:password-too-short]]'); + } + if (password.length > 512) { + throw new Error('[[error:password-too-long]]'); + } + const strength = zxcvbn(password); + if (strength.score < minStrength) { + throw new Error('[[user:weak-password]]'); + } + }; + User.uniqueUsername = async function (userData) { + let numTries = 0; + let { + username + } = userData; + while (true) { + const exists = await meta.userOrGroupExists(username); + if (!exists) { + return numTries ? username : null; + } + username = `${userData.username} ${numTries.toString(32)}`; + numTries += 1; + } + }; +}; \ No newline at end of file diff --git a/lib/user/data.js b/lib/user/data.js new file mode 100644 index 0000000000..acfdb8fd4d --- /dev/null +++ b/lib/user/data.js @@ -0,0 +1,301 @@ +'use strict'; + +const validator = require('validator'); +const nconf = require('nconf'); +const _ = require('lodash'); +const db = require('../database'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const relative_path = nconf.get('relative_path'); +const intFields = ['uid', 'postcount', 'topiccount', 'reputation', 'profileviews', 'banned', 'banned:expire', 'email:confirmed', 'joindate', 'lastonline', 'lastqueuetime', 'lastposttime', 'followingCount', 'followerCount', 'blocksCount', 'passwordExpiry', 'mutedUntil']; +module.exports = function (User) { + const fieldWhitelist = ['uid', 'username', 'userslug', 'email', 'email:confirmed', 'joindate', 'lastonline', 'picture', 'icon:bgColor', 'fullname', 'location', 'birthday', 'website', 'aboutme', 'signature', 'uploadedpicture', 'profileviews', 'reputation', 'postcount', 'topiccount', 'lastposttime', 'banned', 'banned:expire', 'status', 'flags', 'followerCount', 'followingCount', 'cover:url', 'cover:position', 'groupTitle', 'mutedUntil', 'mutedReason']; + User.guestData = { + uid: 0, + username: '[[global:guest]]', + displayname: '[[global:guest]]', + userslug: '', + fullname: '[[global:guest]]', + email: '', + 'icon:text': '?', + 'icon:bgColor': '#aaa', + groupTitle: '', + groupTitleArray: [], + status: 'offline', + reputation: 0, + 'email:confirmed': 0 + }; + let iconBackgrounds; + User.getUsersFields = async function (uids, fields) { + if (!Array.isArray(uids) || !uids.length) { + return []; + } + uids = uids.map(uid => isNaN(uid) ? 0 : parseInt(uid, 10)); + const fieldsToRemove = []; + fields = fields.slice(); + ensureRequiredFields(fields, fieldsToRemove); + const uniqueUids = _.uniq(uids).filter(uid => uid > 0); + const results = await plugins.hooks.fire('filter:user.whitelistFields', { + uids: uids, + whitelist: fieldWhitelist.slice() + }); + if (!fields.length) { + fields = results.whitelist; + } else { + fields = fields.filter(value => value !== 'password'); + } + const users = await db.getObjectsFields(uniqueUids.map(uid => `user:${uid}`), fields); + const result = await plugins.hooks.fire('filter:user.getFields', { + uids: uniqueUids, + users: users, + fields: fields + }); + result.users.forEach((user, index) => { + if (uniqueUids[index] > 0 && !user.uid) { + user.oldUid = uniqueUids[index]; + } + }); + await modifyUserData(result.users, fields, fieldsToRemove); + return uidsToUsers(uids, uniqueUids, result.users); + }; + function ensureRequiredFields(fields, fieldsToRemove) { + function addField(field) { + if (!fields.includes(field)) { + fields.push(field); + fieldsToRemove.push(field); + } + } + if (fields.length && !fields.includes('uid')) { + fields.push('uid'); + } + if (fields.includes('picture')) { + addField('uploadedpicture'); + } + if (fields.includes('status')) { + addField('lastonline'); + } + if (fields.includes('banned') && !fields.includes('banned:expire')) { + addField('banned:expire'); + } + if (fields.includes('username') && !fields.includes('fullname')) { + addField('fullname'); + } + } + function uidsToUsers(uids, uniqueUids, usersData) { + const uidToUser = _.zipObject(uniqueUids, usersData); + const users = uids.map(uid => { + const user = uidToUser[uid] || { + ...User.guestData + }; + if (!parseInt(user.uid, 10)) { + user.username = user.hasOwnProperty('oldUid') && parseInt(user.oldUid, 10) ? '[[global:former-user]]' : '[[global:guest]]'; + user.displayname = user.username; + } + if (uid === -1) { + user.uid = -1; + } + return user; + }); + return users; + } + User.getUserField = async function (uid, field) { + const user = await User.getUserFields(uid, [field]); + return user ? user[field] : null; + }; + User.getUserFields = async function (uid, fields) { + const users = await User.getUsersFields([uid], fields); + return users ? users[0] : null; + }; + User.getUserData = async function (uid) { + const users = await User.getUsersData([uid]); + return users ? users[0] : null; + }; + User.getUsersData = async function (uids) { + return await User.getUsersFields(uids, []); + }; + User.hidePrivateData = async function (users, callerUID) { + let single = false; + if (!Array.isArray(users)) { + users = [users]; + single = true; + } + const [userSettings, isAdmin, isGlobalModerator] = await Promise.all([User.getMultipleUserSettings(users.map(user => user.uid)), User.isAdministrator(callerUID), User.isGlobalModerator(callerUID)]); + users = await Promise.all(users.map(async (userData, idx) => { + const _userData = { + ...userData + }; + const isSelf = parseInt(callerUID, 10) === parseInt(_userData.uid, 10); + const privilegedOrSelf = isAdmin || isGlobalModerator || isSelf; + if (!privilegedOrSelf && (!userSettings[idx].showemail || meta.config.hideEmail)) { + _userData.email = ''; + } + if (!privilegedOrSelf && (!userSettings[idx].showfullname || meta.config.hideFullname)) { + _userData.fullname = ''; + } + return _userData; + })); + return single ? users.pop() : users; + }; + async function modifyUserData(users, requestedFields, fieldsToRemove) { + let uidToSettings = {}; + if (meta.config.showFullnameAsDisplayName) { + const uids = users.map(user => user.uid); + uidToSettings = _.zipObject(uids, await db.getObjectsFields(uids.map(uid => `user:${uid}:settings`), ['showfullname'])); + } + if (!iconBackgrounds) { + iconBackgrounds = await User.getIconBackgrounds(); + } + const unbanUids = []; + users.forEach(user => { + if (!user) { + return; + } + db.parseIntFields(user, intFields, requestedFields); + if (user.hasOwnProperty('username')) { + parseDisplayName(user, uidToSettings); + user.username = validator.escape(user.username ? user.username.toString() : ''); + } + if (user.hasOwnProperty('email')) { + user.email = validator.escape(user.email ? user.email.toString() : ''); + } + if (!user.uid) { + for (const [key, value] of Object.entries(User.guestData)) { + user[key] = value; + } + user.picture = User.getDefaultAvatar(); + } + if (user.hasOwnProperty('groupTitle')) { + parseGroupTitle(user); + } + if (user.picture && user.picture === user.uploadedpicture) { + user.uploadedpicture = user.picture.startsWith('http') ? user.picture : relative_path + user.picture; + user.picture = user.uploadedpicture; + } else if (user.uploadedpicture) { + user.uploadedpicture = user.uploadedpicture.startsWith('http') ? user.uploadedpicture : relative_path + user.uploadedpicture; + } + if (meta.config.defaultAvatar && !user.picture) { + user.picture = User.getDefaultAvatar(); + } + if (user.hasOwnProperty('status') && user.hasOwnProperty('lastonline')) { + user.status = User.getStatus(user); + } + for (let i = 0; i < fieldsToRemove.length; i += 1) { + user[fieldsToRemove[i]] = undefined; + } + if (requestedFields.includes('picture') && user.username && user.uid && !meta.config.defaultAvatar) { + if (!iconBackgrounds.includes(user['icon:bgColor'])) { + const nameAsIndex = Array.from(user.username).reduce((cur, next) => cur + next.charCodeAt(), 0); + user['icon:bgColor'] = iconBackgrounds[nameAsIndex % iconBackgrounds.length]; + } + user['icon:text'] = (user.username[0] || '').toUpperCase(); + } + if (user.hasOwnProperty('joindate')) { + user.joindateISO = utils.toISOString(user.joindate); + } + if (user.hasOwnProperty('lastonline')) { + user.lastonlineISO = utils.toISOString(user.lastonline) || user.joindateISO; + } + if (user.hasOwnProperty('mutedUntil')) { + user.muted = user.mutedUntil > Date.now(); + } + if (user.hasOwnProperty('banned') || user.hasOwnProperty('banned:expire')) { + const result = User.bans.calcExpiredFromUserData(user); + user.banned = result.banned; + const unban = result.banned && result.banExpired; + user.banned_until = unban ? 0 : user['banned:expire']; + user.banned_until_readable = user.banned_until && !unban ? utils.toISOString(user.banned_until) : 'Not Banned'; + if (unban) { + unbanUids.push(user.uid); + user.banned = false; + } + } + }); + if (unbanUids.length) { + await User.bans.unban(unbanUids, '[[user:info.ban-expired]]'); + } + return await plugins.hooks.fire('filter:users.get', users); + } + function parseDisplayName(user, uidToSettings) { + let showfullname = parseInt(meta.config.showfullname, 10) === 1; + if (uidToSettings[user.uid]) { + if (parseInt(uidToSettings[user.uid].showfullname, 10) === 0) { + showfullname = false; + } else if (parseInt(uidToSettings[user.uid].showfullname, 10) === 1) { + showfullname = true; + } + } + user.displayname = validator.escape(String(meta.config.showFullnameAsDisplayName && showfullname && user.fullname ? user.fullname : user.username)); + } + function parseGroupTitle(user) { + try { + user.groupTitleArray = JSON.parse(user.groupTitle); + } catch (err) { + if (user.groupTitle) { + user.groupTitleArray = [user.groupTitle]; + } else { + user.groupTitle = ''; + user.groupTitleArray = []; + } + } + if (!Array.isArray(user.groupTitleArray)) { + if (user.groupTitleArray) { + user.groupTitleArray = [user.groupTitleArray]; + } else { + user.groupTitleArray = []; + } + } + if (!meta.config.allowMultipleBadges && user.groupTitleArray.length) { + user.groupTitleArray = [user.groupTitleArray[0]]; + } + } + User.getIconBackgrounds = async () => { + if (iconBackgrounds) { + return iconBackgrounds; + } + const _iconBackgrounds = ['#f44336', '#e91e63', '#9c27b0', '#673ab7', '#3f51b5', '#2196f3', '#009688', '#1b5e20', '#33691e', '#827717', '#e65100', '#ff5722', '#795548', '#607d8b']; + const data = await plugins.hooks.fire('filter:user.iconBackgrounds', { + iconBackgrounds: _iconBackgrounds + }); + iconBackgrounds = data.iconBackgrounds; + return iconBackgrounds; + }; + User.getDefaultAvatar = function () { + if (!meta.config.defaultAvatar) { + return ''; + } + return meta.config.defaultAvatar.startsWith('http') ? meta.config.defaultAvatar : relative_path + meta.config.defaultAvatar; + }; + User.setUserField = async function (uid, field, value) { + await User.setUserFields(uid, { + [field]: value + }); + }; + User.setUserFields = async function (uid, data) { + await db.setObject(`user:${uid}`, data); + for (const [field, value] of Object.entries(data)) { + plugins.hooks.fire('action:user.set', { + uid, + field, + value, + type: 'set' + }); + } + }; + User.incrementUserFieldBy = async function (uid, field, value) { + return await incrDecrUserFieldBy(uid, field, value, 'increment'); + }; + User.decrementUserFieldBy = async function (uid, field, value) { + return await incrDecrUserFieldBy(uid, field, -value, 'decrement'); + }; + async function incrDecrUserFieldBy(uid, field, value, type) { + const newValue = await db.incrObjectFieldBy(`user:${uid}`, field, value); + plugins.hooks.fire('action:user.set', { + uid: uid, + field: field, + value: newValue, + type: type + }); + return newValue; + } +}; \ No newline at end of file diff --git a/lib/user/delete.js b/lib/user/delete.js new file mode 100644 index 0000000000..4c9885e4c0 --- /dev/null +++ b/lib/user/delete.js @@ -0,0 +1,155 @@ +'use strict'; + +const async = require('async'); +const _ = require('lodash'); +const path = require('path'); +const nconf = require('nconf'); +const { + rimraf +} = require('rimraf'); +const db = require('../database'); +const posts = require('../posts'); +const flags = require('../flags'); +const topics = require('../topics'); +const groups = require('../groups'); +const messaging = require('../messaging'); +const plugins = require('../plugins'); +const batch = require('../batch'); +module.exports = function (User) { + const deletesInProgress = {}; + User.delete = async (callerUid, uid) => { + await User.deleteContent(callerUid, uid); + return await User.deleteAccount(uid); + }; + User.deleteContent = async function (callerUid, uid) { + if (parseInt(uid, 10) <= 0) { + throw new Error('[[error:invalid-uid]]'); + } + if (deletesInProgress[uid]) { + throw new Error('[[error:already-deleting]]'); + } + deletesInProgress[uid] = 'user.delete'; + await deletePosts(callerUid, uid); + await deleteTopics(callerUid, uid); + await deleteUploads(callerUid, uid); + await deleteQueued(uid); + delete deletesInProgress[uid]; + }; + async function deletePosts(callerUid, uid) { + await batch.processSortedSet(`uid:${uid}:posts`, async pids => { + await posts.purge(pids, callerUid); + }, { + alwaysStartAt: 0, + batch: 500 + }); + } + async function deleteTopics(callerUid, uid) { + await batch.processSortedSet(`uid:${uid}:topics`, async ids => { + await async.eachSeries(ids, async tid => { + await topics.purge(tid, callerUid); + }); + }, { + alwaysStartAt: 0 + }); + } + async function deleteUploads(callerUid, uid) { + const uploads = await db.getSortedSetMembers(`uid:${uid}:uploads`); + await User.deleteUpload(callerUid, uid, uploads); + } + async function deleteQueued(uid) { + let deleteIds = []; + await batch.processSortedSet('post:queue', async ids => { + const data = await db.getObjects(ids.map(id => `post:queue:${id}`)); + const userQueuedIds = data.filter(d => parseInt(d.uid, 10) === parseInt(uid, 10)).map(d => d.id); + deleteIds = deleteIds.concat(userQueuedIds); + }, { + batch: 500 + }); + await async.eachSeries(deleteIds, posts.removeFromQueue); + } + async function removeFromSortedSets(uid) { + await db.sortedSetsRemove(['users:joindate', 'users:postcount', 'users:reputation', 'users:banned', 'users:banned:expire', 'users:flags', 'users:online', 'digest:day:uids', 'digest:week:uids', 'digest:biweek:uids', 'digest:month:uids'], uid); + } + User.deleteAccount = async function (uid) { + if (deletesInProgress[uid] === 'user.deleteAccount') { + throw new Error('[[error:already-deleting]]'); + } + deletesInProgress[uid] = 'user.deleteAccount'; + await removeFromSortedSets(uid); + const userData = await db.getObject(`user:${uid}`); + if (!userData || !userData.username) { + delete deletesInProgress[uid]; + throw new Error('[[error:no-user]]'); + } + await plugins.hooks.fire('static:user.delete', { + uid: uid, + userData: userData + }); + await deleteVotes(uid); + await deleteChats(uid); + await User.auth.revokeAllSessions(uid); + const keys = [`uid:${uid}:notifications:read`, `uid:${uid}:notifications:unread`, `uid:${uid}:bookmarks`, `uid:${uid}:tids_read`, `uid:${uid}:tids_unread`, `uid:${uid}:blocked_uids`, `user:${uid}:settings`, `user:${uid}:usernames`, `user:${uid}:emails`, `uid:${uid}:topics`, `uid:${uid}:posts`, `uid:${uid}:chats`, `uid:${uid}:chats:unread`, `uid:${uid}:chat:rooms`, `uid:${uid}:chat:rooms:unread`, `uid:${uid}:chat:rooms:read`, `uid:${uid}:upvote`, `uid:${uid}:downvote`, `uid:${uid}:flag:pids`, `uid:${uid}:sessions`, `invitation:uid:${uid}`]; + const bulkRemove = [['username:uid', userData.username], ['username:sorted', `${userData.username.toLowerCase()}:${uid}`], ['userslug:uid', userData.userslug], ['fullname:uid', userData.fullname]]; + if (userData.email) { + bulkRemove.push(['email:uid', userData.email.toLowerCase()]); + bulkRemove.push(['email:sorted', `${userData.email.toLowerCase()}:${uid}`]); + } + if (userData.fullname) { + bulkRemove.push(['fullname:sorted', `${userData.fullname.toLowerCase()}:${uid}`]); + } + await Promise.all([db.sortedSetRemoveBulk(bulkRemove), db.decrObjectField('global', 'userCount'), db.deleteAll(keys), db.setRemove('invitation:uids', uid), deleteUserIps(uid), deleteUserFromFollowers(uid), deleteUserFromFollowedTopics(uid), deleteUserFromIgnoredTopics(uid), deleteUserFromFollowedTags(uid), deleteImages(uid), groups.leaveAllGroups(uid), flags.resolveFlag('user', uid, uid), User.reset.cleanByUid(uid), User.email.expireValidation(uid)]); + await db.deleteAll([`followers:${uid}`, `following:${uid}`, `user:${uid}`, `uid:${uid}:followed_tags`, `uid:${uid}:followed_tids`, `uid:${uid}:ignored_tids`]); + delete deletesInProgress[uid]; + return userData; + }; + async function deleteUserFromFollowedTopics(uid) { + const tids = await db.getSortedSetRange(`uid:${uid}:followed_tids`, 0, -1); + await db.setsRemove(tids.map(tid => `tid:${tid}:followers`), uid); + } + async function deleteUserFromIgnoredTopics(uid) { + const tids = await db.getSortedSetRange(`uid:${uid}:ignored_tids`, 0, -1); + await db.setsRemove(tids.map(tid => `tid:${tid}:ignorers`), uid); + } + async function deleteUserFromFollowedTags(uid) { + const tags = await db.getSortedSetRange(`uid:${uid}:followed_tags`, 0, -1); + await db.sortedSetsRemove(tags.map(tag => `tag:${tag}:followers`), uid); + } + async function deleteVotes(uid) { + const [upvotedPids, downvotedPids] = await Promise.all([db.getSortedSetRange(`uid:${uid}:upvote`, 0, -1), db.getSortedSetRange(`uid:${uid}:downvote`, 0, -1)]); + const pids = _.uniq(upvotedPids.concat(downvotedPids).filter(Boolean)); + await async.eachSeries(pids, async pid => { + await posts.unvote(pid, uid); + }); + } + async function deleteChats(uid) { + const roomIds = await db.getSortedSetRange([`uid:${uid}:chat:rooms`, `chat:rooms:public`], 0, -1); + await messaging.leaveRooms(uid, roomIds); + } + async function deleteUserIps(uid) { + const ips = await db.getSortedSetRange(`uid:${uid}:ip`, 0, -1); + await db.sortedSetsRemove(ips.map(ip => `ip:${ip}:uid`), uid); + await db.delete(`uid:${uid}:ip`); + } + async function deleteUserFromFollowers(uid) { + const [followers, following] = await Promise.all([db.getSortedSetRange(`followers:${uid}`, 0, -1), db.getSortedSetRange(`following:${uid}`, 0, -1)]); + async function updateCount(uids, name, fieldName) { + await batch.processArray(uids, async uids => { + const counts = await db.sortedSetsCard(uids.map(uid => name + uid)); + const bulkSet = counts.map((count, index) => [`user:${uids[index]}`, { + [fieldName]: count || 0 + }]); + await db.setObjectBulk(bulkSet); + }, { + batch: 500 + }); + } + const followingSets = followers.map(uid => `following:${uid}`); + const followerSets = following.map(uid => `followers:${uid}`); + await db.sortedSetsRemove(followerSets.concat(followingSets), uid); + await Promise.all([updateCount(following, 'followers:', 'followerCount'), updateCount(followers, 'following:', 'followingCount')]); + } + async function deleteImages(uid) { + const folder = path.join(nconf.get('upload_path'), 'profile', `uid-${uid}`); + await rimraf(folder); + } +}; \ No newline at end of file diff --git a/lib/user/digest.js b/lib/user/digest.js new file mode 100644 index 0000000000..60d2070043 --- /dev/null +++ b/lib/user/digest.js @@ -0,0 +1,186 @@ +'use strict'; + +const winston = require('winston'); +const nconf = require('nconf'); +const db = require('../database'); +const batch = require('../batch'); +const meta = require('../meta'); +const user = require('./index'); +const topics = require('../topics'); +const messaging = require('../messaging'); +const plugins = require('../plugins'); +const emailer = require('../emailer'); +const utils = require('../utils'); +const Digest = module.exports; +const baseUrl = nconf.get('base_url'); +Digest.execute = async function (payload) { + const digestsDisabled = meta.config.disableEmailSubscriptions === 1; + if (digestsDisabled) { + winston.info(`[user/jobs] Did not send digests (${payload.interval}) because subscription system is disabled.`); + return false; + } + let { + subscribers + } = payload; + if (!subscribers) { + subscribers = await Digest.getSubscribers(payload.interval); + } + if (!subscribers.length) { + return false; + } + try { + winston.info(`[user/jobs] Digest (${payload.interval}) scheduling completed (${subscribers.length} subscribers). Sending emails; this may take some time...`); + await Digest.send({ + interval: payload.interval, + subscribers: subscribers + }); + winston.info(`[user/jobs] Digest (${payload.interval}) complete.`); + return true; + } catch (err) { + winston.error(`[user/jobs] Could not send digests (${payload.interval})\n${err.stack}`); + throw err; + } +}; +Digest.getUsersInterval = async uids => { + let single = false; + if (!Array.isArray(uids) && !isNaN(parseInt(uids, 10))) { + uids = [uids]; + single = true; + } + const settings = await db.getObjects(uids.map(uid => `user:${uid}:settings`)); + const interval = uids.map((uid, index) => settings[index] && settings[index].dailyDigestFreq || false); + return single ? interval[0] : interval; +}; +Digest.getSubscribers = async function (interval) { + let subscribers = []; + await batch.processSortedSet('users:joindate', async uids => { + const settings = await user.getMultipleUserSettings(uids); + let subUids = []; + settings.forEach(hash => { + if (hash.dailyDigestFreq === interval) { + subUids.push(hash.uid); + } + }); + subUids = await user.bans.filterBanned(subUids); + subscribers = subscribers.concat(subUids); + }, { + interval: 1000, + batch: 500 + }); + const results = await plugins.hooks.fire('filter:digest.subscribers', { + interval: interval, + subscribers: subscribers + }); + return results.subscribers; +}; +Digest.send = async function (data) { + let emailsSent = 0; + if (!data || !data.subscribers || !data.subscribers.length) { + return emailsSent; + } + let errorLogged = false; + await batch.processArray(data.subscribers, async uids => { + let userData = await user.getUsersFields(uids, ['uid', 'email', 'email:confirmed', 'username', 'userslug', 'lastonline']); + userData = userData.filter(u => u && u.email && (meta.config.includeUnverifiedEmails || u['email:confirmed'])); + if (!userData.length) { + return; + } + await Promise.all(userData.map(async userObj => { + const [publicRooms, notifications, topics] = await Promise.all([getUnreadPublicRooms(userObj.uid), user.notifications.getUnreadInterval(userObj.uid, data.interval), getTermTopics(data.interval, userObj.uid)]); + const unreadNotifs = notifications.filter(Boolean); + if (!unreadNotifs.length && !topics.top.length && !topics.popular.length && !topics.recent.length && !publicRooms.length) { + return; + } + unreadNotifs.forEach(n => { + if (n.image && !n.image.startsWith('http')) { + n.image = baseUrl + n.image; + } + if (n.path) { + n.notification_url = n.path.startsWith('http') ? n.path : baseUrl + n.path; + } + }); + emailsSent += 1; + const now = new Date(); + await emailer.send('digest', userObj.uid, { + subject: `[[email:digest.subject, ${now.getFullYear()}/${now.getMonth() + 1}/${now.getDate()}]]`, + username: userObj.username, + userslug: userObj.userslug, + notifications: unreadNotifs, + publicRooms: publicRooms, + recent: topics.recent, + topTopics: topics.top, + popularTopics: topics.popular, + interval: data.interval, + showUnsubscribe: true + }).catch(err => { + if (!errorLogged) { + winston.error(`[user/jobs] Could not send digest email\n[emailer.send] ${err.stack}`); + errorLogged = true; + } + }); + })); + if (data.interval !== 'alltime') { + const now = Date.now(); + await db.sortedSetAdd('digest:delivery', userData.map(() => now), userData.map(u => u.uid)); + } + }, { + interval: 1000, + batch: 100 + }); + winston.info(`[user/jobs] Digest (${data.interval}) sending completed. ${emailsSent} emails sent.`); + return emailsSent; +}; +Digest.getDeliveryTimes = async (start, stop) => { + const count = await db.sortedSetCard('users:joindate'); + const uids = await user.getUidsFromSet('users:joindate', start, stop); + if (!uids.length) { + return []; + } + const [scores, settings] = await Promise.all([db.sortedSetScores('digest:delivery', uids), Digest.getUsersInterval(uids)]); + let userData = await user.getUsersFields(uids, ['username', 'picture']); + userData = userData.map((user, idx) => { + user.lastDelivery = scores[idx] ? new Date(scores[idx]).toISOString() : '[[admin/manage/digest:null]]'; + user.setting = settings[idx]; + return user; + }); + return { + users: userData, + count: count + }; +}; +async function getTermTopics(term, uid) { + const data = await topics.getSortedTopics({ + uid: uid, + start: 0, + stop: 199, + term: term, + sort: 'posts', + teaserPost: 'first' + }); + data.topics = data.topics.filter(topic => topic && !topic.deleted); + const popular = data.topics.filter(t => t.postcount > 1).sort((a, b) => b.postcount - a.postcount).slice(0, 10); + const popularTids = popular.map(t => t.tid); + const top = data.topics.filter(t => t.votes > 0 && !popularTids.includes(t.tid)).sort((a, b) => b.votes - a.votes).slice(0, 10); + const topTids = top.map(t => t.tid); + const recent = data.topics.filter(t => !topTids.includes(t.tid) && !popularTids.includes(t.tid)).sort((a, b) => b.lastposttime - a.lastposttime).slice(0, 10); + [...top, ...popular, ...recent].forEach(topicObj => { + if (topicObj) { + if (topicObj.teaser && topicObj.teaser.content && topicObj.teaser.content.length > 255) { + topicObj.teaser.content = `${topicObj.teaser.content.slice(0, 255)}...`; + } + const user = topicObj.hasOwnProperty('teaser') && topicObj.teaser && topicObj.teaser.user ? topicObj.teaser.user : topicObj.user; + if (user && user.picture && utils.isRelativeUrl(user.picture)) { + user.picture = baseUrl + user.picture; + } + } + }); + return { + top, + popular, + recent + }; +} +async function getUnreadPublicRooms(uid) { + const publicRooms = await messaging.getPublicRooms(uid, uid); + return publicRooms.filter(r => r && r.unread); +} \ No newline at end of file diff --git a/lib/user/email.js b/lib/user/email.js new file mode 100644 index 0000000000..f3bd52f450 --- /dev/null +++ b/lib/user/email.js @@ -0,0 +1,186 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const user = require('./index'); +const utils = require('../utils'); +const plugins = require('../plugins'); +const db = require('../database'); +const meta = require('../meta'); +const emailer = require('../emailer'); +const groups = require('../groups'); +const events = require('../events'); +const UserEmail = module.exports; +UserEmail.exists = async function (email) { + const uid = await user.getUidByEmail(email.toLowerCase()); + return !!uid; +}; +UserEmail.available = async function (email) { + const exists = await db.isSortedSetMember('email:uid', email.toLowerCase()); + return !exists; +}; +UserEmail.remove = async function (uid, sessionId) { + const email = await user.getUserField(uid, 'email'); + if (!email) { + return; + } + await Promise.all([user.setUserFields(uid, { + email: '', + 'email:confirmed': 0 + }), db.sortedSetRemove('email:uid', email.toLowerCase()), db.sortedSetRemove('email:sorted', `${email.toLowerCase()}:${uid}`), user.email.expireValidation(uid), sessionId ? user.auth.revokeAllSessions(uid, sessionId) : Promise.resolve(), events.log({ + targetUid: uid, + type: 'email-change', + email, + newEmail: '' + })]); +}; +UserEmail.getEmailForValidation = async uid => { + let email = ''; + const code = await db.get(`confirm:byUid:${uid}`); + const confirmObj = await db.getObject(`confirm:${code}`); + if (confirmObj && confirmObj.email && parseInt(uid, 10) === parseInt(confirmObj.uid, 10)) { + email = confirmObj.email; + } + if (!email) { + email = await user.getUserField(uid, 'email'); + } + return email; +}; +UserEmail.isValidationPending = async (uid, email) => { + const code = await db.get(`confirm:byUid:${uid}`); + const confirmObj = await db.getObject(`confirm:${code}`); + return !!(confirmObj && (!email || email === confirmObj.email) && Date.now() < parseInt(confirmObj.expires, 10)); +}; +UserEmail.getValidationExpiry = async uid => { + const code = await db.get(`confirm:byUid:${uid}`); + const confirmObj = await db.getObject(`confirm:${code}`); + return confirmObj ? Math.max(0, confirmObj.expires - Date.now()) : null; +}; +UserEmail.expireValidation = async uid => { + const keys = [`confirm:byUid:${uid}`]; + const code = await db.get(`confirm:byUid:${uid}`); + if (code) { + keys.push(`confirm:${code}`); + } + await db.deleteAll(keys); +}; +UserEmail.canSendValidation = async (uid, email) => { + const pending = await UserEmail.isValidationPending(uid, email); + if (!pending) { + return true; + } + const ttl = await UserEmail.getValidationExpiry(uid); + const max = meta.config.emailConfirmExpiry * 60 * 60 * 1000; + const interval = meta.config.emailConfirmInterval * 60 * 1000; + return (ttl || Date.now()) + interval < max; +}; +UserEmail.sendValidationEmail = async function (uid, options) { + if (meta.config.sendValidationEmail !== 1) { + winston.verbose(`[user/email] Validation email for uid ${uid} not sent due to config settings`); + return; + } + options = options || {}; + if (typeof options === 'string') { + options = { + email: options + }; + } + const confirm_code = utils.generateUUID(); + const confirm_link = `${nconf.get('url')}/confirm/${confirm_code}`; + const { + emailConfirmInterval, + emailConfirmExpiry + } = meta.config; + if (!options.email || !options.email.length) { + options.email = await user.getUserField(uid, 'email'); + } + if (!options.email) { + return; + } + if (!options.force && !(await UserEmail.canSendValidation(uid, options.email))) { + throw new Error(`[[error:confirm-email-already-sent, ${emailConfirmInterval}]]`); + } + const username = await user.getUserField(uid, 'username'); + const data = await plugins.hooks.fire('filter:user.verify', { + uid, + username, + confirm_link, + confirm_code: await plugins.hooks.fire('filter:user.verify.code', confirm_code), + email: options.email, + subject: options.subject || '[[email:email.verify-your-email.subject]]', + template: options.template || 'verify-email' + }); + await UserEmail.expireValidation(uid); + await db.set(`confirm:byUid:${uid}`, confirm_code); + await db.setObject(`confirm:${confirm_code}`, { + email: options.email.toLowerCase(), + uid: uid, + expires: Date.now() + emailConfirmExpiry * 60 * 60 * 1000 + }); + winston.verbose(`[user/email] Validation email for uid ${uid} sent to ${options.email}`); + events.log({ + type: 'email-confirmation-sent', + uid, + confirm_code, + ...options + }); + if (plugins.hooks.hasListeners('action:user.verify')) { + plugins.hooks.fire('action:user.verify', { + uid: uid, + data: data + }); + } else { + await emailer.send(data.template, uid, data); + } + return confirm_code; +}; +UserEmail.confirmByCode = async function (code, sessionId) { + const confirmObj = await db.getObject(`confirm:${code}`); + if (!confirmObj || !confirmObj.uid || !confirmObj.email) { + throw new Error('[[error:invalid-data]]'); + } + if (!confirmObj.expires || Date.now() > parseInt(confirmObj.expires, 10)) { + throw new Error('[[error:confirm-email-expired]]'); + } + const oldUid = await db.sortedSetScore('email:uid', confirmObj.email.toLowerCase()); + if (oldUid) { + await UserEmail.remove(oldUid, sessionId); + } + const oldEmail = await user.getUserField(confirmObj.uid, 'email'); + if (oldEmail && confirmObj.email !== oldEmail) { + await UserEmail.remove(confirmObj.uid, sessionId); + } else { + await user.auth.revokeAllSessions(confirmObj.uid, sessionId); + } + await user.setUserField(confirmObj.uid, 'email', confirmObj.email); + await Promise.all([UserEmail.confirmByUid(confirmObj.uid), db.delete(`confirm:${code}`), events.log({ + type: 'email-change', + oldEmail, + newEmail: confirmObj.email, + targetUid: confirmObj.uid + })]); +}; +UserEmail.confirmByUid = async function (uid, callerUid = 0) { + if (!(parseInt(uid, 10) > 0)) { + throw new Error('[[error:invalid-uid]]'); + } + callerUid = callerUid || uid; + const currentEmail = await user.getUserField(uid, 'email'); + if (!currentEmail) { + throw new Error('[[error:invalid-email]]'); + } + const oldUid = await db.sortedSetScore('email:uid', currentEmail.toLowerCase()); + if (oldUid && oldUid !== parseInt(uid, 10)) { + throw new Error('[[error:email-taken]]'); + } + const confirmedEmails = await db.getSortedSetRangeByScore(`email:uid`, 0, -1, uid, uid); + if (confirmedEmails.length) { + await db.sortedSetsRemoveRangeByScore([`email:uid`], uid, uid); + await db.sortedSetRemoveBulk(confirmedEmails.map(email => [`email:sorted`, `${email.toLowerCase()}:${uid}`])); + } + await Promise.all([db.sortedSetAddBulk([['email:uid', uid, currentEmail.toLowerCase()], ['email:sorted', 0, `${currentEmail.toLowerCase()}:${uid}`], [`user:${uid}:emails`, Date.now(), `${currentEmail}:${Date.now()}:${callerUid}`]]), user.setUserField(uid, 'email:confirmed', 1), groups.join('verified-users', uid), groups.leave('unverified-users', uid), user.email.expireValidation(uid), user.reset.cleanByUid(uid)]); + await plugins.hooks.fire('action:user.email.confirmed', { + uid: uid, + email: currentEmail + }); +}; \ No newline at end of file diff --git a/lib/user/follow.js b/lib/user/follow.js new file mode 100644 index 0000000000..4beab7d0d5 --- /dev/null +++ b/lib/user/follow.js @@ -0,0 +1,69 @@ +'use strict'; + +const plugins = require('../plugins'); +const db = require('../database'); +module.exports = function (User) { + User.follow = async function (uid, followuid) { + await toggleFollow('follow', uid, followuid); + }; + User.unfollow = async function (uid, unfollowuid) { + await toggleFollow('unfollow', uid, unfollowuid); + }; + async function toggleFollow(type, uid, theiruid) { + if (parseInt(uid, 10) <= 0 || parseInt(theiruid, 10) <= 0) { + throw new Error('[[error:invalid-uid]]'); + } + if (parseInt(uid, 10) === parseInt(theiruid, 10)) { + throw new Error('[[error:you-cant-follow-yourself]]'); + } + const [exists, isFollowing] = await Promise.all([User.exists(theiruid), User.isFollowing(uid, theiruid)]); + if (!exists) { + throw new Error('[[error:no-user]]'); + } + await plugins.hooks.fire('filter:user.toggleFollow', { + type, + uid, + theiruid, + isFollowing + }); + if (type === 'follow') { + if (isFollowing) { + throw new Error('[[error:already-following]]'); + } + const now = Date.now(); + await db.sortedSetAddBulk([[`following:${uid}`, now, theiruid], [`followers:${theiruid}`, now, uid]]); + } else { + if (!isFollowing) { + throw new Error('[[error:not-following]]'); + } + await db.sortedSetRemoveBulk([[`following:${uid}`, theiruid], [`followers:${theiruid}`, uid]]); + } + const [followingCount, followerCount] = await Promise.all([db.sortedSetCard(`following:${uid}`), db.sortedSetCard(`followers:${theiruid}`)]); + await Promise.all([User.setUserField(uid, 'followingCount', followingCount), User.setUserField(theiruid, 'followerCount', followerCount)]); + } + User.getFollowing = async function (uid, start, stop) { + return await getFollow(uid, 'following', start, stop); + }; + User.getFollowers = async function (uid, start, stop) { + return await getFollow(uid, 'followers', start, stop); + }; + async function getFollow(uid, type, start, stop) { + if (parseInt(uid, 10) <= 0) { + return []; + } + const uids = await db.getSortedSetRevRange(`${type}:${uid}`, start, stop); + const data = await plugins.hooks.fire(`filter:user.${type}`, { + uids: uids, + uid: uid, + start: start, + stop: stop + }); + return await User.getUsers(data.uids, uid); + } + User.isFollowing = async function (uid, theirid) { + if (parseInt(uid, 10) <= 0 || parseInt(theirid, 10) <= 0) { + return false; + } + return await db.isSortedSetMember(`following:${uid}`, theirid); + }; +}; \ No newline at end of file diff --git a/lib/user/index.js b/lib/user/index.js new file mode 100644 index 0000000000..572f528c74 --- /dev/null +++ b/lib/user/index.js @@ -0,0 +1,202 @@ +'use strict'; + +const _ = require('lodash'); +const groups = require('../groups'); +const plugins = require('../plugins'); +const db = require('../database'); +const privileges = require('../privileges'); +const categories = require('../categories'); +const meta = require('../meta'); +const utils = require('../utils'); +const User = module.exports; +User.email = require('./email'); +User.notifications = require('./notifications'); +User.reset = require('./reset'); +User.digest = require('./digest'); +User.interstitials = require('./interstitials'); +require('./data')(User); +require('./auth')(User); +require('./bans')(User); +require('./create')(User); +require('./posts')(User); +require('./topics')(User); +require('./categories')(User); +require('./follow')(User); +require('./profile')(User); +require('./admin')(User); +require('./delete')(User); +require('./settings')(User); +require('./search')(User); +require('./jobs')(User); +require('./picture')(User); +require('./approval')(User); +require('./invite')(User); +require('./password')(User); +require('./info')(User); +require('./online')(User); +require('./blocks')(User); +require('./uploads')(User); +User.exists = async function (uids) { + return await (Array.isArray(uids) ? db.isSortedSetMembers('users:joindate', uids) : db.isSortedSetMember('users:joindate', uids)); +}; +User.existsBySlug = async function (userslug) { + if (Array.isArray(userslug)) { + const uids = await User.getUidsByUserslugs(userslug); + return uids.map(uid => !!uid); + } + const uid = await User.getUidByUserslug(userslug); + return !!uid; +}; +User.getUidsFromSet = async function (set, start, stop) { + if (set === 'users:online') { + const count = parseInt(stop, 10) === -1 ? stop : stop - start + 1; + const now = Date.now(); + return await db.getSortedSetRevRangeByScore(set, start, count, '+inf', now - meta.config.onlineCutoff * 60000); + } + return await db.getSortedSetRevRange(set, start, stop); +}; +User.getUsersFromSet = async function (set, uid, start, stop) { + const uids = await User.getUidsFromSet(set, start, stop); + return await User.getUsers(uids, uid); +}; +User.getUsersWithFields = async function (uids, fields, uid) { + let results = await plugins.hooks.fire('filter:users.addFields', { + fields: fields + }); + results.fields = _.uniq(results.fields); + const userData = await User.getUsersFields(uids, results.fields); + results = await plugins.hooks.fire('filter:userlist.get', { + users: userData, + uid: uid + }); + return results.users; +}; +User.getUsers = async function (uids, uid) { + const userData = await User.getUsersWithFields(uids, ['uid', 'username', 'userslug', 'picture', 'status', 'postcount', 'reputation', 'email:confirmed', 'lastonline', 'flags', 'banned', 'banned:expire', 'joindate'], uid); + return User.hidePrivateData(userData, uid); +}; +User.getStatus = function (userData) { + if (userData.uid <= 0) { + return 'offline'; + } + const isOnline = Date.now() - userData.lastonline < meta.config.onlineCutoff * 60000; + return isOnline ? userData.status || 'online' : 'offline'; +}; +User.getUidByUsername = async function (username) { + if (!username) { + return 0; + } + return await db.sortedSetScore('username:uid', username); +}; +User.getUidsByUsernames = async function (usernames) { + return await db.sortedSetScores('username:uid', usernames); +}; +User.getUidByUserslug = async function (userslug) { + if (!userslug) { + return 0; + } + return await db.sortedSetScore('userslug:uid', userslug); +}; +User.getUidsByUserslugs = async function (userslugs) { + return await db.sortedSetScores('userslug:uid', userslugs); +}; +User.getUsernamesByUids = async function (uids) { + const users = await User.getUsersFields(uids, ['username']); + return users.map(user => user.username); +}; +User.getUsernameByUserslug = async function (slug) { + const uid = await User.getUidByUserslug(slug); + return await User.getUserField(uid, 'username'); +}; +User.getUidByEmail = async function (email) { + return await db.sortedSetScore('email:uid', email.toLowerCase()); +}; +User.getUidsByEmails = async function (emails) { + emails = emails.map(email => email && email.toLowerCase()); + return await db.sortedSetScores('email:uid', emails); +}; +User.getUsernameByEmail = async function (email) { + const uid = await db.sortedSetScore('email:uid', String(email).toLowerCase()); + return await User.getUserField(uid, 'username'); +}; +User.isModerator = async function (uid, cid) { + return await privileges.users.isModerator(uid, cid); +}; +User.isModeratorOfAnyCategory = async function (uid) { + const cids = await User.getModeratedCids(uid); + return Array.isArray(cids) ? !!cids.length : false; +}; +User.isAdministrator = async function (uid) { + return await privileges.users.isAdministrator(uid); +}; +User.isGlobalModerator = async function (uid) { + return await privileges.users.isGlobalModerator(uid); +}; +User.getPrivileges = async function (uid) { + return await utils.promiseParallel({ + isAdmin: User.isAdministrator(uid), + isGlobalModerator: User.isGlobalModerator(uid), + isModeratorOfAnyCategory: User.isModeratorOfAnyCategory(uid) + }); +}; +User.isPrivileged = async function (uid) { + if (!(parseInt(uid, 10) > 0)) { + return false; + } + const results = await User.getPrivileges(uid); + return results ? results.isAdmin || results.isGlobalModerator || results.isModeratorOfAnyCategory : false; +}; +User.isAdminOrGlobalMod = async function (uid) { + const [isAdmin, isGlobalMod] = await Promise.all([User.isAdministrator(uid), User.isGlobalModerator(uid)]); + return isAdmin || isGlobalMod; +}; +User.isAdminOrSelf = async function (callerUid, uid) { + await isSelfOrMethod(callerUid, uid, User.isAdministrator); +}; +User.isAdminOrGlobalModOrSelf = async function (callerUid, uid) { + await isSelfOrMethod(callerUid, uid, User.isAdminOrGlobalMod); +}; +User.isPrivilegedOrSelf = async function (callerUid, uid) { + await isSelfOrMethod(callerUid, uid, User.isPrivileged); +}; +async function isSelfOrMethod(callerUid, uid, method) { + if (parseInt(callerUid, 10) === parseInt(uid, 10)) { + return; + } + const isPass = await method(callerUid); + if (!isPass) { + throw new Error('[[error:no-privileges]]'); + } +} +User.getAdminsandGlobalMods = async function () { + const results = await groups.getMembersOfGroups(['administrators', 'Global Moderators']); + return await User.getUsersData(_.union(...results)); +}; +User.getAdminsandGlobalModsandModerators = async function () { + const results = await Promise.all([groups.getMembers('administrators', 0, -1), groups.getMembers('Global Moderators', 0, -1), User.getModeratorUids()]); + return await User.getUsersData(_.union(...results)); +}; +User.getFirstAdminUid = async function () { + return (await db.getSortedSetRange('group:administrators:members', 0, 0))[0]; +}; +User.getModeratorUids = async function () { + const cids = await categories.getAllCidsFromSet('categories:cid'); + const uids = await categories.getModeratorUids(cids); + return _.union(...uids); +}; +User.getModeratedCids = async function (uid) { + if (parseInt(uid, 10) <= 0) { + return []; + } + const cids = await categories.getAllCidsFromSet('categories:cid'); + const isMods = await User.isModerator(uid, cids); + return cids.filter((cid, index) => cid && isMods[index]); +}; +User.addInterstitials = function (callback) { + plugins.hooks.register('core', { + hook: 'filter:register.interstitial', + method: [User.interstitials.email, User.interstitials.gdpr, User.interstitials.tou] + }); + callback(); +}; +require('../promisify')(User); \ No newline at end of file diff --git a/lib/user/info.js b/lib/user/info.js new file mode 100644 index 0000000000..1985a007a4 --- /dev/null +++ b/lib/user/info.js @@ -0,0 +1,135 @@ +'use strict'; + +const _ = require('lodash'); +const validator = require('validator'); +const db = require('../database'); +const posts = require('../posts'); +const topics = require('../topics'); +const utils = require('../utils'); +const plugins = require('../plugins'); +const Flags = require('../flags'); +module.exports = function (User) { + User.getLatestBanInfo = async function (uid) { + const record = await db.getSortedSetRevRange(`uid:${uid}:bans:timestamp`, 0, 0); + if (!record.length) { + throw new Error('no-ban-info'); + } + const banInfo = await db.getObject(record[0]); + const expire = parseInt(banInfo.expire, 10); + const expire_readable = utils.toISOString(expire); + return { + uid: uid, + timestamp: banInfo.timestamp, + banned_until: expire, + expiry: expire, + banned_until_readable: expire_readable, + expiry_readable: expire_readable, + reason: validator.escape(String(banInfo.reason || '')) + }; + }; + User.getModerationHistory = async function (uid) { + let [flags, bans, mutes] = await Promise.all([db.getSortedSetRevRangeWithScores(`flags:byTargetUid:${uid}`, 0, 19), db.getSortedSetRevRange([`uid:${uid}:bans:timestamp`, `uid:${uid}:unbans:timestamp`], 0, 19), db.getSortedSetRevRange([`uid:${uid}:mutes:timestamp`, `uid:${uid}:unmutes:timestamp`], 0, 19)]); + const keys = flags.map(flagObj => `flag:${flagObj.value}`); + const payload = await db.getObjectsFields(keys, ['flagId', 'type', 'targetId', 'datetime']); + [flags, bans, mutes] = await Promise.all([getFlagMetadata(payload), formatBanMuteData(bans, '[[user:info.banned-no-reason]]'), formatBanMuteData(mutes, '[[user:info.muted-no-reason]]')]); + return { + flags: flags, + bans: bans, + mutes: mutes + }; + }; + User.getHistory = async function (set) { + const data = await db.getSortedSetRevRangeWithScores(set, 0, -1); + data.forEach(set => { + set.timestamp = set.score; + set.timestampISO = utils.toISOString(set.score); + const parts = set.value.split(':'); + set.value = validator.escape(String(parts[0])); + set.byUid = validator.escape(String(parts[2] || '')); + delete set.score; + }); + const uids = _.uniq(data.map(d => d && d.byUid).filter(Boolean)); + const usersData = await User.getUsersFields(uids, ['uid', 'username', 'userslug', 'picture']); + const uidToUser = _.zipObject(uids, usersData); + data.forEach(d => { + if (d.byUid) { + d.byUser = uidToUser[d.byUid]; + } + }); + return data; + }; + async function getFlagMetadata(flags) { + const postFlags = flags.filter(flag => flag && flag.type === 'post'); + const reports = await Promise.all(flags.map(flag => Flags.getReports(flag.flagId))); + flags.forEach((flag, idx) => { + if (flag) { + flag.timestamp = parseInt(flag.datetime, 10); + flag.timestampISO = utils.toISOString(flag.datetime); + flag.reports = reports[idx]; + } + }); + const pids = postFlags.map(flagObj => parseInt(flagObj.targetId, 10)); + const postData = await posts.getPostsFields(pids, ['tid']); + const tids = postData.map(post => post.tid); + const topicData = await topics.getTopicsFields(tids, ['title']); + postFlags.forEach((flagObj, idx) => { + flagObj.pid = flagObj.targetId; + if (!tids[idx]) { + flagObj.targetPurged = true; + } + return _.extend(flagObj, topicData[idx]); + }); + return flags; + } + async function formatBanMuteData(keys, noReasonLangKey) { + const data = await db.getObjects(keys); + const uids = data.map(d => d.fromUid); + const usersData = await User.getUsersFields(uids, ['uid', 'username', 'userslug', 'picture']); + return data.map((banObj, index) => { + banObj.user = usersData[index]; + banObj.until = parseInt(banObj.expire, 10); + banObj.untilISO = utils.toISOString(banObj.until); + banObj.timestampISO = utils.toISOString(banObj.timestamp); + banObj.reason = validator.escape(String(banObj.reason || '')) || noReasonLangKey; + return banObj; + }); + } + User.getModerationNotes = async function (uid, start, stop) { + const noteIds = await db.getSortedSetRevRange(`uid:${uid}:moderation:notes`, start, stop); + return await User.getModerationNotesByIds(uid, noteIds); + }; + User.getModerationNotesByIds = async (uid, noteIds) => { + const keys = noteIds.map(id => `uid:${uid}:moderation:note:${id}`); + const notes = await db.getObjects(keys); + const uids = []; + notes.forEach((note, idx) => { + if (note) { + note.id = noteIds[idx]; + uids.push(note.uid); + note.timestampISO = utils.toISOString(note.timestamp); + } + }); + const userData = await User.getUsersFields(uids, ['uid', 'username', 'userslug', 'picture']); + await Promise.all(notes.map(async (note, index) => { + if (note) { + note.rawNote = validator.escape(String(note.note)); + note.note = await plugins.hooks.fire('filter:parse.raw', String(note.note)); + note.user = userData[index]; + } + })); + return notes; + }; + User.appendModerationNote = async ({ + uid, + noteData + }) => { + await db.sortedSetAdd(`uid:${uid}:moderation:notes`, noteData.timestamp, noteData.timestamp); + await db.setObject(`uid:${uid}:moderation:note:${noteData.timestamp}`, noteData); + }; + User.setModerationNote = async ({ + uid, + noteData + }) => { + await db.setObject(`uid:${uid}:moderation:note:${noteData.timestamp}`, noteData); + }; +}; \ No newline at end of file diff --git a/lib/user/interstitials.js b/lib/user/interstitials.js new file mode 100644 index 0000000000..2a74553af5 --- /dev/null +++ b/lib/user/interstitials.js @@ -0,0 +1,175 @@ +'use strict'; + +const winston = require('winston'); +const util = require('util'); +const user = require('.'); +const db = require('../database'); +const meta = require('../meta'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const utils = require('../utils'); +const sleep = util.promisify(setTimeout); +const Interstitials = module.exports; +Interstitials.get = async (req, userData) => plugins.hooks.fire('filter:register.interstitial', { + req, + userData, + interstitials: [] +}); +Interstitials.email = async data => { + if (!data.userData) { + throw new Error('[[error:invalid-data]]'); + } + if (!data.userData.updateEmail) { + return data; + } + const [hasPassword, hasPending] = await Promise.all([user.hasPassword(data.userData.uid), user.email.isValidationPending(data.userData.uid)]); + let email; + if (data.userData.uid) { + email = await user.getUserField(data.userData.uid, 'email'); + } + data.interstitials.push({ + template: 'partials/email_update', + data: { + email, + requireEmailAddress: meta.config.requireEmailAddress, + issuePasswordChallenge: hasPassword, + hasPending + }, + callback: async (userData, formData) => { + if (formData.email) { + formData.email = String(formData.email).trim(); + } + if (userData.uid) { + const isSelf = parseInt(userData.uid, 10) === parseInt(data.req.uid, 10); + const [isPasswordCorrect, canEdit, { + email: current, + 'email:confirmed': confirmed + }, { + allowed, + error + }] = await Promise.all([user.isPasswordCorrect(userData.uid, formData.password, data.req.ip), privileges.users.canEdit(data.req.uid, userData.uid), user.getUserFields(userData.uid, ['email', 'email:confirmed']), plugins.hooks.fire('filter:user.saveEmail', { + uid: userData.uid, + email: formData.email, + registration: false, + allowed: true, + error: '[[error:invalid-email]]' + })]); + if (!isPasswordCorrect) { + await sleep(2000); + } + if (formData.email && formData.email.length) { + if (!allowed || !utils.isEmailValid(formData.email)) { + throw new Error(error); + } + if (formData.email === current) { + if (confirmed) { + throw new Error('[[error:email-nochange]]'); + } else if (!(await user.email.canSendValidation(userData.uid, current))) { + throw new Error(`[[error:confirm-email-already-sent, ${meta.config.emailConfirmInterval}]]`); + } + } + if (canEdit) { + if (hasPassword && !isPasswordCorrect) { + throw new Error('[[error:invalid-password]]'); + } + await user.email.sendValidationEmail(userData.uid, { + email: formData.email, + force: true + }).catch(err => { + winston.error(`[user.interstitials.email] Validation email failed to send\n[emailer.send] ${err.stack}`); + }); + if (isSelf) { + data.req.session.emailChanged = 1; + } + } else { + throw new Error('[[error:no-privileges]]'); + } + } else { + if (meta.config.requireEmailAddress) { + throw new Error('[[error:invalid-email]]'); + } + if (current.length && (!hasPassword || hasPassword && isPasswordCorrect)) { + await user.email.remove(userData.uid, isSelf ? data.req.session.id : null); + } + } + } else { + const { + allowed, + error + } = await plugins.hooks.fire('filter:user.saveEmail', { + uid: null, + email: formData.email, + registration: true, + allowed: true, + error: '[[error:invalid-email]]' + }); + if (!allowed || meta.config.requireEmailAddress && !(formData.email && formData.email.length)) { + throw new Error(error); + } + userData.email = formData.email; + } + delete userData.updateEmail; + } + }); + return data; +}; +Interstitials.gdpr = async function (data) { + if (!meta.config.gdpr_enabled || data.userData && data.userData.gdpr_consent) { + return data; + } + if (!data.userData) { + throw new Error('[[error:invalid-data]]'); + } + if (data.userData.uid) { + const consented = await db.getObjectField(`user:${data.userData.uid}`, 'gdpr_consent'); + if (parseInt(consented, 10)) { + return data; + } + } + data.interstitials.push({ + template: 'partials/gdpr_consent', + data: { + digestFrequency: meta.config.dailyDigestFreq, + digestEnabled: meta.config.dailyDigestFreq !== 'off' + }, + callback: function (userData, formData, next) { + if (formData.gdpr_agree_data === 'on' && formData.gdpr_agree_email === 'on') { + userData.gdpr_consent = true; + } + next(userData.gdpr_consent ? null : new Error('[[register:gdpr-consent-denied]]')); + } + }); + return data; +}; +Interstitials.tou = async function (data) { + if (!data.userData) { + throw new Error('[[error:invalid-data]]'); + } + if (!meta.config.termsOfUse || data.userData.acceptTos) { + return data; + } + if (data.userData.uid) { + const accepted = await db.getObjectField(`user:${data.userData.uid}`, 'acceptTos'); + if (parseInt(accepted, 10)) { + return data; + } + } + const termsOfUse = await plugins.hooks.fire('filter:parse.post', { + postData: { + content: meta.config.termsOfUse || '' + } + }); + data.interstitials.push({ + template: 'partials/acceptTos', + data: { + termsOfUse: termsOfUse.postData.content + }, + callback: function (userData, formData, next) { + if (formData['agree-terms'] === 'on') { + userData.acceptTos = true; + } + next(userData.acceptTos ? null : new Error('[[register:terms-of-use-error]]')); + } + }); + return data; +}; \ No newline at end of file diff --git a/lib/user/invite.js b/lib/user/invite.js new file mode 100644 index 0000000000..a32dd03f2b --- /dev/null +++ b/lib/user/invite.js @@ -0,0 +1,152 @@ +'use strict'; + +const async = require('async'); +const nconf = require('nconf'); +const validator = require('validator'); +const db = require('../database'); +const meta = require('../meta'); +const emailer = require('../emailer'); +const groups = require('../groups'); +const translator = require('../translator'); +const utils = require('../utils'); +const plugins = require('../plugins'); +module.exports = function (User) { + User.getInvites = async function (uid) { + const emails = await db.getSetMembers(`invitation:uid:${uid}`); + return emails.map(email => validator.escape(String(email))); + }; + User.getInvitesNumber = async function (uid) { + return await db.setCount(`invitation:uid:${uid}`); + }; + User.getInvitingUsers = async function () { + return await db.getSetMembers('invitation:uids'); + }; + User.getAllInvites = async function () { + const uids = await User.getInvitingUsers(); + const invitations = await async.map(uids, User.getInvites); + return invitations.map((invites, index) => ({ + uid: uids[index], + invitations: invites + })); + }; + User.sendInvitationEmail = async function (uid, email, groupsToJoin) { + if (!uid) { + throw new Error('[[error:invalid-uid]]'); + } + const email_exists = await User.getUidByEmail(email); + if (email_exists) { + return true; + } + const invitation_exists = await db.exists(`invitation:uid:${uid}:invited:${email}`); + if (invitation_exists) { + throw new Error('[[error:email-invited]]'); + } + const data = await prepareInvitation(uid, email, groupsToJoin); + await emailer.sendToEmail('invitation', email, meta.config.defaultLang, data); + plugins.hooks.fire('action:user.invite', { + uid, + email, + groupsToJoin + }); + }; + User.verifyInvitation = async function (query) { + if (!query.token) { + if (meta.config.registrationType.startsWith('admin-')) { + throw new Error('[[register:invite.error-admin-only]]'); + } else { + throw new Error('[[register:invite.error-invite-only]]'); + } + } + const token = await db.getObjectField(`invitation:token:${query.token}`, 'token'); + if (!token || token !== query.token) { + throw new Error('[[register:invite.error-invalid-data]]'); + } + }; + User.confirmIfInviteEmailIsUsed = async function (token, enteredEmail, uid) { + if (!enteredEmail) { + return; + } + const email = await db.getObjectField(`invitation:token:${token}`, 'email'); + if (email && email === enteredEmail) { + await User.setUserField(uid, 'email', email); + await User.email.confirmByUid(uid); + } + }; + User.joinGroupsFromInvitation = async function (uid, token) { + let groupsToJoin = await db.getObjectField(`invitation:token:${token}`, 'groupsToJoin'); + try { + groupsToJoin = JSON.parse(groupsToJoin); + } catch (e) { + return; + } + if (!groupsToJoin || groupsToJoin.length < 1) { + return; + } + await groups.join(groupsToJoin, uid); + }; + User.deleteInvitation = async function (invitedBy, email) { + const invitedByUid = await User.getUidByUsername(invitedBy); + if (!invitedByUid) { + throw new Error('[[error:invalid-username]]'); + } + const token = await db.get(`invitation:uid:${invitedByUid}:invited:${email}`); + await Promise.all([deleteFromReferenceList(invitedByUid, email), db.setRemove(`invitation:invited:${email}`, token), db.delete(`invitation:token:${token}`)]); + }; + User.deleteInvitationKey = async function (registrationEmail, token) { + if (registrationEmail) { + const uids = await User.getInvitingUsers(); + await Promise.all(uids.map(uid => deleteFromReferenceList(uid, registrationEmail))); + const tokens = await db.getSetMembers(`invitation:invited:${registrationEmail}`); + const keysToDelete = [`invitation:invited:${registrationEmail}`].concat(tokens.map(token => `invitation:token:${token}`)); + await db.deleteAll(keysToDelete); + } + if (token) { + const invite = await db.getObject(`invitation:token:${token}`); + if (!invite) { + return; + } + await deleteFromReferenceList(invite.inviter, invite.email); + await db.deleteAll([`invitation:invited:${invite.email}`, `invitation:token:${token}`]); + } + }; + async function deleteFromReferenceList(uid, email) { + await Promise.all([db.setRemove(`invitation:uid:${uid}`, email), db.delete(`invitation:uid:${uid}:invited:${email}`)]); + const count = await db.setCount(`invitation:uid:${uid}`); + if (count === 0) { + await db.setRemove('invitation:uids', uid); + } + } + async function prepareInvitation(uid, email, groupsToJoin) { + const inviterExists = await User.exists(uid); + if (!inviterExists) { + throw new Error('[[error:invalid-uid]]'); + } + const token = utils.generateUUID(); + const registerLink = `${nconf.get('url')}/register?token=${token}`; + const expireDays = meta.config.inviteExpiration; + const expireIn = expireDays * 86400000; + await db.setAdd(`invitation:uid:${uid}`, email); + await db.setAdd('invitation:uids', uid); + await db.set(`invitation:uid:${uid}:invited:${email}`, token); + await db.setAdd(`invitation:invited:${email}`, token); + await db.setObject(`invitation:token:${token}`, { + email, + token, + groupsToJoin: JSON.stringify(groupsToJoin), + inviter: uid + }); + await db.pexpireAt(`invitation:token:${token}`, Date.now() + expireIn); + const username = await User.getUserField(uid, 'username'); + const title = meta.config.title || meta.config.browserTitle || 'NodeBB'; + const subject = await translator.translate(`[[email:invite, ${title}]]`, meta.config.defaultLang); + return { + ...emailer._defaultPayload, + site_title: title, + registerLink: registerLink, + subject: subject, + username: username, + template: 'invitation', + expireDays: expireDays + }; + } +}; \ No newline at end of file diff --git a/lib/user/jobs.js b/lib/user/jobs.js new file mode 100644 index 0000000000..f52f912716 --- /dev/null +++ b/lib/user/jobs.js @@ -0,0 +1,60 @@ +'use strict'; + +const winston = require('winston'); +const cronJob = require('cron').CronJob; +const db = require('../database'); +const meta = require('../meta'); +const jobs = {}; +module.exports = function (User) { + User.startJobs = function () { + winston.verbose('[user/jobs] (Re-)starting jobs...'); + let { + digestHour + } = meta.config; + if (isNaN(digestHour)) { + digestHour = 17; + } else if (digestHour > 23 || digestHour < 0) { + digestHour = 0; + } + User.stopJobs(); + startDigestJob('digest.daily', `0 ${digestHour} * * *`, 'day'); + startDigestJob('digest.weekly', `0 ${digestHour} * * 0`, 'week'); + startDigestJob('digest.monthly', `0 ${digestHour} 1 * *`, 'month'); + jobs['reset.clean'] = new cronJob('0 0 * * *', User.reset.clean, null, true); + winston.verbose('[user/jobs] Starting job (reset.clean)'); + winston.verbose(`[user/jobs] jobs started`); + }; + function startDigestJob(name, cronString, term) { + jobs[name] = new cronJob(cronString, async () => { + winston.verbose(`[user/jobs] Digest job (${name}) started.`); + try { + if (name === 'digest.weekly') { + const counter = await db.increment('biweeklydigestcounter'); + if (counter % 2) { + await User.digest.execute({ + interval: 'biweek' + }); + } + } + await User.digest.execute({ + interval: term + }); + } catch (err) { + winston.error(err.stack); + } + }, null, true); + winston.verbose(`[user/jobs] Starting job (${name})`); + } + User.stopJobs = function () { + let terminated = 0; + for (const jobId of Object.keys(jobs)) { + winston.verbose(`[user/jobs] Terminating job (${jobId})`); + jobs[jobId].stop(); + delete jobs[jobId]; + terminated += 1; + } + if (terminated > 0) { + winston.verbose(`[user/jobs] ${terminated} jobs terminated`); + } + }; +}; \ No newline at end of file diff --git a/lib/user/jobs/export-posts.js b/lib/user/jobs/export-posts.js new file mode 100644 index 0000000000..7bf85eed3e --- /dev/null +++ b/lib/user/jobs/export-posts.js @@ -0,0 +1,44 @@ +'use strict'; + +const nconf = require('nconf'); +nconf.argv().env({ + separator: '__' +}); +const fs = require('fs'); +const path = require('path'); +const json2csvAsync = require('json2csv').parseAsync; +process.env.NODE_ENV = process.env.NODE_ENV || 'production'; +const configFile = path.resolve(__dirname, '../../../', nconf.any(['config', 'CONFIG']) || 'config.json'); +const prestart = require('../../prestart'); +prestart.loadConfig(configFile); +prestart.setupWinston(); +const db = require('../../database'); +const batch = require('../../batch'); +process.on('message', async msg => { + if (msg && msg.uid) { + await db.init(); + const targetUid = msg.uid; + const filePath = path.join(__dirname, '../../../build/export', `${targetUid}_posts.csv`); + const posts = require('../../posts'); + let payload = []; + await batch.processSortedSet(`uid:${targetUid}:posts`, async pids => { + let postData = await posts.getPostsData(pids); + postData = postData.filter(Boolean).map(post => { + post.content = `"${String(post.content || '').replace(/\n/g, '\\n').replace(/"/g, '\\"')}"`; + return post; + }); + payload = payload.concat(postData); + }, { + batch: 500, + interval: 1000 + }); + const fields = payload.length ? Object.keys(payload[0]) : []; + const opts = { + fields + }; + const csv = await json2csvAsync(payload, opts); + await fs.promises.writeFile(filePath, csv); + await db.close(); + process.exit(0); + } +}); \ No newline at end of file diff --git a/lib/user/jobs/export-profile.js b/lib/user/jobs/export-profile.js new file mode 100644 index 0000000000..ab1b1f697d --- /dev/null +++ b/lib/user/jobs/export-profile.js @@ -0,0 +1,92 @@ +'use strict'; + +const nconf = require('nconf'); +nconf.argv().env({ + separator: '__' +}); +const fs = require('fs'); +const path = require('path'); +const _ = require('lodash'); +process.env.NODE_ENV = process.env.NODE_ENV || 'production'; +const configFile = path.resolve(__dirname, '../../../', nconf.any(['config', 'CONFIG']) || 'config.json'); +const prestart = require('../../prestart'); +prestart.loadConfig(configFile); +prestart.setupWinston(); +const db = require('../../database'); +const batch = require('../../batch'); +process.on('message', async msg => { + if (msg && msg.uid) { + await db.init(); + await db.initSessionStore(); + const targetUid = msg.uid; + const profileFile = `${targetUid}_profile.json`; + const profilePath = path.join(__dirname, '../../../build/export', profileFile); + const user = require('../index'); + const [userData, userSettings, ips, sessions, usernames, emails, bookmarks, watchedTopics, upvoted, downvoted, following] = await Promise.all([db.getObject(`user:${targetUid}`), db.getObject(`user:${targetUid}:settings`), user.getIPs(targetUid, 9), user.auth.getSessions(targetUid), user.getHistory(`user:${targetUid}:usernames`), user.getHistory(`user:${targetUid}:emails`), getSetData(`uid:${targetUid}:bookmarks`, 'post:', targetUid), getSetData(`uid:${targetUid}:followed_tids`, 'topic:', targetUid), getSetData(`uid:${targetUid}:upvote`, 'post:', targetUid), getSetData(`uid:${targetUid}:downvote`, 'post:', targetUid), getSetData(`following:${targetUid}`, 'user:', targetUid)]); + delete userData.password; + let chatData = []; + await batch.processSortedSet(`uid:${targetUid}:chat:rooms`, async roomIds => { + const result = await Promise.all(roomIds.map(roomId => getRoomMessages(targetUid, roomId))); + chatData = chatData.concat(_.flatten(result)); + }, { + batch: 100, + interval: 1000 + }); + await fs.promises.writeFile(profilePath, JSON.stringify({ + user: userData, + settings: userSettings, + ips: ips, + sessions: sessions, + usernames: usernames, + emails: emails, + messages: chatData, + bookmarks: bookmarks, + watchedTopics: watchedTopics, + upvoted: upvoted, + downvoted: downvoted, + following: following + }, null, 4)); + await db.close(); + process.exit(0); + } +}); +async function getRoomMessages(uid, roomId) { + const batch = require('../../batch'); + let data = []; + await batch.processSortedSet(`chat:room:${roomId}:mids`, async mids => { + const messageData = await db.getObjects(mids.map(mid => `message:${mid}`)); + data = data.concat(messageData.filter(m => m && m.fromuid === uid && !m.system).map(m => ({ + content: m.content, + timestamp: m.timestamp + }))); + }, { + batch: 500, + interval: 1000 + }); + return data; +} +async function getSetData(set, keyPrefix, uid) { + const privileges = require('../../privileges'); + const batch = require('../../batch'); + let data = []; + await batch.processSortedSet(set, async ids => { + if (keyPrefix === 'post:') { + ids = await privileges.posts.filter('topics:read', ids, uid); + } else if (keyPrefix === 'topic:') { + ids = await privileges.topics.filterTids('topics:read', ids, uid); + } + let objData = await db.getObjects(ids.map(id => keyPrefix + id)); + if (keyPrefix === 'post:') { + objData = objData.map(o => _.pick(o, ['pid', 'content', 'timestamp'])); + } else if (keyPrefix === 'topic:') { + objData = objData.map(o => _.pick(o, ['tid', 'title', 'timestamp'])); + } else if (keyPrefix === 'user:') { + objData = objData.map(o => _.pick(o, ['uid', 'username'])); + } + data = data.concat(objData); + }, { + batch: 500, + interval: 1000 + }); + return data; +} \ No newline at end of file diff --git a/lib/user/jobs/export-uploads.js b/lib/user/jobs/export-uploads.js new file mode 100644 index 0000000000..e1b21b0693 --- /dev/null +++ b/lib/user/jobs/export-uploads.js @@ -0,0 +1,64 @@ +'use strict'; + +const nconf = require('nconf'); +nconf.argv().env({ + separator: '__' +}); +const fs = require('fs'); +const path = require('path'); +const archiver = require('archiver'); +const winston = require('winston'); +process.env.NODE_ENV = process.env.NODE_ENV || 'production'; +const configFile = path.resolve(__dirname, '../../../', nconf.any(['config', 'CONFIG']) || 'config.json'); +const prestart = require('../../prestart'); +prestart.loadConfig(configFile); +prestart.setupWinston(); +const db = require('../../database'); +process.on('message', async msg => { + if (msg && msg.uid) { + await db.init(); + const targetUid = msg.uid; + const archivePath = path.join(__dirname, '../../../build/export', `${targetUid}_uploads.zip`); + const rootDirectory = path.join(__dirname, '../../../public/uploads/'); + const user = require('../index'); + const archive = archiver('zip', { + zlib: { + level: 9 + } + }); + archive.on('warning', err => { + switch (err.code) { + case 'ENOENT': + winston.warn(`[user/export/uploads] File not found: ${err.path}`); + break; + default: + winston.warn(`[user/export/uploads] Unexpected warning: ${err.message}`); + break; + } + }); + archive.on('error', err => { + const trimPath = function (path) { + return path.replace(rootDirectory, ''); + }; + switch (err.code) { + case 'EACCES': + winston.error(`[user/export/uploads] File inaccessible: ${trimPath(err.path)}`); + break; + default: + winston.error(`[user/export/uploads] Unable to construct archive: ${err.message}`); + break; + } + }); + const output = fs.createWriteStream(archivePath); + output.on('close', async () => { + await db.close(); + process.exit(0); + }); + archive.pipe(output); + winston.verbose(`[user/export/uploads] Collating uploads for uid ${targetUid}`); + await user.collateUploads(targetUid, archive); + const profileUploadPath = path.join(nconf.get('upload_path'), `profile/uid-${targetUid}`); + archive.directory(profileUploadPath, 'profile'); + archive.finalize(); + } +}); \ No newline at end of file diff --git a/lib/user/notifications.js b/lib/user/notifications.js new file mode 100644 index 0000000000..880365557e --- /dev/null +++ b/lib/user/notifications.js @@ -0,0 +1,225 @@ +'use strict'; + +const winston = require('winston'); +const _ = require('lodash'); +const db = require('../database'); +const meta = require('../meta'); +const notifications = require('../notifications'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const translator = require('../translator'); +const user = require('./index'); +const utils = require('../utils'); +const UserNotifications = module.exports; +UserNotifications.get = async function (uid) { + if (parseInt(uid, 10) <= 0) { + return { + read: [], + unread: [] + }; + } + let unread = await getNotificationsFromSet(`uid:${uid}:notifications:unread`, uid, 0, 49); + unread = unread.filter(Boolean); + let read = []; + if (unread.length < 50) { + read = await getNotificationsFromSet(`uid:${uid}:notifications:read`, uid, 0, 49 - unread.length); + } + return await plugins.hooks.fire('filter:user.notifications.get', { + uid, + read: read.filter(Boolean), + unread: unread + }); +}; +async function filterNotifications(nids, filter) { + if (!filter) { + return nids; + } + const keys = nids.map(nid => `notifications:${nid}`); + const notifications = await db.getObjectsFields(keys, ['nid', 'type']); + return notifications.filter(n => n && n.nid && n.type === filter).map(n => n.nid); +} +UserNotifications.getAll = async function (uid, filter) { + const nids = await getAllNids(uid); + return await filterNotifications(nids, filter); +}; +UserNotifications.getAllWithCounts = async function (uid, filter) { + const nids = await getAllNids(uid); + const keys = nids.map(nid => `notifications:${nid}`); + let notifications = await db.getObjectsFields(keys, ['nid', 'type']); + const counts = {}; + notifications.forEach(n => { + if (n && n.type) { + counts[n.type] = counts[n.type] || 0; + counts[n.type] += 1; + } + }); + if (filter) { + notifications = notifications.filter(n => n && n.nid && n.type === filter); + } + return { + counts, + nids: notifications.map(n => n.nid) + }; +}; +async function getAllNids(uid) { + let nids = await db.getSortedSetRevRange([`uid:${uid}:notifications:unread`, `uid:${uid}:notifications:read`], 0, -1); + nids = _.uniq(nids); + const exists = await db.isSortedSetMembers('notifications', nids); + const deleteNids = []; + nids = nids.filter((nid, index) => { + if (!nid || !exists[index]) { + deleteNids.push(nid); + } + return nid && exists[index]; + }); + await deleteUserNids(deleteNids, uid); + return nids; +} +async function deleteUserNids(nids, uid) { + await db.sortedSetRemove([`uid:${uid}:notifications:read`, `uid:${uid}:notifications:unread`], nids); +} +async function getNotificationsFromSet(set, uid, start, stop) { + const nids = await db.getSortedSetRevRange(set, start, stop); + return await UserNotifications.getNotifications(nids, uid); +} +UserNotifications.getNotifications = async function (nids, uid) { + if (!Array.isArray(nids) || !nids.length) { + return []; + } + const [notifObjs, hasRead, userSettings] = await Promise.all([notifications.getMultiple(nids), db.isSortedSetMembers(`uid:${uid}:notifications:read`, nids), user.getSettings(uid)]); + const deletedNids = []; + let notificationData = notifObjs.filter((notification, index) => { + if (!notification || !notification.nid) { + deletedNids.push(nids[index]); + } + if (notification) { + notification.read = hasRead[index]; + notification.readClass = !notification.read ? 'unread' : ''; + } + return notification; + }); + await deleteUserNids(deletedNids, uid); + notificationData = await notifications.merge(notificationData); + await Promise.all(notificationData.map(async n => { + if (n && n.bodyShort) { + n.bodyShort = await translator.translate(n.bodyShort, userSettings.userLang); + } + })); + const result = await plugins.hooks.fire('filter:user.notifications.getNotifications', { + uid: uid, + notifications: notificationData + }); + return result && result.notifications; +}; +UserNotifications.getUnreadInterval = async function (uid, interval) { + const dayInMs = 1000 * 60 * 60 * 24; + const times = { + day: dayInMs, + week: 7 * dayInMs, + month: 30 * dayInMs + }; + if (!times[interval]) { + return []; + } + const min = Date.now() - times[interval]; + const nids = await db.getSortedSetRevRangeByScore(`uid:${uid}:notifications:unread`, 0, 20, '+inf', min); + return await UserNotifications.getNotifications(nids, uid); +}; +UserNotifications.getDailyUnread = async function (uid) { + return await UserNotifications.getUnreadInterval(uid, 'day'); +}; +UserNotifications.getUnreadCount = async function (uid) { + if (parseInt(uid, 10) <= 0) { + return 0; + } + let nids = await db.getSortedSetRevRange(`uid:${uid}:notifications:unread`, 0, 99); + nids = await notifications.filterExists(nids); + const keys = nids.map(nid => `notifications:${nid}`); + const notifData = await db.getObjectsFields(keys, ['mergeId']); + const mergeIds = notifData.map(n => n.mergeId); + let count = mergeIds.reduce((count, mergeId, idx, arr) => { + if (mergeId === null || idx === arr.indexOf(mergeId)) { + count += 1; + } + return count; + }, 0); + ({ + count + } = await plugins.hooks.fire('filter:user.notifications.getCount', { + uid, + count + })); + return count; +}; +UserNotifications.getUnreadByField = async function (uid, field, values) { + const nids = await db.getSortedSetRevRange(`uid:${uid}:notifications:unread`, 0, 99); + if (!nids.length) { + return []; + } + const keys = nids.map(nid => `notifications:${nid}`); + const notifData = await db.getObjectsFields(keys, ['nid', field]); + const valuesSet = new Set(values.map(value => String(value))); + return notifData.filter(n => n && n[field] && valuesSet.has(String(n[field]))).map(n => n.nid); +}; +UserNotifications.deleteAll = async function (uid) { + if (parseInt(uid, 10) <= 0) { + return; + } + await db.deleteAll([`uid:${uid}:notifications:unread`, `uid:${uid}:notifications:read`]); +}; +UserNotifications.sendTopicNotificationToFollowers = async function (uid, topicData, postData) { + try { + let followers = await db.getSortedSetRange(`followers:${uid}`, 0, -1); + followers = await privileges.categories.filterUids('read', topicData.cid, followers); + if (!followers.length) { + return; + } + let { + title + } = topicData; + if (title) { + title = utils.decodeHTMLEntities(title); + title = title.replace(/,/g, '\\,'); + } + const notifObj = await notifications.create({ + type: 'new-topic', + bodyShort: translator.compile('notifications:user-posted-topic', postData.user.displayname, title), + bodyLong: postData.content, + pid: postData.pid, + path: `/post/${postData.pid}`, + nid: `tid:${postData.tid}:uid:${uid}`, + tid: postData.tid, + from: uid + }); + await notifications.push(notifObj, followers); + } catch (err) { + winston.error(err.stack); + } +}; +UserNotifications.sendWelcomeNotification = async function (uid) { + if (!meta.config.welcomeNotification) { + return; + } + const path = meta.config.welcomeLink ? meta.config.welcomeLink : '#'; + const notifObj = await notifications.create({ + bodyShort: meta.config.welcomeNotification, + path: path, + nid: `welcome_${uid}`, + from: meta.config.welcomeUid ? meta.config.welcomeUid : null + }); + await notifications.push(notifObj, [uid]); +}; +UserNotifications.sendNameChangeNotification = async function (uid, username) { + const notifObj = await notifications.create({ + bodyShort: `[[user:username-taken-workaround, ${username}]]`, + image: 'brand:logo', + nid: `username_taken:${uid}`, + datetime: Date.now() + }); + await notifications.push(notifObj, uid); +}; +UserNotifications.pushCount = async function (uid) { + const websockets = require('../socket.io'); + const count = await UserNotifications.getUnreadCount(uid); + websockets.in(`uid_${uid}`).emit('event:notifications.updateCount', count); +}; \ No newline at end of file diff --git a/lib/user/online.js b/lib/user/online.js new file mode 100644 index 0000000000..cff79115e3 --- /dev/null +++ b/lib/user/online.js @@ -0,0 +1,46 @@ +'use strict'; + +const db = require('../database'); +const topics = require('../topics'); +const plugins = require('../plugins'); +const meta = require('../meta'); +module.exports = function (User) { + User.updateLastOnlineTime = async function (uid) { + if (!(parseInt(uid, 10) > 0)) { + return; + } + const userData = await db.getObjectFields(`user:${uid}`, ['userslug', 'status', 'lastonline']); + const now = Date.now(); + if (!userData.userslug || userData.status === 'offline' || now - parseInt(userData.lastonline, 10) < 300000) { + return; + } + await User.setUserField(uid, 'lastonline', now); + }; + User.updateOnlineUsers = async function (uid) { + if (!(parseInt(uid, 10) > 0)) { + return; + } + const [exists, userOnlineTime] = await Promise.all([User.exists(uid), db.sortedSetScore('users:online', uid)]); + const now = Date.now(); + if (!exists || now - parseInt(userOnlineTime, 10) < 300000) { + return; + } + await User.onUserOnline(uid, now); + topics.pushUnreadCount(uid); + }; + User.onUserOnline = async (uid, timestamp) => { + await db.sortedSetAdd('users:online', timestamp, uid); + plugins.hooks.fire('action:user.online', { + uid, + timestamp + }); + }; + User.isOnline = async function (uid) { + const now = Date.now(); + const isArray = Array.isArray(uid); + uid = isArray ? uid : [uid]; + const lastonline = await db.sortedSetScores('users:online', uid); + const isOnline = uid.map((uid, index) => now - lastonline[index] < meta.config.onlineCutoff * 60000); + return isArray ? isOnline : isOnline[0]; + }; +}; \ No newline at end of file diff --git a/lib/user/password.js b/lib/user/password.js new file mode 100644 index 0000000000..504ef30362 --- /dev/null +++ b/lib/user/password.js @@ -0,0 +1,38 @@ +'use strict'; + +const nconf = require('nconf'); +const db = require('../database'); +const Password = require('../password'); +module.exports = function (User) { + User.hashPassword = async function (password) { + if (!password) { + return password; + } + return await Password.hash(nconf.get('bcrypt_rounds') || 12, password); + }; + User.isPasswordCorrect = async function (uid, password, ip) { + password = password || ''; + let { + password: hashedPassword, + 'password:shaWrapped': shaWrapped + } = await db.getObjectFields(`user:${uid}`, ['password', 'password:shaWrapped']); + if (!hashedPassword) { + hashedPassword = ''; + } + try { + User.isPasswordValid(password, 0); + } catch (e) { + return false; + } + await User.auth.logAttempt(uid, ip); + const ok = await Password.compare(password, hashedPassword, !!parseInt(shaWrapped, 10)); + if (ok) { + await User.auth.clearLoginAttempts(uid); + } + return ok; + }; + User.hasPassword = async function (uid) { + const hashedPassword = await db.getObjectField(`user:${uid}`, 'password'); + return !!hashedPassword; + }; +}; \ No newline at end of file diff --git a/lib/user/picture.js b/lib/user/picture.js new file mode 100644 index 0000000000..40b292a01c --- /dev/null +++ b/lib/user/picture.js @@ -0,0 +1,189 @@ +'use strict'; + +const winston = require('winston'); +const mime = require('mime'); +const path = require('path'); +const nconf = require('nconf'); +const db = require('../database'); +const file = require('../file'); +const image = require('../image'); +const meta = require('../meta'); +module.exports = function (User) { + User.getAllowedProfileImageExtensions = function () { + const exts = User.getAllowedImageTypes().map(type => mime.getExtension(type)); + if (exts.includes('jpeg')) { + exts.push('jpg'); + } + return exts; + }; + User.getAllowedImageTypes = function () { + return ['image/png', 'image/jpeg', 'image/bmp', 'image/gif']; + }; + User.updateCoverPosition = async function (uid, position) { + if (!/^[\d.]+%\s[\d.]+%$/.test(position)) { + winston.warn(`[user/updateCoverPosition] Invalid position received: ${position}`); + throw new Error('[[error:invalid-data]]'); + } + await User.setUserField(uid, 'cover:position', position); + }; + User.updateCoverPicture = async function (data) { + const picture = { + name: 'profileCover', + uid: data.uid + }; + try { + if (!data.imageData && data.position) { + return await User.updateCoverPosition(data.uid, data.position); + } + validateUpload(data, meta.config.maximumCoverImageSize, ['image/png', 'image/jpeg', 'image/bmp']); + picture.path = await image.writeImageDataToTempFile(data.imageData); + const extension = file.typeToExtension(image.mimeFromBase64(data.imageData)); + const filename = `${data.uid}-profilecover-${Date.now()}${extension}`; + const uploadData = await image.uploadImage(filename, `profile/uid-${data.uid}`, picture); + await deleteCurrentPicture(data.uid, 'cover:url'); + await User.setUserField(data.uid, 'cover:url', uploadData.url); + if (data.position) { + await User.updateCoverPosition(data.uid, data.position); + } + return { + url: uploadData.url + }; + } finally { + await file.delete(picture.path); + } + }; + User.uploadCroppedPictureFile = async function (data) { + const userPhoto = data.file; + if (!meta.config.allowProfileImageUploads) { + throw new Error('[[error:profile-image-uploads-disabled]]'); + } + if (userPhoto.size > meta.config.maximumProfileImageSize * 1024) { + throw new Error(`[[error:file-too-big, ${meta.config.maximumProfileImageSize}]]`); + } + if (!userPhoto.type || !User.getAllowedImageTypes().includes(userPhoto.type)) { + throw new Error('[[error:invalid-image]]'); + } + const extension = file.typeToExtension(userPhoto.type); + if (!extension) { + throw new Error('[[error:invalid-image-extension]]'); + } + const newPath = await convertToPNG(userPhoto.path); + await image.resizeImage({ + path: newPath, + width: meta.config.profileImageDimension, + height: meta.config.profileImageDimension + }); + const filename = generateProfileImageFilename(data.uid, extension); + const uploadedImage = await image.uploadImage(filename, `profile/uid-${data.uid}`, { + uid: data.uid, + path: newPath, + name: 'profileAvatar' + }); + await deleteCurrentPicture(data.uid, 'uploadedpicture'); + await User.updateProfile(data.callerUid, { + uid: data.uid, + uploadedpicture: uploadedImage.url, + picture: uploadedImage.url + }, ['uploadedpicture', 'picture']); + return uploadedImage; + }; + User.uploadCroppedPicture = async function (data) { + const picture = { + name: 'profileAvatar', + uid: data.uid + }; + try { + if (!meta.config.allowProfileImageUploads) { + throw new Error('[[error:profile-image-uploads-disabled]]'); + } + validateUpload(data, meta.config.maximumProfileImageSize, User.getAllowedImageTypes()); + const extension = file.typeToExtension(image.mimeFromBase64(data.imageData)); + if (!extension) { + throw new Error('[[error:invalid-image-extension]]'); + } + picture.path = await image.writeImageDataToTempFile(data.imageData); + picture.path = await convertToPNG(picture.path); + await image.resizeImage({ + path: picture.path, + width: meta.config.profileImageDimension, + height: meta.config.profileImageDimension + }); + const filename = generateProfileImageFilename(data.uid, extension); + const uploadedImage = await image.uploadImage(filename, `profile/uid-${data.uid}`, picture); + await deleteCurrentPicture(data.uid, 'uploadedpicture'); + await User.updateProfile(data.callerUid, { + uid: data.uid, + uploadedpicture: uploadedImage.url, + picture: uploadedImage.url + }, ['uploadedpicture', 'picture']); + return uploadedImage; + } finally { + await file.delete(picture.path); + } + }; + async function deleteCurrentPicture(uid, field) { + if (meta.config['profile:keepAllUserImages']) { + return; + } + await deletePicture(uid, field); + } + async function deletePicture(uid, field) { + const uploadPath = await getPicturePath(uid, field); + if (uploadPath) { + await file.delete(uploadPath); + } + } + function validateUpload(data, maxSize, allowedTypes) { + if (!data.imageData) { + throw new Error('[[error:invalid-data]]'); + } + const size = image.sizeFromBase64(data.imageData); + if (size > maxSize * 1024) { + throw new Error(`[[error:file-too-big, ${maxSize}]]`); + } + const type = image.mimeFromBase64(data.imageData); + if (!type || !allowedTypes.includes(type)) { + throw new Error('[[error:invalid-image]]'); + } + } + async function convertToPNG(path) { + const convertToPNG = meta.config['profile:convertProfileImageToPNG'] === 1; + if (!convertToPNG) { + return path; + } + const newPath = await image.normalise(path); + await file.delete(path); + return newPath; + } + function generateProfileImageFilename(uid, extension) { + const convertToPNG = meta.config['profile:convertProfileImageToPNG'] === 1; + return `${uid}-profileavatar-${Date.now()}${convertToPNG ? '.png' : extension}`; + } + User.removeCoverPicture = async function (data) { + await deletePicture(data.uid, 'cover:url'); + await db.deleteObjectFields(`user:${data.uid}`, ['cover:url', 'cover:position']); + }; + User.removeProfileImage = async function (uid) { + const userData = await User.getUserFields(uid, ['uploadedpicture', 'picture']); + await deletePicture(uid, 'uploadedpicture'); + await User.setUserFields(uid, { + uploadedpicture: '', + picture: userData.uploadedpicture === userData.picture ? '' : userData.picture + }); + return userData; + }; + User.getLocalCoverPath = async function (uid) { + return getPicturePath(uid, 'cover:url'); + }; + User.getLocalAvatarPath = async function (uid) { + return getPicturePath(uid, 'uploadedpicture'); + }; + async function getPicturePath(uid, field) { + const value = await User.getUserField(uid, field); + if (!value || !value.startsWith(`${nconf.get('relative_path')}/assets/uploads/profile/uid-${uid}`)) { + return false; + } + const filename = value.split('/').pop(); + return path.join(nconf.get('upload_path'), `profile/uid-${uid}`, filename); + } +}; \ No newline at end of file diff --git a/lib/user/posts.js b/lib/user/posts.js new file mode 100644 index 0000000000..a59777b5ca --- /dev/null +++ b/lib/user/posts.js @@ -0,0 +1,112 @@ +'use strict'; + +const db = require('../database'); +const meta = require('../meta'); +const privileges = require('../privileges'); +const plugins = require('../plugins'); +const groups = require('../groups'); +module.exports = function (User) { + User.isReadyToPost = async function (uid, cid) { + await isReady(uid, cid, 'lastposttime'); + }; + User.isReadyToQueue = async function (uid, cid) { + await isReady(uid, cid, 'lastqueuetime'); + }; + User.checkMuted = async function (uid) { + const now = Date.now(); + const mutedUntil = await User.getUserField(uid, 'mutedUntil'); + if (mutedUntil > now) { + let muteLeft = (mutedUntil - now) / (1000 * 60); + if (muteLeft > 60) { + muteLeft = (muteLeft / 60).toFixed(0); + throw new Error(`[[error:user-muted-for-hours, ${muteLeft}]]`); + } else { + throw new Error(`[[error:user-muted-for-minutes, ${muteLeft.toFixed(0)}]]`); + } + } + }; + async function isReady(uid, cid, field) { + if (parseInt(uid, 10) === 0) { + return; + } + const [userData, isAdminOrMod, isMemberOfExempt] = await Promise.all([User.getUserFields(uid, ['uid', 'mutedUntil', 'joindate', 'email', 'reputation'].concat([field])), privileges.categories.isAdminOrMod(cid, uid), groups.isMemberOfAny(uid, meta.config.groupsExemptFromNewUserRestrictions)]); + if (!userData.uid) { + throw new Error('[[error:no-user]]'); + } + if (isAdminOrMod) { + return; + } + await User.checkMuted(uid); + const { + shouldIgnoreDelays + } = await plugins.hooks.fire('filter:user.posts.isReady', { + shouldIgnoreDelays: false, + user: userData, + cid, + field, + isAdminOrMod, + isMemberOfExempt + }); + if (shouldIgnoreDelays) { + return; + } + const now = Date.now(); + if (now - userData.joindate < meta.config.initialPostDelay * 1000) { + throw new Error(`[[error:user-too-new, ${meta.config.initialPostDelay}]]`); + } + const lasttime = userData[field] || 0; + if (!isMemberOfExempt && meta.config.newbiePostDelay > 0 && meta.config.newbieReputationThreshold > userData.reputation && now - lasttime < meta.config.newbiePostDelay * 1000) { + if (meta.config.newbiewPostDelay % 60 === 0) { + throw new Error(`[[error:too-many-posts-newbie-minutes, ${Math.floor(meta.config.newbiePostDelay / 60)}, ${meta.config.newbieReputationThreshold}]]`); + } else { + throw new Error(`[[error:too-many-posts-newbie, ${meta.config.newbiePostDelay}, ${meta.config.newbieReputationThreshold}]]`); + } + } else if (now - lasttime < meta.config.postDelay * 1000) { + throw new Error(`[[error:too-many-posts, ${meta.config.postDelay}]]`); + } + } + User.onNewPostMade = async function (postData) { + const lastposttime = postData.timestamp > Date.now() ? Date.now() : postData.timestamp; + await Promise.all([User.addPostIdToUser(postData), User.setUserField(postData.uid, 'lastposttime', lastposttime), User.updateLastOnlineTime(postData.uid)]); + }; + User.addPostIdToUser = async function (postData) { + await db.sortedSetsAdd([`uid:${postData.uid}:posts`, `cid:${postData.cid}:uid:${postData.uid}:pids`], postData.timestamp, postData.pid); + await User.updatePostCount(postData.uid); + }; + User.updatePostCount = async uids => { + uids = Array.isArray(uids) ? uids : [uids]; + const exists = await User.exists(uids); + uids = uids.filter((uid, index) => exists[index]); + if (uids.length) { + const counts = await db.sortedSetsCard(uids.map(uid => `uid:${uid}:posts`)); + await Promise.all([db.setObjectBulk(uids.map((uid, index) => [`user:${uid}`, { + postcount: counts[index] + }])), db.sortedSetAdd('users:postcount', counts, uids)]); + } + }; + User.incrementUserPostCountBy = async function (uid, value) { + return await incrementUserFieldAndSetBy(uid, 'postcount', 'users:postcount', value); + }; + User.incrementUserReputationBy = async function (uid, value) { + return await incrementUserFieldAndSetBy(uid, 'reputation', 'users:reputation', value); + }; + User.incrementUserFlagsBy = async function (uid, value) { + return await incrementUserFieldAndSetBy(uid, 'flags', 'users:flags', value); + }; + async function incrementUserFieldAndSetBy(uid, field, set, value) { + value = parseInt(value, 10); + if (!value || !field || !(parseInt(uid, 10) > 0)) { + return; + } + const exists = await User.exists(uid); + if (!exists) { + return; + } + const newValue = await User.incrementUserFieldBy(uid, field, value); + await db.sortedSetAdd(set, newValue, uid); + return newValue; + } + User.getPostIds = async function (uid, start, stop) { + return await db.getSortedSetRevRange(`uid:${uid}:posts`, start, stop); + }; +}; \ No newline at end of file diff --git a/lib/user/profile.js b/lib/user/profile.js new file mode 100644 index 0000000000..a7750e4714 --- /dev/null +++ b/lib/user/profile.js @@ -0,0 +1,283 @@ +'use strict'; + +const _ = require('lodash'); +const validator = require('validator'); +const winston = require('winston'); +const utils = require('../utils'); +const slugify = require('../slugify'); +const meta = require('../meta'); +const db = require('../database'); +const groups = require('../groups'); +const plugins = require('../plugins'); +module.exports = function (User) { + User.updateProfile = async function (uid, data, extraFields) { + let fields = ['username', 'email', 'fullname', 'website', 'location', 'groupTitle', 'birthday', 'signature', 'aboutme']; + if (Array.isArray(extraFields)) { + fields = _.uniq(fields.concat(extraFields)); + } + if (!data.uid) { + throw new Error('[[error:invalid-update-uid]]'); + } + const updateUid = data.uid; + const result = await plugins.hooks.fire('filter:user.updateProfile', { + uid: uid, + data: data, + fields: fields + }); + fields = result.fields; + data = result.data; + await validateData(uid, data); + const oldData = await User.getUserFields(updateUid, fields); + const updateData = {}; + await Promise.all(fields.map(async field => { + if (!(data[field] !== undefined && typeof data[field] === 'string')) { + return; + } + data[field] = data[field].trim(); + if (field === 'email') { + return await updateEmail(updateUid, data.email); + } else if (field === 'username') { + return await updateUsername(updateUid, data.username, uid); + } else if (field === 'fullname') { + return await updateFullname(updateUid, data.fullname); + } + updateData[field] = data[field]; + })); + if (Object.keys(updateData).length) { + await User.setUserFields(updateUid, updateData); + } + plugins.hooks.fire('action:user.updateProfile', { + uid: uid, + data: data, + fields: fields, + oldData: oldData + }); + return await User.getUserFields(updateUid, ['email', 'username', 'userslug', 'picture', 'icon:text', 'icon:bgColor']); + }; + async function validateData(callerUid, data) { + await isEmailValid(data); + await isUsernameAvailable(data, data.uid); + await isWebsiteValid(callerUid, data); + await isAboutMeValid(callerUid, data); + await isSignatureValid(callerUid, data); + isFullnameValid(data); + isLocationValid(data); + isBirthdayValid(data); + isGroupTitleValid(data); + } + async function isEmailValid(data) { + if (!data.email) { + return; + } + data.email = data.email.trim(); + if (!utils.isEmailValid(data.email)) { + throw new Error('[[error:invalid-email]]'); + } + } + async function isUsernameAvailable(data, uid) { + if (!data.username) { + return; + } + data.username = data.username.trim(); + let userData; + if (uid) { + userData = await User.getUserFields(uid, ['username', 'userslug']); + if (userData.username === data.username) { + return; + } + } + if (data.username.length < meta.config.minimumUsernameLength) { + throw new Error('[[error:username-too-short]]'); + } + if (data.username.length > meta.config.maximumUsernameLength) { + throw new Error('[[error:username-too-long]]'); + } + const userslug = slugify(data.username); + if (!utils.isUserNameValid(data.username) || !userslug) { + throw new Error('[[error:invalid-username]]'); + } + if (uid && userslug === userData.userslug) { + return; + } + const exists = await User.existsBySlug(userslug); + if (exists) { + throw new Error('[[error:username-taken]]'); + } + const { + error + } = await plugins.hooks.fire('filter:username.check', { + username: data.username, + error: undefined + }); + if (error) { + throw error; + } + } + User.checkUsername = async username => isUsernameAvailable({ + username + }); + async function isWebsiteValid(callerUid, data) { + if (!data.website) { + return; + } + if (data.website.length > 255) { + throw new Error('[[error:invalid-website]]'); + } + await User.checkMinReputation(callerUid, data.uid, 'min:rep:website'); + } + async function isAboutMeValid(callerUid, data) { + if (!data.aboutme) { + return; + } + if (data.aboutme !== undefined && data.aboutme.length > meta.config.maximumAboutMeLength) { + throw new Error(`[[error:about-me-too-long, ${meta.config.maximumAboutMeLength}]]`); + } + await User.checkMinReputation(callerUid, data.uid, 'min:rep:aboutme'); + } + async function isSignatureValid(callerUid, data) { + if (!data.signature) { + return; + } + const signature = data.signature.replace(/\r\n/g, '\n'); + if (signature.length > meta.config.maximumSignatureLength) { + throw new Error(`[[error:signature-too-long, ${meta.config.maximumSignatureLength}]]`); + } + await User.checkMinReputation(callerUid, data.uid, 'min:rep:signature'); + } + function isFullnameValid(data) { + if (data.fullname && (validator.isURL(data.fullname) || data.fullname.length > 255)) { + throw new Error('[[error:invalid-fullname]]'); + } + } + function isLocationValid(data) { + if (data.location && (validator.isURL(data.location) || data.location.length > 255)) { + throw new Error('[[error:invalid-location]]'); + } + } + function isBirthdayValid(data) { + if (!data.birthday) { + return; + } + const result = new Date(data.birthday); + if (result && result.toString() === 'Invalid Date') { + throw new Error('[[error:invalid-birthday]]'); + } + } + function isGroupTitleValid(data) { + function checkTitle(title) { + if (title === 'registered-users' || groups.isPrivilegeGroup(title)) { + throw new Error('[[error:invalid-group-title]]'); + } + } + if (!data.groupTitle) { + return; + } + let groupTitles = []; + if (validator.isJSON(data.groupTitle)) { + groupTitles = JSON.parse(data.groupTitle); + if (!Array.isArray(groupTitles)) { + throw new Error('[[error:invalid-group-title]]'); + } + groupTitles.forEach(title => checkTitle(title)); + } else { + groupTitles = [data.groupTitle]; + checkTitle(data.groupTitle); + } + if (!meta.config.allowMultipleBadges && groupTitles.length > 1) { + data.groupTitle = JSON.stringify(groupTitles[0]); + } + } + User.checkMinReputation = async function (callerUid, uid, setting) { + const isSelf = parseInt(callerUid, 10) === parseInt(uid, 10); + if (!isSelf || meta.config['reputation:disabled']) { + return; + } + const reputation = await User.getUserField(uid, 'reputation'); + if (reputation < meta.config[setting]) { + throw new Error(`[[error:not-enough-reputation-${setting.replace(/:/g, '-')}, ${meta.config[setting]}]]`); + } + }; + async function updateEmail(uid, newEmail) { + let oldEmail = await db.getObjectField(`user:${uid}`, 'email'); + oldEmail = oldEmail || ''; + if (oldEmail === newEmail) { + return; + } + if (newEmail) { + await User.email.sendValidationEmail(uid, { + email: newEmail, + force: 1 + }).catch(err => winston.error(`[user.create] Validation email failed to send\n[emailer.send] ${err.stack}`)); + } + } + async function updateUsername(uid, newUsername, callerUid) { + if (!newUsername) { + return; + } + const userData = await db.getObjectFields(`user:${uid}`, ['username', 'userslug']); + if (userData.username === newUsername) { + return; + } + const newUserslug = slugify(newUsername); + const now = Date.now(); + await Promise.all([updateUidMapping('username', uid, newUsername, userData.username), updateUidMapping('userslug', uid, newUserslug, userData.userslug), db.sortedSetAdd(`user:${uid}:usernames`, now, `${newUsername}:${now}:${callerUid}`)]); + await db.sortedSetRemove('username:sorted', `${userData.username.toLowerCase()}:${uid}`); + await db.sortedSetAdd('username:sorted', 0, `${newUsername.toLowerCase()}:${uid}`); + } + async function updateUidMapping(field, uid, value, oldValue) { + if (value === oldValue) { + return; + } + await db.sortedSetRemove(`${field}:uid`, oldValue); + await User.setUserField(uid, field, value); + if (value) { + await db.sortedSetAdd(`${field}:uid`, uid, value); + } + } + async function updateFullname(uid, newFullname) { + const fullname = await db.getObjectField(`user:${uid}`, 'fullname'); + await updateUidMapping('fullname', uid, newFullname, fullname); + if (newFullname !== fullname) { + if (fullname) { + await db.sortedSetRemove('fullname:sorted', `${fullname.toLowerCase()}:${uid}`); + } + if (newFullname) { + await db.sortedSetAdd('fullname:sorted', 0, `${newFullname.toLowerCase()}:${uid}`); + } + } + } + User.changePassword = async function (uid, data) { + if (uid <= 0 || !data || !data.uid) { + throw new Error('[[error:invalid-uid]]'); + } + User.isPasswordValid(data.newPassword); + const [isAdmin, hasPassword] = await Promise.all([User.isAdministrator(uid), User.hasPassword(uid)]); + if (meta.config['password:disableEdit'] && !isAdmin) { + throw new Error('[[error:no-privileges]]'); + } + const isSelf = parseInt(uid, 10) === parseInt(data.uid, 10); + if (!isAdmin && !isSelf) { + throw new Error('[[user:change-password-error-privileges]]'); + } + await plugins.hooks.fire('filter:password.check', { + password: data.newPassword, + uid: data.uid + }); + if (isSelf && hasPassword) { + const correct = await User.isPasswordCorrect(data.uid, data.currentPassword, data.ip); + if (!correct) { + throw new Error('[[user:change-password-error-wrong-current]]'); + } + } + const hashedPassword = await User.hashPassword(data.newPassword); + await Promise.all([User.setUserFields(data.uid, { + password: hashedPassword, + 'password:shaWrapped': 1, + rss_token: utils.generateUUID() + }), User.reset.cleanByUid(data.uid), User.reset.updateExpiry(data.uid), User.auth.revokeAllSessions(data.uid), User.email.expireValidation(data.uid)]); + plugins.hooks.fire('action:password.change', { + uid: uid, + targetUid: data.uid + }); + }; +}; \ No newline at end of file diff --git a/lib/user/reset.js b/lib/user/reset.js new file mode 100644 index 0000000000..ff0c2fe9e9 --- /dev/null +++ b/lib/user/reset.js @@ -0,0 +1,138 @@ +'use strict'; + +const nconf = require('nconf'); +const winston = require('winston'); +const user = require('./index'); +const groups = require('../groups'); +const utils = require('../utils'); +const batch = require('../batch'); +const db = require('../database'); +const meta = require('../meta'); +const emailer = require('../emailer'); +const Password = require('../password'); +const plugins = require('../plugins'); +const UserReset = module.exports; +const twoHours = 7200000; +UserReset.minSecondsBetweenEmails = 60; +UserReset.validate = async function (code) { + const uid = await db.getObjectField('reset:uid', code); + if (!uid) { + return false; + } + const issueDate = await db.sortedSetScore('reset:issueDate', code); + return parseInt(issueDate, 10) > Date.now() - twoHours; +}; +UserReset.generate = async function (uid) { + const code = utils.generateUUID(); + await UserReset.cleanByUid(uid); + await Promise.all([db.setObjectField('reset:uid', code, uid), db.sortedSetAdd('reset:issueDate', Date.now(), code)]); + return code; +}; +UserReset.send = async function (email) { + const uid = await user.getUidByEmail(email); + if (!uid) { + throw new Error('[[error:invalid-email]]'); + } + await lockReset(uid, '[[error:reset-rate-limited]]'); + try { + await canGenerate(uid); + await db.sortedSetAdd('reset:issueDate:uid', Date.now(), uid); + const code = await UserReset.generate(uid); + await emailer.send('reset', uid, { + reset_link: `${nconf.get('url')}/reset/${code}`, + subject: '[[email:password-reset-requested]]', + template: 'reset', + uid: uid + }).catch(err => winston.error(`[emailer.send] ${err.stack}`)); + return code; + } finally { + db.deleteObjectField('locks', `reset${uid}`); + } +}; +async function lockReset(uid, error) { + const value = `reset${uid}`; + const count = await db.incrObjectField('locks', value); + if (count > 1) { + throw new Error(error); + } + return value; +} +async function canGenerate(uid) { + const score = await db.sortedSetScore('reset:issueDate:uid', uid); + if (score > Date.now() - UserReset.minSecondsBetweenEmails * 1000) { + throw new Error('[[error:reset-rate-limited]]'); + } +} +UserReset.commit = async function (code, password) { + user.isPasswordValid(password); + const validated = await UserReset.validate(code); + if (!validated) { + throw new Error('[[error:reset-code-not-valid]]'); + } + const uid = await db.getObjectField('reset:uid', code); + if (!uid) { + throw new Error('[[error:reset-code-not-valid]]'); + } + const userData = await db.getObjectFields(`user:${uid}`, ['password', 'passwordExpiry', 'password:shaWrapped', 'username']); + await plugins.hooks.fire('filter:password.check', { + password: password, + uid + }); + const ok = await Password.compare(password, userData.password, !!parseInt(userData['password:shaWrapped'], 10)); + if (ok) { + throw new Error('[[error:reset-same-password]]'); + } + const hash = await user.hashPassword(password); + const data = { + password: hash, + 'password:shaWrapped': 1 + }; + const isPasswordExpired = userData.passwordExpiry && userData.passwordExpiry < Date.now(); + if (!isPasswordExpired) { + data['email:confirmed'] = 1; + await groups.join('verified-users', uid); + await groups.leave('unverified-users', uid); + } + await Promise.all([user.setUserFields(uid, data), db.deleteObjectField('reset:uid', code), db.sortedSetRemoveBulk([['reset:issueDate', code], ['reset:issueDate:uid', uid]]), user.reset.updateExpiry(uid), user.auth.resetLockout(uid), user.auth.revokeAllSessions(uid), user.email.expireValidation(uid)]); +}; +UserReset.updateExpiry = async function (uid) { + const expireDays = meta.config.passwordExpiryDays; + if (expireDays > 0) { + const oneDay = 1000 * 60 * 60 * 24; + const expiry = Date.now() + oneDay * expireDays; + await user.setUserField(uid, 'passwordExpiry', expiry); + } else { + await db.deleteObjectField(`user:${uid}`, 'passwordExpiry'); + } +}; +UserReset.clean = async function () { + const tokens = await db.getSortedSetRangeByScore('reset:issueDate', 0, -1, '-inf', Date.now() - twoHours); + if (!tokens.length) { + return; + } + winston.verbose(`[UserReset.clean] Removing ${tokens.length} reset tokens from database`); + await cleanTokens(tokens); +}; +UserReset.cleanByUid = async function (uid) { + const tokensToClean = []; + uid = parseInt(uid, 10); + await batch.processSortedSet('reset:issueDate', async tokens => { + const results = await db.getObjectFields('reset:uid', tokens); + for (const [code, result] of Object.entries(results)) { + if (parseInt(result, 10) === uid) { + tokensToClean.push(code); + } + } + }, { + batch: 500 + }); + if (!tokensToClean.length) { + winston.verbose(`[UserReset.cleanByUid] No tokens found for uid (${uid}).`); + return; + } + winston.verbose(`[UserReset.cleanByUid] Found ${tokensToClean.length} token(s), removing...`); + await Promise.all([cleanTokens(tokensToClean), db.deleteObjectField('locks', `reset${uid}`)]); +}; +async function cleanTokens(tokens) { + await Promise.all([db.deleteObjectFields('reset:uid', tokens), db.sortedSetRemove('reset:issueDate', tokens)]); +} \ No newline at end of file diff --git a/lib/user/search.js b/lib/user/search.js new file mode 100644 index 0000000000..cdf6e45b73 --- /dev/null +++ b/lib/user/search.js @@ -0,0 +1,143 @@ +'use strict'; + +const _ = require('lodash'); +const meta = require('../meta'); +const plugins = require('../plugins'); +const db = require('../database'); +const groups = require('../groups'); +const utils = require('../utils'); +module.exports = function (User) { + const filterFnMap = { + online: user => user.status !== 'offline' && Date.now() - user.lastonline < 300000, + flagged: user => parseInt(user.flags, 10) > 0, + verified: user => !!user['email:confirmed'], + unverified: user => !user['email:confirmed'] + }; + const filterFieldMap = { + online: ['status', 'lastonline'], + flagged: ['flags'], + verified: ['email:confirmed'], + unverified: ['email:confirmed'] + }; + User.search = async function (data) { + const query = data.query || ''; + const searchBy = data.searchBy || 'username'; + const page = data.page || 1; + const uid = data.uid || 0; + const paginate = data.hasOwnProperty('paginate') ? data.paginate : true; + const startTime = process.hrtime(); + let uids = []; + if (searchBy === 'ip') { + uids = await searchByIP(query); + } else if (searchBy === 'uid') { + uids = [query]; + } else { + const searchMethod = data.findUids || findUids; + uids = await searchMethod(query, searchBy, data.hardCap); + } + uids = await filterAndSortUids(uids, data); + const result = await plugins.hooks.fire('filter:users.search', { + uids: uids, + uid: uid + }); + uids = result.uids; + const searchResult = { + matchCount: uids.length + }; + if (paginate) { + const resultsPerPage = data.resultsPerPage || meta.config.userSearchResultsPerPage; + const start = Math.max(0, page - 1) * resultsPerPage; + const stop = start + resultsPerPage; + searchResult.pageCount = Math.ceil(uids.length / resultsPerPage); + uids = uids.slice(start, stop); + } + const [userData, blocks] = await Promise.all([User.getUsers(uids, uid), User.blocks.list(uid)]); + if (blocks.length) { + userData.forEach(user => { + if (user) { + user.isBlocked = blocks.includes(user.uid); + } + }); + } + searchResult.timing = (process.elapsedTimeSince(startTime) / 1000).toFixed(2); + searchResult.users = userData.filter(user => user && user.uid > 0); + return searchResult; + }; + async function findUids(query, searchBy, hardCap) { + if (!query) { + return []; + } + query = String(query).toLowerCase(); + const min = query; + const max = query.substr(0, query.length - 1) + String.fromCharCode(query.charCodeAt(query.length - 1) + 1); + const resultsPerPage = meta.config.userSearchResultsPerPage; + hardCap = hardCap || resultsPerPage * 10; + const data = await db.getSortedSetRangeByLex(`${searchBy}:sorted`, min, max, 0, hardCap); + const uids = data.map(data => data.split(':').pop()); + return uids; + } + async function filterAndSortUids(uids, data) { + uids = uids.filter(uid => parseInt(uid, 10)); + let filters = data.filters || []; + filters = Array.isArray(filters) ? filters : [data.filters]; + const fields = []; + if (data.sortBy) { + fields.push(data.sortBy); + } + filters.forEach(filter => { + if (filterFieldMap[filter]) { + fields.push(...filterFieldMap[filter]); + } + }); + if (data.groupName) { + const isMembers = await groups.isMembers(uids, data.groupName); + uids = uids.filter((uid, index) => isMembers[index]); + } + if (!fields.length) { + return uids; + } + if (filters.includes('banned') || filters.includes('notbanned')) { + const isMembersOfBanned = await groups.isMembers(uids, groups.BANNED_USERS); + const checkBanned = filters.includes('banned'); + uids = uids.filter((uid, index) => checkBanned ? isMembersOfBanned[index] : !isMembersOfBanned[index]); + } + fields.push('uid'); + let userData = await User.getUsersFields(uids, fields); + filters.forEach(filter => { + if (filterFnMap[filter]) { + userData = userData.filter(filterFnMap[filter]); + } + }); + if (data.sortBy) { + sortUsers(userData, data.sortBy, data.sortDirection); + } + return userData.map(user => user.uid); + } + function sortUsers(userData, sortBy, sortDirection) { + if (!userData || !userData.length) { + return; + } + sortDirection = sortDirection || 'desc'; + const direction = sortDirection === 'desc' ? 1 : -1; + const isNumeric = utils.isNumber(userData[0][sortBy]); + if (isNumeric) { + userData.sort((u1, u2) => direction * (u2[sortBy] - u1[sortBy])); + } else { + userData.sort((u1, u2) => { + if (u1[sortBy] < u2[sortBy]) { + return direction * -1; + } else if (u1[sortBy] > u2[sortBy]) { + return direction * 1; + } + return 0; + }); + } + } + async function searchByIP(ip) { + const ipKeys = await db.scan({ + match: `ip:${ip}*` + }); + const uids = await db.getSortedSetRevRange(ipKeys, 0, -1); + return _.uniq(uids); + } +}; \ No newline at end of file diff --git a/lib/user/settings.js b/lib/user/settings.js new file mode 100644 index 0000000000..dd496f8136 --- /dev/null +++ b/lib/user/settings.js @@ -0,0 +1,153 @@ +'use strict'; + +const validator = require('validator'); +const meta = require('../meta'); +const db = require('../database'); +const plugins = require('../plugins'); +const notifications = require('../notifications'); +const languages = require('../languages'); +module.exports = function (User) { + const spiderDefaultSettings = { + usePagination: 1, + topicPostSort: 'oldest_to_newest', + postsPerPage: 20, + topicsPerPage: 20 + }; + User.getSettings = async function (uid) { + if (parseInt(uid, 10) <= 0) { + const isSpider = parseInt(uid, 10) === -1; + return await onSettingsLoaded(uid, isSpider ? spiderDefaultSettings : {}); + } + let settings = await db.getObject(`user:${uid}:settings`); + settings = settings || {}; + settings.uid = uid; + return await onSettingsLoaded(uid, settings); + }; + User.getMultipleUserSettings = async function (uids) { + if (!Array.isArray(uids) || !uids.length) { + return []; + } + const keys = uids.map(uid => `user:${uid}:settings`); + let settings = await db.getObjects(keys); + settings = settings.map((userSettings, index) => { + userSettings = userSettings || {}; + userSettings.uid = uids[index]; + return userSettings; + }); + return await Promise.all(settings.map(s => onSettingsLoaded(s.uid, s))); + }; + async function onSettingsLoaded(uid, settings) { + const data = await plugins.hooks.fire('filter:user.getSettings', { + uid: uid, + settings: settings + }); + settings = data.settings; + const defaultTopicsPerPage = meta.config.topicsPerPage; + const defaultPostsPerPage = meta.config.postsPerPage; + settings.showemail = parseInt(getSetting(settings, 'showemail', 0), 10) === 1; + settings.showfullname = parseInt(getSetting(settings, 'showfullname', 0), 10) === 1; + settings.openOutgoingLinksInNewTab = parseInt(getSetting(settings, 'openOutgoingLinksInNewTab', 0), 10) === 1; + settings.dailyDigestFreq = getSetting(settings, 'dailyDigestFreq', 'off'); + settings.usePagination = parseInt(getSetting(settings, 'usePagination', 0), 10) === 1; + settings.topicsPerPage = Math.min(meta.config.maxTopicsPerPage, settings.topicsPerPage ? parseInt(settings.topicsPerPage, 10) : defaultTopicsPerPage, defaultTopicsPerPage); + settings.postsPerPage = Math.min(meta.config.maxPostsPerPage, settings.postsPerPage ? parseInt(settings.postsPerPage, 10) : defaultPostsPerPage, defaultPostsPerPage); + settings.userLang = settings.userLang || meta.config.defaultLang || 'en-GB'; + settings.acpLang = settings.acpLang || settings.userLang; + settings.topicPostSort = getSetting(settings, 'topicPostSort', 'oldest_to_newest'); + settings.categoryTopicSort = getSetting(settings, 'categoryTopicSort', 'recently_replied'); + settings.followTopicsOnCreate = parseInt(getSetting(settings, 'followTopicsOnCreate', 1), 10) === 1; + settings.followTopicsOnReply = parseInt(getSetting(settings, 'followTopicsOnReply', 0), 10) === 1; + settings.upvoteNotifFreq = getSetting(settings, 'upvoteNotifFreq', 'all'); + settings.restrictChat = parseInt(getSetting(settings, 'restrictChat', 0), 10) === 1; + settings.topicSearchEnabled = parseInt(getSetting(settings, 'topicSearchEnabled', 0), 10) === 1; + settings.updateUrlWithPostIndex = parseInt(getSetting(settings, 'updateUrlWithPostIndex', 1), 10) === 1; + settings.bootswatchSkin = validator.escape(String(settings.bootswatchSkin || '')); + settings.homePageRoute = validator.escape(String(settings.homePageRoute || '')).replace(///g, '/'); + settings.scrollToMyPost = parseInt(getSetting(settings, 'scrollToMyPost', 1), 10) === 1; + settings.categoryWatchState = getSetting(settings, 'categoryWatchState', 'notwatching'); + const notificationTypes = await notifications.getAllNotificationTypes(); + notificationTypes.forEach(notificationType => { + settings[notificationType] = getSetting(settings, notificationType, 'notification'); + }); + return settings; + } + function getSetting(settings, key, defaultValue) { + if (settings[key] || settings[key] === 0) { + return settings[key]; + } else if (meta.config[key] || meta.config[key] === 0) { + return meta.config[key]; + } + return defaultValue; + } + User.saveSettings = async function (uid, data) { + const maxPostsPerPage = meta.config.maxPostsPerPage || 20; + if (!data.postsPerPage || parseInt(data.postsPerPage, 10) <= 1 || parseInt(data.postsPerPage, 10) > maxPostsPerPage) { + throw new Error(`[[error:invalid-pagination-value, 2, ${maxPostsPerPage}]]`); + } + const maxTopicsPerPage = meta.config.maxTopicsPerPage || 20; + if (!data.topicsPerPage || parseInt(data.topicsPerPage, 10) <= 1 || parseInt(data.topicsPerPage, 10) > maxTopicsPerPage) { + throw new Error(`[[error:invalid-pagination-value, 2, ${maxTopicsPerPage}]]`); + } + const languageCodes = await languages.listCodes(); + if (data.userLang && !languageCodes.includes(data.userLang)) { + throw new Error('[[error:invalid-language]]'); + } + if (data.acpLang && !languageCodes.includes(data.acpLang)) { + throw new Error('[[error:invalid-language]]'); + } + data.userLang = data.userLang || meta.config.defaultLang; + plugins.hooks.fire('action:user.saveSettings', { + uid: uid, + settings: data + }); + const settings = { + showemail: data.showemail, + showfullname: data.showfullname, + openOutgoingLinksInNewTab: data.openOutgoingLinksInNewTab, + dailyDigestFreq: data.dailyDigestFreq || 'off', + usePagination: data.usePagination, + topicsPerPage: Math.min(data.topicsPerPage, parseInt(maxTopicsPerPage, 10) || 20), + postsPerPage: Math.min(data.postsPerPage, parseInt(maxPostsPerPage, 10) || 20), + userLang: data.userLang || meta.config.defaultLang, + acpLang: data.acpLang || meta.config.defaultLang, + followTopicsOnCreate: data.followTopicsOnCreate, + followTopicsOnReply: data.followTopicsOnReply, + restrictChat: data.restrictChat, + topicSearchEnabled: data.topicSearchEnabled, + updateUrlWithPostIndex: data.updateUrlWithPostIndex, + homePageRoute: ((data.homePageRoute === 'custom' ? data.homePageCustom : data.homePageRoute) || '').replace(/^\//, ''), + scrollToMyPost: data.scrollToMyPost, + upvoteNotifFreq: data.upvoteNotifFreq, + bootswatchSkin: data.bootswatchSkin, + categoryWatchState: data.categoryWatchState, + categoryTopicSort: data.categoryTopicSort, + topicPostSort: data.topicPostSort + }; + const notificationTypes = await notifications.getAllNotificationTypes(); + notificationTypes.forEach(notificationType => { + if (data[notificationType]) { + settings[notificationType] = data[notificationType]; + } + }); + const result = await plugins.hooks.fire('filter:user.saveSettings', { + uid: uid, + settings: settings, + data: data + }); + await db.setObject(`user:${uid}:settings`, result.settings); + await User.updateDigestSetting(uid, data.dailyDigestFreq); + return await User.getSettings(uid); + }; + User.updateDigestSetting = async function (uid, dailyDigestFreq) { + await db.sortedSetsRemove(['digest:day:uids', 'digest:week:uids', 'digest:month:uids'], uid); + if (['day', 'week', 'biweek', 'month'].includes(dailyDigestFreq)) { + await db.sortedSetAdd(`digest:${dailyDigestFreq}:uids`, Date.now(), uid); + } + }; + User.setSetting = async function (uid, key, value) { + if (parseInt(uid, 10) <= 0) { + return; + } + await db.setObjectField(`user:${uid}:settings`, key, value); + }; +}; \ No newline at end of file diff --git a/lib/user/topics.js b/lib/user/topics.js new file mode 100644 index 0000000000..c12d9759a9 --- /dev/null +++ b/lib/user/topics.js @@ -0,0 +1,11 @@ +'use strict'; + +const db = require('../database'); +module.exports = function (User) { + User.getIgnoredTids = async function (uid, start, stop) { + return await db.getSortedSetRevRange(`uid:${uid}:ignored_tids`, start, stop); + }; + User.addTopicIdToUser = async function (uid, tid, timestamp) { + await Promise.all([db.sortedSetAdd(`uid:${uid}:topics`, timestamp, tid), User.incrementUserFieldBy(uid, 'topiccount', 1)]); + }; +}; \ No newline at end of file diff --git a/lib/user/uploads.js b/lib/user/uploads.js new file mode 100644 index 0000000000..c30b0335e4 --- /dev/null +++ b/lib/user/uploads.js @@ -0,0 +1,66 @@ +'use strict'; + +const path = require('path'); +const nconf = require('nconf'); +const winston = require('winston'); +const crypto = require('crypto'); +const db = require('../database'); +const posts = require('../posts'); +const file = require('../file'); +const batch = require('../batch'); +const md5 = filename => crypto.createHash('md5').update(filename).digest('hex'); +const _getFullPath = relativePath => path.resolve(nconf.get('upload_path'), relativePath); +const _validatePath = async relativePaths => { + if (typeof relativePaths === 'string') { + relativePaths = [relativePaths]; + } else if (!Array.isArray(relativePaths)) { + throw new Error(`[[error:wrong-parameter-type, relativePaths, ${typeof relativePaths}, array]]`); + } + const fullPaths = relativePaths.map(path => _getFullPath(path)); + const exists = await Promise.all(fullPaths.map(async fullPath => file.exists(fullPath))); + if (!fullPaths.every(fullPath => fullPath.startsWith(nconf.get('upload_path'))) || !exists.every(Boolean)) { + throw new Error('[[error:invalid-path]]'); + } +}; +module.exports = function (User) { + User.associateUpload = async (uid, relativePath) => { + await _validatePath(relativePath); + await Promise.all([db.sortedSetAdd(`uid:${uid}:uploads`, Date.now(), relativePath), db.setObjectField(`upload:${md5(relativePath)}`, 'uid', uid)]); + }; + User.deleteUpload = async function (callerUid, uid, uploadNames) { + if (typeof uploadNames === 'string') { + uploadNames = [uploadNames]; + } else if (!Array.isArray(uploadNames)) { + throw new Error(`[[error:wrong-parameter-type, uploadNames, ${typeof uploadNames}, array]]`); + } + await _validatePath(uploadNames); + const [isUsersUpload, isAdminOrGlobalMod] = await Promise.all([db.isSortedSetMembers(`uid:${callerUid}:uploads`, uploadNames), User.isAdminOrGlobalMod(callerUid)]); + if (!isAdminOrGlobalMod && !isUsersUpload.every(Boolean)) { + throw new Error('[[error:no-privileges]]'); + } + await batch.processArray(uploadNames, async uploadNames => { + const fullPaths = uploadNames.map(path => _getFullPath(path)); + await Promise.all(fullPaths.map(async (fullPath, idx) => { + winston.verbose(`[user/deleteUpload] Deleting ${uploadNames[idx]}`); + await Promise.all([file.delete(fullPath), file.delete(file.appendToFileName(fullPath, '-resized'))]); + await Promise.all([db.sortedSetRemove(`uid:${uid}:uploads`, uploadNames[idx]), db.delete(`upload:${md5(uploadNames[idx])}`)]); + })); + const pids = await db.getSortedSetsMembers(uploadNames.map(relativePath => `upload:${md5(relativePath)}:pids`)); + await Promise.all(pids.map(async (pids, idx) => Promise.all(pids.map(async pid => posts.uploads.dissociate(pid, uploadNames[idx]))))); + }, { + batch: 50 + }); + }; + User.collateUploads = async function (uid, archive) { + await batch.processSortedSet(`uid:${uid}:uploads`, (files, next) => { + files.forEach(file => { + archive.file(_getFullPath(file), { + name: path.basename(file) + }); + }); + setImmediate(next); + }, { + batch: 100 + }); + }; +}; \ No newline at end of file diff --git a/lib/utils.js b/lib/utils.js new file mode 100644 index 0000000000..9a0667afaf --- /dev/null +++ b/lib/utils.js @@ -0,0 +1,63 @@ +'use strict'; + +const crypto = require('crypto'); +const nconf = require('nconf'); +const path = require('node:path'); +process.profile = function (operation, start) { + console.log('%s took %d milliseconds', operation, process.elapsedTimeSince(start)); +}; +process.elapsedTimeSince = function (start) { + const diff = process.hrtime(start); + return diff[0] * 1e3 + diff[1] / 1e6; +}; +const utils = { + ...require('../public/src/utils.common') +}; +utils.getLanguage = function () { + const meta = require('./meta'); + return meta.config && meta.config.defaultLang ? meta.config.defaultLang : 'en-GB'; +}; +utils.generateUUID = function () { + let rnd = crypto.randomBytes(16); + rnd[6] = rnd[6] & 0x0f | 0x40; + rnd[8] = rnd[8] & 0x3f | 0x80; + rnd = rnd.toString('hex').match(/(.{8})(.{4})(.{4})(.{4})(.{12})/); + rnd.shift(); + return rnd.join('-'); +}; +utils.getSass = function () { + try { + const sass = require('sass-embedded'); + return sass; + } catch (_err) { + return require('sass'); + } +}; +utils.getFontawesomePath = function () { + let packageName = '@fortawesome/fontawesome-free'; + if (nconf.get('fontawesome:pro') === true) { + packageName = '@fortawesome/fontawesome-pro'; + } + const pathToMainFile = require.resolve(packageName); + const fontawesomePath = path.dirname(path.dirname(pathToMainFile)); + return fontawesomePath; +}; +utils.getFontawesomeStyles = function () { + let styles = nconf.get('fontawesome:styles') || '*'; + if ([...styles][0] === '*') { + styles = ['solid', 'brands', 'regular']; + if (nconf.get('fontawesome:pro')) { + styles.push('light', 'thin', 'sharp', 'duotone'); + } + } + if (!Array.isArray(styles)) { + styles = [styles]; + } + return styles; +}; +utils.getFontawesomeVersion = function () { + const fontawesomePath = utils.getFontawesomePath(); + const packageJson = require(path.join(fontawesomePath, 'package.json')); + return packageJson.version; +}; +module.exports = utils; \ No newline at end of file diff --git a/lib/webserver.js b/lib/webserver.js new file mode 100644 index 0000000000..13d161d221 --- /dev/null +++ b/lib/webserver.js @@ -0,0 +1,291 @@ +'use strict'; + +const fs = require('fs'); +const util = require('util'); +const path = require('path'); +const nconf = require('nconf'); +const express = require('express'); +const chalk = require('chalk'); +const app = express(); +app.renderAsync = util.promisify((tpl, data, callback) => app.render(tpl, data, callback)); +let server; +const winston = require('winston'); +const flash = require('connect-flash'); +const bodyParser = require('body-parser'); +const cookieParser = require('cookie-parser'); +const session = require('express-session'); +const useragent = require('express-useragent'); +const favicon = require('serve-favicon'); +const detector = require('@nodebb/spider-detector'); +const helmet = require('helmet'); +const Benchpress = require('benchpressjs'); +const db = require('./database'); +const analytics = require('./analytics'); +const file = require('./file'); +const emailer = require('./emailer'); +const meta = require('./meta'); +const logger = require('./logger'); +const plugins = require('./plugins'); +const flags = require('./flags'); +const topicEvents = require('./topics/events'); +const privileges = require('./privileges'); +const routes = require('./routes'); +const auth = require('./routes/authentication'); +const helpers = require('./helpers'); +if (nconf.get('ssl')) { + server = require('https').createServer({ + key: fs.readFileSync(nconf.get('ssl').key), + cert: fs.readFileSync(nconf.get('ssl').cert) + }, app); +} else { + server = require('http').createServer(app); +} +module.exports.server = server; +module.exports.app = app; +server.on('error', err => { + if (err.code === 'EADDRINUSE') { + winston.error(`NodeBB address in use, exiting...\n${err.stack}`); + } else { + winston.error(err.stack); + } + throw err; +}); +const connections = {}; +server.on('connection', conn => { + const key = `${conn.remoteAddress}:${conn.remotePort}`; + connections[key] = conn; + conn.on('close', () => { + delete connections[key]; + }); +}); +exports.destroy = function (callback) { + server.close(callback); + for (const connection of Object.values(connections)) { + connection.destroy(); + } +}; +exports.getConnectionCount = function () { + return Object.keys(connections).length; +}; +exports.listen = async function () { + emailer.registerApp(app); + setupExpressApp(app); + helpers.register(); + logger.init(app); + await initializeNodeBB(); + winston.info('🎉 NodeBB Ready'); + require('./socket.io').server.emit('event:nodebb.ready', {}); + plugins.hooks.fire('action:nodebb.ready'); + await listen(); +}; +async function initializeNodeBB() { + const middleware = require('./middleware'); + await meta.themes.setupPaths(); + await plugins.init(app, middleware); + await plugins.hooks.fire('static:assets.prepare', {}); + await plugins.hooks.fire('static:app.preload', { + app: app, + middleware: middleware + }); + await routes(app, middleware); + await privileges.init(); + await meta.blacklist.load(); + await flags.init(); + await analytics.init(); + await topicEvents.init(); + if (nconf.get('runJobs')) { + await require('./widgets').moveMissingAreasToDrafts(); + } +} +function setupExpressApp(app) { + const middleware = require('./middleware'); + const pingController = require('./controllers/ping'); + const relativePath = nconf.get('relative_path'); + const viewsDir = nconf.get('views_dir'); + app.engine('tpl', (filepath, data, next) => { + filepath = filepath.replace(/\.tpl$/, '.js'); + Benchpress.__express(filepath, data, next); + }); + app.set('view engine', 'tpl'); + app.set('views', viewsDir); + app.set('json spaces', global.env === 'development' ? 4 : 0); + app.use(flash()); + app.enable('view cache'); + if (global.env !== 'development') { + app.enable('cache'); + app.enable('minification'); + } + if (meta.config.useCompression) { + const compression = require('compression'); + app.use(compression()); + } + if (relativePath) { + app.use((req, res, next) => { + if (!req.path.startsWith(relativePath)) { + return require('./controllers/helpers').redirect(res, req.path); + } + next(); + }); + } + app.get(`${relativePath}/ping`, pingController.ping); + app.get(`${relativePath}/sping`, pingController.ping); + setupFavicon(app); + app.use(`${relativePath}/apple-touch-icon`, middleware.routeTouchIcon); + configureBodyParser(app); + app.use(cookieParser(nconf.get('secret'))); + app.use(useragent.express()); + app.use(detector.middleware()); + app.use(session({ + store: db.sessionStore, + secret: nconf.get('secret'), + key: nconf.get('sessionKey'), + cookie: setupCookie(), + resave: nconf.get('sessionResave') || false, + saveUninitialized: nconf.get('sessionSaveUninitialized') || false + })); + setupHelmet(app); + app.use(middleware.addHeaders); + app.use(middleware.processRender); + auth.initialize(app, middleware); + const als = require('./als'); + const apiHelpers = require('./api/helpers'); + app.use((req, res, next) => { + als.run({ + uid: req.uid, + req: apiHelpers.buildReqObject(req) + }, next); + }); + const toobusy = require('toobusy-js'); + toobusy.maxLag(meta.config.eventLoopLagThreshold); + toobusy.interval(meta.config.eventLoopInterval); +} +function setupHelmet(app) { + const options = { + contentSecurityPolicy: false, + crossOriginOpenerPolicy: { + policy: meta.config['cross-origin-opener-policy'] + }, + crossOriginResourcePolicy: { + policy: meta.config['cross-origin-resource-policy'] + }, + referrerPolicy: { + policy: 'strict-origin-when-cross-origin' + }, + crossOriginEmbedderPolicy: !!meta.config['cross-origin-embedder-policy'] + }; + if (meta.config['hsts-enabled']) { + options.hsts = { + maxAge: Math.max(0, meta.config['hsts-maxage']), + includeSubDomains: !!meta.config['hsts-subdomains'], + preload: !!meta.config['hsts-preload'] + }; + } + try { + app.use(helmet(options)); + } catch (err) { + winston.error(`[startup] unable to initialize helmet \n${err.stack}`); + } +} +function setupFavicon(app) { + let faviconPath = meta.config['brand:favicon'] || 'favicon.ico'; + faviconPath = path.join(nconf.get('base_dir'), 'public', faviconPath.replace(/assets\/uploads/, 'uploads')); + if (file.existsSync(faviconPath)) { + app.use(nconf.get('relative_path'), favicon(faviconPath)); + } +} +function configureBodyParser(app) { + const urlencodedOpts = nconf.get('bodyParser:urlencoded') || {}; + if (!urlencodedOpts.hasOwnProperty('extended')) { + urlencodedOpts.extended = true; + } + app.use(bodyParser.urlencoded(urlencodedOpts)); + const jsonOpts = nconf.get('bodyParser:json') || {}; + app.use(bodyParser.json(jsonOpts)); +} +function setupCookie() { + const cookie = meta.configs.cookie.get(); + const ttl = meta.getSessionTTLSeconds() * 1000; + cookie.maxAge = ttl; + return cookie; +} +async function listen() { + let port = nconf.get('port'); + const isSocket = isNaN(port) && !Array.isArray(port); + const socketPath = isSocket ? nconf.get('port') : ''; + if (Array.isArray(port)) { + if (!port.length) { + winston.error('[startup] empty ports array in config.json'); + process.exit(); + } + winston.warn('[startup] If you want to start nodebb on multiple ports please use loader.js'); + winston.warn(`[startup] Defaulting to first port in array, ${port[0]}`); + port = port[0]; + if (!port) { + winston.error('[startup] Invalid port, exiting'); + process.exit(); + } + } + port = parseInt(port, 10); + if (port !== 80 && port !== 443 || nconf.get('trust_proxy') === true) { + winston.info('🤝 Enabling \'trust proxy\''); + app.enable('trust proxy'); + } + if ((port === 80 || port === 443) && process.env.NODE_ENV !== 'development') { + winston.info('Using ports 80 and 443 is not recommend; use a proxy instead. See README.md'); + } + const bind_address = nconf.get('bind_address') === '0.0.0.0' || !nconf.get('bind_address') ? '0.0.0.0' : nconf.get('bind_address'); + const args = isSocket ? [socketPath] : [port, bind_address]; + let oldUmask; + if (isSocket) { + oldUmask = process.umask('0000'); + try { + await exports.testSocket(socketPath); + } catch (err) { + winston.error(`[startup] NodeBB was unable to secure domain socket access (${socketPath})\n${err.stack}`); + throw err; + } + } + return new Promise((resolve, reject) => { + server.listen(...args.concat([function (err) { + const onText = `${isSocket ? socketPath : `${bind_address}:${port}`}`; + if (err) { + winston.error(`[startup] NodeBB was unable to listen on: ${chalk.yellow(onText)}`); + reject(err); + } + winston.info(`📡 NodeBB is now listening on: ${chalk.yellow(onText)}`); + winston.info(`🔗 Canonical URL: ${chalk.yellow(nconf.get('url'))}`); + if (oldUmask) { + process.umask(oldUmask); + } + resolve(); + }])); + }); +} +exports.testSocket = async function (socketPath) { + if (typeof socketPath !== 'string') { + throw new Error(`invalid socket path : ${socketPath}`); + } + const net = require('net'); + const file = require('./file'); + const exists = await file.exists(socketPath); + if (!exists) { + return; + } + return new Promise((resolve, reject) => { + const testSocket = new net.Socket(); + testSocket.on('error', err => { + if (err.code !== 'ECONNREFUSED') { + return reject(err); + } + fs.unlink(socketPath, err => { + if (err) reject(err);else resolve(); + }); + }); + testSocket.connect({ + path: socketPath + }, () => { + reject(new Error('port-in-use')); + }); + }); +}; +require('./promisify')(exports); \ No newline at end of file diff --git a/lib/widgets/admin.js b/lib/widgets/admin.js new file mode 100644 index 0000000000..4bac3e9cce --- /dev/null +++ b/lib/widgets/admin.js @@ -0,0 +1,67 @@ +'use strict'; + +const webserver = require('../webserver'); +const plugins = require('../plugins'); +const groups = require('../groups'); +const index = require('./index'); +const admin = module.exports; +admin.get = async function () { + const [areas, availableWidgets] = await Promise.all([admin.getAreas(), getAvailableWidgets()]); + return { + templates: buildTemplatesFromAreas(areas), + areas: areas, + availableWidgets: availableWidgets + }; +}; +admin.getAreas = async function () { + const areas = await index.getAvailableAreas(); + areas.push({ + name: 'Draft Zone', + template: 'global', + location: 'drafts' + }); + const areaData = await Promise.all(areas.map(area => index.getArea(area.template, area.location))); + areas.forEach((area, i) => { + area.data = areaData[i]; + }); + return areas; +}; +async function getAvailableWidgets() { + const [availableWidgets, adminTemplate] = await Promise.all([plugins.hooks.fire('filter:widgets.getWidgets', []), renderAdminTemplate()]); + availableWidgets.forEach(w => { + w.content += adminTemplate; + }); + return availableWidgets; +} +async function renderAdminTemplate() { + const groupsData = await groups.getNonPrivilegeGroups('groups:createtime', 0, -1); + groupsData.sort((a, b) => b.system - a.system); + return await webserver.app.renderAsync('admin/partials/widget-settings', { + groups: groupsData + }); +} +function buildTemplatesFromAreas(areas) { + const templates = []; + const list = {}; + let index = 0; + areas.forEach(area => { + if (typeof list[area.template] === 'undefined') { + list[area.template] = index; + templates.push({ + template: area.template, + areas: [], + widgetCount: 0 + }); + index += 1; + } + templates[list[area.template]].areas.push({ + name: area.name, + location: area.location + }); + if (area.location !== 'drafts') { + templates[list[area.template]].widgetCount += area.data.length; + } + }); + return templates; +} +require('../promisify')(admin); \ No newline at end of file diff --git a/lib/widgets/index.js b/lib/widgets/index.js new file mode 100644 index 0000000000..4ae1bef6a3 --- /dev/null +++ b/lib/widgets/index.js @@ -0,0 +1,269 @@ +'use strict'; + +const winston = require('winston'); +const _ = require('lodash'); +const Benchpress = require('benchpressjs'); +const plugins = require('../plugins'); +const groups = require('../groups'); +const translator = require('../translator'); +const db = require('../database'); +const apiController = require('../controllers/api'); +const meta = require('../meta'); +const widgets = module.exports; +widgets.render = async function (uid, options) { + if (!options.template) { + throw new Error('[[error:invalid-data]]'); + } + const data = await widgets.getWidgetDataForTemplates(['global', options.template]); + delete data.global.drafts; + const locations = _.uniq(Object.keys(data.global).concat(Object.keys(data[options.template]))); + const widgetData = await Promise.all(locations.map(location => renderLocation(location, data, uid, options))); + const returnData = {}; + locations.forEach((location, i) => { + if (Array.isArray(widgetData[i]) && widgetData[i].length) { + returnData[location] = widgetData[i].filter(widget => widget && widget.html); + } + }); + return returnData; +}; +async function renderLocation(location, data, uid, options) { + const widgetsAtLocation = (data[options.template][location] || []).concat(data.global[location] || []); + if (!widgetsAtLocation.length) { + return []; + } + const renderedWidgets = await Promise.all(widgetsAtLocation.map(widget => renderWidget(widget, uid, options, location))); + return renderedWidgets; +} +async function renderWidget(widget, uid, options, location) { + if (!widget || !widget.data || !!widget.data['hide-mobile'] && options.req.useragent.isMobile) { + return; + } + const isVisible = await widgets.checkVisibility(widget.data, uid); + if (!isVisible) { + return; + } + let config = options.res.locals.config || {}; + if (options.res.locals.isAPI) { + config = await apiController.loadConfig(options.req); + } + const userLang = config.userLang || meta.config.defaultLang || 'en-GB'; + const templateData = _.assign({}, options.templateData, { + config: config + }); + const data = await plugins.hooks.fire(`filter:widget.render:${widget.widget}`, { + uid: uid, + area: options, + templateData: templateData, + data: widget.data, + req: options.req, + res: options.res, + location + }); + if (!data) { + return; + } + let { + html + } = data; + if (widget.data.container && widget.data.container.match('{body}')) { + html = await Benchpress.compileRender(widget.data.container, { + title: widget.data.title, + body: html, + template: data.templateData && data.templateData.template + }); + } + if (html) { + html = await translator.translate(html, userLang); + } + return { + html + }; +} +widgets.checkVisibility = async function (data, uid) { + let isVisible = true; + let isHidden = false; + if (data.groups.length) { + isVisible = await groups.isMemberOfAny(uid, data.groups); + } + if (data.groupsHideFrom.length) { + isHidden = await groups.isMemberOfAny(uid, data.groupsHideFrom); + } + const isExpired = data.startDate && Date.now() < new Date(data.startDate).getTime() || data.endDate && Date.now() > new Date(data.endDate).getTime(); + return isVisible && !isHidden && !isExpired; +}; +widgets.getWidgetDataForTemplates = async function (templates) { + const keys = templates.map(tpl => `widgets:${tpl}`); + const data = await db.getObjects(keys); + const returnData = {}; + templates.forEach((template, index) => { + returnData[template] = returnData[template] || {}; + const templateWidgetData = data[index] || {}; + const locations = Object.keys(templateWidgetData); + locations.forEach(location => { + if (templateWidgetData && templateWidgetData[location]) { + try { + returnData[template][location] = parseWidgetData(templateWidgetData[location]); + } catch (err) { + winston.error(`can not parse widget data. template: ${template} location: ${location}`); + returnData[template][location] = []; + } + } else { + returnData[template][location] = []; + } + }); + }); + return returnData; +}; +widgets.getArea = async function (template, location) { + const result = await db.getObjectField(`widgets:${template}`, location); + if (!result) { + return []; + } + return parseWidgetData(result); +}; +function parseWidgetData(data) { + const widgets = JSON.parse(data); + widgets.forEach(widget => { + if (widget) { + widget.data.groups = widget.data.groups || []; + if (widget.data.groups && !Array.isArray(widget.data.groups)) { + widget.data.groups = [widget.data.groups]; + } + widget.data.groupsHideFrom = widget.data.groupsHideFrom || []; + if (widget.data.groupsHideFrom && !Array.isArray(widget.data.groupsHideFrom)) { + widget.data.groupsHideFrom = [widget.data.groupsHideFrom]; + } + } + }); + return widgets; +} +widgets.setArea = async function (area) { + if (!area.location || !area.template) { + throw new Error('Missing location and template data'); + } + await db.setObjectField(`widgets:${area.template}`, area.location, JSON.stringify(area.widgets)); +}; +widgets.setAreas = async function (areas) { + const templates = {}; + areas.forEach(area => { + if (!area.location || !area.template) { + throw new Error('Missing location and template data'); + } + templates[area.template] = templates[area.template] || {}; + templates[area.template][area.location] = JSON.stringify(area.widgets); + }); + await db.setObjectBulk(Object.keys(templates).map(tpl => [`widgets:${tpl}`, templates[tpl]])); +}; +widgets.getAvailableAreas = async function () { + const defaultAreas = [{ + name: 'Global Header', + template: 'global', + location: 'header' + }, { + name: 'Global Footer', + template: 'global', + location: 'footer' + }, { + name: 'Global Sidebar', + template: 'global', + location: 'sidebar' + }, { + name: 'Group Page (Left)', + template: 'groups/details.tpl', + location: 'left' + }, { + name: 'Group Page (Right)', + template: 'groups/details.tpl', + location: 'right' + }, { + name: 'Chat Header', + template: 'chats.tpl', + location: 'header' + }, { + name: 'Chat Sidebar', + template: 'chats.tpl', + location: 'sidebar' + }]; + return await plugins.hooks.fire('filter:widgets.getAreas', defaultAreas); +}; +widgets.saveLocationsOnThemeReset = async function () { + const locations = {}; + const available = await widgets.getAvailableAreas(); + for (const area of available) { + const widgetsAtLocation = await widgets.getArea(area.template, area.location); + if (widgetsAtLocation.length) { + locations[area.template] = locations[area.template] || []; + if (!locations[area.template].includes(area.location)) { + locations[area.template].push(area.location); + } + } + } + if (Object.keys(locations).length) { + await db.set('widgets:draft:locations', JSON.stringify(locations)); + } +}; +widgets.moveMissingAreasToDrafts = async function () { + const locationsObj = await db.get('widgets:draft:locations'); + if (!locationsObj) { + return; + } + try { + const locations = JSON.parse(locationsObj); + const [available, draftWidgets] = await Promise.all([widgets.getAvailableAreas(), widgets.getArea('global', 'drafts')]); + let saveDraftWidgets = draftWidgets || []; + for (const [template, tplLocations] of Object.entries(locations)) { + for (const location of tplLocations) { + const locationExists = available.find(area => area.template === template && area.location === location); + if (!locationExists) { + const widgetsAtLocation = await widgets.getArea(template, location); + saveDraftWidgets = saveDraftWidgets.concat(widgetsAtLocation); + await widgets.setArea({ + template, + location, + widgets: [] + }); + } + } + } + await widgets.setArea({ + template: 'global', + location: 'drafts', + widgets: saveDraftWidgets + }); + } catch (err) { + winston.error(err.stack); + } finally { + await db.delete('widgets:draft:locations'); + } +}; +widgets.reset = async function () { + const [areas, drafts] = await Promise.all([widgets.getAvailableAreas(), widgets.getArea('global', 'drafts')]); + let saveDrafts = drafts || []; + for (const area of areas) { + const areaData = await widgets.getArea(area.template, area.location); + saveDrafts = saveDrafts.concat(areaData); + area.widgets = []; + await widgets.setArea(area); + } + await widgets.setArea({ + template: 'global', + location: 'drafts', + widgets: saveDrafts + }); +}; +widgets.resetTemplate = async function (template) { + const area = await db.getObject(`widgets:${template}.tpl`); + if (area) { + const toBeDrafted = _.flatMap(Object.values(area), value => JSON.parse(value)); + await db.delete(`widgets:${template}.tpl`); + let draftWidgets = await db.getObjectField('widgets:global', 'drafts'); + draftWidgets = JSON.parse(draftWidgets).concat(toBeDrafted); + await db.setObjectField('widgets:global', 'drafts', JSON.stringify(draftWidgets)); + } +}; +widgets.resetTemplates = async function (templates) { + for (const template of templates) { + await widgets.resetTemplate(template); + } +}; +require('../promisify')(widgets); \ No newline at end of file diff --git a/src/cli/index.js b/src/cli/index.js index e6f0485585..e472d70d1e 100644 --- a/src/cli/index.js +++ b/src/cli/index.js @@ -111,7 +111,7 @@ prestart.versionCheck(); if (!configExists && process.argv[2] !== 'setup') { require('./setup').webInstall(); - return; + // return; } if (configExists) {