From 04f910ee03e3e0ba9fcb4265bb6b799449718060 Mon Sep 17 00:00:00 2001 From: ginnyTheCat Date: Sat, 6 Aug 2022 05:56:26 +0200 Subject: [PATCH] Blurhash support (#701) * Generate blurhash client side * Make blurhash generation faster * Simple blurhash display support * Make image display simpler * Support non square images * Don't attach video blurhash to thumbnail * Add video display support * Ignore alt tag missing warning Co-authored-by: Ajay Bura <32841439+ajbura@users.noreply.github.com> --- package-lock.json | 46 +++++++++++++++++++++++---- package.json | 2 ++ src/app/molecules/media/Media.jsx | 24 ++++++++++---- src/app/molecules/media/Media.scss | 31 +++++++++++------- src/app/molecules/message/Message.jsx | 4 +++ src/client/state/RoomsInput.js | 32 +++++++++++++++++-- 6 files changed, 110 insertions(+), 29 deletions(-) diff --git a/package-lock.json b/package-lock.json index 29a0520fa..abc3d3303 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz", "@tippyjs/react": "^4.2.6", "babel-polyfill": "^6.26.0", + "blurhash": "^1.1.5", "browser-encrypt-attachment": "^0.3.0", "dateformat": "^5.0.3", "emojibase-data": "^7.0.1", @@ -34,6 +35,7 @@ "prop-types": "^15.8.1", "react": "^17.0.2", "react-autosize-textarea": "^7.1.0", + "react-blurhash": "^0.1.3", "react-dnd": "^15.1.2", "react-dnd-html5-backend": "^15.1.3", "react-dom": "^17.0.2", @@ -3014,6 +3016,8 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz", "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -3029,7 +3033,9 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/ajv-keywords": { "version": "3.5.2", @@ -3558,6 +3564,11 @@ "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", "dev": true }, + "node_modules/blurhash": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/blurhash/-/blurhash-1.1.5.tgz", + "integrity": "sha512-a+LO3A2DfxTaTztsmkbLYmUzUeApi0LZuKalwbNmqAHR6HhJGMt1qSV/R3wc+w4DL28holjqO3Bg74aUGavGjg==" + }, "node_modules/bmp-js": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", @@ -11540,6 +11551,15 @@ "react-dom": "^0.14.0 || ^15.0.0 || ^16.0.0" } }, + "node_modules/react-blurhash": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/react-blurhash/-/react-blurhash-0.1.3.tgz", + "integrity": "sha512-Q9lqbXg92NU6/2DoIl/cBM8YWL+Z4X66OiG4aT9ozOgjBwx104LHFCH5stf6aF+s0Q9Wf310Ul+dG+VXJltmPg==", + "peerDependencies": { + "blurhash": "^1.1.1", + "react": ">=15" + } + }, "node_modules/react-dnd": { "version": "15.1.2", "resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-15.1.2.tgz", @@ -16495,15 +16515,14 @@ "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", "dev": true, - "requires": { - "ajv": "^8.0.0" - }, + "requires": {}, "dependencies": { "ajv": { - "version": "8.9.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz", + "version": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz", "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -16515,7 +16534,9 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true + "dev": true, + "optional": true, + "peer": true } } }, @@ -16950,6 +16971,11 @@ } } }, + "blurhash": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/blurhash/-/blurhash-1.1.5.tgz", + "integrity": "sha512-a+LO3A2DfxTaTztsmkbLYmUzUeApi0LZuKalwbNmqAHR6HhJGMt1qSV/R3wc+w4DL28holjqO3Bg74aUGavGjg==" + }, "bmp-js": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", @@ -22933,6 +22959,12 @@ "prop-types": "^15.5.6" } }, + "react-blurhash": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/react-blurhash/-/react-blurhash-0.1.3.tgz", + "integrity": "sha512-Q9lqbXg92NU6/2DoIl/cBM8YWL+Z4X66OiG4aT9ozOgjBwx104LHFCH5stf6aF+s0Q9Wf310Ul+dG+VXJltmPg==", + "requires": {} + }, "react-dnd": { "version": "15.1.2", "resolved": "https://registry.npmjs.org/react-dnd/-/react-dnd-15.1.2.tgz", diff --git a/package.json b/package.json index 698da860a..9a633d3b6 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz", "@tippyjs/react": "^4.2.6", "babel-polyfill": "^6.26.0", + "blurhash": "^1.1.5", "browser-encrypt-attachment": "^0.3.0", "dateformat": "^5.0.3", "emojibase-data": "^7.0.1", @@ -40,6 +41,7 @@ "prop-types": "^15.8.1", "react": "^17.0.2", "react-autosize-textarea": "^7.1.0", + "react-blurhash": "^0.1.3", "react-dnd": "^15.1.2", "react-dnd-html5-backend": "^15.1.3", "react-dom": "^17.0.2", diff --git a/src/app/molecules/media/Media.jsx b/src/app/molecules/media/Media.jsx index c4b4a1712..5f081b914 100644 --- a/src/app/molecules/media/Media.jsx +++ b/src/app/molecules/media/Media.jsx @@ -4,6 +4,7 @@ import './Media.scss'; import encrypt from 'browser-encrypt-attachment'; +import { BlurhashCanvas } from 'react-blurhash'; import Text from '../../atoms/text/Text'; import IconButton from '../../atoms/button/IconButton'; import Spinner from '../../atoms/spinner/Spinner'; @@ -154,7 +155,7 @@ File.propTypes = { }; function Image({ - name, width, height, link, file, type, + name, width, height, link, file, type, blurhash, }) { const [url, setUrl] = useState(null); @@ -175,6 +176,7 @@ function Image({
+ { blurhash && } { url !== null && {name}}
@@ -185,6 +187,7 @@ Image.defaultProps = { width: null, height: null, type: '', + blurhash: '', }; Image.propTypes = { name: PropTypes.string.isRequired, @@ -193,6 +196,7 @@ Image.propTypes = { link: PropTypes.string.isRequired, file: PropTypes.shape({}), type: PropTypes.string, + blurhash: PropTypes.string, }; function Sticker({ @@ -278,8 +282,8 @@ Audio.propTypes = { }; function Video({ - name, link, thumbnail, - width, height, file, type, thumbnailFile, thumbnailType, + name, link, thumbnail, thumbnailFile, thumbnailType, + width, height, file, type, blurhash, }) { const [isLoading, setIsLoading] = useState(false); const [url, setUrl] = useState(null); @@ -315,10 +319,14 @@ function Video({
+ { url === null && blurhash && } + { url === null && thumbUrl !== null && ( + /* eslint-disable-next-line jsx-a11y/alt-text */ + + )} { url === null && isLoading && } { url === null && !isLoading && } { url !== null && ( @@ -336,20 +344,22 @@ Video.defaultProps = { height: null, file: null, thumbnail: null, - type: '', thumbnailType: null, thumbnailFile: null, + type: '', + blurhash: null, }; Video.propTypes = { name: PropTypes.string.isRequired, link: PropTypes.string.isRequired, thumbnail: PropTypes.string, + thumbnailFile: PropTypes.shape({}), + thumbnailType: PropTypes.string, width: PropTypes.number, height: PropTypes.number, file: PropTypes.shape({}), type: PropTypes.string, - thumbnailFile: PropTypes.shape({}), - thumbnailType: PropTypes.string, + blurhash: PropTypes.string, }; export { diff --git a/src/app/molecules/media/Media.scss b/src/app/molecules/media/Media.scss index 16cf8f7e2..b26b232a1 100644 --- a/src/app/molecules/media/Media.scss +++ b/src/app/molecules/media/Media.scss @@ -33,6 +33,8 @@ font-size: 0; line-height: 0; + position: relative; + display: flex; justify-content: center; align-items: center; @@ -42,34 +44,39 @@ background-size: cover; } -.sticker-container { - display: inline-flex; - max-width: 128px; - width: 100%; - & img { +.image-container, +.video-container { + & img, + & canvas { + position: absolute; + max-width: unset !important; width: 100% !important; + height: 100%; + border-radius: 0 !important; + margin: 0 !important; } } -.image-container { +.sticker-container { + display: inline-flex; + max-width: 128px; + width: 100%; & img { - max-width: unset !important; width: 100% !important; - border-radius: 0 !important; - margin: 0 !important; } } .video-container { & .ic-btn-surface { background-color: var(--bg-surface-low); + position: absolute; } video { - width: 100% + width: 100%; } } .audio-container { audio { - width: 100% + width: 100%; } -} \ No newline at end of file +} diff --git a/src/app/molecules/message/Message.jsx b/src/app/molecules/message/Message.jsx index 49337bdc1..e94e5a460 100644 --- a/src/app/molecules/message/Message.jsx +++ b/src/app/molecules/message/Message.jsx @@ -610,6 +610,8 @@ function genMediaContent(mE) { let msgType = mE.getContent()?.msgtype; if (mE.getType() === 'm.sticker') msgType = 'm.sticker'; + const blurhash = mContent?.info?.['xyz.amorgan.blurhash']; + switch (msgType) { case 'm.file': return ( @@ -629,6 +631,7 @@ function genMediaContent(mE) { link={mx.mxcUrlToHttp(mediaMXC)} file={isEncryptedFile ? mContent.file : null} type={mContent.info?.mimetype} + blurhash={blurhash} /> ); case 'm.sticker': @@ -666,6 +669,7 @@ function genMediaContent(mE) { height={typeof mContent.info?.h === 'number' ? mContent.info?.h : null} file={isEncryptedFile ? mContent.file : null} type={mContent.info?.mimetype} + blurhash={blurhash} /> ); default: diff --git a/src/client/state/RoomsInput.js b/src/client/state/RoomsInput.js index 2377c8d05..814255429 100644 --- a/src/client/state/RoomsInput.js +++ b/src/client/state/RoomsInput.js @@ -3,12 +3,34 @@ import { micromark } from 'micromark'; import { gfm, gfmHtml } from 'micromark-extension-gfm'; import encrypt from 'browser-encrypt-attachment'; import { math } from 'micromark-extension-math'; +import { encode } from 'blurhash'; import { getShortcodeToEmoji } from '../../app/organisms/emoji-board/custom-emoji'; import { mathExtensionHtml, spoilerExtension, spoilerExtensionHtml } from '../../util/markdown'; import { getImageDimension } from '../../util/common'; import cons from './cons'; import settings from './settings'; +const blurhashField = 'xyz.amorgan.blurhash'; + +function encodeBlurhash(img) { + const canvas = document.createElement('canvas'); + canvas.width = 100; + canvas.height = 100; + const context = canvas.getContext('2d'); + context.drawImage(img, 0, 0, canvas.width, canvas.height); + const data = context.getImageData(0, 0, canvas.width, canvas.height); + return encode(data.data, data.width, data.height, 4, 4); +} + +function loadImage(url) { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = (err) => reject(err); + img.src = url; + }); +} + function loadVideo(videoFile) { return new Promise((resolve, reject) => { const video = document.createElement('video'); @@ -300,10 +322,11 @@ class RoomsInput extends EventEmitter { let uploadData = null; if (fileType === 'image') { - const imgDimension = await getImageDimension(file); + const img = await loadImage(URL.createObjectURL(file)); - info.w = imgDimension.w; - info.h = imgDimension.h; + info.w = img.width; + info.h = img.height; + info[blurhashField] = encodeBlurhash(img); content.msgtype = 'm.image'; content.body = file.name || 'Image'; @@ -313,8 +336,11 @@ class RoomsInput extends EventEmitter { try { const video = await loadVideo(file); + info.w = video.videoWidth; info.h = video.videoHeight; + info[blurhashField] = encodeBlurhash(video); + const thumbnailData = await getVideoThumbnail(video, video.videoWidth, video.videoHeight, 'image/jpeg'); const thumbnailUploadData = await this.uploadFile(roomId, thumbnailData.thumbnail); info.thumbnail_info = thumbnailData.info;