From 2fbb78956421d9b270d33cb30513281ae144c1eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcel=20M=C3=BCller?= Date: Tue, 20 Aug 2024 11:37:34 +0200 Subject: [PATCH] Add blurhash as placeholder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marcel Müller --- NextcloudTalk.xcodeproj/project.pbxproj | 4 + .../BaseChatTableViewCell+File.swift | 21 ++- NextcloudTalk/BlurHashDecode.swift | 151 ++++++++++++++++++ NextcloudTalk/NCMessageFileParameter.h | 1 + NextcloudTalk/NCMessageFileParameter.m | 1 + 5 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 NextcloudTalk/BlurHashDecode.swift diff --git a/NextcloudTalk.xcodeproj/project.pbxproj b/NextcloudTalk.xcodeproj/project.pbxproj index c8a9a888a..a20fa46b2 100644 --- a/NextcloudTalk.xcodeproj/project.pbxproj +++ b/NextcloudTalk.xcodeproj/project.pbxproj @@ -69,6 +69,7 @@ 1F205C572CEFA01900AAA673 /* OutOfOfficeView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1F205C562CEFA01900AAA673 /* OutOfOfficeView.swift */; }; 1F205D412CFC6DD300AAA673 /* AvatarProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1F205D402CFC6DCF00AAA673 /* AvatarProtocol.swift */; }; 1F205D442CFC70AD00AAA673 /* AvatarProtocol.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1F205D402CFC6DCF00AAA673 /* AvatarProtocol.swift */; }; + 1F21A0502C747FC500ED8C0C /* BlurHashDecode.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1F21A04F2C747FC500ED8C0C /* BlurHashDecode.swift */; }; 1F24B5A228E0648600654457 /* ReferenceGithubView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1F24B5A128E0648600654457 /* ReferenceGithubView.swift */; }; 1F24B5A428E0649200654457 /* ReferenceGithubView.xib in Resources */ = {isa = PBXBuildFile; fileRef = 1F24B5A328E0649200654457 /* ReferenceGithubView.xib */; }; 1F35F8E22AEEBAF900044BDA /* InputbarViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1F5A24322ADA77DA009939FE /* InputbarViewController.swift */; }; @@ -706,6 +707,7 @@ 1F21A06B2C77869600ED8C0C /* sr */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = sr; path = sr.lproj/InfoPlist.strings; sourceTree = ""; }; 1F21A06C2C77869600ED8C0C /* sr */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = sr; path = sr.lproj/Localizable.strings; sourceTree = ""; }; 1F21A06D2C77869600ED8C0C /* sr */ = {isa = PBXFileReference; lastKnownFileType = text.plist.stringsdict; name = sr; path = sr.lproj/Localizable.stringsdict; sourceTree = ""; }; + 1F21A04F2C747FC500ED8C0C /* BlurHashDecode.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = BlurHashDecode.swift; sourceTree = ""; }; 1F24B5A128E0648600654457 /* ReferenceGithubView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ReferenceGithubView.swift; sourceTree = ""; }; 1F24B5A328E0649200654457 /* ReferenceGithubView.xib */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.xib; path = ReferenceGithubView.xib; sourceTree = ""; }; 1F35F8FA2AEEDBC600044BDA /* ChatViewControllerExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatViewControllerExtension.swift; sourceTree = ""; }; @@ -1695,6 +1697,7 @@ 2CB997C32A052449003C41AC /* EmojiAvatarPickerViewController.swift */, 2CB997C42A052449003C41AC /* EmojiAvatarPickerViewController.xib */, 1F1B0F352BDD8B9C003FD766 /* NCActivityIndicator.swift */, + 1F21A04F2C747FC500ED8C0C /* BlurHashDecode.swift */, ); name = "User Interface"; sourceTree = ""; @@ -2875,6 +2878,7 @@ 2C0574851EDD9E8E00D9E7F2 /* AppDelegate.m in Sources */, 2C4987BD21E640E20060AC27 /* CallKitManager.m in Sources */, 1F1B0F4C2BE18FF3003FD766 /* CustomPresentableNavigationController.swift in Sources */, + 1F21A0502C747FC500ED8C0C /* BlurHashDecode.swift in Sources */, 2C4446F3265D51A600DF1DBC /* NCPushNotificationsUtils.m in Sources */, 2C0424902CA32D45004772F6 /* BaseChatTableViewCell+Audio.swift in Sources */, 2C1ABDE5257F883400AEDFB6 /* ABContact.m in Sources */, diff --git a/NextcloudTalk/BaseChatTableViewCell+File.swift b/NextcloudTalk/BaseChatTableViewCell+File.swift index 16052b034..7f057eda8 100644 --- a/NextcloudTalk/BaseChatTableViewCell+File.swift +++ b/NextcloudTalk/BaseChatTableViewCell+File.swift @@ -123,14 +123,29 @@ extension BaseChatTableViewCell { return } + let isVideoFile = NCUtils.isVideo(fileType: message.file().mimetype) + let isMediaFile = isVideoFile || NCUtils.isImage(fileType: message.file().mimetype) + + var placeholderImage: UIImage? + var previewImageHeight: CGFloat? + // In case we can determine the height before requesting the preview, adjust the imageView constraints accordingly if file.previewImageHeight > 0 { - self.filePreviewImageViewHeightConstraint?.constant = CGFloat(file.previewImageHeight) + previewImageHeight = CGFloat(file.previewImageHeight) } else { let estimatedPreviewHeight = BaseChatTableViewCell.getEstimatedPreviewSize(for: message) if estimatedPreviewHeight > 0 { - self.filePreviewImageViewHeightConstraint?.constant = estimatedPreviewHeight + previewImageHeight = estimatedPreviewHeight + } + } + + if let previewImageHeight { + self.filePreviewImageViewHeightConstraint?.constant = previewImageHeight + + if let blurhash = message.file()?.blurhash { + // TODO: Need to determine width as well, we currently only store the height in some cases + placeholderImage = .init(blurHash: blurhash, size: .init(width: 250, height: previewImageHeight)) } } @@ -177,7 +192,7 @@ extension BaseChatTableViewCell { let requestedHeight = Int(3 * fileMessageCellFileMaxPreviewHeight) guard let previewRequest = NCAPIController.sharedInstance().createPreviewRequest(forFile: file.parameterId, withMaxHeight: requestedHeight, using: account) else { return } - self.filePreviewImageView?.setImageWith(previewRequest, placeholderImage: nil, success: { [weak self] _, _, image in + self.filePreviewImageView?.setImageWith(previewRequest, placeholderImage: placeholderImage, success: { [weak self] _, _, image in guard let self, let imageView = self.filePreviewImageView else { return } imageView.image = image diff --git a/NextcloudTalk/BlurHashDecode.swift b/NextcloudTalk/BlurHashDecode.swift new file mode 100644 index 000000000..13e746f17 --- /dev/null +++ b/NextcloudTalk/BlurHashDecode.swift @@ -0,0 +1,151 @@ +// +// SPDX-FileCopyrightText: 2018 Wolt Enterprises +// SPDX-License-Identifier: MIT +// + +import UIKit + +extension UIImage { + public convenience init?(blurHash: String, size: CGSize, punch: Float = 1) { + guard blurHash.count >= 6 else { return nil } + + let sizeFlag = String(blurHash[0]).decode83() + let numY = (sizeFlag / 9) + 1 + let numX = (sizeFlag % 9) + 1 + + let quantisedMaximumValue = String(blurHash[1]).decode83() + let maximumValue = Float(quantisedMaximumValue + 1) / 166 + + guard blurHash.count == 4 + 2 * numX * numY else { return nil } + + let colours: [(Float, Float, Float)] = (0 ..< numX * numY).map { i in + if i == 0 { + let value = String(blurHash[2 ..< 6]).decode83() + return decodeDC(value) + } else { + let value = String(blurHash[4 + i * 2 ..< 4 + i * 2 + 2]).decode83() + return decodeAC(value, maximumValue: maximumValue * punch) + } + } + + let width = Int(size.width) + let height = Int(size.height) + let bytesPerRow = width * 3 + guard let data = CFDataCreateMutable(kCFAllocatorDefault, bytesPerRow * height) else { return nil } + CFDataSetLength(data, bytesPerRow * height) + guard let pixels = CFDataGetMutableBytePtr(data) else { return nil } + + for y in 0 ..< height { + for x in 0 ..< width { + var r: Float = 0 + var g: Float = 0 + var b: Float = 0 + + for j in 0 ..< numY { + for i in 0 ..< numX { + let basis = cos(Float.pi * Float(x) * Float(i) / Float(width)) * cos(Float.pi * Float(y) * Float(j) / Float(height)) + let colour = colours[i + j * numX] + r += colour.0 * basis + g += colour.1 * basis + b += colour.2 * basis + } + } + + let intR = UInt8(linearTosRGB(r)) + let intG = UInt8(linearTosRGB(g)) + let intB = UInt8(linearTosRGB(b)) + + pixels[3 * x + 0 + y * bytesPerRow] = intR + pixels[3 * x + 1 + y * bytesPerRow] = intG + pixels[3 * x + 2 + y * bytesPerRow] = intB + } + } + + let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.none.rawValue) + + guard let provider = CGDataProvider(data: data) else { return nil } + guard let cgImage = CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 24, bytesPerRow: bytesPerRow, + space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: bitmapInfo, provider: provider, decode: nil, shouldInterpolate: true, intent: .defaultIntent) else { return nil } + + self.init(cgImage: cgImage) + } +} + +private func decodeDC(_ value: Int) -> (Float, Float, Float) { + let intR = value >> 16 + let intG = (value >> 8) & 255 + let intB = value & 255 + return (sRGBToLinear(intR), sRGBToLinear(intG), sRGBToLinear(intB)) +} + +private func decodeAC(_ value: Int, maximumValue: Float) -> (Float, Float, Float) { + let quantR = value / (19 * 19) + let quantG = (value / 19) % 19 + let quantB = value % 19 + + let rgb = ( + signPow((Float(quantR) - 9) / 9, 2) * maximumValue, + signPow((Float(quantG) - 9) / 9, 2) * maximumValue, + signPow((Float(quantB) - 9) / 9, 2) * maximumValue + ) + + return rgb +} + +private func signPow(_ value: Float, _ exp: Float) -> Float { + return copysign(pow(abs(value), exp), value) +} + +private func linearTosRGB(_ value: Float) -> Int { + let v = max(0, min(1, value)) + if v <= 0.0031308 { return Int(v * 12.92 * 255 + 0.5) } + else { return Int((1.055 * pow(v, 1 / 2.4) - 0.055) * 255 + 0.5) } +} + +private func sRGBToLinear(_ value: Type) -> Float { + let v = Float(Int64(value)) / 255 + if v <= 0.04045 { return v / 12.92 } + else { return pow((v + 0.055) / 1.055, 2.4) } +} + +private let encodeCharacters: [String] = { + return "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~".map { String($0) } +}() + +private let decodeCharacters: [String: Int] = { + var dict: [String: Int] = [:] + for (index, character) in encodeCharacters.enumerated() { + dict[character] = index + } + return dict +}() + +extension String { + func decode83() -> Int { + var value: Int = 0 + for character in self { + if let digit = decodeCharacters[String(character)] { + value = value * 83 + digit + } + } + return value + } +} + +private extension String { + subscript (offset: Int) -> Character { + return self[index(startIndex, offsetBy: offset)] + } + + subscript (bounds: CountableClosedRange) -> Substring { + let start = index(startIndex, offsetBy: bounds.lowerBound) + let end = index(startIndex, offsetBy: bounds.upperBound) + return self[start...end] + } + + subscript (bounds: CountableRange) -> Substring { + let start = index(startIndex, offsetBy: bounds.lowerBound) + let end = index(startIndex, offsetBy: bounds.upperBound) + return self[start..