Merge branch 'mem_issue' of github.com:muxinc/swift-upload-sdk into m…

…em_issue

Example/SwiftUploadSDKExample/SwiftUploadSDKExampleUnitTests/MemoryTests.swift

@@ -0,0 +1,148 @@
+//
+//  SwiftUploadSDKExampleUnitTests.swift
+//  SwiftUploadSDKExampleUnitTests
+//
+//  Created by Tomislav Kordic on 22.2.24..
+//
+
+import XCTest
+@testable import MuxUploadSDK
+
+import MuxUploadSDK
+
+final class MemoryTests: XCTestCase {
+
+    private let myServerBackend = FakeBackend(urlSession: URLSession(configuration: URLSessionConfiguration.default))
+
+    func memoryFootprint() -> mach_vm_size_t? {
+        // The `TASK_VM_INFO_COUNT` and `TASK_VM_INFO_REV1_COUNT` macros are too
+        // complex for the Swift C importer, so we have to define them ourselves.
+        let TASK_VM_INFO_COUNT = mach_msg_type_number_t(MemoryLayout<task_vm_info_data_t>.size / MemoryLayout<integer_t>.size)
+        let TASK_VM_INFO_REV1_COUNT = mach_msg_type_number_t(MemoryLayout.offset(of: \task_vm_info_data_t.min_address)! / MemoryLayout<integer_t>.size)
+        var info = task_vm_info_data_t()
+        var count = TASK_VM_INFO_COUNT
+        let kr = withUnsafeMutablePointer(to: &info) { infoPtr in
+            infoPtr.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { intPtr in
+                task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), intPtr, &count)
+            }
+        }
+        guard
+            kr == KERN_SUCCESS,
+            count >= TASK_VM_INFO_REV1_COUNT
+        else { return nil }
+        return info.phys_footprint
+    }
+
+    func getUploadFilePath() -> URL? {
+        let bundle = Bundle(for: MemoryTests.self)
+        let fileManager = FileManager.default
+        let cwd = bundle.bundlePath
+        guard let content = try? fileManager.contentsOfDirectory(atPath: cwd) else {
+            XCTFail("five_min.mov file not found")
+            return nil
+        }
+        guard let videoURL = try? bundle.url(forResource: "five_min", withExtension: "mov") else {
+            XCTFail("five_min.mov file not found")
+            return nil
+        }
+        return videoURL
+    }
+
+    func testChunkWorkerMemoryUsage() async throws {
+        let chunkSizeInBytes = 6 * 1024 * 1024
+        let videoURL = getUploadFilePath()
+        let uploadURL = try await self.myServerBackend.createDirectUpload()
+        let chunkedFile = ChunkedFile(chunkSize: chunkSizeInBytes)
+        try chunkedFile.openFile(fileURL: videoURL!)
+        try chunkedFile.seekTo(byte: 0)
+        let startMemory = memoryFootprint()
+        repeat {
+            let chunk = try chunkedFile.readNextChunk().get()
+            if (chunk.size() == 0) {
+                break;
+            }
+            let chunkProgress = Progress(totalUnitCount: Int64(chunk.size()))
+            let worker = ChunkWorker(
+                uploadURL: uploadURL,
+                fileChunk: chunk,
+                chunkProgress: chunkProgress,
+                maxRetries: 3
+            )
+            // The problem is in get task it retain the copy of the chunk pointer
+            let task = worker.getTask()
+
+//            let chunkResult = try await task.value
+//            Swift.print("Completed Chunk:\n \(String(describing: chunkResult))")
+//            task.cancel()
+        } while (true)
+        let endMemory = memoryFootprint()
+        if ((startMemory! * 2) < endMemory!) {
+            XCTFail("We have mem leak, started with \(startMemory!) bytes, ended up with \(endMemory!) bytes")
+        }
+    }
+
+    func testChunkedFileMemoryUsage() throws {
+        let videoURL = getUploadFilePath()
+        let chunkSizeInBytes = 6 * 1024 * 1024
+        let chunkedFile = ChunkedFile(chunkSize: chunkSizeInBytes)
+        try chunkedFile.openFile(fileURL: videoURL!)
+        try chunkedFile.seekTo(byte: 0)
+        let startMemory = memoryFootprint()
+        repeat {
+            let chunk = try chunkedFile.readNextChunk().get()
+            Swift.print("Got chunk at position: \(chunk.startByte)")
+            if (chunk.size() == 0) {
+                break;
+            }
+        } while (true)
+        let endMemory = memoryFootprint()
+        if ((startMemory! * 2) < endMemory!) {
+            XCTFail("We have mem leak, started with \(startMemory!) bytes, ended up with \(endMemory!) bytes")
+        }
+    }
+
+    func testLargeUpload() async throws {
+        // Construct custom upload options to upload a file in 6MB chunks
+        let chunkSizeInBytes = 6 * 1024 * 1024
+        let options = DirectUploadOptions(
+            inputStandardization: .skipped,
+            chunkSizeInBytes: chunkSizeInBytes,
+            retryLimitPerChunk: 5
+        )
+        let putURL = try await self.myServerBackend.createDirectUpload()
+        let videoURL = getUploadFilePath()
+
+        let muxDirectUpload = DirectUpload(
+            uploadURL: putURL,
+            inputFileURL: videoURL!,
+            options: options
+        )
+
+        muxDirectUpload.progressHandler = { state in
+            // TODO: print progress, print memory usage
+            Swift.print("Upload progress: " + (state.progress?.fractionCompleted.description)!)
+        }
+        let expectation = XCTestExpectation(description: "Upload task done(completed or failed)")
+        muxDirectUpload.resultHandler = { result in
+            switch result {
+            case .success(let success):
+                Swift.print("File uploaded successfully ")
+                expectation.fulfill()
+            case .failure(let error):
+                Swift.print("Failed to upload file")
+                expectation.fulfill()
+            }
+        }
+        Swift.print("Starting upload video")
+        muxDirectUpload.start()
+
+        let result = await XCTWaiter().fulfillment(of: [expectation], timeout: 300.0)
+        switch result {
+            //        case .completed:    XCTAssertEqual(muxDirectUpload.complete, true)
+        case .timedOut:     XCTFail()
+        default:            XCTFail()
+        }
+        Swift.print("All done !!!")
+    }
+
+}

Mux-Upload-SDK.podspec

@@ -1,7 +1,7 @@
 Pod::Spec.new do |s|
   s.name             = 'Mux-Upload-SDK'
   s.module_name      = 'MuxUploadSDK'
-  s.version          = '0.6.0'
+  s.version          = '0.7.0'
   s.summary          = 'Upload video to Mux.'
   s.description      = 'A library for uploading video to Mux. Similar to UpChunk, but for iOS.'
 

Sources/MuxUploadSDK/InternalUtilities/ChunkedFile.swift

@@ -41,10 +41,47 @@ class ChunkedFile {
     func readNextChunk() -> Result<FileChunk, Error> {
         SDKLogger.logger?.info("--readNextChunk(): called")
         do {
-            guard fileHandle != nil else {
+            guard let fileHandle else {
                 return Result.failure(ChunkedFileError.invalidState("readNextChunk() called but the file was not open"))
             }
-            return try Result.success(doReadNextChunk())
+
+            guard let fileURL = fileURL else {
+                return Result.failure(ChunkedFileError.invalidState("Missing file url."))
+            }
+            var data : Data?
+            try autoreleasepool {
+                data = try fileHandle.read(upToCount: chunkSize)
+            }
+
+            let fileSize = try fileManager.fileSizeOfItem(
+                atPath: fileURL.path
+            )
+
+            guard let data = data else {
+                // Called while already at the end of the file. We read zero bytes, "ending" at the end of the file
+                return .success(
+                    FileChunk(
+                        startByte: fileSize,
+                        endByte: fileSize,
+                        totalFileSize: fileSize,
+                        chunkData: Data(capacity: 0)
+                    )
+                )
+            }
+
+            let chunkLength = data.count
+            let updatedFilePosition = filePos + UInt64(chunkLength)
+
+            let chunk = FileChunk(
+                startByte: self.filePos,
+                endByte: updatedFilePosition,
+                totalFileSize: fileSize,
+                chunkData: data
+            )
+
+            state?.filePosition = updatedFilePosition
+
+            return .success(chunk)
         } catch {
             return Result.failure(ChunkedFileError.fileHandle(error))
         }

Sources/MuxUploadSDK/PublicAPI/SemanticVersion.swift

@@ -12,7 +12,7 @@ public struct SemanticVersion {
     /// Major version component.
     public static let major = 0
     /// Minor version component.
-    public static let minor = 6
+    public static let minor = 7
     /// Patch version component.
     public static let patch = 0
 

Sources/MuxUploadSDK/Upload/ChunkWorker.swift

@@ -7,12 +7,13 @@
 
 import Foundation
 
-/// Uploads a single chunk. Starts an internal Task on creation, which you can get with ``getTask``
-///  This class takes care of retries and backoff on a per-chunk basis
-///  This class provides no thread safety to the outside world
+/// Uploads a single chunk. Starts an internal Task on creation, 
+/// which can be accessed with ``makeUploadTaskIfNeeded``.
+///
+/// This class takes care of retries and backoff on a per-chunk basis
+/// This class provides no thread safety to the outside world
 class ChunkWorker {
     let uploadURL: URL
-    let chunk: FileChunk
     let maxRetries: Int
     let chunkProgress: Progress
 
@@ -33,11 +34,12 @@ class ChunkWorker {
         self.progressDelegate = delegatePair
     }
 
-    func getTask() -> Task<Success, Error> {
-
+    func makeUploadTaskIfNeeded(
+        chunk: FileChunk
+    ) -> Task<Success, Error> {
         guard let uploadTask else {
             chunkStartTime = Date().timeIntervalSince1970
-            let uploadTask = makeUploadTask()
+            let uploadTask = makeUploadTask(chunk: chunk)
             self.uploadTask = uploadTask
             return uploadTask
         }
@@ -51,12 +53,14 @@ class ChunkWorker {
         }
     }
 
-    private func makeUploadTask() -> Task<Success, Error> {
+    private func makeUploadTask(
+        chunk: FileChunk
+    ) -> Task<Success, Error> {
         return Task { [self] in
             var retries = 0
             var requestError: Error?
             let repsonseValidator = ChunkResponseValidator()
-    
+
             repeat {
                 do {
                     let chunkActor = ChunkActor(
@@ -145,21 +149,25 @@ class ChunkWorker {
     struct Success : Sendable {
         let finalState: Update
         let tries: Int
-        // TODO: Also AF Response
     }
 
-    convenience init(uploadInfo: UploadInfo, fileChunk: FileChunk, chunkProgress: Progress) {
+    convenience init(
+        uploadInfo: UploadInfo,
+        chunkProgress: Progress
+    ) {
         self.init(
             uploadURL: uploadInfo.uploadURL,
-            fileChunk: fileChunk,
             chunkProgress: chunkProgress,
             maxRetries: uploadInfo.options.transport.retryLimitPerChunk
         )
     }
 
-    init(uploadURL: URL, fileChunk: FileChunk, chunkProgress: Progress, maxRetries: Int) {
+    init(
+        uploadURL: URL,
+        chunkProgress: Progress,
+        maxRetries: Int
+    ) {
         self.uploadURL = uploadURL
-        self.chunk = fileChunk
         self.maxRetries = maxRetries
         self.chunkProgress = chunkProgress
     }

Sources/MuxUploadSDK/Upload/ChunkedFileUploader.swift

@@ -431,17 +431,27 @@ fileprivate actor Worker {
         var readBytes: Int
         repeat {
             try Task.checkCancellation()
-
-            let chunk = try chunkedFile.readNextChunk().get()
+
+            guard case let Result.success(chunk) = chunkedFile.readNextChunk() else {
+                // TODO: report error accurately
+                throw ChunkWorker.ChunkWorkerError.init(
+                    lastSeenProgress: ChunkWorker.Update(
+                        progress: overallProgress,
+                        bytesSinceLastUpdate: 0,
+                        chunkStartTime: Date().timeIntervalSince1970,
+                        eventTime: Date().timeIntervalSince1970
+                    ), 
+                    reason: nil
+                )
+            }
+
             readBytes = chunk.size()
 
             let wideChunkSize = Int64(chunk.size())
             let chunkProgress = Progress(totalUnitCount: wideChunkSize)
-            //overallProgress.addChild(chunkProgress, withPendingUnitCount: wideChunkSize)
 
             let chunkWorker = ChunkWorker(
                 uploadURL: uploadInfo.uploadURL,
-                fileChunk: chunk,
                 chunkProgress: chunkProgress,
                 maxRetries: uploadInfo.options.transport.retryLimitPerChunk
             )
@@ -458,7 +468,11 @@ fileprivate actor Worker {
             // Problem is in line bellow, task will retain the reference to a chunk read from file and will
             // not release it until the for loop is exited, we need to find a way to implicitly release task memory
             // withouth breaking the for loop.
-            let chunkResult = try await chunkWorker.getTask().value
+//            let chunkResult = try await chunkWorker.getTask().value
+
+            let chunkResult = try await chunkWorker.makeUploadTaskIfNeeded(
+                chunk: chunk
+            ).value
             SDKLogger.logger?.info("Completed Chunk:\n \(String(describing: chunkResult))")
         } while (readBytes == uploadInfo.options.transport.chunkSizeInBytes)
 

Tests/MuxUploadSDKTests/Upload Tests/ChunkWorkerTests.swift

@@ -8,8 +8,7 @@
 import XCTest
 @testable import MuxUploadSDK
 
-final class ChunkWorker: XCTestCase {
-
+class ChunkWorkerTests: XCTestCase {
     func testResponseValidatorRetryCodes() throws {
         let validator = ChunkResponseValidator()
         for code in ChunkResponseValidator.retryableHTTPStatusCodes {