diff --git a/examples/README.md b/examples/README.md index 7fa3ab2a..b92401d3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -5,3 +5,4 @@ HERE Data SDK for TypeScript contains several examples that demonstrate some of - [Authorization example](authorization-example/README.md) – build the Data SDK using webpack and work with the APIs from `@here/olp-sdk-dataservice-api`. You can use this example application to find the list of groups that you have access to as well as create groups. - [React App example](react-app-example/README.md) – use the Data SDK and React in a browser and learn how to work with the following modules: `olp-sdk-authentication`, `olp-sdk-dataservice-read`, and `olp-sdk-dataservice-write`. - [Node.js example](nodejs-example/README.md) – work with the Data SDK in Node.js and learn how to get data from versioned layers and save it to files. +- [MultiPartUploadWrapper example](multipart-upload-wrapper-example/README.md) – use the Data SDK to upload and publish a large amount of data in a browser. diff --git a/examples/multipart-upload-wrapper-example/README.md b/examples/multipart-upload-wrapper-example/README.md new file mode 100644 index 00000000..580daa84 --- /dev/null +++ b/examples/multipart-upload-wrapper-example/README.md @@ -0,0 +1,16 @@ +# Publish large data files in a browser + +This example app uses an HTML and JS UI and an HTTP server with mocked routes to simulate the real `DataStore` class. +You can use this app to learn how to work with the `MultipartUploadWrapper` class and publish large files of more than 8 GB to a versioned layer without loading them in memory. + +## Setup + +This example does not need any setup. Nevertheless, make sure you installed Node.js. + +## Run + +1. Start the server. + ``` + node server.js + ``` +2. In your favorite browser, open `http://localhost:8080/`. diff --git a/examples/multipart-upload-wrapper-example/server.js b/examples/multipart-upload-wrapper-example/server.js new file mode 100644 index 00000000..46ea5eee --- /dev/null +++ b/examples/multipart-upload-wrapper-example/server.js @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2021 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +"use strict"; + +const http = require("http"); +const fs = require("fs"); + +const HOST = "127.0.0.1"; +const PORT = 8080; +const RESPONSE_DELAY_MS = 300; + +const routing = { + "/": ` + + + + + + + +
+ + + + + + + `, + "/ui.js": fs.readFileSync("./ui.js").toString(), + "/oauth2/token": () => ({ + accessToken: "mocked-access-token", + tokenType: "bearer", + expiresIn: 99999999, + }), + "/api/lookup/resources/hrn:here:data::mocked:catalog/apis": () => [ + { + api: "blob", + version: "v1", + baseURL: `http://${HOST}:${PORT}/blobstore/v1`, + parameters: {}, + }, + { + api: "publish", + version: "v2", + baseURL: `http://${HOST}:${PORT}/publish/v2`, + parameters: {}, + }, + ], + "/publish/v2/publications": () => ({ + catalogId: "catalog", + catalogVersion: 999, + details: { + expires: 9999999999999, + message: "", + modified: 9999999999999, + started: 9999999999999, + state: "initialized", + }, + id: "mocked-publication-id", + layerIds: ["mocked-layer"], + versionDependencies: [], + }), + "/blobstore/v1/layers/mocked-layer/data/mocked-datahandle/multiparts": () => ({ + links: { + status: { + href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token`, + method: "GET", + }, + delete: { + href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token`, + method: "DELETE", + }, + uploadPart: { + href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token/parts`, + method: "POST", + }, + complete: { + href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token`, + method: "PUT", + }, + }, + }), + "/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token/parts": ( + req, + res + ) => { + res.setHeader("ETag", Math.random() * 10000); + res.setHeader("Access-Control-Expose-Headers", "ETag"); + return { + status: res.statusCode, + }; + }, + "/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token": () => ({ + ok: true, + }), + "/publish/v2/layers/mocked-layer/publications/mocked-publication-id/partitions": () => ({ + ok: true, + }), + "/publish/v2/publications/mocked-publication-id": () => ({ + ok: true, + }), +}; + +const types = { + object: JSON.stringify, + string: (s) => s, + number: (val) => `${val}`, + undefined: () => "Not Found", + function: (fn, req, res) => JSON.stringify(fn(req, res)), +}; + +const server = http.createServer((req, res) => { + console.dir({ type: "Reguest", method: req.method, url: req.url }); + const queryIndex = req.url.indexOf("?"); + const adaptedUrl = + queryIndex !== -1 ? req.url.substr(0, queryIndex) : req.url; + + const data = routing[adaptedUrl]; + const type = typeof data; + const serialiser = types[type]; + const result = serialiser(data, req, res); + + res.setHeader("Access-Control-Allow-Origin", "*"); + res.setHeader( + "Access-Control-Allow-Headers", + "authorization,content-type, ETag" + ); + res.setHeader("Access-Control-Allow-Methods", "*"); + + res.statusCode = type === "undefined" ? 404 : 200; + setTimeout(() => { + res.end(result); + }, RESPONSE_DELAY_MS); +}); + +server.listen(PORT, HOST, () => { + console.log(`Server running at http://${HOST}:${PORT}`); +}); diff --git a/examples/multipart-upload-wrapper-example/ui.js b/examples/multipart-upload-wrapper-example/ui.js new file mode 100644 index 00000000..aa2f5215 --- /dev/null +++ b/examples/multipart-upload-wrapper-example/ui.js @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2021 HERE Europe B.V. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * License-Filename: LICENSE + */ + +"use strict"; + +const appContainer = document.getElementById("app"); +const abortController = new AbortController(); + +// Create `UserAuth` and set it up to use our mocked server. +const userAuth = new UserAuth({ + tokenRequester: requestToken, + customUrl: "http://127.0.0.1:8080/oauth2/token", + credentials: { + accessKeyId: "mocked-access-key-id", + accessKeySecret: "mocked-access-key-secret", + }, +}); + +// Create `OlpClientSettings` and set it up to use our mocked server. +const settings = new OlpClientSettings({ + environment: "http://127.0.0.1:8080/api/lookup", + getToken: () => userAuth.getToken(), +}); + +/** + * ****** Example of using HERE Data SDK for TypeScript in plain JS ******* + */ +async function upload(file) { + // Set up and draw a progress bar. + const progress = document.createElement("div"); + const progressBar = document.createElement("progress"); + appContainer.appendChild(progressBar); + appContainer.appendChild(progress); + + // Set up the mocked data. + const catalogHrn = "hrn:here:data::mocked:catalog"; + const layerId = "mocked-layer"; + const datahandle = "mocked-datahandle"; + const contentType = "text/plain"; + + /** + * Create the `DataStoreRequest` builder. + * You need it to make requests to the Publish API. + */ + const publishRequestBuilder = await RequestFactory.create( + "publish", + "v2", + settings, + HRN.fromString(catalogHrn), + abortController.signal + ); + + // Initialize the new publication by sending a request to the Publish API. + const publication = await PublishApi.initPublication(publishRequestBuilder, { + body: { + layerIds: [layerId], + }, + }); + + /** + * Set up callbacks to subscribe to the uploading progress and progress bar drawing. + */ + let totalFileSize = 0; + let uploadedSize = 0; + + const onStart = (event) => { + totalFileSize = event.dataSize; + progressBar.setAttribute("max", `${totalFileSize}`); + progress.innerHTML = `Processing...`; + }; + + const onStatus = (status) => { + uploadedSize += status.chunkSize; + progressBar.setAttribute("value", `${uploadedSize}`); + progress.innerHTML = `Uploaded to Blob V1 ${uploadedSize} of ${totalFileSize} bytes. + Chunks ${status.uploadedChunks} of ${status.totalChunks}`; + }; + + // Initialize `MultiPartUploadWrapper`. + const wrapper = new MultiPartUploadWrapper( + { + blobVersion: "v1", + catalogHrn, + contentType, + handle: datahandle, + layerId, + onStart, + onStatus, + }, + settings + ); + + // Upload the file. + await wrapper.upload(file, abortController.signal); + + // Upload metadata of a new partition by sending a request to the Publish API. + await PublishApi.uploadPartitions(publishRequestBuilder, { + layerId, + publicationId: publication.id, + body: { + partitions: [ + { + partition: file.name, + dataHandle: datahandle, + dataSize: totalFileSize, + }, + ], + }, + }); + + // Submit the new publication by sending a request to the Publish API. + await PublishApi.submitPublication(publishRequestBuilder, { + publicationId: publication.id, + }); + + progress.innerHTML += "\nDone!"; +} + +/** + * ************** UI ****************** + */ +const abortButton = document.getElementById("abort"); +abortButton.onclick = () => { + abortController.abort(); +}; + +const label = document.createElement("label"); +label.innerHTML = "Publish to Blob V1: "; +label.setAttribute("for", "uploadToBlobV1"); +appContainer.appendChild(label); + +const input = document.createElement("input"); +input.setAttribute("type", "file"); +input.setAttribute("name", "uploadToBlobV1"); +input.setAttribute("onchange", "upload(this.files[0])"); +appContainer.appendChild(input);