An example web app. (#460)

An example web app shows how to use Here Data SDK for TypeScript to upload and publish large data in a browser. Resolves: OLPEDGE-2501 Relates-To: OLPEDGE-2456 Signed-off-by: Oleksii Zubko <[email protected]>
heremaps · May 14, 2021 · 99c3031 · 99c3031
1 parent 2de0e17
commit 99c3031
Show file tree

Hide file tree

Showing 4 changed files with 324 additions and 0 deletions.
diff --git a/examples/README.md b/examples/README.md
@@ -5,3 +5,4 @@ HERE Data SDK for TypeScript contains several examples that demonstrate some of
 - [Authorization example](authorization-example/README.md) – build the Data SDK using webpack and work with the APIs from `@here/olp-sdk-dataservice-api`. You can use this example application to find the list of groups that you have access to as well as create groups.
 - [React App example](react-app-example/README.md) – use the Data SDK and React in a browser and learn how to work with the following modules: `olp-sdk-authentication`, `olp-sdk-dataservice-read`, and `olp-sdk-dataservice-write`.
 - [Node.js example](nodejs-example/README.md) – work with the Data SDK in Node.js and learn how to get data from versioned layers and save it to files.
+- [MultiPartUploadWrapper example](multipart-upload-wrapper-example/README.md) – use the Data SDK to upload and publish a large amount of data in a browser.
diff --git a/examples/multipart-upload-wrapper-example/README.md b/examples/multipart-upload-wrapper-example/README.md
@@ -0,0 +1,16 @@
+# Publish large data files in a browser
+
+This example app uses an HTML and JS UI and an HTTP server with mocked routes to simulate the real `DataStore` class.
+You can use this app to learn how to work with the `MultipartUploadWrapper` class and publish large files of more than 8 GB to a versioned layer without loading them in memory.
+
+## Setup
+
+This example does not need any setup. Nevertheless, make sure you installed Node.js.
+
+## Run
+
+1. Start the server.
+   ```
+   node server.js
+   ```
+2. In your favorite browser, open `http://localhost:8080/`.
diff --git a/examples/multipart-upload-wrapper-example/server.js b/examples/multipart-upload-wrapper-example/server.js
@@ -0,0 +1,155 @@
+/*
+ * Copyright (C) 2021 HERE Europe B.V.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+"use strict";
+
+const http = require("http");
+const fs = require("fs");
+
+const HOST = "127.0.0.1";
+const PORT = 8080;
+const RESPONSE_DELAY_MS = 300;
+
+const routing = {
+  "/": `<!DOCTYPE html>
+  <html lang="en">
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta name="title" content="MultiPartUpload example webapp">
+  <meta name=”description” content="MultiPartUpload example webapp">
+  <body>
+      <button id="abort">Abort</button>
+      <div id="app"></div>
+      <script src="https://unpkg.com/@here/olp-sdk-core/bundle.umd.min.js"></script> 
+      <script src="https://unpkg.com/@here/olp-sdk-authentication/bundle.umd.min.js"></script> 
+      <script src="https://unpkg.com/@here/olp-sdk-dataservice-api/bundle.umd.min.js"></script> 
+      <script src="https://unpkg.com/@here/olp-sdk-dataservice-write/bundle.umd.min.js"></script> 
+      <script src="ui.js"></script>
+  </body>
+  </html>`,
+  "/ui.js": fs.readFileSync("./ui.js").toString(),
+  "/oauth2/token": () => ({
+    accessToken: "mocked-access-token",
+    tokenType: "bearer",
+    expiresIn: 99999999,
+  }),
+  "/api/lookup/resources/hrn:here:data::mocked:catalog/apis": () => [
+    {
+      api: "blob",
+      version: "v1",
+      baseURL: `http://${HOST}:${PORT}/blobstore/v1`,
+      parameters: {},
+    },
+    {
+      api: "publish",
+      version: "v2",
+      baseURL: `http://${HOST}:${PORT}/publish/v2`,
+      parameters: {},
+    },
+  ],
+  "/publish/v2/publications": () => ({
+    catalogId: "catalog",
+    catalogVersion: 999,
+    details: {
+      expires: 9999999999999,
+      message: "",
+      modified: 9999999999999,
+      started: 9999999999999,
+      state: "initialized",
+    },
+    id: "mocked-publication-id",
+    layerIds: ["mocked-layer"],
+    versionDependencies: [],
+  }),
+  "/blobstore/v1/layers/mocked-layer/data/mocked-datahandle/multiparts": () => ({
+    links: {
+      status: {
+        href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token`,
+        method: "GET",
+      },
+      delete: {
+        href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token`,
+        method: "DELETE",
+      },
+      uploadPart: {
+        href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token/parts`,
+        method: "POST",
+      },
+      complete: {
+        href: `http://${HOST}:${PORT}/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token`,
+        method: "PUT",
+      },
+    },
+  }),
+  "/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token/parts": (
+    req,
+    res
+  ) => {
+    res.setHeader("ETag", Math.random() * 10000);
+    res.setHeader("Access-Control-Expose-Headers", "ETag");
+    return {
+      status: res.statusCode,
+    };
+  },
+  "/blobstore/v1/catalogs/hrn:here:data::mocked:catalog/layers/mocked-layer/data/mocked-datahandle/multiparts/mocked-blob-token": () => ({
+    ok: true,
+  }),
+  "/publish/v2/layers/mocked-layer/publications/mocked-publication-id/partitions": () => ({
+    ok: true,
+  }),
+  "/publish/v2/publications/mocked-publication-id": () => ({
+    ok: true,
+  }),
+};
+
+const types = {
+  object: JSON.stringify,
+  string: (s) => s,
+  number: (val) => `${val}`,
+  undefined: () => "Not Found",
+  function: (fn, req, res) => JSON.stringify(fn(req, res)),
+};
+
+const server = http.createServer((req, res) => {
+  console.dir({ type: "Reguest", method: req.method, url: req.url });
+  const queryIndex = req.url.indexOf("?");
+  const adaptedUrl =
+    queryIndex !== -1 ? req.url.substr(0, queryIndex) : req.url;
+
+  const data = routing[adaptedUrl];
+  const type = typeof data;
+  const serialiser = types[type];
+  const result = serialiser(data, req, res);
+
+  res.setHeader("Access-Control-Allow-Origin", "*");
+  res.setHeader(
+    "Access-Control-Allow-Headers",
+    "authorization,content-type, ETag"
+  );
+  res.setHeader("Access-Control-Allow-Methods", "*");
+
+  res.statusCode = type === "undefined" ? 404 : 200;
+  setTimeout(() => {
+    res.end(result);
+  }, RESPONSE_DELAY_MS);
+});
+
+server.listen(PORT, HOST, () => {
+  console.log(`Server running at http://${HOST}:${PORT}`);
+});
diff --git a/examples/multipart-upload-wrapper-example/ui.js b/examples/multipart-upload-wrapper-example/ui.js
@@ -0,0 +1,152 @@
+/*
+ * Copyright (C) 2021 HERE Europe B.V.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * License-Filename: LICENSE
+ */
+
+"use strict";
+
+const appContainer = document.getElementById("app");
+const abortController = new AbortController();
+
+// Create `UserAuth` and set it up to use our mocked server.
+const userAuth = new UserAuth({
+  tokenRequester: requestToken,
+  customUrl: "http://127.0.0.1:8080/oauth2/token",
+  credentials: {
+    accessKeyId: "mocked-access-key-id",
+    accessKeySecret: "mocked-access-key-secret",
+  },
+});
+
+// Create `OlpClientSettings` and set it up to use our mocked server.
+const settings = new OlpClientSettings({
+  environment: "http://127.0.0.1:8080/api/lookup",
+  getToken: () => userAuth.getToken(),
+});
+
+/**
+ * ****** Example of using HERE Data SDK for TypeScript in plain JS *******
+ */
+async function upload(file) {
+  // Set up and draw a progress bar.
+  const progress = document.createElement("div");
+  const progressBar = document.createElement("progress");
+  appContainer.appendChild(progressBar);
+  appContainer.appendChild(progress);
+
+  // Set up the mocked data.
+  const catalogHrn = "hrn:here:data::mocked:catalog";
+  const layerId = "mocked-layer";
+  const datahandle = "mocked-datahandle";
+  const contentType = "text/plain";
+
+  /**
+   * Create the `DataStoreRequest` builder.
+   * You need it to make requests to the Publish API.
+   */
+  const publishRequestBuilder = await RequestFactory.create(
+    "publish",
+    "v2",
+    settings,
+    HRN.fromString(catalogHrn),
+    abortController.signal
+  );
+
+  // Initialize the new publication by sending a request to the Publish API.
+  const publication = await PublishApi.initPublication(publishRequestBuilder, {
+    body: {
+      layerIds: [layerId],
+    },
+  });
+
+  /**
+   * Set up callbacks to subscribe to the uploading progress and progress bar drawing.
+   */
+  let totalFileSize = 0;
+  let uploadedSize = 0;
+
+  const onStart = (event) => {
+    totalFileSize = event.dataSize;
+    progressBar.setAttribute("max", `${totalFileSize}`);
+    progress.innerHTML = `Processing...`;
+  };
+
+  const onStatus = (status) => {
+    uploadedSize += status.chunkSize;
+    progressBar.setAttribute("value", `${uploadedSize}`);
+    progress.innerHTML = `Uploaded to Blob V1 ${uploadedSize} of ${totalFileSize} bytes.
+    Chunks ${status.uploadedChunks} of ${status.totalChunks}`;
+  };
+
+  // Initialize `MultiPartUploadWrapper`.
+  const wrapper = new MultiPartUploadWrapper(
+    {
+      blobVersion: "v1",
+      catalogHrn,
+      contentType,
+      handle: datahandle,
+      layerId,
+      onStart,
+      onStatus,
+    },
+    settings
+  );
+
+  // Upload the file.
+  await wrapper.upload(file, abortController.signal);
+
+  // Upload metadata of a new partition by sending a request to the Publish API.
+  await PublishApi.uploadPartitions(publishRequestBuilder, {
+    layerId,
+    publicationId: publication.id,
+    body: {
+      partitions: [
+        {
+          partition: file.name,
+          dataHandle: datahandle,
+          dataSize: totalFileSize,
+        },
+      ],
+    },
+  });
+
+  // Submit the new publication by sending a request to the Publish API.
+  await PublishApi.submitPublication(publishRequestBuilder, {
+    publicationId: publication.id,
+  });
+
+  progress.innerHTML += "\nDone!";
+}
+
+/**
+ * ************** UI ******************
+ */
+const abortButton = document.getElementById("abort");
+abortButton.onclick = () => {
+  abortController.abort();
+};
+
+const label = document.createElement("label");
+label.innerHTML = "Publish to Blob V1: ";
+label.setAttribute("for", "uploadToBlobV1");
+appContainer.appendChild(label);
+
+const input = document.createElement("input");
+input.setAttribute("type", "file");
+input.setAttribute("name", "uploadToBlobV1");
+input.setAttribute("onchange", "upload(this.files[0])");
+appContainer.appendChild(input);