v0.12.0 (#77)

ArweaveTeam · Jun 5, 2021 · f6d5d37 · f6d5d37
1 parent bbcec1e
commit f6d5d37
Show file tree

Hide file tree

Showing 9 changed files with 74 additions and 26 deletions.
diff --git a/.env.dev b/.env.dev
@@ -19,4 +19,6 @@ INDICES=["App-Name", "app", "domain", "namespace"]
 
 CACHING=1
 CACHE_FOLDER=/gateway/cache
-CACHE_OFFSET=0
+CACHE_OFFSET=0
+
+MANIFEST_PREFIX=http://localhost:3000
diff --git a/.env.docker b/.env.docker
@@ -19,4 +19,6 @@ INDICES=["App-Name", "app", "domain", "namespace"]
 
 CACHING=1
 CACHE_FOLDER=/gateway/cache
-CACHE_OFFSET=0
+CACHE_OFFSET=0
+
+MANIFEST_PREFIX=https://gateway.amplify.host
diff --git a/README.md b/README.md
@@ -47,6 +47,8 @@ INDICES=["App-Name", "app", "domain", "namespace"]
 CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
+
+MANIFEST_PREFIX=https://gateway.amplify.host
 ```
 
 Make sure you copy this configuration to `.env`.

diff --git a/docs/DEV.md b/docs/DEV.md
@@ -67,6 +67,8 @@ INDICES=["App-Name", "app", "domain", "namespace"]
 CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
+
+MANIFEST_PREFIX=https://gateway.amplify.host
 ```
 
 Make sure you copy this configuration to `.env`.

diff --git a/docs/README.md b/docs/README.md
@@ -50,6 +50,8 @@ INDICES=["App-Name", "app", "domain", "namespace"]
 CACHING=1
 CACHE_FOLDER=/gateway/cache
 CACHE_OFFSET=0
+
+MANIFEST_PREFIX=https://gateway.amplify.host
 ```
 
 Make sure you copy this configuration to `.env`.

diff --git a/src/Gateway.ts b/src/Gateway.ts
@@ -10,7 +10,7 @@ import {graphServer} from './graphql/server.graphql';
 import {statusRoute} from './route/status.route';
 import {syncRoute} from './route/sync.route';
 import {proxyRoute} from './route/proxy.route';
-import {dataRouteRegex, dataHeadRoute, dataRoute} from './route/data.route';
+import {dataRouteRegex, dataRoute} from './route/data.route';
 import {peerRoute} from './route/peer.route';
 import {koiLogger, koiLogsRoute, koiLogsRawRoute} from './route/koi.route';
 import {startSync} from './database/sync.database';
@@ -32,7 +32,6 @@ export function start() {
   app.get('/', statusRoute);
   app.get('/status', syncRoute);
 
-  app.head(dataRouteRegex, dataHeadRoute);
   app.get(dataRouteRegex, dataRoute);
 
   graphServer({introspection: true, playground: true}).applyMiddleware({app, path: '/graphql'});

diff --git a/src/database/sync.database.ts b/src/database/sync.database.ts
@@ -29,6 +29,7 @@ export let SIGKILL: boolean = false;
 export let bar: ProgressBar;
 export let topHeight = 0;
 export let currentHeight = 0;
+export let timer = setTimeout(() => {}, 0);
 
 export function configureSyncBar(start: number, end: number) {
   bar = new ProgressBar(
@@ -75,6 +76,12 @@ export async function startSync() {
 }
 
 export async function parallelize(height: number) {
+  clearTimeout(timer);
+  timer = setTimeout(() => {
+    log.info('[database] sync timed out, restarting server');
+    process.exit();
+  }, 300 * 1000);
+
   currentHeight = height;
 
   if (height >= topHeight) {
@@ -83,8 +90,9 @@ export async function parallelize(height: number) {
     const nodeInfo = await getNodeInfo();
     if (nodeInfo.height > topHeight) {
       log.info(`[database] updated height from ${topHeight} to ${nodeInfo.height} syncing new blocks`);
+      topHeight = nodeInfo.height;
     }
-    topHeight = nodeInfo.height;
+
     await parallelize(height);
   } else {
     const batch = [];
@@ -149,7 +157,7 @@ export async function storeBlock(height: number, retry: number = 0) {
     }
   } catch (error) {
     if (SIGKILL === false) {
-      if (retry >= 3) {
+      if (retry >= 25) {
         log.info(`[snapshot] there were problems retrieving ${height}, restarting the server`);
         process.exit();
       } else {

diff --git a/src/route/data.route.ts b/src/route/data.route.ts
@@ -1,13 +1,18 @@
-import {exists} from 'fs-jetpack';
+import {config} from 'dotenv';
+import {read, exists} from 'fs-jetpack';
 import {Request, Response} from 'express';
-import {stringToBip39, stringToHash} from '../utility/bip39.utility';
+import {ManifestV1} from '../types/manifest.types';
+import {log} from '../utility/log.utility';
 import {transaction as getTransaction, tagValue} from '../query/transaction.query';
 import {cacheFolder, cacheFile, cacheAnsFile} from '../caching/file.caching';
 
+config();
 
 export const dataRouteRegex = /^\/?([a-zA-Z0-9-_]{43})\/?$|^\/?([a-zA-Z0-9-_]{43})\/(.*)$/i;
 export const pathRegex = /^\/?([a-z0-9-_]{43})/i;
 
+export const manifestPrefix = process.env.MANIFEST_PREFIX || 'https://gateway.amplify.host';
+
 export async function dataHeadRoute(req: Request, res: Response) {
   const path = req.path.match(pathRegex) || [];
   const transaction = path.length > 1 ? path[1] : '';
@@ -23,36 +28,54 @@ export async function dataHeadRoute(req: Request, res: Response) {
 export async function dataRoute(req: Request, res: Response) {
   const path = req.path.match(pathRegex) || [];
   const transaction = path.length > 1 ? path[1] : '';
-  const hostname = req.hostname;
-
-  if (hostname !== 'localhost' && process.env.MANIFESTS === '1') {
-    const subdomain = process.env.BIP39 === '1' ? stringToBip39(transaction) : stringToHash(transaction);
-
-    if (hostname.indexOf(subdomain) === -1) {
-      return res.redirect(308, `http://${subdomain}.${hostname}/${transaction}`);
-    }
-  }
 
   try {
     const metadata = await getTransaction(transaction);
     const contentType = tagValue(metadata.tags, 'Content-Type');
     const ans102 = tagValue(metadata.tags, 'Bundle-Type') === 'ANS-102';
 
-    res.setHeader('content-type', contentType);
-
     if (ans102) {
       await cacheAnsFile(transaction);
     } else {
       await cacheFile(transaction);
-    }
-  } catch (error) {
 
-  }
+      if (exists(`${cacheFolder}/${transaction}`)) {
+        if (contentType === 'application/x.arweave-manifest+json') {
+          res.setHeader('content-type', 'text/html');
 
-  if (exists(`${cacheFolder}/${transaction}`)) {
-    res.status(200);
-    res.sendFile(`${cacheFolder}/${transaction}`);
-  } else {
+          const manifestFile = read(`${cacheFolder}/${transaction}`) || '{}';
+          const manifest: ManifestV1 = JSON.parse(manifestFile.toString());
+
+          const manifestIndex = manifest.index.path;
+          const manifestIndexTransaction = manifest.paths[manifestIndex].id;
+
+          const cachePaths = Object.keys(manifest.paths).map((key) => cacheFile(manifest.paths[key].id));
+          await Promise.all(cachePaths);
+
+          if (exists(`${cacheFolder}/${manifestIndexTransaction}`)) {
+            let manifestHtml = read(`${cacheFolder}/${manifestIndexTransaction}`) || '';
+
+            Object.keys(manifest.paths).map((key) => {
+              const id = manifest.paths[key].id;
+              manifestHtml = manifestHtml.split(key).join(`${manifestPrefix}/${id}`);
+            });
+
+            res.status(200);
+            res.send(manifestHtml);
+          } else {
+            throw new Error('Could not parse manifest html file');
+          }
+        } else {
+          res.setHeader('content-type', contentType);
+
+          res.status(200);
+          res.sendFile(`${cacheFolder}/${transaction}`);
+        }
+      }
+    }
+  } catch (error) {
+    log.error(`[route] error generating response for ${transaction}`);
+    console.error(error);
     res.status(500);
     res.json({status: 'ERROR', message: 'Could not retrieve transaction'});
   }

diff --git a/src/types/manifest.types.ts b/src/types/manifest.types.ts
@@ -0,0 +1,8 @@
+export interface ManifestV1 {
+    manifest: string;
+    version: string;
+    index: {
+        path: string;
+    };
+    paths: any;
+}