diff --git a/.env.dev b/.env.dev index 6b6f644..d30749e 100644 --- a/.env.dev +++ b/.env.dev @@ -19,4 +19,6 @@ INDICES=["App-Name", "app", "domain", "namespace"] CACHING=1 CACHE_FOLDER=/gateway/cache -CACHE_OFFSET=0 \ No newline at end of file +CACHE_OFFSET=0 + +MANIFEST_PREFIX=http://localhost:3000 \ No newline at end of file diff --git a/.env.docker b/.env.docker index 6dabdd2..c79f244 100644 --- a/.env.docker +++ b/.env.docker @@ -19,4 +19,6 @@ INDICES=["App-Name", "app", "domain", "namespace"] CACHING=1 CACHE_FOLDER=/gateway/cache -CACHE_OFFSET=0 \ No newline at end of file +CACHE_OFFSET=0 + +MANIFEST_PREFIX=https://gateway.amplify.host \ No newline at end of file diff --git a/README.md b/README.md index 0918f3a..abb383b 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ INDICES=["App-Name", "app", "domain", "namespace"] CACHING=1 CACHE_FOLDER=/gateway/cache CACHE_OFFSET=0 + +MANIFEST_PREFIX=https://gateway.amplify.host ``` Make sure you copy this configuration to `.env`. diff --git a/docs/DEV.md b/docs/DEV.md index c4e3e14..558845b 100755 --- a/docs/DEV.md +++ b/docs/DEV.md @@ -67,6 +67,8 @@ INDICES=["App-Name", "app", "domain", "namespace"] CACHING=1 CACHE_FOLDER=/gateway/cache CACHE_OFFSET=0 + +MANIFEST_PREFIX=https://gateway.amplify.host ``` Make sure you copy this configuration to `.env`. diff --git a/docs/README.md b/docs/README.md index b889a21..92cd187 100755 --- a/docs/README.md +++ b/docs/README.md @@ -50,6 +50,8 @@ INDICES=["App-Name", "app", "domain", "namespace"] CACHING=1 CACHE_FOLDER=/gateway/cache CACHE_OFFSET=0 + +MANIFEST_PREFIX=https://gateway.amplify.host ``` Make sure you copy this configuration to `.env`. diff --git a/src/Gateway.ts b/src/Gateway.ts index a00a77f..3e36a73 100755 --- a/src/Gateway.ts +++ b/src/Gateway.ts @@ -10,7 +10,7 @@ import {graphServer} from './graphql/server.graphql'; import {statusRoute} from './route/status.route'; import {syncRoute} from './route/sync.route'; import {proxyRoute} from './route/proxy.route'; -import {dataRouteRegex, dataHeadRoute, dataRoute} from './route/data.route'; +import {dataRouteRegex, dataRoute} from './route/data.route'; import {peerRoute} from './route/peer.route'; import {koiLogger, koiLogsRoute, koiLogsRawRoute} from './route/koi.route'; import {startSync} from './database/sync.database'; @@ -32,7 +32,6 @@ export function start() { app.get('/', statusRoute); app.get('/status', syncRoute); - app.head(dataRouteRegex, dataHeadRoute); app.get(dataRouteRegex, dataRoute); graphServer({introspection: true, playground: true}).applyMiddleware({app, path: '/graphql'}); diff --git a/src/database/sync.database.ts b/src/database/sync.database.ts index 401c6ac..0894037 100644 --- a/src/database/sync.database.ts +++ b/src/database/sync.database.ts @@ -29,6 +29,7 @@ export let SIGKILL: boolean = false; export let bar: ProgressBar; export let topHeight = 0; export let currentHeight = 0; +export let timer = setTimeout(() => {}, 0); export function configureSyncBar(start: number, end: number) { bar = new ProgressBar( @@ -75,6 +76,12 @@ export async function startSync() { } export async function parallelize(height: number) { + clearTimeout(timer); + timer = setTimeout(() => { + log.info('[database] sync timed out, restarting server'); + process.exit(); + }, 300 * 1000); + currentHeight = height; if (height >= topHeight) { @@ -83,8 +90,9 @@ export async function parallelize(height: number) { const nodeInfo = await getNodeInfo(); if (nodeInfo.height > topHeight) { log.info(`[database] updated height from ${topHeight} to ${nodeInfo.height} syncing new blocks`); + topHeight = nodeInfo.height; } - topHeight = nodeInfo.height; + await parallelize(height); } else { const batch = []; @@ -149,7 +157,7 @@ export async function storeBlock(height: number, retry: number = 0) { } } catch (error) { if (SIGKILL === false) { - if (retry >= 3) { + if (retry >= 25) { log.info(`[snapshot] there were problems retrieving ${height}, restarting the server`); process.exit(); } else { diff --git a/src/route/data.route.ts b/src/route/data.route.ts index e90c4d5..4f3b738 100644 --- a/src/route/data.route.ts +++ b/src/route/data.route.ts @@ -1,13 +1,18 @@ -import {exists} from 'fs-jetpack'; +import {config} from 'dotenv'; +import {read, exists} from 'fs-jetpack'; import {Request, Response} from 'express'; -import {stringToBip39, stringToHash} from '../utility/bip39.utility'; +import {ManifestV1} from '../types/manifest.types'; +import {log} from '../utility/log.utility'; import {transaction as getTransaction, tagValue} from '../query/transaction.query'; import {cacheFolder, cacheFile, cacheAnsFile} from '../caching/file.caching'; +config(); export const dataRouteRegex = /^\/?([a-zA-Z0-9-_]{43})\/?$|^\/?([a-zA-Z0-9-_]{43})\/(.*)$/i; export const pathRegex = /^\/?([a-z0-9-_]{43})/i; +export const manifestPrefix = process.env.MANIFEST_PREFIX || 'https://gateway.amplify.host'; + export async function dataHeadRoute(req: Request, res: Response) { const path = req.path.match(pathRegex) || []; const transaction = path.length > 1 ? path[1] : ''; @@ -23,36 +28,54 @@ export async function dataHeadRoute(req: Request, res: Response) { export async function dataRoute(req: Request, res: Response) { const path = req.path.match(pathRegex) || []; const transaction = path.length > 1 ? path[1] : ''; - const hostname = req.hostname; - - if (hostname !== 'localhost' && process.env.MANIFESTS === '1') { - const subdomain = process.env.BIP39 === '1' ? stringToBip39(transaction) : stringToHash(transaction); - - if (hostname.indexOf(subdomain) === -1) { - return res.redirect(308, `http://${subdomain}.${hostname}/${transaction}`); - } - } try { const metadata = await getTransaction(transaction); const contentType = tagValue(metadata.tags, 'Content-Type'); const ans102 = tagValue(metadata.tags, 'Bundle-Type') === 'ANS-102'; - res.setHeader('content-type', contentType); - if (ans102) { await cacheAnsFile(transaction); } else { await cacheFile(transaction); - } - } catch (error) { - } + if (exists(`${cacheFolder}/${transaction}`)) { + if (contentType === 'application/x.arweave-manifest+json') { + res.setHeader('content-type', 'text/html'); - if (exists(`${cacheFolder}/${transaction}`)) { - res.status(200); - res.sendFile(`${cacheFolder}/${transaction}`); - } else { + const manifestFile = read(`${cacheFolder}/${transaction}`) || '{}'; + const manifest: ManifestV1 = JSON.parse(manifestFile.toString()); + + const manifestIndex = manifest.index.path; + const manifestIndexTransaction = manifest.paths[manifestIndex].id; + + const cachePaths = Object.keys(manifest.paths).map((key) => cacheFile(manifest.paths[key].id)); + await Promise.all(cachePaths); + + if (exists(`${cacheFolder}/${manifestIndexTransaction}`)) { + let manifestHtml = read(`${cacheFolder}/${manifestIndexTransaction}`) || ''; + + Object.keys(manifest.paths).map((key) => { + const id = manifest.paths[key].id; + manifestHtml = manifestHtml.split(key).join(`${manifestPrefix}/${id}`); + }); + + res.status(200); + res.send(manifestHtml); + } else { + throw new Error('Could not parse manifest html file'); + } + } else { + res.setHeader('content-type', contentType); + + res.status(200); + res.sendFile(`${cacheFolder}/${transaction}`); + } + } + } + } catch (error) { + log.error(`[route] error generating response for ${transaction}`); + console.error(error); res.status(500); res.json({status: 'ERROR', message: 'Could not retrieve transaction'}); } diff --git a/src/types/manifest.types.ts b/src/types/manifest.types.ts new file mode 100644 index 0000000..3e2de09 --- /dev/null +++ b/src/types/manifest.types.ts @@ -0,0 +1,8 @@ +export interface ManifestV1 { + manifest: string; + version: string; + index: { + path: string; + }; + paths: any; +}