Skip to content

Commit

Permalink
Merge pull request #62 from tkiapril/chore/remove-dataproxy
Browse files Browse the repository at this point in the history
Remove dataproxy
  • Loading branch information
tkiapril authored Aug 18, 2023
2 parents 98f1380 + 4acb196 commit 39c5c66
Show file tree
Hide file tree
Showing 27 changed files with 80 additions and 5,207 deletions.
25 changes: 2 additions & 23 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,11 @@
**/.env
/prisma/dev.db
/prisma/dev.db-journal
/prisma/dev.db-shm
/prisma/dev.db-wal
**/.vscode
/dev.crt
/dev.key
/dataproxy/schema.prisma
/generated
/docker-compose-data
.DS_Store

# Adapted from /dataproxy/.gitignore
**/.yarn/*
!**/.yarn/patches
!**/.yarn/plugins
!**/.yarn/releases
!**/.yarn/versions

**/coverage
**/coverage.json

# misc
**/.DS_Store

# debug
**/npm-debug.log*
**/yarn-debug.log*
**/yarn-error.log*
**/.pnpm-debug.log*

# typescript
**/*.tsbuildinfo
2 changes: 0 additions & 2 deletions .env.defaults
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
# default env file, to prevent Deno dotenv from complaining for optional values

DATABASE_URL="('DATABASE_URL'_was_not_provided)"
DIRECT_URL="('DIRECT_URL'_was_not_provided)"
CHAIN_DEFINITION_URL="('CHAIN_DEFINITION_URL'_was_not_provided)"
AMQP_BROKER_URL="('AMQP_BROKER_URL'_was_not_provided)"
API_URL="('API_URL'_was_not_provided)"
API_BEHIND_REVERSE_PROXY="false"
WEBUI_URL="('WEBUI_URL'_was_not_provided)"
BLOCK_FINALITY="('BLOCK_FINALITY'_was_not_provided)"
DATAPROXY_INTERNAL_PORT="0"
LOG_LEVEL="WARNING"
1 change: 0 additions & 1 deletion .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
# See the documentation for all the connection string options: https://pris.ly/d/connection-strings

DATABASE_URL="prisma://localhost:8088?api_key=x5aol207TyRdhHuanAXlpTgVWDcK8jm2rVKJOQGbwaMqfvZbmGJnIbwBjUxYaBrFx1XOfyF4" # arbitrary api key
DIRECT_URL="file:./dev.db"
CHAIN_DEFINITION_URL="file:./sample/chainDefinitions/goerli.json"
AMQP_BROKER_URL="amqp://localhost"
API_URL="http://localhost:8000"
Expand Down
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ node_modules
.env
/prisma/dev.db
/prisma/dev.db-journal
/prisma/dev.db-shm
/prisma/dev.db-wal
/.vscode/launch.json
/dev.crt
/dev.key
/dataproxy/schema.prisma
/generated
/docker-compose-data
.DS_Store
19 changes: 1 addition & 18 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,19 @@ RUN apk add npm && \
> prisma/schema.prisma.new && \
mv prisma/schema.prisma.new prisma/schema.prisma && \
deno task prisma format && \
deno task prisma-generate
deno task prisma generate

FROM common as observer
RUN deno cache observer.ts
# --unsafely-ignore-certificate-errors should be included to use data proxy
ENTRYPOINT [ "deno", "run", "--allow-env", "--allow-read", "--allow-net", "--allow-ffi", "observer.ts" ]

FROM common as emitter
RUN deno cache emitter.ts
# --unsafely-ignore-certificate-errors should be included to use data proxy
ENTRYPOINT [ "deno", "run", "--allow-env", "--allow-read", "--allow-net", "--allow-ffi", "emitter.ts" ]

FROM common as api
ENV API_URL="http://0.0.0.0:80"
EXPOSE 80
# --unsafely-ignore-certificate-errors should be included to use data proxy
ENTRYPOINT [ "deno", "run", "--allow-env", "--allow-read", "--allow-net", "--allow-ffi", "api.ts" ]

FROM common as web
Expand All @@ -38,17 +35,3 @@ RUN apk add git && \
ENV WEBUI_URL="http://0.0.0.0:80"
EXPOSE 80
ENTRYPOINT [ "deno", "run", "--allow-read", "--allow-env", "--allow-sys", "--allow-run", "../scripts/run-with-env.ts", "deno", "run", "--allow-env", "--allow-read", "--allow-write", "--allow-net", "--allow-run", "--allow-ffi", "main.ts" ]

FROM alpine:3.18.2 as dataproxy-builder
WORKDIR /Corvette
RUN apk add openssl && \
openssl req -x509 -nodes -days 3650 -subj "/CN=localhost" -newkey rsa:4096 -keyout dev.key -out dev.crt

FROM common as dataproxy
COPY --from=dataproxy-builder /Corvette/dev.key /Corvette/dev.crt /Corvette/
RUN awk '/datasource[[:space:]]+[^[:space:]]+[[:space:]]*\{/{ print; print "directUrl = env(\"DIRECT_URL\")" }' \
prisma/schema.prisma > prisma/schema.prisma.new && \
mv prisma/schema.prisma.new prisma/schema.prisma && \
deno run --allow-env --allow-read --allow-write --allow-run dataproxy.ts generate
EXPOSE 8088
ENTRYPOINT [ "deno", "run", "--allow-env", "--allow-read", "--allow-write", "--allow-net", "--allow-run", "--unsafely-ignore-certificate-errors=localhost", "dataproxy.ts" ]
39 changes: 10 additions & 29 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,8 @@ To run Corvette, you must have Deno and Node.js with corepack enabled installed
Before you can run the components, you must apply the database migrations to
your DB. Adjust the database provider in `prisma/schema.prisma`, copy the
configuration from the example `.env.example` to `.env`, and configure
`DATABASE_URL` and `DIRECT_URL` accordingly. For SQLite, leave the DATABASE_URL
as is. For PostgreSQL, provide the same values to the `DATABASE_URL` and
`DIRECT_URL`. Only SQLite (for development only) and PostgreSQL has been tested
at the moment.
`DATABASE_URL` accordingly. Only SQLite (for development only) and PostgreSQL
has been tested at the moment.

After database configuration, run `deno task prisma db push`. Your database
will be populated with required tables.
Expand All @@ -43,18 +41,12 @@ A convenience script (`app.ts`) for running all the components (except for web
at the moment) is provided, along with a deno task (`deno task serve`) that
runs this script with required permissions. After applying the database
migrations and configuring `.env`, start the components with `deno task serve`.
The convenience script will also launch an instance of embedded Prisma data
proxy if using SQLite as the DB, to mitigate an unconfirmed bug in Prisma where
the client will crash intermittently if the database is accessed frequently
(probably due to improper lock handling.) The database will automatically be
connected with the default configuration. The script will also launch an
instance of an embedded AMQP broker (https://deno.land/x/lop/mod.ts) and a
webhook receiver for testing (`testWebhookReceiver.ts`) on http://localhost:8888.

The web component can be started with `pushd web; deno task start; popd` (hot
reload) or
`deno run -A --unsafely-ignore-certificate-errors=localhost web/main.ts` (flag
required for SQLite as DB, see remarks below in [Running components independently & for production](#Running-components-independently-&-for-production).)
The database will automatically be connected with the default configuration.
The script will also launch an instance of an embedded AMQP broker
(https://deno.land/x/lop/mod.ts) and a webhook receiver for testing
(`testWebhookReceiver.ts`) on http://localhost:8888.

The web component can be started with `deno task serve-web`.

The components will be started using the configuration defined in `.env` file
by default. The configuration may also be overrided by providing the
Expand All @@ -72,19 +64,8 @@ be run simultaneously to achieve failsafe and/or load balancing.

Before running the components, you must generate the Prisma client. You must do
this each time you change the `prisma/schema.prisma` file or Prisma version.
Generate the client with `deno task prisma-generate`. Prisma client specific to
Prisma data proxy must be generated if you are using SQLite as the database
(not recommended for production,) and the aforementioned task will detect the
type of the database and generate the appropriate type of the client for you.

You must also prepare the database server and configure the components
accordingly. In case you are using SQLite, you must run the data proxy by your
own. The data proxy requires TLS certificates, which can be generated with
`deno run -A generate-cert`. To use self-signed TLS certificates, you must
provide the following flag to the components (including the data proxy itself):
`--unsafely-ignore-certificate-errors=localhost`. You can run the data proxy
with the following command:
`deno run -A --unsafely-ignore-certificate-errors=localhost dataproxy.ts`.
Generate the client with `deno task prisma generate`. You must also prepare the
database and configure the components accordingly.

You also need an AMQP 0-9-1 compliant message broker (such as RabbitMQ.) For
testing, you may also use [lop](https://deno.land/x/lop/mod.ts), which is run
Expand Down
43 changes: 5 additions & 38 deletions app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,10 @@ import { parseOptions } from "amqp/src/amqp_connect_options.ts";
import { broker } from "https://deno.land/x/[email protected]/mod.ts";

import { api } from "./api.ts";
import { dataproxy, generateDataproxy } from "./dataproxy.ts";
import { emitter } from "./emitter.ts";
import { AmqpBrokerUrlEnvKey, combinedEnv } from "./envUtils.ts";
import {
ApiLoggerName,
DataproxyLoggerName,
defaultLogFormatter,
DevLoggerName,
EmitterLoggerName,
Expand All @@ -23,30 +21,24 @@ import {
WebLoggerName,
} from "./logUtils.ts";
import { observer } from "./observer.ts";
import { getSchemaPath, shouldUseDataproxy } from "./prismaSchemaUtils.ts";
import {
block,
type CleanupFunction,
runWithAmqp,
runWithChainDefinition,
runWithPrisma,
} from "./runHelpers.ts";
import { testWebhookReceiver } from "./testWebhookReceiver.ts";

async function prepareAndMain() {
await new Deno.Command("deno", {
args: ["task", "prisma-generate"],
const status = await new Deno.Command("deno", {
args: ["task", "prisma", "db", "push"],
stdout: "inherit",
stderr: "inherit",
}).spawn().status;
await new Deno.Command("deno", {
if (!status.success) Deno.exit(status.code);
new Deno.Command("deno", {
args: [
"run",
...(
await shouldUseDataproxy()
? ["--unsafely-ignore-certificate-errors=localhost"]
: []
),
"--allow-read",
"--allow-write",
"--allow-env",
Expand All @@ -62,7 +54,7 @@ async function prepareAndMain() {
cwd: Deno.cwd(),
uid: Deno.uid() !== null ? Deno.uid()! : undefined,
gid: Deno.gid() !== null ? Deno.gid()! : undefined,
}).spawn().status;
}).spawn().ref();
}

async function main() {
Expand Down Expand Up @@ -95,10 +87,6 @@ async function main() {
level: "DEBUG",
handlers: ["console"],
},
[DataproxyLoggerName]: {
level: "DEBUG",
handlers: ["console"],
},
[TestWebhookReceiverLoggerName]: {
level: "INFO",
handlers: ["console"],
Expand All @@ -112,24 +100,6 @@ async function main() {

const logger = getLogger(DevLoggerName);

let useDataproxy: boolean;
try {
useDataproxy = await shouldUseDataproxy();
} catch (e) {
logger.critical(
`Irrecoverable error, could not load ${
getSchemaPath({ useParams: true })
}.`,
);
throw e;
}
if (useDataproxy) {
logger.warning(
"Using data proxy, as sqlite is being used for the database.",
);
await generateDataproxy();
}

logger.debug(
`Serving AMQP Broker, URL: ${combinedEnv[AmqpBrokerUrlEnvKey]}.`,
);
Expand All @@ -138,8 +108,6 @@ async function main() {
hostname: amqpOptions.hostname,
port: amqpOptions.port,
});
let cleanupDataProxy: CleanupFunction | undefined;
if (useDataproxy) ({ cleanup: cleanupDataProxy } = await dataproxy());
try {
await runWithChainDefinition((chain) => ({
runningPromise: runWithPrisma(async (prisma) => {
Expand Down Expand Up @@ -176,7 +144,6 @@ async function main() {
}),
}));
} finally {
if (cleanupDataProxy !== undefined) await cleanupDataProxy();
abortBroker();
}
}
Expand Down
Loading

0 comments on commit 39c5c66

Please sign in to comment.