From dc1cda007e6143c413c23a32e7dd614be643c63c Mon Sep 17 00:00:00 2001 From: matea16 Date: Fri, 4 Aug 2023 09:41:52 +0200 Subject: [PATCH 1/2] Add Memgraph integration SDK CLI --- .../integration-sdk-cli/src/commands/index.ts | 1 + .../src/commands/memgraph.ts | 88 +++++++++ packages/integration-sdk-cli/src/index.ts | 2 + .../src/memgraph/README.md | 30 +++ .../integration-sdk-cli/src/memgraph/index.ts | 2 + .../src/memgraph/memgraphGraphStore.ts | 181 ++++++++++++++++++ .../src/memgraph/memgraphUtilities.ts | 79 ++++++++ .../src/memgraph/uploadToMemgraph.ts | 64 +++++++ .../src/memgraph/wipeMemgraph.ts | 68 +++++++ 9 files changed, 515 insertions(+) create mode 100644 packages/integration-sdk-cli/src/commands/memgraph.ts create mode 100644 packages/integration-sdk-cli/src/memgraph/README.md create mode 100644 packages/integration-sdk-cli/src/memgraph/index.ts create mode 100644 packages/integration-sdk-cli/src/memgraph/memgraphGraphStore.ts create mode 100644 packages/integration-sdk-cli/src/memgraph/memgraphUtilities.ts create mode 100644 packages/integration-sdk-cli/src/memgraph/uploadToMemgraph.ts create mode 100644 packages/integration-sdk-cli/src/memgraph/wipeMemgraph.ts diff --git a/packages/integration-sdk-cli/src/commands/index.ts b/packages/integration-sdk-cli/src/commands/index.ts index e351e643b..8b364b9a1 100644 --- a/packages/integration-sdk-cli/src/commands/index.ts +++ b/packages/integration-sdk-cli/src/commands/index.ts @@ -6,6 +6,7 @@ export * from './run'; export * from './document'; export * from './visualize-types'; export * from './validate-question-file'; +export * from './memgraph'; export * from './neo4j'; export * from './visualize-dependencies'; export * from './generate-integration-graph-schema'; diff --git a/packages/integration-sdk-cli/src/commands/memgraph.ts b/packages/integration-sdk-cli/src/commands/memgraph.ts new file mode 100644 index 000000000..c4c48baa9 --- /dev/null +++ b/packages/integration-sdk-cli/src/commands/memgraph.ts @@ -0,0 +1,88 @@ +import * as commander from 'commander'; +import path from 'path'; + +import dotenv from 'dotenv'; +import dotenvExpand from 'dotenv-expand'; + +import * as log from '../log'; +import { uploadToMemgraph, wipeMemgraphByID, wipeAllMemgraph } from '../memgraph'; + +export function memgraph() { + dotenvExpand(dotenv.config()); + + const program = new commander.Command(); + program.description( + `Suite of memgraph commands. Options are currently 'memgraph push', 'memgraph wipe', and 'memgraph wipe-all'`, + ); + const memgraphCommand = program.command('memgraph'); + memgraphCommand + .command('push') + .description('upload collected entities and relationships to local Memgraph') + .option( + '-d, --data-dir ', + 'path to collected entities and relationships', + path.resolve(process.cwd(), '.j1-integration'), + ) + .option( + '-i, --integration-instance-id ', + '_integrationInstanceId assigned to uploaded entities', + 'defaultLocalInstanceID', + ) + .option( + '-db, --database-name ', + 'optional database to push data to (only available for enterprise Memgraph databases)', + 'memgraph', + ) + .action(async (options) => { + log.info(`Beginning data upload to local Memgraph instance`); + // Point `fileSystem.ts` functions to expected location relative to + // integration project path. + const finalDir = path.resolve(process.cwd(), options.dataDir); + process.env.JUPITERONE_INTEGRATION_STORAGE_DIRECTORY = finalDir; + + await uploadToMemgraph({ + pathToData: finalDir, + integrationInstanceID: options.integrationInstanceId, + memgraphDatabase: options.databaseName, + }); + log.info(`Data uploaded to local Memgraph instance`); + }); + + memgraphCommand + .command('wipe') + .description( + 'wipe entities and relationships for a given integrationInstanceID in the Memgraph database', + ) + .option( + '-i, --integration-instance-id ', + '_integrationInstanceId assigned to uploaded entities', + 'defaultLocalInstanceID', + ) + .option( + '-db, --database-name ', + 'optional database to wipe data from (only available for enterprise Memgraph databases)', + 'memgraph', + ) + .action(async (options) => { + await wipeMemgraphByID({ + integrationInstanceID: options.integrationInstanceId, + memgraphDatabase: options.databaseName, + }); + }); + + memgraphCommand + .command('wipe-all') + .description('wipe all entities and relationships in the Memgraph database') + .option( + '-db, --database-name ', + 'optional database to wipe data from (only available for enterprise Memgraph databases)', + 'memgraph', + ) + .action(async (options) => { + await wipeAllMemgraph({ + memgraphDatabase: options.databaseName, + }); + }); + + return memgraphCommand; +} diff --git a/packages/integration-sdk-cli/src/index.ts b/packages/integration-sdk-cli/src/index.ts index 736a671aa..496317d2c 100644 --- a/packages/integration-sdk-cli/src/index.ts +++ b/packages/integration-sdk-cli/src/index.ts @@ -9,6 +9,7 @@ import { visualize, visualizeTypes, validateQuestionFile, + memgraph, neo4j, visualizeDependencies, generateIntegrationGraphSchemaCommand, @@ -27,6 +28,7 @@ export function createCli() { .addCommand(visualizeTypes()) .addCommand(document()) .addCommand(validateQuestionFile()) + .addCommand(memgraph()) .addCommand(neo4j()) .addCommand(visualizeDependencies()) .addCommand(generateIntegrationGraphSchemaCommand()) diff --git a/packages/integration-sdk-cli/src/memgraph/README.md b/packages/integration-sdk-cli/src/memgraph/README.md new file mode 100644 index 000000000..65dbc653c --- /dev/null +++ b/packages/integration-sdk-cli/src/memgraph/README.md @@ -0,0 +1,30 @@ +# Memgraph JupiterOne CLI Command + +## Installation + +This command assumes you have three additional values stored in your local .env +file: NEO4J_URI NEO4J_USER NEO4J_PASSWORD + +This can be used for uploading to local or remote Memgraph databases. For +easy access to a local Memgraph instance, you can launch one via a Memgraph provided +Docker image with the command: + +``` +docker run \ + -p 3000:3000 -p 7444:7444 -p 7687:7687 \ + -d \ + -v mg_lib:/var/lib/memgraph \ + -v mg_log:/var/log/memgraph \ + -v mg_etc:/etc/memgraph \ + memgraph/memgraph-platform +``` + +## Usage + +Data is still collected in the same way as before with a call to `yarn start`. + +Once data has been collected, you can run `j1-integration memgraph push`. This will +push data to the Memgraph server listed in the MEMGRAPH_URI .env parameter. If running +locally, you can then access data in the Memgraph database by visiting +http://localhost:3000. Alternatively, you can download Memgraph Lab at the +https://memgraph.com/lab. diff --git a/packages/integration-sdk-cli/src/memgraph/index.ts b/packages/integration-sdk-cli/src/memgraph/index.ts new file mode 100644 index 000000000..6cdf69d62 --- /dev/null +++ b/packages/integration-sdk-cli/src/memgraph/index.ts @@ -0,0 +1,2 @@ +export * from './uploadToMemgraph'; +export * from './wipeMemgraph'; diff --git a/packages/integration-sdk-cli/src/memgraph/memgraphGraphStore.ts b/packages/integration-sdk-cli/src/memgraph/memgraphGraphStore.ts new file mode 100644 index 000000000..2d474bd55 --- /dev/null +++ b/packages/integration-sdk-cli/src/memgraph/memgraphGraphStore.ts @@ -0,0 +1,181 @@ +import { Entity, Relationship } from '@jupiterone/integration-sdk-core'; +import { + sanitizeValue, + buildPropertyParameters, + sanitizePropertyName, + getFromTypeLabel, + getToTypeLabel, +} from './memgraphUtilities'; + +import * as memgraph from 'neo4j-driver'; + +export interface MemgraphGraphObjectStoreParams { + uri: string; + username: string; + password: string; + integrationInstanceID: string; + session?: memgraph.Session; + database?: string; +} + +export class MemgraphGraphStore { + private memgraphDriver: memgraph.Driver; + private persistedSession: memgraph.Session; + private databaseName = 'memgraph'; + private typeList = new Set(); + private integrationInstanceID: string; + + constructor(params: MemgraphGraphObjectStoreParams) { + if (params.session) { + this.persistedSession = params.session; + } else { + this.memgraphDriver = memgraph.driver( + params.uri, + memgraph.auth.basic(params.username, params.password), + ); + } + this.integrationInstanceID = params.integrationInstanceID; + if (params.database) { + this.databaseName = params.database; + } + } + + private async runCypherCommand( + cypherCommand: string, + cypherParameters?: any, + ): Promise { + if (this.persistedSession) { + const result = await this.persistedSession.run(cypherCommand); + return result; + } else { + const session = this.memgraphDriver.session({ + database: this.databaseName, + defaultAccessMode: memgraph.session.WRITE, + }); + const result = await session.run(cypherCommand, cypherParameters); + await session.close(); + return result; + } + } + + async addEntities(newEntities: Entity[]) { + const nodeAlias: string = 'entityNode'; + const promiseArray: Promise[] = []; + for (const entity of newEntities) { + let classLabels = ''; + if (entity._class) { + if (typeof entity._class === 'string') { + classLabels += `:${sanitizePropertyName(entity._class)}`; + } else { + for (const className of entity._class) { + classLabels += `:${sanitizePropertyName(className)}`; + } + } + } + if (!this.typeList.has(entity._type)) { + await this.runCypherCommand(`CREATE INDEX ON :${entity._type}(_key);`); + await this.runCypherCommand(`CREATE INDEX ON :${entity._type}(_integrationInstanceID);`); + this.typeList.add(entity._type); + } + const sanitizedType = sanitizePropertyName(entity._type); + const propertyParameters = buildPropertyParameters(entity); + const finalKeyValue = sanitizeValue(entity._key.toString()); + const buildCommand = ` + MERGE (${nodeAlias} {_key: $finalKeyValue, _integrationInstanceID: $integrationInstanceID}) + SET ${nodeAlias} += $propertyParameters + SET ${nodeAlias}:${sanitizedType}${classLabels};`; + promiseArray.push( + this.runCypherCommand(buildCommand, { + propertyParameters: propertyParameters, + finalKeyValue: finalKeyValue, + integrationInstanceID: this.integrationInstanceID, + }), + ); + } + await Promise.all(promiseArray); + } + + async addRelationships(newRelationships: Relationship[]) { + const promiseArray: Promise[] = []; + for (const relationship of newRelationships) { + const relationshipAlias: string = 'relationship'; + const propertyParameters = buildPropertyParameters(relationship); + + let startEntityKey = ''; + let endEntityKey = ''; + + if (relationship._fromEntityKey) { + startEntityKey = sanitizeValue(relationship._fromEntityKey.toString()); + } + if (relationship._toEntityKey) { + endEntityKey = sanitizeValue(relationship._toEntityKey.toString()); + } + + //Attempt to get start and end types + const startEntityTypeLabel = getFromTypeLabel(relationship); + const endEntityTypeLabel = getToTypeLabel(relationship); + + if (relationship._mapping) { + //Mapped Relationship + if (relationship._mapping['skipTargetCreation'] === false) { + const targetEntity = relationship._mapping['targetEntity']; + //Create target entity first + const tempEntity: Entity = { + ...targetEntity, + _class: targetEntity._class, + _key: sanitizeValue( + relationship._key.replace( + relationship._mapping['sourceEntityKey'], + '', + ), + ), + _type: targetEntity._type, + }; + await this.addEntities([tempEntity]); + } + startEntityKey = sanitizeValue( + relationship._mapping['sourceEntityKey'], + ); + endEntityKey = sanitizeValue( + relationship._key.replace( + relationship._mapping['sourceEntityKey'], + '', + ), + ); + } + + const sanitizedRelationshipClass = sanitizePropertyName( + relationship._class, + ); + + const buildCommand = ` + MERGE (start${startEntityTypeLabel} {_key: $startEntityKey, _integrationInstanceID: $integrationInstanceID}) + MERGE (end${endEntityTypeLabel} {_key: $endEntityKey, _integrationInstanceID: $integrationInstanceID}) + MERGE (start)-[${relationshipAlias}:${sanitizedRelationshipClass}]->(end) + SET ${relationshipAlias} += $propertyParameters;`; + promiseArray.push( + this.runCypherCommand(buildCommand, { + propertyParameters: propertyParameters, + startEntityKey: startEntityKey, + endEntityKey: endEntityKey, + integrationInstanceID: this.integrationInstanceID, + }), + ); + } + await Promise.all(promiseArray); + } + + async wipeInstanceIdData() { + const wipeCypherCommand = `MATCH (n {_integrationInstanceID: '${this.integrationInstanceID}'}) DETACH DELETE n`; + await this.runCypherCommand(wipeCypherCommand); + } + + async wipeDatabase() { + const wipeCypherCommand = `MATCH (n) DETACH DELETE n`; + await this.runCypherCommand(wipeCypherCommand); + } + + async close() { + await this.memgraphDriver.close(); + } +} diff --git a/packages/integration-sdk-cli/src/memgraph/memgraphUtilities.ts b/packages/integration-sdk-cli/src/memgraph/memgraphUtilities.ts new file mode 100644 index 000000000..812e79e02 --- /dev/null +++ b/packages/integration-sdk-cli/src/memgraph/memgraphUtilities.ts @@ -0,0 +1,79 @@ +import { + Relationship, + RelationshipMapping, +} from '@jupiterone/integration-sdk-core'; + +export function startsWithNumeric(str: string): boolean { + return /^\d/.test(str); +} + +export function sanitizePropertyName(propertyName: string): string { + let sanitizedName = ''; + if (startsWithNumeric(propertyName)) { + sanitizedName += 'n'; + } + sanitizedName += propertyName; + sanitizedName = sanitizedName.replace( + /[\s!@#$%^&*()\-=+\\|'";:/?.,><`~\t\n[\]{}]/g, + '_', + ); + return sanitizedName; +} + +export function sanitizeValue(value: string): string { + return value.replace(/\\*"/gi, '\\"'); +} + +export function buildPropertyParameters(propList: Object) { + const propertyParameters = {}; + + for (const key in propList) { + const propVal = propList[key]; + + if (key === '_rawData') { + // stringify JSON in rawData so we can store it. + propertyParameters[key] = `"${JSON.stringify(propVal)}"`; + } else { + // Sanitize out characters that aren't allowed in property names + const propertyName = sanitizePropertyName(key); + + if (propVal === undefined || propVal === null) { + // Ignore properties that have the value `undefined` or `null`. + continue; + } + + // If we're dealing with a number or boolean, leave alone, otherwise + // wrap in single quotes to convert to a string and escape all + // other single quotes so they don't terminate strings prematurely. + if (typeof propVal == 'number' || typeof propVal == 'boolean') { + propertyParameters[propertyName] = propVal; + } else { + propertyParameters[propertyName] = sanitizeValue(propVal.toString()); + } + } + } + + return propertyParameters; +} + +// Start and end type helper functions. Prepends a : to any nonempty results for +// immediate use in a Memgraph command. +export function getFromTypeLabel(relationship: Relationship): String { + if (relationship._fromType) { + return ':' + relationship._fromType.toString(); + } + return ''; +} + +export function getToTypeLabel(relationship: Relationship): String { + if (relationship._toType) { + return ':' + relationship._toType.toString(); + } else if ( + (relationship._mapping as RelationshipMapping)?.targetEntity?._type + ) { + return ( + ':' + (relationship._mapping as RelationshipMapping).targetEntity._type + ); + } + return ''; +} diff --git a/packages/integration-sdk-cli/src/memgraph/uploadToMemgraph.ts b/packages/integration-sdk-cli/src/memgraph/uploadToMemgraph.ts new file mode 100644 index 000000000..fad0eb71e --- /dev/null +++ b/packages/integration-sdk-cli/src/memgraph/uploadToMemgraph.ts @@ -0,0 +1,64 @@ +import { MemgraphGraphStore } from './memgraphGraphStore'; +import { + iterateParsedGraphFiles, + isDirectoryPresent, +} from '@jupiterone/integration-sdk-runtime'; +import { FlushedGraphObjectData } from '@jupiterone/integration-sdk-runtime/src/storage/types'; + +type UploadToMemgraphParams = { + pathToData: string; + integrationInstanceID: string; + memgraphUri?: string; + memgraphUser?: string; + memgraphPassword?: string; + memgraphDatabase?: string; +}; + +export async function uploadToMemgraph({ + pathToData, + integrationInstanceID, + memgraphUri = process.env.MEMGRAPH_URI, + memgraphUser = process.env.MEMGRAPH_USER, + memgraphPassword = process.env.MEMGRAPH_PASSWORD, + memgraphDatabase, +}: UploadToMemgraphParams) { + if (!memgraphUri || !memgraphUser || !memgraphPassword) { + throw new Error( + 'ERROR: must provide login information in function call or include MEMGRAPH_URI, MEMGRAPH_USER, and MEMGRAPH_PASSWORD files in your .env file!', + ); + } + if (!(await isDirectoryPresent(pathToData))) { + throw new Error('ERROR: graph directory does not exist!'); + } + + const store = new MemgraphGraphStore({ + uri: memgraphUri, + username: memgraphUser, + password: memgraphPassword, + integrationInstanceID: integrationInstanceID, + database: memgraphDatabase, + }); + + async function handleGraphObjectEntityFiles( + parsedData: FlushedGraphObjectData, + ) { + if (parsedData.entities) await store.addEntities(parsedData.entities); + } + + async function handleGraphObjectRelationshipFiles( + parsedData: FlushedGraphObjectData, + ) { + if (parsedData.relationships) + await store.addRelationships(parsedData.relationships); + } + + try { + await iterateParsedGraphFiles(handleGraphObjectEntityFiles, pathToData); + await iterateParsedGraphFiles( + handleGraphObjectRelationshipFiles, + pathToData, + ); + } finally { + await store.close(); + } +} diff --git a/packages/integration-sdk-cli/src/memgraph/wipeMemgraph.ts b/packages/integration-sdk-cli/src/memgraph/wipeMemgraph.ts new file mode 100644 index 000000000..c420daa93 --- /dev/null +++ b/packages/integration-sdk-cli/src/memgraph/wipeMemgraph.ts @@ -0,0 +1,68 @@ +import { MemgraphGraphStore } from './memgraphGraphStore'; + +type WipeMemgraphParams = { + integrationInstanceID: string; + memgraphUri?: string; + memgraphUser?: string; + memgraphPassword?: string; + memgraphDatabase?: string; +}; +export async function wipeMemgraphByID({ + integrationInstanceID, + memgraphUri = process.env.MEMGRAPH_URI, + memgraphUser = process.env.MEMGRAPH_USER, + memgraphPassword = process.env.MEMGRAPH_PASSWORD, + memgraphDatabase, +}: WipeMemgraphParams) { + if (!memgraphUri || !memgraphUser || !memgraphPassword) { + throw new Error( + 'ERROR: must provide login information in function call or include MEMGRAPH_URI, MEMGRAPH_USER, and MEMGRAPH_PASSWORD files in your .env file!', + ); + } + + const store = new MemgraphGraphStore({ + uri: memgraphUri, + username: memgraphUser, + password: memgraphPassword, + integrationInstanceID: integrationInstanceID, + database: memgraphDatabase, + }); + try { + await store.wipeInstanceIdData(); + } finally { + await store.close(); + } +} + +type WipeAllMemgraphParams = { + memgraphUri?: string; + memgraphUser?: string; + memgraphPassword?: string; + memgraphDatabase?: string; +}; + +export async function wipeAllMemgraph({ + memgraphUri = process.env.MEMGRAPH_URI, + memgraphUser = process.env.MEMGRAPH_USER, + memgraphPassword = process.env.MEMGRAPH_PASSWORD, + memgraphDatabase, +}: WipeAllMemgraphParams) { + if (!memgraphUri || !memgraphUser || !memgraphPassword) { + throw new Error( + 'ERROR: must provide login information in function call or include MEMGRAPH_URI, MEMGRAPH_USER, and MEMGRAPH_PASSWORD files in your .env file!', + ); + } + + const store = new MemgraphGraphStore({ + uri: memgraphUri, + username: memgraphUser, + password: memgraphPassword, + integrationInstanceID: '', + database: memgraphDatabase, + }); + try { + await store.wipeDatabase(); + } finally { + await store.close(); + } +} From ead0dac9135afe4d1e3f8ffbfeab4bd82b7fe9c7 Mon Sep 17 00:00:00 2001 From: matea16 Date: Thu, 21 Sep 2023 15:04:25 +0200 Subject: [PATCH 2/2] fix .env value names --- packages/integration-sdk-cli/src/memgraph/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/integration-sdk-cli/src/memgraph/README.md b/packages/integration-sdk-cli/src/memgraph/README.md index 65dbc653c..86f07b9fe 100644 --- a/packages/integration-sdk-cli/src/memgraph/README.md +++ b/packages/integration-sdk-cli/src/memgraph/README.md @@ -3,7 +3,7 @@ ## Installation This command assumes you have three additional values stored in your local .env -file: NEO4J_URI NEO4J_USER NEO4J_PASSWORD +file: MEMGRAPH_URI MEMGRAPH_USER MEMGRAPH_PASSWORD This can be used for uploading to local or remote Memgraph databases. For easy access to a local Memgraph instance, you can launch one via a Memgraph provided