Merge pull request #307 from ardriveapp/dev

Release CLI v1.19.0

README.md

@@ -766,6 +766,14 @@ NOTE: To upload to the root of a drive, specify its root folder ID as the parent
 ardrive drive-info -d "c7f87712-b54e-4491-bc96-1c5fa7b1da50" | jq -r '.rootFolderId'
 ```
 
+### IPFS CID Tagging
+
+Certain nodes on the Arweave network may be running the [IPFS+Arweave bridge](https://arweave.medium.com/arweave-ipfs-persistence-for-the-interplanetary-file-system-9f12981c36c3). Tagging your file upload transaction with its IPFS v1 CID value in the 'IPFS-Add' tag may allow you to take advantage of this system. It can also be helpful for finding data on Arweave via GQL based on its CID. To include the CID tag on your file uploads, you may use the '--add-ipfs-tag' flag:
+
+```shell
+ardrive upload-file --add-ipfs-tag --local-path /path/to/file.txt  --parent-folder-id "9af694f6-4cfc-4eee-88a8-1b02704760c0" -w /path/to/wallet.json
+```
+
 ### Progress Logging of Transaction Uploads
 
 Progress logging of transaction uploads to stderr can be enabled by setting the `ARDRIVE_PROGRESS_LOG` environment variable to `1`:

package.json

@@ -1,18 +1,19 @@
 {
 	"name": "ardrive-cli",
-	"version": "1.18.0",
+	"version": "1.19.0",
 	"description": "The ArDrive Command Line Interface (CLI is a Node.js application for terminal-based ArDrive workflows. It also offers utility operations for securely interacting with Arweave wallets and inspecting various Arweave blockchain conditions.",
 	"main": "./lib/index.js",
 	"bin": {
 		"ardrive": "./lib/index.js"
 	},
 	"types": "./lib/index.d.ts",
 	"dependencies": {
-		"ardrive-core-js": "1.16.0",
+		"ardrive-core-js": "1.17.0",
 		"arweave": "1.11.4",
 		"axios": "^0.21.1",
 		"bn.js": "^5.2.1",
 		"commander": "^8.2.0",
+		"ipfs-only-hash": "^4.0.0",
 		"lodash": "^4.17.21",
 		"prompts": "^2.4.0"
 	},

src/CLICommand/cli_command.test.ts

@@ -7,7 +7,9 @@ import {
 	DriveNameParameter,
 	UnsafeDrivePasswordParameter,
 	SeedPhraseParameter,
-	WalletFileParameter
+	WalletFileParameter,
+	IPFSParameter,
+	PrivateParameter
 } from '../parameter_declarations';
 import { CliApiObject } from './cli';
 import { baseArgv } from './test_constants';
@@ -48,6 +50,15 @@ const parsedCommandOptionsBothSpecified = {
 	[WalletFileParameter]: nonEmptyValue,
 	[SeedPhraseParameter]: nonEmptyValue
 };
+const commandDescriptorIPFSAndPrivate: CommandDescriptor = {
+	name: testingCommandName,
+	parameters: [IPFSParameter, PrivateParameter],
+	action: new CLIAction(dummyAction)
+};
+const parsedCommandOptionsIPFSAndPrivate = {
+	[PrivateParameter]: true,
+	[IPFSParameter]: true
+};
 
 class TestCliApiObject {
 	constructor(private readonly program: CliApiObject = new Command() as CliApiObject) {}
@@ -103,7 +114,10 @@ describe('CLICommand class', () => {
 				commandDescriptorForbiddenWalletFileAndSeedPhrase,
 				parsedCommandOptionsBothSpecified
 			);
-		}).to.throw();
+		}).to.throw('Parameter walletFile cannot be used in conjunction with seedPhrase');
+		expect(function () {
+			assertConjunctionParameters(commandDescriptorIPFSAndPrivate, parsedCommandOptionsIPFSAndPrivate);
+		}).to.throw('Parameter addIpfsTag cannot be used in conjunction with private');
 	});
 
 	it('No colliding parameters', () => {

src/CLICommand/parameters_helper.test.ts

@@ -28,7 +28,9 @@ import {
 	DryRunParameter,
 	MetaDataFileParameter,
 	MetaDataGqlTagsParameter,
-	MetadataJsonParameter
+	MetadataJsonParameter,
+	IPFSParameter,
+	DataGqlTagsParameter
 } from '../parameter_declarations';
 import '../parameter_declarations';
 import { CLIAction } from './action';
@@ -704,7 +706,8 @@ describe('ParametersHelper class', () => {
 
 				expect(metaDataResult).to.deep.equal({
 					metaDataGqlTags: { tag: 'val' },
-					metaDataJson: { 'json field': true }
+					metaDataJson: { 'json field': true },
+					dataGqlTags: { 'IPFS-Add': 'HASH DATA' }
 				});
 			});
 		});
@@ -732,6 +735,29 @@ describe('ParametersHelper class', () => {
 			});
 		});
 
+		it('returns the expected custom metadata with the --data-gql-tags parameter', async () => {
+			const cmd = declareCommandWithParams(program, [DataGqlTagsParameter]);
+
+			CLICommand.parse(program, [
+				...baseArgv,
+				testCommandName,
+				'--data-gql-tags',
+				'data Tag',
+				'Val 1',
+				'data Tag 2',
+				'Val 2'
+			]);
+
+			await cmd.action.then((options) => {
+				const parameters = new ParametersHelper(options);
+				const metaDataResult = parameters.getCustomMetaData();
+
+				expect(metaDataResult).to.deep.equal({
+					dataGqlTags: { 'data Tag 2': 'Val 2', 'data Tag': 'Val 1' }
+				});
+			});
+		});
+
 		it('returns the expected custom metadata with the --metadata-json parameter', async () => {
 			const cmd = declareCommandWithParams(program, [MetadataJsonParameter]);
 
@@ -752,4 +778,48 @@ describe('ParametersHelper class', () => {
 			});
 		});
 	});
+
+	describe('getCustomMetaDataWithIpfsCid method', () => {
+		it('returns a valid custom metadata object that contains the IPFS-Add data tag', async () => {
+			const cmd = declareCommandWithParams(program, [IPFSParameter]);
+
+			CLICommand.parse(program, [...baseArgv, testCommandName, '--add-ipfs-tag']);
+
+			await cmd.action.then(async (options) => {
+				const parameters = new ParametersHelper(options);
+				const metaDataResult = await parameters.getCustomMetaDataWithIpfsCid({
+					localFilePath: 'tests/stub_files/file_to_be_uploaded.txt'
+				});
+
+				expect(metaDataResult.dataGqlTags).to.deep.equal({
+					'IPFS-Add': 'QmPqAhyw2FcvTj9yAKF6rC5LvF2L5xGBLBQUHsMgiNPn2y'
+				});
+			});
+		});
+
+		it('throws if IPFS-Add is present in the custom metadata of dataTx', async () => {
+			const cmd = declareCommandWithParams(program, [IPFSParameter, MetaDataFileParameter]);
+
+			CLICommand.parse(program, [
+				...baseArgv,
+				testCommandName,
+				'--add-ipfs-tag',
+				'--metadata-file',
+				'tests/stub_files/custom_metadata.json'
+			]);
+
+			await cmd.action
+				.then(async (options) => {
+					const parameters = new ParametersHelper(options);
+					await parameters.getCustomMetaDataWithIpfsCid({
+						localFilePath: 'tests/stub_files/file_to_be_uploaded.txt'
+					});
+				})
+				.catch((err) => {
+					expect(err.message).to.equal(
+						`You cannot pass the --add-ipfs-tag flag and set the custom IPFS-Add metadata item. Found: { 'IPFS-Add': "HASH DATA"}`
+					);
+				});
+		});
+	});
 });

src/CLICommand/parameters_helper.ts

@@ -17,7 +17,8 @@ import {
 	DryRunParameter,
 	MetaDataFileParameter,
 	MetaDataGqlTagsParameter,
-	MetadataJsonParameter
+	MetadataJsonParameter,
+	DataGqlTagsParameter
 } from '../parameter_declarations';
 import { cliWalletDao } from '..';
 import passwordPrompt from 'prompts';
@@ -44,6 +45,7 @@ import {
 	CustomMetaDataJsonFields
 } from 'ardrive-core-js';
 import { JWKInterface } from 'arweave/node/lib/wallet';
+import { deriveIpfsCid } from '../utils/ipfs_utils';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 type ParameterOptions = any;
@@ -57,6 +59,12 @@ interface GetDriveKeyParams {
 	useCache?: boolean;
 }
 
+interface GetCustomMetaDataWithIpfsCidParameter {
+	localFilePath: FilePath;
+}
+
+type FilePath = string;
+
 /**
  * @type {ParametersHelper}
  * A class that assists with handling Commander options during common ArDrive CLI workflows
@@ -288,6 +296,31 @@ export class ParametersHelper {
 		return mapFunc(value);
 	}
 
+	async getCustomMetaDataWithIpfsCid({
+		localFilePath
+	}: GetCustomMetaDataWithIpfsCidParameter): Promise<CustomMetaData> {
+		const customMetaDataClone: CustomMetaData = Object.assign({}, this.getCustomMetaData());
+
+		const customIpfsTag = customMetaDataClone.dataGqlTags?.['IPFS-Add'];
+		if (customIpfsTag) {
+			throw new Error(
+				`You cannot pass the --add-ipfs-tag flag and set the custom IPFS-Add metadata item. Found: { 'IPFS-Add': ${JSON.stringify(
+					customIpfsTag
+				)}}`
+			);
+		} else {
+			const fileContent = fs.readFileSync(localFilePath);
+			const cidHash = await deriveIpfsCid(fileContent);
+
+			if (!customMetaDataClone.dataGqlTags) {
+				customMetaDataClone.dataGqlTags = {};
+			}
+			customMetaDataClone.dataGqlTags['IPFS-Add'] = cidHash;
+		}
+
+		return customMetaDataClone;
+	}
+
 	public getCustomMetaData(): CustomMetaData | undefined {
 		const metaDataPath = this.getParameterValue(MetaDataFileParameter);
 
@@ -304,6 +337,13 @@ export class ParametersHelper {
 				Object.assign(customMetaData, { metaDataGqlTags });
 			}
 
+			const dataGqlTags = this.mapMetaDataArrayToCustomMetaDataShape(
+				this.getParameterValue<string[]>(DataGqlTagsParameter)
+			);
+			if (dataGqlTags) {
+				Object.assign(customMetaData, { dataGqlTags });
+			}
+
 			const metaDataJson = this.getParameterValue<string>(MetadataJsonParameter);
 
 			if (metaDataJson) {

src/commands/upload_file.ts

@@ -14,7 +14,8 @@ import {
 	LocalCSVParameter,
 	GatewayParameter,
 	CustomContentTypeParameter,
-	CustomMetaDataParameters
+	CustomMetaDataParameters,
+	IPFSParameter
 } from '../parameter_declarations';
 import { fileAndFolderUploadConflictPrompts } from '../prompts';
 import { ERROR_EXIT_CODE, SUCCESS_EXIT_CODE } from '../CLICommand/error_codes';
@@ -82,34 +83,47 @@ function getFilesFromCSV(parameters: ParametersHelper): UploadPathParameter[] |
 	return fileParameters;
 }
 
-function getFileList(parameters: ParametersHelper, parentFolderId: FolderID): UploadPathParameter[] | undefined {
+async function getFileList(
+	parameters: ParametersHelper,
+	parentFolderId: FolderID
+): Promise<UploadPathParameter[] | undefined> {
 	const localPaths = parameters.getParameterValue<string[]>(LocalPathsParameter);
 	if (!localPaths) {
 		return undefined;
 	}
 	const customContentType = parameters.getParameterValue(CustomContentTypeParameter);
 	const customMetaData = parameters.getCustomMetaData();
 
-	const localPathsToUpload = localPaths.map((filePath: FilePath) => {
-		const wrappedEntity = wrapFileOrFolder(filePath, customContentType, customMetaData);
+	const localPathsToUpload = localPaths.map(async (filePath: FilePath) => {
+		const customMetaDataWithIipfsFlag = await (async function () {
+			const ipfsFlag = parameters.getParameterValue(IPFSParameter);
+			return ipfsFlag
+				? await parameters.getCustomMetaDataWithIpfsCid({ localFilePath: filePath })
+				: customMetaData;
+		})();
+		const wrappedEntity = wrapFileOrFolder(filePath, customContentType, customMetaDataWithIipfsFlag);
 
 		return {
 			parentFolderId,
 			wrappedEntity
 		};
 	});
 
-	return localPathsToUpload;
+	return Promise.all(localPathsToUpload);
 }
 
-function getSingleFile(parameters: ParametersHelper, parentFolderId: FolderID): UploadPathParameter[] {
+async function getSingleFile(parameters: ParametersHelper, parentFolderId: FolderID): Promise<UploadPathParameter[]> {
 	// NOTE: Single file is the last possible use case. Throw exception if the parameter isn't found.
 	const localFilePath =
 		parameters.getParameterValue(LocalFilePathParameter_DEPRECATED) ??
 		parameters.getRequiredParameterValue<string>(LocalPathParameter);
 
 	const customContentType = parameters.getParameterValue(CustomContentTypeParameter);
-	const customMetaData = parameters.getCustomMetaData();
+	const customMetaData = await (async function () {
+		const customMetaData = parameters.getCustomMetaData();
+		const ipfsFlag = parameters.getParameterValue(IPFSParameter);
+		return ipfsFlag ? await parameters.getCustomMetaDataWithIpfsCid({ localFilePath }) : customMetaData;
+	})();
 
 	const wrappedEntity = wrapFileOrFolder(localFilePath, customContentType, customMetaData);
 	const singleParameter = {
@@ -140,7 +154,8 @@ new CLICommand({
 		LocalFilePathParameter_DEPRECATED,
 		LocalFilesParameter_DEPRECATED,
 		BoostParameter,
-		GatewayParameter
+		GatewayParameter,
+		IPFSParameter
 	],
 	action: new CLIAction(async function action(options) {
 		const parameters = new ParametersHelper(options);
@@ -155,7 +170,7 @@ new CLICommand({
 			// Determine list of files to upload and destinations from parameter list
 			// First check the multi-file input case
 			const parentFolderId: FolderID = parameters.getRequiredParameterValue(ParentFolderIdParameter, EID);
-			const fileList = getFileList(parameters, parentFolderId);
+			const fileList = await getFileList(parameters, parentFolderId);
 			if (fileList) {
 				return fileList;
 			}

src/parameter_declarations.ts

@@ -41,14 +41,15 @@ export const LocalCSVParameter = 'localCsv';
 export const WithKeysParameter = 'withKeys';
 export const GatewayParameter = 'gateway';
 export const CustomContentTypeParameter = 'contentType';
+export const IPFSParameter = 'addIpfsTag';
 export const DataGqlTagsParameter = 'dataGqlTags';
 export const MetaDataFileParameter = 'metadataFile';
 export const MetaDataGqlTagsParameter = 'metadataGqlTags';
 export const MetadataJsonParameter = 'metadataJson';
 
 // Aggregates for convenience
 export const CustomMetaDataParameters = [
-	// DataGqlTagsParameter,
+	DataGqlTagsParameter,
 	MetaDataFileParameter,
 	MetaDataGqlTagsParameter,
 	MetadataJsonParameter
@@ -96,7 +97,8 @@ export const AllParameters = [
 	TxFilePathParameter,
 	UnsafeDrivePasswordParameter,
 	WalletFileParameter,
-	WithKeysParameter
+	WithKeysParameter,
+	IPFSParameter
 ] as const;
 export type ParameterName = typeof AllParameters[number];
 
@@ -482,6 +484,15 @@ Parameter.declare({
 		'(OPTIONAL) Provide a custom content type to all files within the upload to be used by the gateway to display the content'
 });
 
+Parameter.declare({
+	name: IPFSParameter,
+	aliases: ['--add-ipfs-tag'],
+	description:
+		'(OPTIONAL) Computes the v1 IPFS content identifier (CID) for each file and sets it as the "IPFS-Add" tag value of each\'s respective file data transaction',
+	type: 'boolean',
+	forbiddenConjunctionParameters: [PrivateParameter, UnsafeDrivePasswordParameter]
+});
+
 Parameter.declare({
 	name: MetaDataFileParameter,
 	aliases: ['--metadata-file'],
@@ -507,12 +518,11 @@ Parameter.declare({
 	forbiddenConjunctionParameters: [MetaDataFileParameter]
 });
 
-// TODO: PE-1534
-// Parameter.declare({
-// 	name: DataGqlTagsParameter,
-// 	aliases: ['--data-gql-tags'],
-// 	type: 'array',
-// 	description:
-// 		'(OPTIONAL) A mapping of custom metadata in the `"TAG_NAME" "TAG_VALUE"` format to be applied to the GQL Tags of all Data Transactions created. Must be an even number of string values to determine custom metadata. Can NOT be used in conjunction with --metadata-file',
-// 	forbiddenConjunctionParameters: [MetaDataFileParameter]
-// });
+Parameter.declare({
+	name: DataGqlTagsParameter,
+	aliases: ['--data-gql-tags'],
+	type: 'array',
+	description:
+		'(OPTIONAL) A list of custom Arweave tag name and value pairs in the format `"TAG_NAME" "TAG_VALUE"` that will be applied to all file data transactions created during an invocation. Must be an even number of string values. Can NOT be used in conjunction with --metadata-file',
+	forbiddenConjunctionParameters: [MetaDataFileParameter]
+});