From efc79b293c1a03dee857868b73580786642d53f9 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Tue, 16 Jul 2024 16:06:32 +0200 Subject: [PATCH 01/23] CLDSRRV-546: update packages --- package.json | 6 +++++- yarn.lock | 9 +++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 88d99d1dcd..1d4e815bdf 100644 --- a/package.json +++ b/package.json @@ -19,8 +19,9 @@ }, "homepage": "https://github.com/scality/S3#readme", "dependencies": { + "@fastify/busboy": "^2.1.1", "@hapi/joi": "^17.1.0", - "arsenal": "git+https://github.com/scality/arsenal#7.70.29", + "arsenal": "git+https://github.com/scality/arsenal#61984fbac3721d72cfc05a0cba0a0965c158008d", "async": "~2.5.0", "aws-sdk": "2.905.0", "azure-storage": "^2.1.0", @@ -60,6 +61,9 @@ }, "scripts": { "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/", + "ft_post": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/object/post.js", + "ft_post_aws": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/object/post-copy.js", + "ft_post_unit": "CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit/api/callPostMethod.js", "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/", "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/bucket", "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support", diff --git a/yarn.lock b/yarn.lock index 57f5a07451..9b594ab03f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16,6 +16,11 @@ enabled "2.0.x" kuler "^2.0.0" +"@fastify/busboy@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" + integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== + "@gar/promisify@^1.0.1": version "1.1.3" resolved "https://registry.yarnpkg.com/@gar/promisify/-/promisify-1.1.3.tgz#555193ab2e3bb3b6adc3d551c9c030d9e860daf6" @@ -499,9 +504,9 @@ arraybuffer.slice@~0.0.7: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#7.70.29": +"arsenal@git+https://github.com/scality/arsenal#61984fbac3721d72cfc05a0cba0a0965c158008d": version "7.70.29" - resolved "git+https://github.com/scality/arsenal#a643a3e6ccbc49327339a285de1d4cb17afcd171" + resolved "git+https://github.com/scality/arsenal#61984fbac3721d72cfc05a0cba0a0965c158008d" dependencies: "@js-sdsl/ordered-set" "^4.4.2" "@types/async" "^3.2.12" From 6f82826984a3aa48c1129e4dfd3a992ba0a00320 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Tue, 16 Jul 2024 16:13:49 +0200 Subject: [PATCH 02/23] CLDSRRV-546: add POST object api handler --- lib/api/api.js | 85 +++++++++---------- .../authorization/permissionChecks.js | 43 ++++++++++ 2 files changed, 84 insertions(+), 44 deletions(-) diff --git a/lib/api/api.js b/lib/api/api.js index 23039807da..dbf3a980d8 100644 --- a/lib/api/api.js +++ b/lib/api/api.js @@ -52,6 +52,7 @@ const objectGetRetention = require('./objectGetRetention'); const objectGetTagging = require('./objectGetTagging'); const objectHead = require('./objectHead'); const objectPut = require('./objectPut'); +const objectPost = require('./objectPost'); const objectPutACL = require('./objectPutACL'); const objectPutLegalHold = require('./objectPutLegalHold'); const objectPutTagging = require('./objectPutTagging'); @@ -67,7 +68,9 @@ const writeContinue = require('../utilities/writeContinue'); const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders'); const parseCopySource = require('./apiUtils/object/parseCopySource'); const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys'); +const { checkAuthResults } = require('./apiUtils/authorization/permissionChecks'); const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize'); +const { processPostForm } = require('./apiUtils/apiCallers/callPostObject'); const monitoringMap = policies.actionMaps.actionMonitoringMapS3; @@ -142,49 +145,6 @@ const api = { // eslint-disable-next-line no-param-reassign request.apiMethods = apiMethods; - function checkAuthResults(authResults) { - let returnTagCount = true; - const isImplicitDeny = {}; - let isOnlyImplicitDeny = true; - if (apiMethod === 'objectGet') { - // first item checks s3:GetObject(Version) action - if (!authResults[0].isAllowed && !authResults[0].isImplicit) { - log.trace('get object authorization denial from Vault'); - return errors.AccessDenied; - } - // TODO add support for returnTagCount in the bucket policy - // checks - isImplicitDeny[authResults[0].action] = authResults[0].isImplicit; - // second item checks s3:GetObject(Version)Tagging action - if (!authResults[1].isAllowed) { - log.trace('get tagging authorization denial ' + - 'from Vault'); - returnTagCount = false; - } - } else { - for (let i = 0; i < authResults.length; i++) { - isImplicitDeny[authResults[i].action] = true; - if (!authResults[i].isAllowed && !authResults[i].isImplicit) { - // Any explicit deny rejects the current API call - log.trace('authorization denial from Vault'); - return errors.AccessDenied; - } - if (authResults[i].isAllowed) { - // If the action is allowed, the result is not implicit - // Deny. - isImplicitDeny[authResults[i].action] = false; - isOnlyImplicitDeny = false; - } - } - } - // These two APIs cannot use ACLs or Bucket Policies, hence, any - // implicit deny from vault must be treated as an explicit deny. - if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) { - return errors.AccessDenied; - } - return { returnTagCount, isImplicitDeny }; - } - return async.waterfall([ next => auth.server.doAuth( request, log, (err, userInfo, authorizationResults, streamingV4Params) => { @@ -256,7 +216,7 @@ const api = { return callback(err); } if (authorizationResults) { - const checkedResults = checkAuthResults(authorizationResults); + const checkedResults = checkAuthResults(apiMethod, authorizationResults, log); if (checkedResults instanceof Error) { return callback(checkedResults); } @@ -286,6 +246,42 @@ const api = { return this[apiMethod](userInfo, request, log, callback); }); }, + callPostObject(request, response, log, callback) { + request.apiMethod = 'objectPost'; + + const requestContexts = prepareRequestContexts('objectPost', request, + undefined, undefined, undefined); + // Extract all the _apiMethods and store them in an array + const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : []; + // Attach the names to the current request + // eslint-disable-next-line no-param-reassign + request.apiMethods = apiMethods; + + return processPostForm(request, response, requestContexts, log, + (err, userInfo, authorizationResults, streamingV4Params) => { + if (err) { + return callback(err); + } + if (authorizationResults) { + const checkedResults = checkAuthResults(authorizationResults); + if (checkedResults instanceof Error) { + return callback(checkedResults); + } + request.actionImplicitDenies = checkedResults.isImplicitDeny; + } else { + // create an object of keys apiMethods with all values to false: + // for backward compatibility, all apiMethods are allowed by default + // thus it is explicitly allowed, so implicit deny is false + request.actionImplicitDenies = apiMethods.reduce((acc, curr) => { + acc[curr] = false; + return acc; + }, {}); + } + request._response = response; + return objectPost(userInfo, request, streamingV4Params, + log, callback, authorizationResults); + }); + }, bucketDelete, bucketDeleteCors, bucketDeleteEncryption, @@ -337,6 +333,7 @@ const api = { objectCopy, objectHead, objectPut, + objectPost, objectPutACL, objectPutLegalHold, objectPutTagging, diff --git a/lib/api/apiUtils/authorization/permissionChecks.js b/lib/api/apiUtils/authorization/permissionChecks.js index 170488b44b..b4e919edee 100644 --- a/lib/api/apiUtils/authorization/permissionChecks.js +++ b/lib/api/apiUtils/authorization/permissionChecks.js @@ -576,6 +576,48 @@ function validatePolicyConditions(policy) { return null; } +function checkAuthResults(apiMethod, authResults, log) { + let returnTagCount = true; + const isImplicitDeny = {}; + let isOnlyImplicitDeny = true; + if (apiMethod === 'objectGet') { + // first item checks s3:GetObject(Version) action + if (!authResults[0].isAllowed && !authResults[0].isImplicit) { + log.trace('get object authorization denial from Vault'); + return errors.AccessDenied; + } + // TODO add support for returnTagCount in the bucket policy + // checks + isImplicitDeny[authResults[0].action] = authResults[0].isImplicit; + // second item checks s3:GetObject(Version)Tagging action + if (!authResults[1].isAllowed) { + log.trace('get tagging authorization denial ' + + 'from Vault'); + returnTagCount = false; + } + } else { + for (let i = 0; i < authResults.length; i++) { + isImplicitDeny[authResults[i].action] = true; + if (!authResults[i].isAllowed && !authResults[i].isImplicit) { + // Any explicit deny rejects the current API call + log.trace('authorization denial from Vault'); + return errors.AccessDenied; + } + if (authResults[i].isAllowed) { + // If the action is allowed, the result is not implicit + // Deny. + isImplicitDeny[authResults[i].action] = false; + isOnlyImplicitDeny = false; + } + } + } + // These two APIs cannot use ACLs or Bucket Policies, hence, any + // implicit deny from vault must be treated as an explicit deny. + if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) { + return errors.AccessDenied; + } + return { returnTagCount, isImplicitDeny }; +} /** isLifecycleSession - check if it is the Lifecycle assumed role session arn. * @param {string} arn - Amazon resource name - example: @@ -607,6 +649,7 @@ module.exports = { checkObjectAcls, validatePolicyResource, validatePolicyConditions, + checkAuthResults, isLifecycleSession, evaluateBucketPolicyWithIAM, }; From 41b839c0144d4180395c1bccb9ac6f09ee73807f Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Tue, 16 Jul 2024 16:19:43 +0200 Subject: [PATCH 03/23] CLDSRRV-546: add POST object action --- constants.js | 1 + lib/api/apiUtils/apiCallers/callPostObject.js | 180 ++++++++++++++++++ lib/api/objectPost.js | 118 ++++++++++++ 3 files changed, 299 insertions(+) create mode 100644 lib/api/apiUtils/apiCallers/callPostObject.js create mode 100644 lib/api/objectPost.js diff --git a/constants.js b/constants.js index 085103d608..b4178ba42f 100644 --- a/constants.js +++ b/constants.js @@ -215,6 +215,7 @@ const constants = { 'initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload', + 'objectPost', ], }; diff --git a/lib/api/apiUtils/apiCallers/callPostObject.js b/lib/api/apiUtils/apiCallers/callPostObject.js new file mode 100644 index 0000000000..6ca112ce8f --- /dev/null +++ b/lib/api/apiUtils/apiCallers/callPostObject.js @@ -0,0 +1,180 @@ +const { auth, errors } = require('arsenal'); +const busboy = require('@fastify/busboy'); +const writeContinue = require('../../../utilities/writeContinue'); +const fs = require('fs'); +const path = require('path'); +const os = require('os'); + +// per doc: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTForms.html#HTTPPOSTFormDeclaration +const MAX_FIELD_SIZE = 20 * 1024; // 20KB +// per doc: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html +const MAX_KEY_SIZE = 1024; + +async function authenticateRequest(request, requestContexts, log) { + return new Promise(resolve => { + // TODO RING-45960 remove ignore for POST object here + auth.server.doAuth(request, log, (err, userInfo, authorizationResults, streamingV4Params) => + resolve({ userInfo, authorizationResults, streamingV4Params }), 's3', requestContexts); + }); +} + +async function parseFormData(request, response, requestContexts, log) { + /* eslint-disable no-param-reassign */ + const formDataParser = busboy({ headers: request.headers }); + writeContinue(request, response); + + return new Promise((resolve, reject) => { + request.formData = {}; + let totalFieldSize = 0; + let fileEventData = null; + let tempFileStream; + let tempFilePath; + let authResponse; + let fileWrittenPromiseResolve; + let formParserFinishedPromiseResolve; + + const fileWrittenPromise = new Promise((res) => { fileWrittenPromiseResolve = res; }); + const formParserFinishedPromise = new Promise((res) => { formParserFinishedPromiseResolve = res; }); + + formDataParser.on('field', (fieldname, val) => { + totalFieldSize += Buffer.byteLength(val, 'utf8'); + if (totalFieldSize > MAX_FIELD_SIZE) { + return reject(errors.MaxPostPreDataLengthExceeded); + } + const lowerFieldname = fieldname.toLowerCase(); + if (lowerFieldname === 'key') { + if (val.length > MAX_KEY_SIZE) { + return reject(errors.KeyTooLongError); + } else if (val.length === 0) { + return reject(errors.InvalidArgument + .customizeDescription('User key must have a length greater than 0.')); + } + } + request.formData[lowerFieldname] = val; + return undefined; + }); + + formDataParser.on('file', async (fieldname, file, filename, encoding, mimetype) => { + if (fileEventData) { + const err = errors.InvalidArgument + .customizeDescription('POST requires exactly one file upload per request.'); + file.resume(); // Resume the stream to drain and discard the file + if (tempFilePath) { + fs.unlink(tempFilePath, unlinkErr => { + if (unlinkErr) { + log.error('Failed to delete temp file', { error: unlinkErr }); + } + }); + } + return reject(err); + } + + fileEventData = { fieldname, file, filename, encoding, mimetype }; + if (!('key' in request.formData)) { + const err = errors.InvalidArgument + .customizeDescription('Bucket POST must contain a field named ' + + "'key'. If it is specified, please check the order of the fields."); + return reject(err); + } + // Replace `${filename}` with the actual filename + request.formData.key = request.formData.key.replace('${filename}', filename); + try { + // Authenticate request before streaming file + // TODO RING-45960 auth to be properly implemented + authResponse = await authenticateRequest(request, requestContexts, log); + + // Create a temporary file to stream the file data + // This is to finalize validation on form data before storing the file + tempFilePath = path.join(os.tmpdir(), filename); + tempFileStream = fs.createWriteStream(tempFilePath); + + file.pipe(tempFileStream); + + tempFileStream.on('finish', () => { + request.fileEventData = { ...fileEventData, file: tempFilePath }; + fileWrittenPromiseResolve(); + }); + + tempFileStream.on('error', (err) => { + log.trace('Error streaming file to temporary location', { error: err }); + reject(errors.InternalError); + }); + + // Wait for both file writing and form parsing to finish + return Promise.all([fileWrittenPromise, formParserFinishedPromise]) + .then(() => resolve(authResponse)) + .catch(reject); + } catch (err) { + return reject(err); + } + }); + + formDataParser.on('finish', () => { + if (!fileEventData) { + const err = errors.InvalidArgument + .customizeDescription('POST requires exactly one file upload per request.'); + return reject(err); + } + return formParserFinishedPromiseResolve(); + }); + + formDataParser.on('error', (err) => { + log.trace('Error processing form data:', { error: err.description }); + request.unpipe(formDataParser); + // Following observed AWS behaviour + reject(errors.MalformedPOSTRequest); + }); + + request.pipe(formDataParser); + return undefined; + }); +} + +function getFileStat(filePath) { + return new Promise((resolve, reject) => { + fs.stat(filePath, (err, stats) => { + if (err) { + return reject(err); + } + return resolve(stats); + }); + }); +} + +async function processPostForm(request, response, requestContexts, log, callback) { + if (!request.headers || !request.headers['content-type'].includes('multipart/form-data')) { + const contentTypeError = errors.PreconditionFailed + .customizeDescription('Bucket POST must be of the enclosure-type multipart/form-data'); + return process.nextTick(callback, contentTypeError); + } + try { + const { userInfo, authorizationResults, streamingV4Params } = + await parseFormData(request, response, requestContexts, log); + + const fileStat = await getFileStat(request.fileEventData.file); + request.parsedContentLength = fileStat.size; + request.fileEventData.file = fs.createReadStream(request.fileEventData.file); + if (request.formData['content-type']) { + request.headers['content-type'] = request.formData['content-type']; + } else { + request.headers['content-type'] = 'binary/octet-stream'; + } + + const authNames = { accountName: userInfo.getAccountDisplayName() }; + if (userInfo.isRequesterAnIAMUser()) { + authNames.userName = userInfo.getIAMdisplayName(); + } + log.addDefaultFields(authNames); + + return callback(null, userInfo, authorizationResults, streamingV4Params); + } catch (err) { + return callback(err); + } +} + +module.exports = { + authenticateRequest, + parseFormData, + processPostForm, + getFileStat, +}; diff --git a/lib/api/objectPost.js b/lib/api/objectPost.js new file mode 100644 index 0000000000..f1b3b73b99 --- /dev/null +++ b/lib/api/objectPost.js @@ -0,0 +1,118 @@ +const async = require('async'); +const { errors, versioning } = require('arsenal'); + +const collectCorsHeaders = require('../utilities/collectCorsHeaders'); +const createAndStoreObject = require('./apiUtils/object/createAndStoreObject'); +const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); +const { config } = require('../Config'); +const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); +const monitoring = require('../utilities/metrics'); +const writeContinue = require('../utilities/writeContinue'); +const { overheadField } = require('../../constants'); + + +const versionIdUtils = versioning.VersionID; + + +/** + * POST Object in the requested bucket. Steps include: + * validating metadata for authorization, bucket and object existence etc. + * store object data in datastore upon successful authorization + * store object location returned by datastore and + * object's (custom) headers in metadata + * return the result in final callback + * + * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info + * @param {request} request - request object given by router, + * includes normalized headers + * @param {object | undefined } streamingV4Params - if v4 auth, + * object containing accessKey, signatureFromRequest, region, scopeDate, + * timestamp, and credentialScope + * (to be used for streaming v4 auth if applicable) + * @param {object} log - the log request + * @param {Function} callback - final callback to call with the result + * @return {undefined} + */ +function objectPost(authInfo, request, streamingV4Params, log, callback) { + const { + headers, + method, + formData, + bucketName, + } = request; + const requestType = request.apiMethods || 'objectPost'; + const valParams = { + authInfo, + bucketName, + objectKey: formData.key, + requestType, + request, + }; + const canonicalID = authInfo.getCanonicalID(); + + log.trace('owner canonicalID to send to data', { canonicalID }); + return standardMetadataValidateBucketAndObj(valParams, request.actionImplicitDenies, log, + (err, bucket, objMD) => { + const responseHeaders = collectCorsHeaders(headers.origin, + method, bucket); + + // TODO RING-45960 remove accessdenied skip + if (err && !err.AccessDenied) { + log.trace('error processing request', { + error: err, + method: 'metadataValidateBucketAndObj', + }); + monitoring.promMetrics('POST', request.bucketName, err.code, 'postObject'); + return callback(err, responseHeaders); + } + if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { + log.trace('deleted flag on bucket and request ' + + 'from non-owner account'); + monitoring.promMetrics('POST', request.bucketName, 404, 'postObject'); + return callback(errors.NoSuchBucket); + } + + return async.waterfall([ + function objectCreateAndStore(next) { + writeContinue(request, request._response); + return createAndStoreObject(request.bucketName, + bucket, request.formData.key, objMD, authInfo, canonicalID, null, + request, false, streamingV4Params, overheadField, log, next); + }, + ], (err, storingResult) => { + if (err) { + monitoring.promMetrics('POST', request.bucketName, err.code, + 'postObject'); + return callback(err, responseHeaders); + } + setExpirationHeaders(responseHeaders, { + lifecycleConfig: bucket.getLifecycleConfiguration(), + objectParams: { + key: request.key, + date: storingResult.lastModified, + tags: storingResult.tags, + }, + }); + if (storingResult) { + // ETag's hex should always be enclosed in quotes + responseHeaders.Key = request.formData.key; + responseHeaders.location = `/${bucketName}/${request.formData.key}`; + responseHeaders.Bucket = bucketName; + responseHeaders.ETag = `"${storingResult.contentMD5}"`; + } + const vcfg = bucket.getVersioningConfiguration(); + const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; + if (isVersionedObj) { + if (storingResult && storingResult.versionId) { + responseHeaders['x-amz-version-id'] = + versionIdUtils.encode(storingResult.versionId, + config.versionIdEncodingType); + } + } + + return callback(null, responseHeaders); + }); + }); +} + +module.exports = objectPost; From 9d9c4ae66d77726487f7fcb4a18775afb9bacd1a Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Tue, 16 Jul 2024 16:21:44 +0200 Subject: [PATCH 04/23] CLDSRRV-546: update extraneous code for POST object --- lib/api/apiUtils/object/createAndStoreObject.js | 14 ++++++++++++-- lib/api/apiUtils/object/prepareStream.js | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 7dc84089bf..62c600d37b 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -210,8 +210,18 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo, metadataStoreParams.contentMD5 = constants.emptyFileMd5; return next(null, null, null); } - return dataStore(objectKeyContext, cipherBundle, request, size, - streamingV4Params, backendInfo, log, next); + // Object Post receives a file stream. + // This is to be used to store data instead of the request stream itself. + + let stream; + + if (request.apiMethod === 'objectPost') { + stream = request.fileEventData ? request.fileEventData.file : undefined; + } else { + stream = request; + } + + return dataStore(objectKeyContext, cipherBundle, stream, size, streamingV4Params, backendInfo, log, next); }, function processDataResult(dataGetInfo, calculatedHash, next) { if (dataGetInfo === null || dataGetInfo === undefined) { diff --git a/lib/api/apiUtils/object/prepareStream.js b/lib/api/apiUtils/object/prepareStream.js index 7d436dd96b..985ec4efe0 100644 --- a/lib/api/apiUtils/object/prepareStream.js +++ b/lib/api/apiUtils/object/prepareStream.js @@ -13,7 +13,7 @@ const V4Transform = require('../../../auth/streamingV4/V4Transform'); * the type of request requires them */ function prepareStream(stream, streamingV4Params, log, errCb) { - if (stream.headers['x-amz-content-sha256'] === + if (stream && stream.headers && stream.headers['x-amz-content-sha256'] === 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') { if (typeof streamingV4Params !== 'object') { // this might happen if the user provided a valid V2 From b740e13d1513d8a1227219fc866b35ca1eb30a7f Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Tue, 16 Jul 2024 16:25:19 +0200 Subject: [PATCH 05/23] CLDSRRV-546: unit test objectPOST action --- tests/unit/api/objectPost.js | 433 +++++++++++++++++++++++++++++++++++ tests/unit/helpers.js | 13 ++ 2 files changed, 446 insertions(+) create mode 100644 tests/unit/api/objectPost.js diff --git a/tests/unit/api/objectPost.js b/tests/unit/api/objectPost.js new file mode 100644 index 0000000000..1416558193 --- /dev/null +++ b/tests/unit/api/objectPost.js @@ -0,0 +1,433 @@ +const assert = require('assert'); +const async = require('async'); +const moment = require('moment'); +const { errors } = require('arsenal'); +const sinon = require('sinon'); + +const { bucketPut } = require('../../../lib/api/bucketPut'); +const bucketPutObjectLock = require('../../../lib/api/bucketPutObjectLock'); +const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } + = require('../helpers'); +const { ds } = require('arsenal').storage.data.inMemory.datastore; +const metadata = require('../metadataswitch'); +const objectPost = require('../../../lib/api/objectPost'); +const { objectLockTestUtils } = require('../helpers'); +const DummyRequest = require('../DummyRequest'); +const mpuUtils = require('../utils/mpuUtils'); + +const any = sinon.match.any; + +const log = new DummyRequestLogger(); +const canonicalID = 'accessKey1'; +const authInfo = makeAuthInfo(canonicalID); +const bucketName = 'bucketname123'; +const postBody = Buffer.from('I am a body', 'utf8'); +const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; +const mockDate = new Date(2050, 10, 12); +const testPutBucketRequest = new DummyRequest({ + bucketName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', +}); +const testPutBucketRequestLock = new DummyRequest({ + bucketName, + headers: { + 'host': `${bucketName}.s3.amazonaws.com`, + 'x-amz-bucket-object-lock-enabled': 'true', + }, + url: '/', +}); + +const originalputObjectMD = metadata.putObjectMD; +const objectName = 'objectName'; + +let testPostObjectRequest; +const enableVersioningRequest = + versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); +const suspendVersioningRequest = + versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Suspended'); + + +describe('objectPost API', () => { + beforeEach(() => { + cleanup(); + sinon.spy(metadata, 'putObjectMD'); + testPostObjectRequest = new DummyRequest({ + bucketName, + formData: { + key: objectName, + }, + fileEventData: {}, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + }, postBody); + }); + + afterEach(() => { + sinon.restore(); + metadata.putObjectMD = originalputObjectMD; + }); + + it('should return an error if the bucket does not exist', done => { + objectPost(authInfo, testPostObjectRequest, undefined, log, err => { + assert.deepStrictEqual(err, errors.NoSuchBucket); + done(); + }); + }); + + it('should successfully post an object', done => { + const testPostObjectRequest = new DummyRequest({ + bucketName, + formData: { + key: objectName, + }, + fileEventData: {}, + headers: {}, + url: '/', + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, postBody); + + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPost(authInfo, testPostObjectRequest, undefined, log, + (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, + {}, log, (err, md) => { + assert(md); + assert + .strictEqual(md['content-md5'], correctMD5); + done(); + }); + }); + }); + }); + + const mockModes = ['GOVERNANCE', 'COMPLIANCE']; + mockModes.forEach(mockMode => { + it(`should post an object with valid date & ${mockMode} mode`, done => { + const testPostObjectRequest = new DummyRequest({ + bucketName, + formData: { + key: objectName, + }, + fileEventData: {}, + headers: { + 'x-amz-object-lock-retain-until-date': mockDate, + 'x-amz-object-lock-mode': mockMode, + }, + url: '/', + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, postBody); + bucketPut(authInfo, testPutBucketRequestLock, log, () => { + objectPost(authInfo, testPostObjectRequest, undefined, log, + (err, headers) => { + assert.ifError(err); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, + (err, md) => { + const mode = md.retentionMode; + const retainUntilDate = md.retentionDate; + assert.ifError(err); + assert(md); + assert.strictEqual(mode, mockMode); + assert.strictEqual(retainUntilDate, mockDate); + done(); + }); + }); + }); + }); + }); + + const formatTime = time => time.slice(0, 20); + + const testObjectLockConfigs = [ + { + testMode: 'COMPLIANCE', + val: 30, + type: 'Days', + }, + { + testMode: 'GOVERNANCE', + val: 5, + type: 'Years', + }, + ]; + testObjectLockConfigs.forEach(config => { + const { testMode, type, val } = config; + it('should put an object with default retention if object does not ' + + 'have retention configuration but bucket has', done => { + const testPostObjectRequest = new DummyRequest({ + bucketName, + formData: { + key: objectName, + }, + fileEventData: {}, + headers: {}, + url: '/', + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, postBody); + + const testObjLockRequest = { + bucketName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + post: objectLockTestUtils.generateXml(testMode, val, type), + }; + + bucketPut(authInfo, testPutBucketRequestLock, log, () => { + bucketPutObjectLock(authInfo, testObjLockRequest, log, () => { + objectPost(authInfo, testPostObjectRequest, undefined, log, + (err, headers) => { + assert.ifError(err); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, + log, (err, md) => { + const mode = md.retentionMode; + const retainDate = md.retentionDate; + const date = moment(); + const days + = type === 'Days' ? val : val * 365; + const expectedDate + = date.add(days, 'days'); + assert.ifError(err); + assert.strictEqual(mode, testMode); + assert.strictEqual(formatTime(retainDate), + formatTime(expectedDate.toISOString())); + done(); + }); + }); + }); + }); + }); + }); + + + it('should successfully put an object with legal hold ON', done => { + const request = new DummyRequest({ + bucketName, + formData: { + key: objectName, + }, + fileEventData: {}, + headers: { + 'x-amz-object-lock-legal-hold': 'ON', + }, + url: '/', + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, postBody); + + bucketPut(authInfo, testPutBucketRequestLock, log, () => { + objectPost(authInfo, request, undefined, log, (err, headers) => { + assert.ifError(err); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, + (err, md) => { + assert.ifError(err); + assert.strictEqual(md.legalHold, true); + done(); + }); + }); + }); + }); + + it('should successfully put an object with legal hold OFF', done => { + const request = new DummyRequest({ + bucketName, + formData: { + key: objectName, + }, + fileEventData: {}, + headers: { + 'x-amz-object-lock-legal-hold': 'OFF', + }, + url: '/', + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', + }, postBody); + + bucketPut(authInfo, testPutBucketRequestLock, log, () => { + objectPost(authInfo, request, undefined, log, (err, headers) => { + assert.ifError(err); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + metadata.getObjectMD(bucketName, objectName, {}, log, + (err, md) => { + assert.ifError(err); + assert(md); + assert.strictEqual(md.legalHold, false); + done(); + }); + }); + }); + }); + + it('should not leave orphans in data when overwriting an object', done => { + const testPostObjectRequest2 = new DummyRequest({ + bucketName, + formData: { + key: objectName, + }, + fileEventData: {}, + headers: {}, + url: '/', + }, Buffer.from('I am another body', 'utf8')); + + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPost(authInfo, testPostObjectRequest, + undefined, log, () => { + objectPost(authInfo, testPostObjectRequest2, undefined, + log, + () => { + // orphan objects don't get deleted + // until the next tick + // in memory + setImmediate(() => { + // Data store starts at index 1 + assert.strictEqual(ds[0], undefined); + assert.strictEqual(ds[1], undefined); + assert.deepStrictEqual(ds[2].value, + Buffer.from('I am another body', 'utf8')); + done(); + }); + }); + }); + }); + }); + + it('should not leave orphans in data when overwriting an multipart upload object', done => { + bucketPut(authInfo, testPutBucketRequest, log, () => { + mpuUtils.createMPU('default', bucketName, objectName, log, + (err, testUploadId) => { + objectPost(authInfo, testPostObjectRequest, undefined, log, err => { + assert.ifError(err); + sinon.assert.calledWith(metadata.putObjectMD, + any, any, any, sinon.match({ oldReplayId: testUploadId }), any, any); + done(); + }); + }); + }); + }); + + describe('objectPost API with versioning', () => { + beforeEach(() => { + cleanup(); + }); + + const objData = ['foo0', 'foo1', 'foo2'].map(str => + Buffer.from(str, 'utf8')); + const testPostObjectRequests = objData.map(data => versioningTestUtils + .createPostObjectRequest(bucketName, objectName, data)); + + it('should delete latest version when creating new null version ' + + 'if latest version is null version', done => { + async.series([ + callback => bucketPut(authInfo, testPutBucketRequest, log, + callback), + // putting null version by putting obj before versioning configured + callback => objectPost(authInfo, testPostObjectRequests[0], undefined, + log, err => { + versioningTestUtils.assertDataStoreValues(ds, [objData[0]]); + callback(err); + }), + callback => bucketPutVersioning(authInfo, suspendVersioningRequest, + log, callback), + // creating new null version by putting obj after ver suspended + callback => objectPost(authInfo, testPostObjectRequests[1], + undefined, log, err => { + // wait until next tick since mem backend executes + // deletes in the next tick + setImmediate(() => { + // old null version should be deleted + versioningTestUtils.assertDataStoreValues(ds, + [undefined, objData[1]]); + callback(err); + }); + }), + // create another null version + callback => objectPost(authInfo, testPostObjectRequests[2], + undefined, log, err => { + setImmediate(() => { + // old null version should be deleted + versioningTestUtils.assertDataStoreValues(ds, + [undefined, undefined, objData[2]]); + callback(err); + }); + }), + ], done); + }); + + describe('when null version is not the latest version', () => { + const objData = ['foo0', 'foo1', 'foo2'].map(str => + Buffer.from(str, 'utf8')); + const testPostObjectRequests = objData.map(data => versioningTestUtils + .createPostObjectRequest(bucketName, objectName, data)); + beforeEach(done => { + async.series([ + callback => bucketPut(authInfo, testPutBucketRequest, log, + callback), + // putting null version: put obj before versioning configured + callback => objectPost(authInfo, testPostObjectRequests[0], + undefined, log, callback), + callback => bucketPutVersioning(authInfo, + enableVersioningRequest, log, callback), + // put another version: + callback => objectPost(authInfo, testPostObjectRequests[1], + undefined, log, callback), + callback => bucketPutVersioning(authInfo, + suspendVersioningRequest, log, callback), + ], err => { + if (err) { + return done(err); + } + versioningTestUtils.assertDataStoreValues(ds, + objData.slice(0, 2)); + return done(); + }); + }); + + it('should still delete null version when creating new null version', + done => { + objectPost(authInfo, testPostObjectRequests[2], undefined, + log, err => { + assert.ifError(err, `Unexpected err: ${err}`); + setImmediate(() => { + // old null version should be deleted after putting + // new null version + versioningTestUtils.assertDataStoreValues(ds, + [undefined, objData[1], objData[2]]); + done(err); + }); + }); + }); + }); + + it('should return BadDigest error and not leave orphans in data when ' + + 'contentMD5 and completedHash do not match', done => { + const testPostObjectRequests = new DummyRequest({ + bucketName, + formData: { + key: objectName, + }, + fileEventData: {}, + headers: {}, + url: '/', + contentMD5: 'vnR+tLdVF79rPPfF+7YvOg==', + }, Buffer.from('I am another body', 'utf8')); + + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPost(authInfo, testPostObjectRequests, undefined, log, + err => { + assert.deepStrictEqual(err, errors.BadDigest); + // orphan objects don't get deleted + // until the next tick + // in memory + setImmediate(() => { + // Data store starts at index 1 + assert.strictEqual(ds[0], undefined); + assert.strictEqual(ds[1], undefined); + done(); + }); + }); + }); + }); + }); +}); + diff --git a/tests/unit/helpers.js b/tests/unit/helpers.js index d4c017003d..1277e9110f 100644 --- a/tests/unit/helpers.js +++ b/tests/unit/helpers.js @@ -374,6 +374,19 @@ const versioningTestUtils = { }; return new DummyRequest(params, body); }, + createPostObjectRequest: (bucketName, keyName, body) => { + const params = { + bucketName, + formData: { + bucket: bucketName, + key: keyName, + }, + fileEventData: {}, + headers: {}, + url: '/', + }; + return new DummyRequest(params, body); + }, createBucketPutVersioningReq: (bucketName, status) => { const request = { bucketName, From 6ff75dbe886e59c4830e6c93dd791e8faaa7a35e Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Tue, 16 Jul 2024 16:27:25 +0200 Subject: [PATCH 06/23] CLDSRRV-546: functional test POST Object API --- .../aws-node-sdk/test/object/post.js | 906 ++++++++++++++++++ 1 file changed, 906 insertions(+) create mode 100644 tests/functional/aws-node-sdk/test/object/post.js diff --git a/tests/functional/aws-node-sdk/test/object/post.js b/tests/functional/aws-node-sdk/test/object/post.js new file mode 100644 index 0000000000..f4597302bd --- /dev/null +++ b/tests/functional/aws-node-sdk/test/object/post.js @@ -0,0 +1,906 @@ + +const xml2js = require('xml2js'); +const axios = require('axios'); +const crypto = require('crypto'); +const FormData = require('form-data'); +const assert = require('assert'); + +const BucketUtility = require('../../lib/utility/bucket-util'); +const getConfig = require('../support/config'); + +let bucketName; +const filename = 'test-file.txt'; +let fileBuffer; +const region = 'us-east-1'; +let ak; +let sk; +let s3; + +const generateBucketName = () => `test-bucket-${crypto.randomBytes(8).toString('hex')}`; + +const formatDate = (date) => { + const year = date.getUTCFullYear(); + const month = (date.getUTCMonth() + 1).toString().padStart(2, '0'); + const day = date.getUTCDate().toString().padStart(2, '0'); + return `${year}${month}${day}`; +}; + +const getSignatureKey = (key, dateStamp, regionName, serviceName) => { + const kDate = crypto.createHmac('sha256', `AWS4${key}`).update(dateStamp).digest(); + const kRegion = crypto.createHmac('sha256', kDate).update(regionName).digest(); + const kService = crypto.createHmac('sha256', kRegion).update(serviceName).digest(); + const kSigning = crypto.createHmac('sha256', kService).update('aws4_request').digest(); + return kSigning; +}; + +// 'additionalConditions' will also replace existing keys if they are present +const calculateFields = (ak, sk, additionalConditions, bucket = bucketName, key = filename) => { + const service = 's3'; + + const now = new Date(); + const formattedDate = now.toISOString().replace(/[:-]|\.\d{3}/g, ''); + let shortFormattedDate = formatDate(now); + + const credential = `${ak}/${shortFormattedDate}/${region}/${service}/aws4_request`; + const conditionsFields = [ + { bucket }, + { key }, + { 'x-amz-credential': credential }, + { 'x-amz-algorithm': 'AWS4-HMAC-SHA256' }, + { 'x-amz-date': formattedDate }, + ]; + if (additionalConditions) { + additionalConditions.forEach(field => { + const key = Object.keys(field)[0]; + const value = field[key]; + const index = conditionsFields.findIndex(condition => condition.hasOwnProperty(key)); + if (index !== -1) { + conditionsFields[index][key] = value; + if (key === 'x-amz-date') { + shortFormattedDate = value.split('T')[0]; + } + } else { + conditionsFields.push({ [key]: value }); + } + }); + } + const policy = { + expiration: new Date(new Date().getTime() + 60000).toISOString(), + conditions: conditionsFields, + }; + const policyBase64 = Buffer.from(JSON.stringify(policy)).toString('base64'); + + const signingKey = getSignatureKey(sk, shortFormattedDate, region, service); + const signature = crypto.createHmac('sha256', signingKey).update(policyBase64).digest('hex'); + + const returnFields = [ + { name: 'x-amz-credential', value: credential }, + { name: 'x-amz-algorithm', value: 'AWS4-HMAC-SHA256' }, + { name: 'x-amz-signature', value: signature }, + { name: 'x-amz-date', value: formattedDate }, + { name: 'policy', value: policyBase64 }, + { name: 'bucket', value: bucket }, + { name: 'key', value: key }, + ]; + if (!additionalConditions) { + return returnFields; + } + if (additionalConditions) { + additionalConditions.forEach(field => { + const key = Object.keys(field)[0]; + const value = field[key]; + const index = returnFields.findIndex(f => f.name === key); + if (index !== -1) { + returnFields[index].value = value; + } else { + returnFields.push({ name: key, value }); + } + }); + } + return returnFields; +}; + +describe('POST object', () => { + let bucketUtil; + let config; + const testContext = {}; + + before(() => { + config = getConfig('default'); + ak = config.credentials.accessKeyId; + sk = config.credentials.secretAccessKey; + bucketUtil = new BucketUtility('default'); + s3 = bucketUtil.s3; + }); + + beforeEach(done => { + bucketName = generateBucketName(); + const url = `${config.endpoint}/${bucketName}`; + testContext.bucketName = bucketName; + testContext.url = url; + + const fileContent = 'This is a test file'; + fileBuffer = Buffer.from(fileContent); + + // Create the bucket + s3.createBucket({ Bucket: bucketName }, err => { + if (err) { + return done(err); + } + return done(); + }); + }); + + + afterEach(done => { + const { bucketName } = testContext; + + process.stdout.write('Emptying bucket\n'); + bucketUtil.empty(bucketName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucketName); + }) + .then(() => done()) + .catch(err => { + if (err.code !== 'NoSuchBucket') { + process.stdout.write('Error in afterEach\n'); + return done(err); + } + return done(); + }); + }); + + + it('should successfully upload an object using a POST form', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk); + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, { filename }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(response => { + assert.equal(response.status, 204); + assert.equal(response.headers.location, `/${bucketName}/${filename}`); + assert.equal(response.headers.key, filename); + assert.equal(response.headers.bucket, bucketName); + done(); + }) + .catch(err => { + done(err); + }); + }); + }); + + it('should handle error when bucket does not exist', done => { + const fakeBucketName = generateBucketName(); + const tempUrl = `${config.endpoint}/${fakeBucketName}`; + const fields = calculateFields(ak, sk, [], fakeBucketName); + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + const fileContent = 'This is a test file'; + const fileBuffer = Buffer.from(fileContent); + + formData.append('file', fileBuffer, { filename }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(tempUrl, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(() => { + done(new Error('Expected error but got success response')); + }) + .catch(err => { + assert.equal(err.response.status, 404); + done(); + }); + }); + }); + + + it('should successfully upload a larger file to S3 using a POST form', done => { + const { url } = testContext; + const largeFileName = 'large-test-file.txt'; + const largeFileContent = 'This is a larger test file'.repeat(10000); // Simulate a larger file + const largeFileBuffer = Buffer.from(largeFileContent); + + const fields = calculateFields(ak, sk, [{ key: largeFileName }]); + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', largeFileBuffer, { filename: largeFileName }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(response => { + assert.equal(response.status, 204); + s3.listObjectsV2({ Bucket: bucketName }, (err, data) => { + if (err) { + return done(err); + } + + const uploadedFile = data.Contents.find(item => item.Key === largeFileName); + assert(uploadedFile, 'Uploaded file should exist in the bucket'); + assert.equal(uploadedFile.Size, Buffer.byteLength(largeFileContent), 'File size should match'); + + return done(); + }); + }) + .catch(err => { + done(err); + }); + }); + }); + + it('should be able to post an empty file and verify its existence', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk); + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + const emptyFileBuffer = Buffer.from(''); // Create a buffer for an empty file + + formData.append('file', emptyFileBuffer, filename); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(response => { + assert.equal(response.status, 204); + + // Check if the object exists using listObjects + s3.listObjectsV2({ Bucket: bucketName, Prefix: filename }, (err, data) => { + if (err) { + return done(err); + } + + const fileExists = data.Contents.some(item => item.Key === filename); + const file = data.Contents.find(item => item.Key === filename); + + assert(fileExists, 'File should exist in S3'); + assert.equal(file.Size, 0, 'File size should be 0'); + + // Clean up: delete the empty file from S3 + return s3.deleteObject({ Bucket: bucketName, Key: filename }, err => { + if (err) { + return done(err); + } + + return done(); + }); + }); + }) + .catch(err => { + done(err); + }); + }); + }); + + it('should handle error when file is missing', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk); + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(() => { + done(new Error('Expected error but got success response')); + }) + .catch(err => { + assert.equal(err.response.status, 400); + xml2js.parseString(err.response.data, (parseErr, result) => { + if (parseErr) { + return done(parseErr); + } + + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], 'POST requires exactly one file upload per request.'); + return done(); + }); + }); + }); + }); + + it('should handle error when there are multiple files', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk); + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + // Append the same buffer twice to simulate multiple files + formData.append('file', fileBuffer, { filename }); + formData.append('file', fileBuffer, { filename }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(() => { + done(new Error('Expected error but got success response')); + }) + .catch(err => { + assert.equal(err.response.status, 400); + xml2js.parseString(err.response.data, (parseErr, result) => { + if (parseErr) { + return done(parseErr); + } + + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], 'POST requires exactly one file upload per request.'); + return done(); + }); + }); + }); + }); + + + it('should handle error when key is missing', done => { + const { url } = testContext; + // Prep fields then remove the key field + let fields = calculateFields(ak, sk); + fields = fields.filter(e => e.name !== 'key'); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + const fileContent = 'This is a test file'; + const fileBuffer = Buffer.from(fileContent); + + formData.append('file', fileBuffer, { filename }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(() => { + done(new Error('Request should not succeed without key field')); + }) + .catch(err => { + assert.ok(err.response, 'Error should be returned by axios'); + + xml2js.parseString(err.response.data, (parseErr, result) => { + if (parseErr) { + return done(parseErr); + } + + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], + "Bucket POST must contain a field named 'key'. " + + 'If it is specified, please check the order of the fields.'); + return done(); + }); + }); + }); + }); + + it('should handle error when content-type is incorrect', done => { + const { url } = testContext; + // Prep fields then remove the key field + let fields = calculateFields(ak, sk); + fields = fields.filter(e => e.name !== 'key'); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; + headers['content-type'] = 'application/json'; + return axios.post(url, formData, { + headers, + }) + .then(() => { + done(new Error('Request should not succeed wrong content-type')); + }) + .catch(err => { + assert.ok(err.response, 'Error should be returned by axios'); + + xml2js.parseString(err.response.data, (err, result) => { + if (err) { + return done(err); + } + + const error = result.Error; + assert.equal(error.Code[0], 'PreconditionFailed'); + assert.equal(error.Message[0], + 'Bucket POST must be of the enclosure-type multipart/form-data'); + return done(); + }); + }); + }); + }); + + it('should handle error when content-type is missing', done => { + const { url } = testContext; + // Prep fields then remove the key field + let fields = calculateFields(ak, sk); + fields = fields.filter(e => e.name !== 'key'); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; + delete headers['content-type']; + return axios.post(url, formData, { + headers, + }) + .then(() => { + done(new Error('Request should not succeed without correct content-type')); + }) + .catch(err => { + assert.ok(err.response, 'Error should be returned by axios'); + + xml2js.parseString(err.response.data, (err, result) => { + if (err) { + return done(err); + } + + const error = result.Error; + assert.equal(error.Code[0], 'PreconditionFailed'); + assert.equal(error.Message[0], + 'Bucket POST must be of the enclosure-type multipart/form-data'); + return done(); + }); + }); + }); + }); + + it('should upload an object with key slash', done => { + const { url } = testContext; + const slashKey = '/'; + const fields = calculateFields(ak, sk, [{ key: slashKey }]); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(response => { + assert.equal(response.status, 204); + done(); + }) + .catch(err => { + done(err); + }); + }); + }); + + it('should fail to upload an object with key length of 0', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk, [ + { key: '' }, + ]); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + // Use an incorrect content length (e.g., actual length - 20) + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(() => done(new Error('Request should have failed but succeeded'))) + .catch(err => { + // Expecting an error response from the API + assert.equal(err.response.status, 400); + xml2js.parseString(err.response.data, (err, result) => { + if (err) { + return done(err); + } + + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], + 'User key must have a length greater than 0.'); + return done(); + }); + }); + }); + }); + + it('should fail to upload an object with key longer than 1024 bytes', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk, [ + { key: 'a'.repeat(1025) }, + ]); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + // Use an incorrect content length (e.g., actual length - 20) + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(() => { + // The request should fail, so we shouldn't get here + done(new Error('Request should have failed but succeeded')); + }) + .catch(err => { + // Expecting an error response from the API + assert.equal(err.response.status, 400); + xml2js.parseString(err.response.data, (err, result) => { + if (err) { + return done(err); + } + + const error = result.Error; + assert.equal(error.Code[0], 'KeyTooLongError'); + assert.equal(error.Message[0], + 'Your key is too long.'); + return done(); + }); + }); + }); + }); + + it('should replace ${filename} variable in key with the name of the uploaded file', done => { + const { url } = testContext; + const keyTemplate = 'uploads/test/${filename}'; + const fileToUpload = keyTemplate.replace('${filename}', filename); + const fields = calculateFields(ak, sk, [{ key: fileToUpload }]); + const formData = new FormData(); + + fields.forEach(field => { + const value = field.name === 'key' ? keyTemplate : field.value; + formData.append(field.name, value); + }); + + formData.append('file', fileBuffer, filename); + + formData.getLength((err, length) => { + if (err) return done(err); + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(response => { + assert.equal(response.status, 204); + const expectedKey = keyTemplate.replace('${filename}', filename); + + const listParams = { Bucket: bucketName, Prefix: expectedKey }; + return s3.listObjects(listParams, (err, data) => { + if (err) return done(err); + const objectExists = data.Contents.some(item => item.Key === expectedKey); + assert(objectExists, 'Object was not uploaded with the expected key'); + return done(); + }); + }) + .catch(done); + }); + }); + + it('should fail to upload an object with an invalid multipart boundary', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + // Generate the form data with a valid boundary + const validBoundary = formData.getBoundary(); + + // Manually create the payload with an invalid boundary + const invalidBoundary = '----InvalidBoundary'; + const payload = Buffer.concat([ + Buffer.from(`--${invalidBoundary}\r\n`), + Buffer.from(`Content-Disposition: form-data; name="key"\r\n\r\n${filename}\r\n`), + Buffer.from(`--${invalidBoundary}\r\n`), + Buffer.from(`Content-Disposition: form-data; name="file"; filename="${filename}"\r\n`), + Buffer.from('Content-Type: application/octet-stream\r\n\r\n'), + fileBuffer, + Buffer.from(`\r\n--${invalidBoundary}--\r\n`), + ]); + + // Create an axios instance with invalid headers + axios.post(url, payload, { + headers: { + 'Content-Type': `multipart/form-data; boundary=${validBoundary}`, + 'Content-Length': payload.length, + }, + }) + .then(() => { + // The request should fail, so we shouldn't get here + done(new Error('Request should have failed but succeeded')); + }) + .catch(err => { + // Expecting an error response from the API + assert.equal(err.response.status, 400); + done(); + }); + }); + + it('should fail to upload an object with an too small content length header', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + // Use an incorrect content length (e.g., actual length - 20) + const incorrectLength = length - 20; + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': incorrectLength, + }, + }) + .then(() => done(new Error('Request should have failed but succeeded'))) + .catch(err => { + // Expecting an error response from the API + assert.equal(err.response.status, 400); + done(); + }); + }); + }); + + it('should return an error if form data (excluding file) exceeds 20KB', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk); + + // Add additional fields to make form data exceed 20KB + const largeValue = 'A'.repeat(1024); // 1KB value + for (let i = 0; i < 21; i++) { // Add 21 fields of 1KB each to exceed 20KB + fields.push({ name: `field${i}`, value: largeValue }); + } + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + return formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(() => { + done(new Error('Request should not succeed with form data exceeding 20KB')); + }) + .catch(err => { + assert.ok(err.response, 'Error should be returned by axios'); + + xml2js.parseString(err.response.data, (err, result) => { + if (err) { + return done(err); + } + + const error = result.Error; + assert.equal(error.Code[0], 'MaxPostPreDataLengthExceeded'); + assert.equal(error.Message[0], + 'Your POST request fields preceeding the upload file was too large.'); + return done(); + }); + }); + }); + }); + + it('should successfully upload an object with bucket versioning enabled and verify version ID', done => { + const { url } = testContext; + + // Enable versioning on the bucket + const versioningParams = { + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }; + + return s3.putBucketVersioning(versioningParams, (err) => { + if (err) { + return done(err); + } + + const fields = calculateFields(ak, sk, [{ bucket: bucketName }]); + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + return formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(response => { + assert.equal(response.status, 204); + + // Verify version ID is present in the response + const versionId = response.headers['x-amz-version-id']; + assert.ok(versionId, 'Version ID should be present in the response headers'); + done(); + }) + .catch(err => { + done(err); + }); + }); + }); + }); +}); + From 8d59e1cd8e50a2578f345857496d5972db1f6b6c Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 14:21:41 +0200 Subject: [PATCH 07/23] fixup: update arsenal --- lib/api/api.js | 4 ++-- package.json | 5 +---- yarn.lock | 4 ++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/api/api.js b/lib/api/api.js index dbf3a980d8..b79cdc4833 100644 --- a/lib/api/api.js +++ b/lib/api/api.js @@ -246,8 +246,8 @@ const api = { return this[apiMethod](userInfo, request, log, callback); }); }, - callPostObject(request, response, log, callback) { - request.apiMethod = 'objectPost'; + callPostObject(apiMethod, request, response, log, callback) { + request.apiMethod = apiMethod; const requestContexts = prepareRequestContexts('objectPost', request, undefined, undefined, undefined); diff --git a/package.json b/package.json index 1d4e815bdf..328fb8035c 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "dependencies": { "@fastify/busboy": "^2.1.1", "@hapi/joi": "^17.1.0", - "arsenal": "git+https://github.com/scality/arsenal#61984fbac3721d72cfc05a0cba0a0965c158008d", + "arsenal": "git+https://github.com/scality/arsenal#5cd2814b4a128c44ecb3e4ed464610a47adda5d5", "async": "~2.5.0", "aws-sdk": "2.905.0", "azure-storage": "^2.1.0", @@ -61,9 +61,6 @@ }, "scripts": { "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/", - "ft_post": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/object/post.js", - "ft_post_aws": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/object/post-copy.js", - "ft_post_unit": "CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit/api/callPostMethod.js", "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/", "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/bucket", "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support", diff --git a/yarn.lock b/yarn.lock index 9b594ab03f..0f3e1d4972 100644 --- a/yarn.lock +++ b/yarn.lock @@ -504,9 +504,9 @@ arraybuffer.slice@~0.0.7: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#61984fbac3721d72cfc05a0cba0a0965c158008d": +"arsenal@git+https://github.com/scality/arsenal#5cd2814b4a128c44ecb3e4ed464610a47adda5d5": version "7.70.29" - resolved "git+https://github.com/scality/arsenal#61984fbac3721d72cfc05a0cba0a0965c158008d" + resolved "git+https://github.com/scality/arsenal#5cd2814b4a128c44ecb3e4ed464610a47adda5d5" dependencies: "@js-sdsl/ordered-set" "^4.4.2" "@types/async" "^3.2.12" From b954a452e74ec3af91cef8a7eb98e3b57f0c64c1 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 15:04:36 +0200 Subject: [PATCH 08/23] fixup: callPostObject error cleanup --- lib/api/apiUtils/apiCallers/callPostObject.js | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/api/apiUtils/apiCallers/callPostObject.js b/lib/api/apiUtils/apiCallers/callPostObject.js index 6ca112ce8f..0235d482df 100644 --- a/lib/api/apiUtils/apiCallers/callPostObject.js +++ b/lib/api/apiUtils/apiCallers/callPostObject.js @@ -56,8 +56,6 @@ async function parseFormData(request, response, requestContexts, log) { formDataParser.on('file', async (fieldname, file, filename, encoding, mimetype) => { if (fileEventData) { - const err = errors.InvalidArgument - .customizeDescription('POST requires exactly one file upload per request.'); file.resume(); // Resume the stream to drain and discard the file if (tempFilePath) { fs.unlink(tempFilePath, unlinkErr => { @@ -66,15 +64,15 @@ async function parseFormData(request, response, requestContexts, log) { } }); } - return reject(err); + return reject(errors.InvalidArgument + .customizeDescription('POST requires exactly one file upload per request.')); } fileEventData = { fieldname, file, filename, encoding, mimetype }; if (!('key' in request.formData)) { - const err = errors.InvalidArgument + return reject(errors.InvalidArgument .customizeDescription('Bucket POST must contain a field named ' - + "'key'. If it is specified, please check the order of the fields."); - return reject(err); + + "'key'. If it is specified, please check the order of the fields.")); } // Replace `${filename}` with the actual filename request.formData.key = request.formData.key.replace('${filename}', filename); @@ -111,9 +109,8 @@ async function parseFormData(request, response, requestContexts, log) { formDataParser.on('finish', () => { if (!fileEventData) { - const err = errors.InvalidArgument - .customizeDescription('POST requires exactly one file upload per request.'); - return reject(err); + return reject(errors.InvalidArgument + .customizeDescription('POST requires exactly one file upload per request.')); } return formParserFinishedPromiseResolve(); }); @@ -130,11 +127,12 @@ async function parseFormData(request, response, requestContexts, log) { }); } -function getFileStat(filePath) { +function getFileStat(filePath, log) { return new Promise((resolve, reject) => { fs.stat(filePath, (err, stats) => { if (err) { - return reject(err); + log.trace('Error getting file size', { error: err }); + return reject(errors.InternalError); } return resolve(stats); }); @@ -151,7 +149,7 @@ async function processPostForm(request, response, requestContexts, log, callback const { userInfo, authorizationResults, streamingV4Params } = await parseFormData(request, response, requestContexts, log); - const fileStat = await getFileStat(request.fileEventData.file); + const fileStat = await getFileStat(request.fileEventData.file, log); request.parsedContentLength = fileStat.size; request.fileEventData.file = fs.createReadStream(request.fileEventData.file); if (request.formData['content-type']) { From 3443d712e1b616e2f6ce791779fca01c7d1528ab Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 15:14:31 +0200 Subject: [PATCH 09/23] fixup: arsenal package update --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 328fb8035c..6ea9a5cc59 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "dependencies": { "@fastify/busboy": "^2.1.1", "@hapi/joi": "^17.1.0", - "arsenal": "git+https://github.com/scality/arsenal#5cd2814b4a128c44ecb3e4ed464610a47adda5d5", + "arsenal": "git+https://github.com/scality/arsenal#e109d2dcfffd083bf35c0e3db2715484ffab8efd", "async": "~2.5.0", "aws-sdk": "2.905.0", "azure-storage": "^2.1.0", diff --git a/yarn.lock b/yarn.lock index 0f3e1d4972..860b77168a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -504,9 +504,9 @@ arraybuffer.slice@~0.0.7: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#5cd2814b4a128c44ecb3e4ed464610a47adda5d5": +"arsenal@git+https://github.com/scality/arsenal#e109d2dcfffd083bf35c0e3db2715484ffab8efd": version "7.70.29" - resolved "git+https://github.com/scality/arsenal#5cd2814b4a128c44ecb3e4ed464610a47adda5d5" + resolved "git+https://github.com/scality/arsenal#e109d2dcfffd083bf35c0e3db2715484ffab8efd" dependencies: "@js-sdsl/ordered-set" "^4.4.2" "@types/async" "^3.2.12" From 0d40c42b4ba4d847dc1cace636c2349ccea9e368 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 15:28:06 +0200 Subject: [PATCH 10/23] fixup: additional fn test for querry in url --- .../aws-node-sdk/test/object/post.js | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/functional/aws-node-sdk/test/object/post.js b/tests/functional/aws-node-sdk/test/object/post.js index f4597302bd..957b74e859 100644 --- a/tests/functional/aws-node-sdk/test/object/post.js +++ b/tests/functional/aws-node-sdk/test/object/post.js @@ -852,6 +852,51 @@ describe('POST object', () => { }); }); + it('should return an error if a query parameter is present in the URL', done => { + const { url } = testContext; + const queryParam = '?invalidParam=true'; + const invalidUrl = `${url}${queryParam}`; + const fields = calculateFields(ak, sk); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + return formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(invalidUrl, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(() => { + done(new Error('Request should not succeed with an invalid query parameter')); + }) + .catch(err => { + assert.ok(err.response, 'Error should be returned by axios'); + + xml2js.parseString(err.response.data, (err, result) => { + if (err) { + return done(err); + } + + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], 'Query String Parameters not allowed on POST requests.'); + return done(); + }); + }); + }); + }); + it('should successfully upload an object with bucket versioning enabled and verify version ID', done => { const { url } = testContext; From 99b0c26d626a84ba03cd28969660708d3d15e3a0 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 15:57:51 +0200 Subject: [PATCH 11/23] fixup: error messages for trace log in callPostObject handler --- lib/api/apiUtils/apiCallers/callPostObject.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/api/apiUtils/apiCallers/callPostObject.js b/lib/api/apiUtils/apiCallers/callPostObject.js index 0235d482df..6d8f76c39a 100644 --- a/lib/api/apiUtils/apiCallers/callPostObject.js +++ b/lib/api/apiUtils/apiCallers/callPostObject.js @@ -94,7 +94,7 @@ async function parseFormData(request, response, requestContexts, log) { }); tempFileStream.on('error', (err) => { - log.trace('Error streaming file to temporary location', { error: err }); + log.trace('Error streaming file to temporary location', { error: err.message }); reject(errors.InternalError); }); @@ -116,7 +116,7 @@ async function parseFormData(request, response, requestContexts, log) { }); formDataParser.on('error', (err) => { - log.trace('Error processing form data:', { error: err.description }); + log.trace('Error processing form data:', { error: err.message }); request.unpipe(formDataParser); // Following observed AWS behaviour reject(errors.MalformedPOSTRequest); @@ -131,7 +131,7 @@ function getFileStat(filePath, log) { return new Promise((resolve, reject) => { fs.stat(filePath, (err, stats) => { if (err) { - log.trace('Error getting file size', { error: err }); + log.trace('Error getting file size', { error: err.message }); return reject(errors.InternalError); } return resolve(stats); From 252205a76eed2adcfea11fef433f02e1511369f3 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 17:02:24 +0200 Subject: [PATCH 12/23] fixup: jsdoc ref in callPostObject handler --- lib/api/apiUtils/apiCallers/callPostObject.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/api/apiUtils/apiCallers/callPostObject.js b/lib/api/apiUtils/apiCallers/callPostObject.js index 6d8f76c39a..73929f8b76 100644 --- a/lib/api/apiUtils/apiCallers/callPostObject.js +++ b/lib/api/apiUtils/apiCallers/callPostObject.js @@ -5,9 +5,9 @@ const fs = require('fs'); const path = require('path'); const os = require('os'); -// per doc: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTForms.html#HTTPPOSTFormDeclaration +/** @see doc: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTForms.html#HTTPPOSTFormDeclaration */ const MAX_FIELD_SIZE = 20 * 1024; // 20KB -// per doc: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html +/** @see doc: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html */ const MAX_KEY_SIZE = 1024; async function authenticateRequest(request, requestContexts, log) { From 7c183269cdcbb468292f759307a28b4e7c58a210 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 17:14:05 +0200 Subject: [PATCH 13/23] fixup: error to string --- lib/api/apiUtils/apiCallers/callPostObject.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/api/apiUtils/apiCallers/callPostObject.js b/lib/api/apiUtils/apiCallers/callPostObject.js index 73929f8b76..8dc82e164b 100644 --- a/lib/api/apiUtils/apiCallers/callPostObject.js +++ b/lib/api/apiUtils/apiCallers/callPostObject.js @@ -94,7 +94,7 @@ async function parseFormData(request, response, requestContexts, log) { }); tempFileStream.on('error', (err) => { - log.trace('Error streaming file to temporary location', { error: err.message }); + log.trace('Error streaming file to temporary location', { error: err.toString() }); reject(errors.InternalError); }); @@ -116,7 +116,7 @@ async function parseFormData(request, response, requestContexts, log) { }); formDataParser.on('error', (err) => { - log.trace('Error processing form data:', { error: err.message }); + log.trace('Error processing form data:', { error: err.toString() }); request.unpipe(formDataParser); // Following observed AWS behaviour reject(errors.MalformedPOSTRequest); @@ -131,7 +131,7 @@ function getFileStat(filePath, log) { return new Promise((resolve, reject) => { fs.stat(filePath, (err, stats) => { if (err) { - log.trace('Error getting file size', { error: err.message }); + log.trace('Error getting file size', { error: err.toString() }); return reject(errors.InternalError); } return resolve(stats); From 88403e414fe4600e2b4f687b6d065100ff0c8720 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 17:45:48 +0200 Subject: [PATCH 14/23] fixup: fn tests --- tests/functional/aws-node-sdk/test/object/post.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/object/post.js b/tests/functional/aws-node-sdk/test/object/post.js index 957b74e859..41cdf9adf1 100644 --- a/tests/functional/aws-node-sdk/test/object/post.js +++ b/tests/functional/aws-node-sdk/test/object/post.js @@ -197,9 +197,6 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - const fileContent = 'This is a test file'; - const fileBuffer = Buffer.from(fileContent); - formData.append('file', fileBuffer, { filename }); formData.getLength((err, length) => { @@ -674,7 +671,7 @@ describe('POST object', () => { } const error = result.Error; - assert.equal(error.Code[0], 'KeyTooLongError'); + assert.equal(error.Code[0], 'KeyTooLong'); assert.equal(error.Message[0], 'Your key is too long.'); return done(); From 7018ba1967d709175f285a9f6c011e3364beb18f Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 17:46:20 +0200 Subject: [PATCH 15/23] fixup: keytoolong change callPostObject --- lib/api/apiUtils/apiCallers/callPostObject.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/apiUtils/apiCallers/callPostObject.js b/lib/api/apiUtils/apiCallers/callPostObject.js index 8dc82e164b..a4509f593f 100644 --- a/lib/api/apiUtils/apiCallers/callPostObject.js +++ b/lib/api/apiUtils/apiCallers/callPostObject.js @@ -44,7 +44,7 @@ async function parseFormData(request, response, requestContexts, log) { const lowerFieldname = fieldname.toLowerCase(); if (lowerFieldname === 'key') { if (val.length > MAX_KEY_SIZE) { - return reject(errors.KeyTooLongError); + return reject(errors.KeyTooLong); } else if (val.length === 0) { return reject(errors.InvalidArgument .customizeDescription('User key must have a length greater than 0.')); From d09a42d2782b8927d6c1b7ee8d6cec25dcaa21a0 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Wed, 17 Jul 2024 18:09:56 +0200 Subject: [PATCH 16/23] fixup: arsenal package --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 6ea9a5cc59..59f852ac91 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "dependencies": { "@fastify/busboy": "^2.1.1", "@hapi/joi": "^17.1.0", - "arsenal": "git+https://github.com/scality/arsenal#e109d2dcfffd083bf35c0e3db2715484ffab8efd", + "arsenal": "git+https://github.com/scality/arsenal#9e824e2ce7bf62e2070f679b2c2334cf7a1b7c5f", "async": "~2.5.0", "aws-sdk": "2.905.0", "azure-storage": "^2.1.0", diff --git a/yarn.lock b/yarn.lock index 860b77168a..fe55bd0ed5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -504,9 +504,9 @@ arraybuffer.slice@~0.0.7: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#e109d2dcfffd083bf35c0e3db2715484ffab8efd": +"arsenal@git+https://github.com/scality/arsenal#9e824e2ce7bf62e2070f679b2c2334cf7a1b7c5f": version "7.70.29" - resolved "git+https://github.com/scality/arsenal#e109d2dcfffd083bf35c0e3db2715484ffab8efd" + resolved "git+https://github.com/scality/arsenal#9e824e2ce7bf62e2070f679b2c2334cf7a1b7c5f" dependencies: "@js-sdsl/ordered-set" "^4.4.2" "@types/async" "^3.2.12" From 2a7d4cf93ab3259d6a9529dc73702718ba7f4d12 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Thu, 18 Jul 2024 11:40:23 +0200 Subject: [PATCH 17/23] fixup: url encode header response elements --- lib/api/objectPost.js | 3 +- .../aws-node-sdk/test/object/post.js | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/lib/api/objectPost.js b/lib/api/objectPost.js index f1b3b73b99..99086e03a0 100644 --- a/lib/api/objectPost.js +++ b/lib/api/objectPost.js @@ -95,8 +95,7 @@ function objectPost(authInfo, request, streamingV4Params, log, callback) { }); if (storingResult) { // ETag's hex should always be enclosed in quotes - responseHeaders.Key = request.formData.key; - responseHeaders.location = `/${bucketName}/${request.formData.key}`; + responseHeaders.location = `/${bucketName}/${encodeURIComponent(request.formData.key)}`; responseHeaders.Bucket = bucketName; responseHeaders.ETag = `"${storingResult.contentMD5}"`; } diff --git a/tests/functional/aws-node-sdk/test/object/post.js b/tests/functional/aws-node-sdk/test/object/post.js index 41cdf9adf1..7552e3e15c 100644 --- a/tests/functional/aws-node-sdk/test/object/post.js +++ b/tests/functional/aws-node-sdk/test/object/post.js @@ -187,6 +187,41 @@ describe('POST object', () => { }); }); + it('should handle url invalid characters in keys', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk, [{ key: 'key with spaces' }]); + const formData = new FormData(); + const encodedKey = 'key%20with%20spaces'; // Expected URL-encoded key + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, { filename }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(url, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(response => { + assert.equal(response.status, 204); + assert.equal(response.headers.location, `/${bucketName}/${encodedKey}`); + assert.equal(response.headers.bucket, bucketName); + done(); + }) + .catch(err => { + done(err); + }); + }); + }); + it('should handle error when bucket does not exist', done => { const fakeBucketName = generateBucketName(); const tempUrl = `${config.endpoint}/${fakeBucketName}`; From 3aadbdef47f83d9adf387de085c9caf1457145ef Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Thu, 18 Jul 2024 13:53:55 +0200 Subject: [PATCH 18/23] fixup: fn test fixup --- tests/functional/aws-node-sdk/test/object/post.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/functional/aws-node-sdk/test/object/post.js b/tests/functional/aws-node-sdk/test/object/post.js index 7552e3e15c..ce3a53b0ee 100644 --- a/tests/functional/aws-node-sdk/test/object/post.js +++ b/tests/functional/aws-node-sdk/test/object/post.js @@ -177,7 +177,6 @@ describe('POST object', () => { .then(response => { assert.equal(response.status, 204); assert.equal(response.headers.location, `/${bucketName}/${filename}`); - assert.equal(response.headers.key, filename); assert.equal(response.headers.bucket, bucketName); done(); }) From 29ebf3f0eb702a15067c67656f78b3421d30eb9e Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Thu, 18 Jul 2024 14:59:57 +0200 Subject: [PATCH 19/23] fixup: fn test 405 not allowed error --- .../aws-node-sdk/test/object/post.js | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/functional/aws-node-sdk/test/object/post.js b/tests/functional/aws-node-sdk/test/object/post.js index ce3a53b0ee..e0a3093bf3 100644 --- a/tests/functional/aws-node-sdk/test/object/post.js +++ b/tests/functional/aws-node-sdk/test/object/post.js @@ -928,6 +928,53 @@ describe('POST object', () => { }); }); + it('should return 405 Method Not Allowed if objectKey is present with a non-matching query parameter', done => { + const { url } = testContext; + const objectKey = 'someObjectKey'; + const queryParam = '?nonMatchingParam=true'; + const invalidUrl = `${url}/${objectKey}${queryParam}`; + const fields = calculateFields(ak, sk); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, filename); + + return formData.getLength((err, length) => { + if (err) { + return done(err); + } + + return axios.post(invalidUrl, formData, { + headers: { + ...formData.getHeaders(), + 'Content-Length': length, + }, + }) + .then(() => { + done(new Error('Request should not succeed with a non-matching query parameter')); + }) + .catch(err => { + assert.ok(err.response, 'Error should be returned by axios'); + + xml2js.parseString(err.response.data, (err, result) => { + if (err) { + return done(err); + } + + const error = result.Error; + assert.equal(error.Code[0], 'MethodNotAllowed'); + assert.equal(error.Message[0], 'The specified method is not allowed against this resource.'); + return done(); + }); + }); + }); + }); + + it('should successfully upload an object with bucket versioning enabled and verify version ID', done => { const { url } = testContext; From 2a41094c4397633351cf422b741375e69982b5a9 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Fri, 19 Jul 2024 11:39:42 +0200 Subject: [PATCH 20/23] fixup: small fixups --- tests/functional/aws-node-sdk/test/object/post.js | 2 -- tests/unit/helpers.js | 1 - 2 files changed, 3 deletions(-) diff --git a/tests/functional/aws-node-sdk/test/object/post.js b/tests/functional/aws-node-sdk/test/object/post.js index e0a3093bf3..b37b7afda6 100644 --- a/tests/functional/aws-node-sdk/test/object/post.js +++ b/tests/functional/aws-node-sdk/test/object/post.js @@ -177,7 +177,6 @@ describe('POST object', () => { .then(response => { assert.equal(response.status, 204); assert.equal(response.headers.location, `/${bucketName}/${filename}`); - assert.equal(response.headers.bucket, bucketName); done(); }) .catch(err => { @@ -212,7 +211,6 @@ describe('POST object', () => { .then(response => { assert.equal(response.status, 204); assert.equal(response.headers.location, `/${bucketName}/${encodedKey}`); - assert.equal(response.headers.bucket, bucketName); done(); }) .catch(err => { diff --git a/tests/unit/helpers.js b/tests/unit/helpers.js index 1277e9110f..6f39588048 100644 --- a/tests/unit/helpers.js +++ b/tests/unit/helpers.js @@ -378,7 +378,6 @@ const versioningTestUtils = { const params = { bucketName, formData: { - bucket: bucketName, key: keyName, }, fileEventData: {}, From cb7b4feb97ff4109390e60dfa118098422ee291f Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Mon, 22 Jul 2024 12:27:42 +0200 Subject: [PATCH 21/23] fixup: update for content-type checking --- lib/api/apiUtils/apiCallers/callPostObject.js | 48 +- .../aws-node-sdk/test/object/post.js | 1260 ++++++++++++----- 2 files changed, 967 insertions(+), 341 deletions(-) diff --git a/lib/api/apiUtils/apiCallers/callPostObject.js b/lib/api/apiUtils/apiCallers/callPostObject.js index a4509f593f..f69053176b 100644 --- a/lib/api/apiUtils/apiCallers/callPostObject.js +++ b/lib/api/apiUtils/apiCallers/callPostObject.js @@ -9,10 +9,30 @@ const os = require('os'); const MAX_FIELD_SIZE = 20 * 1024; // 20KB /** @see doc: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html */ const MAX_KEY_SIZE = 1024; +const POST_OBJECT_OPTIONAL_FIELDS = [ + 'acl', + 'awsaccesskeyid', + 'bucket', + 'cache-control', + 'content-disposition', + 'content-encoding', + 'content-type', + 'expires', + 'policy', + 'redirect', + 'tagging', + 'success_action_redirect', + 'success_action_status', + 'x-amz-meta-', + 'x-amz-storage-class', + 'x-amz-security-token', + 'x-amz-signgnature', + 'x-amz-website-redirect-location', +]; async function authenticateRequest(request, requestContexts, log) { return new Promise(resolve => { - // TODO RING-45960 remove ignore for POST object here + // TODO RING-45960 remove ignore auth check for POST object here auth.server.doAuth(request, log, (err, userInfo, authorizationResults, streamingV4Params) => resolve({ userInfo, authorizationResults, streamingV4Params }), 's3', requestContexts); }); @@ -20,7 +40,16 @@ async function authenticateRequest(request, requestContexts, log) { async function parseFormData(request, response, requestContexts, log) { /* eslint-disable no-param-reassign */ - const formDataParser = busboy({ headers: request.headers }); + let formDataParser; + try { + formDataParser = busboy({ headers: request.headers }); + } catch (err) { + log.trace('Error creating form data parser', { error: err.toString() }); + return Promise.reject(errors.PreconditionFailed + .customizeDescription('Bucket POST must be of the enclosure-type multipart/form-data')); + } + + // formDataParser = busboy({ headers: request.headers }); writeContinue(request, response); return new Promise((resolve, reject) => { @@ -37,11 +66,15 @@ async function parseFormData(request, response, requestContexts, log) { const formParserFinishedPromise = new Promise((res) => { formParserFinishedPromiseResolve = res; }); formDataParser.on('field', (fieldname, val) => { + // Check if we have exceeded the max size allowed for all fields totalFieldSize += Buffer.byteLength(val, 'utf8'); if (totalFieldSize > MAX_FIELD_SIZE) { return reject(errors.MaxPostPreDataLengthExceeded); } + + // validate the fieldname const lowerFieldname = fieldname.toLowerCase(); + // special handling for key field if (lowerFieldname === 'key') { if (val.length > MAX_KEY_SIZE) { return reject(errors.KeyTooLong); @@ -49,8 +82,12 @@ async function parseFormData(request, response, requestContexts, log) { return reject(errors.InvalidArgument .customizeDescription('User key must have a length greater than 0.')); } + request.formData[lowerFieldname] = val; + } + // add only the recognized fields to the formData object + if (POST_OBJECT_OPTIONAL_FIELDS.some(field => lowerFieldname.startsWith(field))) { + request.formData[lowerFieldname] = val; } - request.formData[lowerFieldname] = val; return undefined; }); @@ -140,11 +177,6 @@ function getFileStat(filePath, log) { } async function processPostForm(request, response, requestContexts, log, callback) { - if (!request.headers || !request.headers['content-type'].includes('multipart/form-data')) { - const contentTypeError = errors.PreconditionFailed - .customizeDescription('Bucket POST must be of the enclosure-type multipart/form-data'); - return process.nextTick(callback, contentTypeError); - } try { const { userInfo, authorizationResults, streamingV4Params } = await parseFormData(request, response, requestContexts, log); diff --git a/tests/functional/aws-node-sdk/test/object/post.js b/tests/functional/aws-node-sdk/test/object/post.js index b37b7afda6..7997af7d74 100644 --- a/tests/functional/aws-node-sdk/test/object/post.js +++ b/tests/functional/aws-node-sdk/test/object/post.js @@ -1,10 +1,12 @@ +// const AWS = require('aws-sdk'); const xml2js = require('xml2js'); const axios = require('axios'); const crypto = require('crypto'); const FormData = require('form-data'); const assert = require('assert'); - +const http = require('http'); +const { URL } = require('url'); const BucketUtility = require('../../lib/utility/bucket-util'); const getConfig = require('../support/config'); @@ -117,7 +119,7 @@ describe('POST object', () => { bucketName = generateBucketName(); const url = `${config.endpoint}/${bucketName}`; testContext.bucketName = bucketName; - testContext.url = url; + testContext.url = new URL(url); const fileContent = 'This is a test file'; fileBuffer = Buffer.from(fileContent); @@ -151,7 +153,6 @@ describe('POST object', () => { }); }); - it('should successfully upload an object using a POST form', done => { const { url } = testContext; const fields = calculateFields(ak, sk); @@ -168,20 +169,35 @@ describe('POST object', () => { return done(err); } - return axios.post(url, formData, { + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, headers: { ...formData.getHeaders(), 'Content-Length': length, }, - }) - .then(response => { - assert.equal(response.status, 204); - assert.equal(response.headers.location, `/${bucketName}/${filename}`); + }; + + const req = http.request(options); + + req.on('response', res => { + try { + assert.equal(res.statusCode, 204); + assert.equal(res.headers.location, `/${bucketName}/${filename}`); done(); - }) - .catch(err => { + } catch (err) { done(err); - }); + } + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; }); }); @@ -202,20 +218,35 @@ describe('POST object', () => { return done(err); } - return axios.post(url, formData, { + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, headers: { ...formData.getHeaders(), 'Content-Length': length, }, - }) - .then(response => { - assert.equal(response.status, 204); - assert.equal(response.headers.location, `/${bucketName}/${encodedKey}`); + }; + + const req = http.request(options); + + req.on('response', res => { + try { + assert.equal(res.statusCode, 204); + assert.equal(res.headers.location, `/${bucketName}/${encodedKey}`); done(); - }) - .catch(err => { + } catch (err) { done(err); - }); + } + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; }); }); @@ -236,23 +267,36 @@ describe('POST object', () => { return done(err); } - return axios.post(tempUrl, formData, { + const parsedUrl = new URL(tempUrl); + + const options = { + method: 'POST', + hostname: parsedUrl.hostname, + port: parsedUrl.port, + path: parsedUrl.pathname + parsedUrl.search, headers: { ...formData.getHeaders(), 'Content-Length': length, }, - }) - .then(() => { - done(new Error('Expected error but got success response')); - }) - .catch(err => { - assert.equal(err.response.status, 404); + }; + + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode === 404) { done(); - }); + } else { + done(new Error('Expected error but got success response')); + } + }); + + req.on('error', done); + + formData.pipe(req); + return undefined; }); }); - it('should successfully upload a larger file to S3 using a POST form', done => { const { url } = testContext; const largeFileName = 'large-test-file.txt'; @@ -273,29 +317,46 @@ describe('POST object', () => { return done(err); } - return axios.post(url, formData, { + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, headers: { ...formData.getHeaders(), 'Content-Length': length, }, - }) - .then(response => { - assert.equal(response.status, 204); + }; + + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode === 204) { s3.listObjectsV2({ Bucket: bucketName }, (err, data) => { if (err) { return done(err); } const uploadedFile = data.Contents.find(item => item.Key === largeFileName); - assert(uploadedFile, 'Uploaded file should exist in the bucket'); - assert.equal(uploadedFile.Size, Buffer.byteLength(largeFileContent), 'File size should match'); - - return done(); + try { + assert(uploadedFile, 'Uploaded file should exist in the bucket'); + assert.equal(uploadedFile.Size, + Buffer.byteLength(largeFileContent), 'File size should match'); + done(); + } catch (err) { + done(err); + } + return undefined; }); - }) - .catch(err => { - done(err); - }); + } else { + done(new Error(`Expected status 204 but got ${res.statusCode}`)); + } + }); + + req.on('error', done); + + formData.pipe(req); + return undefined; }); }); @@ -308,24 +369,30 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - const emptyFileBuffer = Buffer.from(''); // Create a buffer for an empty file + const emptyFileBuffer = Buffer.from(''); - formData.append('file', emptyFileBuffer, filename); + formData.append('file', emptyFileBuffer, { filename }); formData.getLength((err, length) => { if (err) { return done(err); } - return axios.post(url, formData, { + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, headers: { ...formData.getHeaders(), 'Content-Length': length, }, - }) - .then(response => { - assert.equal(response.status, 204); + }; + + const req = http.request(options); + req.on('response', res => { + if (res.statusCode === 204) { // Check if the object exists using listObjects s3.listObjectsV2({ Bucket: bucketName, Prefix: filename }, (err, data) => { if (err) { @@ -335,22 +402,34 @@ describe('POST object', () => { const fileExists = data.Contents.some(item => item.Key === filename); const file = data.Contents.find(item => item.Key === filename); - assert(fileExists, 'File should exist in S3'); - assert.equal(file.Size, 0, 'File size should be 0'); + try { + assert(fileExists, 'File should exist in S3'); + assert.equal(file.Size, 0, 'File size should be 0'); - // Clean up: delete the empty file from S3 - return s3.deleteObject({ Bucket: bucketName, Key: filename }, err => { - if (err) { - return done(err); - } + // Clean up: delete the empty file from S3 + s3.deleteObject({ Bucket: bucketName, Key: filename }, err => { + if (err) { + return done(err); + } - return done(); - }); + return done(); + }); + } catch (err) { + return done(err); + } + return undefined; }); - }) - .catch(err => { - done(err); - }); + } else { + done(new Error(`Expected status 204 but got ${res.statusCode}`)); + } + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; }); }); @@ -368,28 +447,55 @@ describe('POST object', () => { return done(err); } - return axios.post(url, formData, { + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, headers: { ...formData.getHeaders(), 'Content-Length': length, }, - }) - .then(() => { + }; + + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode !== 400) { done(new Error('Expected error but got success response')); - }) - .catch(err => { - assert.equal(err.response.status, 400); - xml2js.parseString(err.response.data, (parseErr, result) => { + return; + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { if (parseErr) { return done(parseErr); } - const error = result.Error; - assert.equal(error.Code[0], 'InvalidArgument'); - assert.equal(error.Message[0], 'POST requires exactly one file upload per request.'); - return done(); + try { + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], 'POST requires exactly one file upload per request.'); + done(); + } catch (err) { + done(err); + } + return undefined; }); }); + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; }); }); @@ -411,32 +517,58 @@ describe('POST object', () => { return done(err); } - return axios.post(url, formData, { + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, headers: { ...formData.getHeaders(), 'Content-Length': length, }, - }) - .then(() => { + }; + + const req = http.request(options); + + // Handle the response + req.on('response', res => { + if (res.statusCode !== 400) { done(new Error('Expected error but got success response')); - }) - .catch(err => { - assert.equal(err.response.status, 400); - xml2js.parseString(err.response.data, (parseErr, result) => { + return; + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { if (parseErr) { return done(parseErr); } - const error = result.Error; - assert.equal(error.Code[0], 'InvalidArgument'); - assert.equal(error.Message[0], 'POST requires exactly one file upload per request.'); - return done(); + try { + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], 'POST requires exactly one file upload per request.'); + return done(); + } catch (err) { + return done(err); + } }); }); + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; }); }); - it('should handle error when key is missing', done => { const { url } = testContext; // Prep fields then remove the key field @@ -459,39 +591,63 @@ describe('POST object', () => { return done(err); } - return axios.post(url, formData, { + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, headers: { ...formData.getHeaders(), 'Content-Length': length, }, - }) - .then(() => { + }; + + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode !== 400) { done(new Error('Request should not succeed without key field')); - }) - .catch(err => { - assert.ok(err.response, 'Error should be returned by axios'); + return; + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); - xml2js.parseString(err.response.data, (parseErr, result) => { + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { if (parseErr) { return done(parseErr); } - const error = result.Error; - assert.equal(error.Code[0], 'InvalidArgument'); - assert.equal(error.Message[0], - "Bucket POST must contain a field named 'key'. " - + 'If it is specified, please check the order of the fields.'); - return done(); + try { + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], + "Bucket POST must contain a field named 'key'. " + + 'If it is specified, please check the order of the fields.'); + return done(); + } catch (err) { + return done(err); + } }); }); + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; }); }); it('should handle error when content-type is incorrect', done => { const { url } = testContext; // Prep fields then remove the key field - let fields = calculateFields(ak, sk); - fields = fields.filter(e => e.name !== 'key'); + const fields = calculateFields(ak, sk); const formData = new FormData(); @@ -499,7 +655,7 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); formData.getLength((err, length) => { if (err) { @@ -509,37 +665,62 @@ describe('POST object', () => { const headers = { ...formData.getHeaders(), 'Content-Length': length, + 'Content-Type': 'application/json', // Incorrect content type }; - headers['content-type'] = 'application/json'; - return axios.post(url, formData, { + + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, headers, - }) - .then(() => { - done(new Error('Request should not succeed wrong content-type')); - }) - .catch(err => { - assert.ok(err.response, 'Error should be returned by axios'); + }; - xml2js.parseString(err.response.data, (err, result) => { - if (err) { - return done(err); + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode !== 412) { // 412 Precondition Failed + done(new Error('Request should not succeed with wrong content-type')); + return; + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { + if (parseErr) { + return done(parseErr); } - const error = result.Error; - assert.equal(error.Code[0], 'PreconditionFailed'); - assert.equal(error.Message[0], - 'Bucket POST must be of the enclosure-type multipart/form-data'); - return done(); + try { + const error = result.Error; + assert.equal(error.Code[0], 'PreconditionFailed'); + assert.equal(error.Message[0], + 'Bucket POST must be of the enclosure-type multipart/form-data'); + return done(); + } catch (err) { + return done(err); + } }); }); + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; }); }); - it('should handle error when content-type is missing', done => { + it('should handle error when content-type is "abc multipart/form-data"', done => { const { url } = testContext; // Prep fields then remove the key field - let fields = calculateFields(ak, sk); - fields = fields.filter(e => e.name !== 'key'); + const fields = calculateFields(ak, sk); const formData = new FormData(); @@ -547,7 +728,7 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); formData.getLength((err, length) => { if (err) { @@ -557,36 +738,62 @@ describe('POST object', () => { const headers = { ...formData.getHeaders(), 'Content-Length': length, + 'Content-Type': 'abc multipart/form-data', // Incorrect content type }; - delete headers['content-type']; - return axios.post(url, formData, { + + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, headers, - }) - .then(() => { - done(new Error('Request should not succeed without correct content-type')); - }) - .catch(err => { - assert.ok(err.response, 'Error should be returned by axios'); + }; - xml2js.parseString(err.response.data, (err, result) => { - if (err) { - return done(err); + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode !== 412) { // 412 Precondition Failed + done(new Error('Request should not succeed with wrong content-type')); + return; + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { + if (parseErr) { + return done(parseErr); } - const error = result.Error; - assert.equal(error.Code[0], 'PreconditionFailed'); - assert.equal(error.Message[0], - 'Bucket POST must be of the enclosure-type multipart/form-data'); - return done(); + try { + const error = result.Error; + assert.equal(error.Code[0], 'PreconditionFailed'); + assert.equal(error.Message[0], + 'Bucket POST must be of the enclosure-type multipart/form-data'); + return done(); + } catch (err) { + return done(err); + } }); }); + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; }); }); - it('should upload an object with key slash', done => { + it('should handle error when content-type is "multipart/form-data abc"', done => { const { url } = testContext; - const slashKey = '/'; - const fields = calculateFields(ak, sk, [{ key: slashKey }]); + // Prep fields then remove the key field + const fields = calculateFields(ak, sk); const formData = new FormData(); @@ -594,80 +801,71 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); formData.getLength((err, length) => { if (err) { return done(err); } - return axios.post(url, formData, { - headers: { - ...formData.getHeaders(), - 'Content-Length': length, - }, - }) - .then(response => { - assert.equal(response.status, 204); - done(); - }) - .catch(err => { - done(err); - }); - }); - }); - - it('should fail to upload an object with key length of 0', done => { - const { url } = testContext; - const fields = calculateFields(ak, sk, [ - { key: '' }, - ]); + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + 'Content-Type': 'multipart/form-data abc', // Incorrect content type + }; - const formData = new FormData(); + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; - fields.forEach(field => { - formData.append(field.name, field.value); - }); + const req = http.request(options); - formData.append('file', fileBuffer, filename); + req.on('response', res => { + if (res.statusCode !== 412) { // 412 Precondition Failed + done(new Error('Request should not succeed with wrong content-type')); + return; + } - formData.getLength((err, length) => { - if (err) { - return done(err); - } + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); - // Use an incorrect content length (e.g., actual length - 20) + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { + if (parseErr) { + return done(parseErr); + } - return axios.post(url, formData, { - headers: { - ...formData.getHeaders(), - 'Content-Length': length, - }, - }) - .then(() => done(new Error('Request should have failed but succeeded'))) - .catch(err => { - // Expecting an error response from the API - assert.equal(err.response.status, 400); - xml2js.parseString(err.response.data, (err, result) => { - if (err) { + try { + const error = result.Error; + assert.equal(error.Code[0], 'PreconditionFailed'); + assert.equal(error.Message[0], + 'Bucket POST must be of the enclosure-type multipart/form-data'); + return done(); + } catch (err) { return done(err); } - - const error = result.Error; - assert.equal(error.Code[0], 'InvalidArgument'); - assert.equal(error.Message[0], - 'User key must have a length greater than 0.'); - return done(); }); }); + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; }); }); - it('should fail to upload an object with key longer than 1024 bytes', done => { + it('should handle error when content-type is missing', done => { const { url } = testContext; - const fields = calculateFields(ak, sk, [ - { key: 'a'.repeat(1025) }, - ]); + const fields = calculateFields(ak, sk); const formData = new FormData(); @@ -675,40 +873,257 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); formData.getLength((err, length) => { if (err) { return done(err); } - // Use an incorrect content length (e.g., actual length - 20) + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; + delete headers['content-type']; // Ensure content-type is missing - return axios.post(url, formData, { - headers: { - ...formData.getHeaders(), - 'Content-Length': length, - }, - }) - .then(() => { - // The request should fail, so we shouldn't get here - done(new Error('Request should have failed but succeeded')); - }) - .catch(err => { - // Expecting an error response from the API - assert.equal(err.response.status, 400); - xml2js.parseString(err.response.data, (err, result) => { - if (err) { + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; + + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode !== 412) { + done(new Error('Request should not succeed without correct content-type')); + return; + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { + if (parseErr) { + return done(parseErr); + } + + try { + const error = result.Error; + assert.equal(error.Code[0], 'PreconditionFailed'); + assert.equal(error.Message[0], + 'Bucket POST must be of the enclosure-type multipart/form-data'); + return done(); + } catch (err) { return done(err); } + }); + }); + }); + + req.on('error', done); + + formData.pipe(req); + return undefined; + }); + }); + + it('should upload an object with key slash', done => { + const { url } = testContext; + const slashKey = '/'; + const fields = calculateFields(ak, sk, [{ key: slashKey }]); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, { filename }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; + + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; + + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode === 204) { + done(); + } else { + done(new Error(`Expected status 204 but got ${res.statusCode}`)); + } + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; + }); + }); + + it('should fail to upload an object with key length of 0', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk, [{ key: '' }]); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, { filename }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; + + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; - const error = result.Error; - assert.equal(error.Code[0], 'KeyTooLong'); - assert.equal(error.Message[0], - 'Your key is too long.'); - return done(); + const req = http.request(options); + + // Handle the response + req.on('response', res => { + if (res.statusCode !== 400) { + done(new Error('Request should have failed but succeeded')); + return; + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { + if (parseErr) { + return done(parseErr); + } + + try { + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], 'User key must have a length greater than 0.'); + return done(); + } catch (err) { + return done(err); + } }); }); + }); + + req.on('error', err => { + done(err); + }); + + formData.pipe(req); + return undefined; + }); + }); + + it('should fail to upload an object with key longer than 1024 bytes', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk, [{ key: 'a'.repeat(1025) }]); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, { filename }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; + + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; + + const req = http.request(options); + + // Handle the response + req.on('response', res => { + if (res.statusCode !== 400) { + done(new Error('Request should have failed but succeeded')); + return; + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { + if (parseErr) { + return done(parseErr); + } + + try { + const error = result.Error; + assert.equal(error.Code[0], 'KeyTooLong'); + assert.equal(error.Message[0], 'Your key is too long.'); + return done(); + } catch (err) { + return done(err); + } + }); + }); + }); + + // Handle any errors during the request + req.on('error', err => { + done(err); + }); + + // Stream the form data into the request + formData.pipe(req); + return undefined; }); }); @@ -724,30 +1139,48 @@ describe('POST object', () => { formData.append(field.name, value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); formData.getLength((err, length) => { if (err) return done(err); - return axios.post(url, formData, { - headers: { - ...formData.getHeaders(), - 'Content-Length': length, - }, - }) - .then(response => { - assert.equal(response.status, 204); - const expectedKey = keyTemplate.replace('${filename}', filename); - - const listParams = { Bucket: bucketName, Prefix: expectedKey }; - return s3.listObjects(listParams, (err, data) => { - if (err) return done(err); - const objectExists = data.Contents.some(item => item.Key === expectedKey); - assert(objectExists, 'Object was not uploaded with the expected key'); - return done(); - }); - }) - .catch(done); + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; + + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; + + const req = http.request(options); + + // Handle the response + req.on('response', res => { + if (res.statusCode !== 204) { + done(new Error(`Expected status 204 but got ${res.statusCode}`)); + return; + } + + const expectedKey = keyTemplate.replace('${filename}', filename); + + const listParams = { Bucket: bucketName, Prefix: expectedKey }; + s3.listObjects(listParams, (err, data) => { + if (err) return done(err); + const objectExists = data.Contents.some(item => item.Key === expectedKey); + assert(objectExists, 'Object was not uploaded with the expected key'); + return done(); + }); + }); + + req.on('error', done); + + formData.pipe(req); + return undefined; }); }); @@ -761,7 +1194,7 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); // Generate the form data with a valid boundary const validBoundary = formData.getBoundary(); @@ -778,25 +1211,93 @@ describe('POST object', () => { Buffer.from(`\r\n--${invalidBoundary}--\r\n`), ]); - // Create an axios instance with invalid headers - axios.post(url, payload, { - headers: { - 'Content-Type': `multipart/form-data; boundary=${validBoundary}`, - 'Content-Length': payload.length, - }, - }) - .then(() => { - // The request should fail, so we shouldn't get here + const headers = { + 'Content-Type': `multipart/form-data; boundary=${validBoundary}`, + 'Content-Length': payload.length, + }; + + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; + + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode !== 400) { done(new Error('Request should have failed but succeeded')); - }) - .catch(err => { - // Expecting an error response from the API - assert.equal(err.response.status, 400); - done(); + return; + } + + assert.equal(res.statusCode, 400); + done(); + }); + + req.on('error', err => { + done(err); + }); + + req.write(payload); + req.end(); + }); + + it('should fail to upload an object with a too small content length header', done => { + const { url } = testContext; + const fields = calculateFields(ak, sk); + + const formData = new FormData(); + + fields.forEach(field => { + formData.append(field.name, field.value); + }); + + formData.append('file', fileBuffer, { filename }); + + formData.getLength((err, length) => { + if (err) { + return done(err); + } + + // Use an incorrect content length (e.g., actual length - 20) + const incorrectLength = length - 20; + + const headers = { + ...formData.getHeaders(), + 'Content-Length': incorrectLength, + }; + + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; + + const req = http.request(options); + + // Handle the response + req.on('response', res => { + if (res.statusCode !== 400) { + return done(new Error('Request should have failed but succeeded')); + } + + return done(); }); + + // Handle any errors during the request + req.on('error', done); + + // Stream the form data into the request + formData.pipe(req); + return undefined; + }); }); - it('should fail to upload an object with an too small content length header', done => { + it.skip('should fail to upload an object with a too big content length header', done => { const { url } = testContext; const fields = calculateFields(ak, sk); @@ -814,7 +1315,7 @@ describe('POST object', () => { } // Use an incorrect content length (e.g., actual length - 20) - const incorrectLength = length - 20; + const incorrectLength = length + 2000; return axios.post(url, formData, { headers: { @@ -847,44 +1348,71 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); - return formData.getLength((err, length) => { + formData.getLength((err, length) => { if (err) { return done(err); } - return axios.post(url, formData, { - headers: { - ...formData.getHeaders(), - 'Content-Length': length, - }, - }) - .then(() => { - done(new Error('Request should not succeed with form data exceeding 20KB')); - }) - .catch(err => { - assert.ok(err.response, 'Error should be returned by axios'); + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; - xml2js.parseString(err.response.data, (err, result) => { - if (err) { - return done(err); + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; + + const req = http.request(options); + + // Handle the response + req.on('response', res => { + if (res.statusCode !== 400) { + return done(new Error('Request should not succeed with form data exceeding 20KB')); + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { + if (parseErr) { + return done(parseErr); } - const error = result.Error; - assert.equal(error.Code[0], 'MaxPostPreDataLengthExceeded'); - assert.equal(error.Message[0], - 'Your POST request fields preceeding the upload file was too large.'); - return done(); + try { + const error = result.Error; + assert.equal(error.Code[0], 'MaxPostPreDataLengthExceeded'); + assert.equal(error.Message[0], + 'Your POST request fields preceeding the upload file was too large.'); + return done(); + } catch (err) { + return done(err); + } }); }); + return undefined; + }); + + req.on('error', done); + + formData.pipe(req); + + return undefined; }); }); it('should return an error if a query parameter is present in the URL', done => { const { url } = testContext; const queryParam = '?invalidParam=true'; - const invalidUrl = `${url}${queryParam}`; + const invalidUrl = new URL(url.toString() + queryParam); const fields = calculateFields(ak, sk); const formData = new FormData(); @@ -893,36 +1421,62 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); - return formData.getLength((err, length) => { + formData.getLength((err, length) => { if (err) { return done(err); } - return axios.post(invalidUrl, formData, { - headers: { - ...formData.getHeaders(), - 'Content-Length': length, - }, - }) - .then(() => { - done(new Error('Request should not succeed with an invalid query parameter')); - }) - .catch(err => { - assert.ok(err.response, 'Error should be returned by axios'); + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; - xml2js.parseString(err.response.data, (err, result) => { - if (err) { - return done(err); + const options = { + method: 'POST', + hostname: invalidUrl.hostname, + port: invalidUrl.port, + path: invalidUrl.pathname + invalidUrl.search, + headers, + }; + + const req = http.request(options); + + // Handle the response + req.on('response', res => { + if (res.statusCode !== 400) { + return done(new Error('Request should not succeed with an invalid query parameter')); + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { + if (parseErr) { + return done(parseErr); } - const error = result.Error; - assert.equal(error.Code[0], 'InvalidArgument'); - assert.equal(error.Message[0], 'Query String Parameters not allowed on POST requests.'); - return done(); + try { + const error = result.Error; + assert.equal(error.Code[0], 'InvalidArgument'); + assert.equal(error.Message[0], 'Query String Parameters not allowed on POST requests.'); + return done(); + } catch (err) { + return done(err); + } }); }); + return undefined; + }); + + req.on('error', done); + + formData.pipe(req); + return undefined; }); }); @@ -930,7 +1484,7 @@ describe('POST object', () => { const { url } = testContext; const objectKey = 'someObjectKey'; const queryParam = '?nonMatchingParam=true'; - const invalidUrl = `${url}/${objectKey}${queryParam}`; + const invalidUrl = new URL(`${url}/${objectKey}${queryParam}`); const fields = calculateFields(ak, sk); const formData = new FormData(); @@ -939,40 +1493,66 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); - return formData.getLength((err, length) => { + formData.getLength((err, length) => { if (err) { return done(err); } - return axios.post(invalidUrl, formData, { - headers: { - ...formData.getHeaders(), - 'Content-Length': length, - }, - }) - .then(() => { - done(new Error('Request should not succeed with a non-matching query parameter')); - }) - .catch(err => { - assert.ok(err.response, 'Error should be returned by axios'); + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; - xml2js.parseString(err.response.data, (err, result) => { - if (err) { - return done(err); + const options = { + method: 'POST', + hostname: invalidUrl.hostname, + port: invalidUrl.port, + path: invalidUrl.pathname + invalidUrl.search, + headers, + }; + + const req = http.request(options); + + // Handle the response + req.on('response', res => { + if (res.statusCode !== 405) { + return done(new Error('Request should not succeed with a non-matching query parameter')); + } + + let responseData = ''; + res.on('data', chunk => { + responseData += chunk; + }); + + res.on('end', () => { + xml2js.parseString(responseData, (parseErr, result) => { + if (parseErr) { + return done(parseErr); } - const error = result.Error; - assert.equal(error.Code[0], 'MethodNotAllowed'); - assert.equal(error.Message[0], 'The specified method is not allowed against this resource.'); - return done(); + try { + const error = result.Error; + assert.equal(error.Code[0], 'MethodNotAllowed'); + assert.equal(error.Message[0], + 'The specified method is not allowed against this resource.'); + return done(); + } catch (err) { + return done(err); + } }); }); + return undefined; + }); + + req.on('error', done); + + formData.pipe(req); + return undefined; }); }); - it('should successfully upload an object with bucket versioning enabled and verify version ID', done => { const { url } = testContext; @@ -996,31 +1576,45 @@ describe('POST object', () => { formData.append(field.name, field.value); }); - formData.append('file', fileBuffer, filename); + formData.append('file', fileBuffer, { filename }); - return formData.getLength((err, length) => { + formData.getLength((err, length) => { if (err) { return done(err); } - return axios.post(url, formData, { - headers: { - ...formData.getHeaders(), - 'Content-Length': length, - }, - }) - .then(response => { - assert.equal(response.status, 204); - - // Verify version ID is present in the response - const versionId = response.headers['x-amz-version-id']; - assert.ok(versionId, 'Version ID should be present in the response headers'); - done(); - }) - .catch(err => { - done(err); - }); + const headers = { + ...formData.getHeaders(), + 'Content-Length': length, + }; + + const options = { + method: 'POST', + hostname: url.hostname, + port: url.port, + path: url.pathname + url.search, + headers, + }; + + const req = http.request(options); + + req.on('response', res => { + if (res.statusCode !== 204) { + return done(new Error(`Expected status 204 but got ${res.statusCode}`)); + } + + // Verify version ID is present in the response headers + const versionId = res.headers['x-amz-version-id']; + assert.ok(versionId, 'Version ID should be present in the response headers'); + return done(); + }); + + req.on('error', done); + + formData.pipe(req); + return undefined; }); + return undefined; }); }); }); From 95140e30c6f3c34a3d5e85f842241216bd722d60 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Mon, 22 Jul 2024 13:00:38 +0200 Subject: [PATCH 22/23] fixup: arsenal package update --- package.json | 5 ++++- yarn.lock | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 59f852ac91..833056a003 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "dependencies": { "@fastify/busboy": "^2.1.1", "@hapi/joi": "^17.1.0", - "arsenal": "git+https://github.com/scality/arsenal#9e824e2ce7bf62e2070f679b2c2334cf7a1b7c5f", + "arsenal": "git+https://github.com/scality/arsenal#4ef5748c028619edff10d6d38b21df43c8d63d88", "async": "~2.5.0", "aws-sdk": "2.905.0", "azure-storage": "^2.1.0", @@ -61,6 +61,9 @@ }, "scripts": { "ft_awssdk": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/", + "ft_post": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/object/post.js", + "ft_post_aws": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/object/post-copy.js", + "ft_post_unit": "CI=true S3BACKEND=mem mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json --recursive tests/unit/api/callPostMethod.js", "ft_awssdk_aws": "cd tests/functional/aws-node-sdk && AWS_ON_AIR=true mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/", "ft_awssdk_buckets": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/bucket", "ft_awssdk_objects_misc": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json test/legacy test/object test/service test/support", diff --git a/yarn.lock b/yarn.lock index fe55bd0ed5..03c97db0c0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -504,9 +504,9 @@ arraybuffer.slice@~0.0.7: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#9e824e2ce7bf62e2070f679b2c2334cf7a1b7c5f": +"arsenal@git+https://github.com/scality/arsenal#4ef5748c028619edff10d6d38b21df43c8d63d88": version "7.70.29" - resolved "git+https://github.com/scality/arsenal#9e824e2ce7bf62e2070f679b2c2334cf7a1b7c5f" + resolved "git+https://github.com/scality/arsenal#4ef5748c028619edff10d6d38b21df43c8d63d88" dependencies: "@js-sdsl/ordered-set" "^4.4.2" "@types/async" "^3.2.12" From 92d927339ced597292a0d8187b344a71c3a40d69 Mon Sep 17 00:00:00 2001 From: Will Toozs Date: Mon, 22 Jul 2024 14:45:57 +0200 Subject: [PATCH 23/23] update response headers --- lib/api/objectPost.js | 4 ++-- tests/functional/aws-node-sdk/test/object/post.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/api/objectPost.js b/lib/api/objectPost.js index 99086e03a0..399da5e5db 100644 --- a/lib/api/objectPost.js +++ b/lib/api/objectPost.js @@ -95,9 +95,9 @@ function objectPost(authInfo, request, streamingV4Params, log, callback) { }); if (storingResult) { // ETag's hex should always be enclosed in quotes - responseHeaders.location = `/${bucketName}/${encodeURIComponent(request.formData.key)}`; - responseHeaders.Bucket = bucketName; responseHeaders.ETag = `"${storingResult.contentMD5}"`; + responseHeaders.location = + `${request.headers.host}/${bucketName}/${encodeURIComponent(request.formData.key)}`; } const vcfg = bucket.getVersioningConfiguration(); const isVersionedObj = vcfg && vcfg.Status === 'Enabled'; diff --git a/tests/functional/aws-node-sdk/test/object/post.js b/tests/functional/aws-node-sdk/test/object/post.js index 7997af7d74..4f3f1908ba 100644 --- a/tests/functional/aws-node-sdk/test/object/post.js +++ b/tests/functional/aws-node-sdk/test/object/post.js @@ -185,7 +185,7 @@ describe('POST object', () => { req.on('response', res => { try { assert.equal(res.statusCode, 204); - assert.equal(res.headers.location, `/${bucketName}/${filename}`); + assert.equal(res.headers.location, `${url.hostname}:${url.port}/${bucketName}/${filename}`); done(); } catch (err) { done(err); @@ -234,7 +234,7 @@ describe('POST object', () => { req.on('response', res => { try { assert.equal(res.statusCode, 204); - assert.equal(res.headers.location, `/${bucketName}/${encodedKey}`); + assert.equal(res.headers.location, `${url.hostname}:${url.port}/${bucketName}/${encodedKey}`); done(); } catch (err) { done(err);