这些是我的package.json:中的相关依赖项
"dependencies": {
"@aws-sdk/client-s3": "^3.52.0",
"@aws-sdk/node-http-handler": "^3.52.0",
"@aws-sdk/s3-request-presigner": "^3.52.0",
"axios": "^0.26.0",
"body-parser": "^1.19.1",
"cors": "^2.8.5",
"express": "^4.17.2",
"express-fileupload": "^1.3.1",
"http-proxy-agent": "^5.0.0",
"lodash": "^4.17.21",
"morgan": "^1.10.0",
"node-fetch": "^2.6.7",
"proxy-agent": "^5.0.0"
}
这是我的代码：
const express = require('express');
const fileUpload = require('express-fileupload');
const cors = require('cors');
const bodyParser = require('body-parser');
const morgan = require('morgan');
const _ = require('lodash');
const path = require('path');
const app = express();
const fs = require('fs')
//s3 relevant imports
const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3");
const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
const fetch = require('node-fetch');
const HttpProxyAgent = require('http-proxy-agent');
let s3 = null
app.listen(port, async () => {
console.log(`App is listening on port ${port}. Static ${root}`)
try {
s3 = new S3Client({
region: 'ap-southeast-2',
credentials: {
accessKeyId: accessKey.data.access_key,
secretAccessKey: accessKey.data.secret_key
}
});
} catch (e) {
console.error(e)
}
});
app.get('/uploadS3Get', async (req, res) => {
try {
const presignedS3Url = await getSignedUrl(s3, new PutObjectCommand({
Bucket: 'xyz-unique-bucketname',
Key: 'test/carlos-test.txt',
}) );
const optionsForFetch = {
method: 'PUT',
body: fs.readFileSync('carlos-test.txt'),
agent: new HttpProxyAgent ('http://username:password@proxy.com:8080')
}
const respFromUpload = await fetch(presignedS3Url, optionsForFetch).catch( err => {
console.log("error catch from fetch")
console.log(err);
return null;
});
console.log("completed so this is the respFrom Upload")
console.log(respFromUpload)
console.log("sending the response back now")
res.status(200).send(respFromUpload);
} catch (e) {
console.log("general catch error")
console.log(e)
res.status(500).send(e);
}
})
我从节点提取中得到200，这是它打印出来的时候：
Response {
size: 0,
timeout: 0,
[Symbol(Body internals)]: {
body: PassThrough {
_readableState: [ReadableState],
_events: [Object: null prototype],
_eventsCount: 5,
_maxListeners: undefined,
_writableState: [WritableState],
allowHalfOpen: true,
[Symbol(kCapture)]: false,
[Symbol(kTransformState)]: [Object]
},
disturbed: false,
error: null
},
[Symbol(Response internals)]: {
url: 'https://xyz-unique-bucketname.s3.ap-southeast-2.amazonaws.com/test/carlos-test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=omitted%2F20220223%2Fap-southeast-2
%2Fs3%2Faws4_request&X-Amz-Date=20220223T022913Z&X-Amz-Expires=900&X-Amz-Signature=sigommitted&X-Amz-SignedHeaders=content-length%3Bhost&x-id=PutObject',
status: 200,
statusText: 'OK',
headers: Headers { [Symbol(map)]: [Object: null prototype] },
counter: 0
}
}
然而，即使使用签名url的节点fetch PUT返回200，文件本身也不在bucket中。
我的代码出了什么问题？为什么AWS SDK说200响应但文件丢失，从而产生误导？
建议：使用他们的createPresignedPost机制，而不是getSignedUrl/PutObjectCommand
服务器端代码类似，但为内容类型匹配和设置最大文件大小等条件提供了更好的功能。参见后预先签署的@aws sdk/s3
const { url, fields } = await createPresignedPost(client, postOpts);
这是到目前为止我为客户端上传的WIP类型脚本代码。createPresignedPost的输出像uploadFileAsFormData({ destUrl: url, formFields: fields, targetFile, method: 'POST' })一样传递
免责声明：自上次成功运行以来，我已经进行了更改，但还没有添加功能测试，所以它可能有问题。(我把它作为一个更大功能的一部分进行了粗略的处理，可能还要一周才能回到这个文件。(
import { createReadStream } from "fs";
import http from "http";
import { stat } from "fs/promises";
import FormData from "form-data";
const DefaultHighWaterMark = 1024 * 1024;
export interface UploadFileAsFormDataArgs {
/**
* File to upload.
* REMINDER: verify it is in an authorized directory before calling (/tmp or a mounted working-data volume)
*/
targetFile: string;
/**
* Upload destination URL. (Often a presigned url)
*
*/
destUrl: string;
/**
* Added as both a request header and as a form field (for the nitpicky unpleasant servers that require it.)
* mime type of file. See https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
*
* NOTE: if sending to object store using presigned url, this is likely REQUIRED
*/
contentType?: string;
/**
* HTTP request method. Usually POST or PUT
*/
method?: string;
extraHeaders?: Record<string, string>;
skipResponseBody?: boolean;
formFields: Record<string, string>;
/**
* Option to explicitly set buffer size for the stream. Normally, the default is 64kb in newer nodejs versions, while browsers use
* closer to 1MB. The default here follows browser-like default: 1024 * 1024.
*/
highWaterMark?: number;
// todo: option for AbortController
// todo: implement progress reporting w/ callback option
}
/**
* Upload a file similar to a browser using FORM submit.
* @param args
*/
export async function uploadFileAsFormData(args: UploadFileAsFormDataArgs): Promise<UploadFileResult> {
const { formFields, highWaterMark, targetFile, destUrl, contentType } = args;
const form = new FormData();
Object.entries(formFields).forEach(([ field, value ]) => {
form.append(field, value);
});
if (contentType) {
form.append('Content-Type', args.contentType); // in case needed... these object store APIs are finicky and pretty shitty about reporting errors (or success).
}
const readStream = createReadStream(targetFile, { highWaterMark: highWaterMark || DefaultHighWaterMark });
const { size } = await stat(targetFile);
const appendFileOpts: FormData.AppendOptions = {
knownLength: size,
};
if (contentType) {
appendFileOpts.contentType = contentType;
}
form.append('file', readStream, appendFileOpts);
const uploadStartedAt = Date.now();
const responseData: { body: string, statusCode?: number, statusMessage?: string, headers?: Record<string, any> } = await new Promise((resolve, reject) => {
form.submit(destUrl, (err: Error | null, resp: http.IncomingMessage) => {
if (err) {
reject(err);
} else {
const { statusCode, statusMessage, headers } = resp;
const buffers = [] as Buffer[];
resp.on('data', (b: Buffer) => {
buffers.push(b);
});
resp.on('end', () => {
resolve({
body: Buffer.concat(buffers).toString(),
statusCode,
statusMessage,
headers,
});
});
resp.on('error', (err) => {
reject(err);
});
resp.resume();
}
});
});
const { statusCode, statusMessage, body, headers } = responseData || {};
return {
sizeBytes: size,
statusCode,
statusMessage,
rawRespBody: body,
responseHeaders: headers,
uploadStartedAt,
uploadFinishedAt: Date.now(),
}
}
export interface UploadFileResult {
sizeBytes: number;
statusCode: number;
statusMessage?: string;
rawRespBody?: string;
responseHeaders?: Record<string, any>;
uploadStartedAt: number;
uploadFinishedAt?: number;
}
上一个答案：
S3不支持分块传输编码，所以应该添加一个Content-Length头，如下所示：
const { stat } = require('fs/promises'); // move up with the others
const { size } = await stat('carlos-test.txt');
const optionsForFetch = {
method: 'PUT',
body: fs.readFileSync('carlos-test.txt'),
headers: { 'Content-Length': size },
agent: new HttpProxyAgent ('http://username:password@proxy.com:8080')
}
令人沮丧的是，S3不支持使用预签名URL进行流式上传。我现在正在四处寻找这样的实体店，但到目前为止，挑选的机会很少
我同意你的200条回复——这是误导。他们的响应主体包含一个xml错误：<Error><Code>NotImplemented</Code><Message>A header you provided implies functionality that is not implemented</Message><Header>Transfer-Encoding</Header>...