使用nodejs Lambda函数获取多个文件并写入AWS S3

46qrfjad  于 2023-01-25  发布在  Node.js
关注(0)|答案(1)|浏览(162)

我有一个从SQS消息中获取的图像url数组。我需要下载图像并将其存储在S3 bucket中。如果下载或存储图像失败,我需要捕获错误,以便将图像推送到另一个SQS队列中,以便稍后重试。
目前我所拥有的是下载和存储图像,但我不知道如何访问fetch和putObject函数的结果,而且我不确定我的方法是否正确,或者是否有更有效/更好/更优雅的方法。
这就是我现在拥有的

const AWS = require("aws-sdk");
const fetch = require("node-fetch")
const s3 = new AWS.S3();

exports.handler = function(event, context) {

    // SQS may invoke with multiple messages
    for (const message of event.Records) {
        const bodyData = JSON.parse(message.body);
        const bucket = 'my_images_bucket';
        const images = bodyData.images;

        let urls = [];

        for (const image of images) {
            urls.push(image);
        }

        let promises = urls.map(image => {
            fetch(image)
                .then((response) => {
                    if (!response.ok) {
                        throw new Error('An error occurred while fetching ' + image + ': ' + response.statusText);
                    }
                    return response;
                })
                .then(async res => {
                    try {
                        const buffer = await res.buffer();

                        console.log(image);

                        // store
                        return s3.putObject(
                            {
                                Bucket: bucket,
                                Key: image,
                                Body: buffer,
                                ContentType: "image/jpeg"
                            }
                        ).promise();

                    } catch (e) {
                        console.log('An error occurred while storing image ' + image + ': ' + e);
                    }
                })
                .catch((error) => {
                    console.error(error);
                });
        });

        Promise.all(promises)
            .then(d => {
                console.log('All images downloaded.');
                console.log('PromiseAll result: ' + d);
            }).catch(e => {
            console.log('Whoops something went wrong!', e);
        });
    }
}

我从这里得到的输出是:

INFO All images downloaded.
INFO PromiseAll result: ,,,,
INFO https://myserver/10658272812/image14.jpg
INFO https://myserver/10658272810/image12.jpg
INFO https://myserver/10658272804/image6.jpg
INFO https://myserver/10658272813/image15.jpg
INFO https://myserver/10658272816/image18.jpg
ni65a41a

ni65a41a1#

我附上了我为类似问题编写的代码。

const s3Put = (filename, data, mime, s3Params = {}) => {
        return new Promise((resolve, reject) => {
            s3.putObject({
                Bucket: bucket,
                Key: filename,
                Body: data,
                ContentType: 
                ...s3Params
            }, (err, data) => {
                if (err) {
                    return reject(err);
                }
                return resolve(data);
            });
        });
    };

    let filePromise = s3Put(to, content, fileMime, s3Params)
                    .then(() => console.log("MyCode"))
                    .catch(err => {
                        const error = {
                            message: `S3: ${(err.pri && err.pri.message) || (err.internal && err.internal.message)}`,
                            to
                        };
                        errors.push(error);
                        return onError(error);
                    });

相关问题