Skip to content

Commit f20f0e2

Browse files
VivianBassVivianBass
authored
Merge pull request #218 from Zendro-dev/217-aws-update
update: migration to aws-sdk v3
2 parents 8ee7592 + 55ccdad commit f20f0e2

File tree

5 files changed

+46
-32
lines changed

5 files changed

+46
-32
lines changed

‎package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@
1414
"test-unit": "mocha ./test/mocha_unit.test.js",
1515
"test-integration": "bash ./test/testenv_cli.sh",
1616
"test-migration": "mocha ./test/mocha_migration.test.js"
17-
1817
},
1918
"author": "Science-DB team",
2019
"license": "GPL-3.0",
2120
"dependencies": {
21+
"@aws-sdk/client-s3": "^3.370.0",
22+
"@aws-sdk/lib-storage": "^3.370.0",
2223
"ajv": "^8.11.0",
23-
"aws-sdk": "^2.1207.0",
2424
"colors": "^1.4.0",
2525
"commander": "^9.4.0",
2626
"dotenv": "^16.0.2",

‎test/mocha_integration_amazon_s3.test.js

+28-11
Original file line numberDiff line numberDiff line change
@@ -2,26 +2,37 @@ const { expect } = require("chai");
22
const delay = require("delay");
33
const path = require("path");
44
const itHelpers = require("./integration_test_misc/integration_test_helpers");
5-
const AWS = require("aws-sdk");
5+
const {
6+
Upload
7+
} = require("@aws-sdk/lib-storage"),
8+
{
9+
S3
10+
} = require("@aws-sdk/client-s3");
611
const fs = require("fs");
712

813
describe("Amazon S3/ Minio - Upload/Read Operations", () => {
914
it("01. Reader: CSV bulkUpload", async () => {
1015
let csvPath = __dirname + "/integration_test_misc/minio/reader.csv";
1116
try {
12-
const s3 = new AWS.S3({
13-
accessKeyId: "sciencedb",
14-
secretAccessKey: "sciencedb",
17+
const s3 = new S3({
18+
region: "eu-central-1",
19+
credentials: {
20+
accessKeyId: "sciencedb",
21+
secretAccessKey: "sciencedb"
22+
},
1523
endpoint: `http://127.0.0.1:9000`,
16-
s3ForcePathStyle: true,
24+
forcePathStyle: true,
1725
signatureVersion: "v4",
1826
});
1927
let file_param = {
2028
Bucket: "sciencedb-development",
2129
Key: "reader.csv",
2230
Body: fs.createReadStream(csvPath),
2331
};
24-
await s3.upload(file_param).promise();
32+
await new Upload({
33+
client: s3,
34+
params: file_param
35+
}).done();
2536
} catch (e) {
2637
throw new Error(e);
2738
}
@@ -652,19 +663,25 @@ describe("Amazon S3/ Minio - Distributed Data Models", () => {
652663
it("01. Reader: CSV bulkUpload", async () => {
653664
let csvPath = __dirname + "/integration_test_misc/minio/dist_reader.csv";
654665
try {
655-
const s3 = new AWS.S3({
656-
accessKeyId: "sciencedb",
657-
secretAccessKey: "sciencedb",
666+
const s3 = new S3({
667+
region: "eu-central-1",
668+
credentials: {
669+
accessKeyId: "sciencedb",
670+
secretAccessKey: "sciencedb"
671+
},
658672
endpoint: `http://127.0.0.1:9000`,
659-
s3ForcePathStyle: true,
673+
forcePathStyle: true,
660674
signatureVersion: "v4",
661675
});
662676
let file_param = {
663677
Bucket: "sciencedb-development",
664678
Key: "dist_reader.csv",
665679
Body: fs.createReadStream(csvPath),
666680
};
667-
await s3.upload(file_param).promise();
681+
await new Upload({
682+
client: s3,
683+
params: file_param
684+
}).done();
668685
} catch (e) {
669686
throw new Error(e);
670687
}

‎test/unit_test_misc/test-describe/amazonS3-unittest.js

+4-5
Original file line numberDiff line numberDiff line change
@@ -60,13 +60,13 @@ static async countRecords(search, benignErrorReporter) {
6060
let num = null;
6161
try {
6262
const s3 = await this.storageHandler;
63-
const result = await s3.selectObjectContent(query).promise();
63+
const result = await s3.selectObjectContent(query);
6464
const events = result.Payload;
6565
for await (const event of events) {
6666
// Check the top-level field to determine which event this is.
6767
if (event.Records) {
6868
// handle Records event
69-
num = event.Records.Payload.toString();
69+
num = Buffer.from(event.Records.Payload).toString();
7070
num = JSON.parse(num.slice(0, num.length - 1)).num;
7171
}
7272
}
@@ -108,14 +108,13 @@ static async readAllCursor(search, order, pagination, benignErrorReporter){
108108
.selectObjectContent({
109109
...params,
110110
Expression: query,
111-
})
112-
.promise();
111+
});
113112
const events = result.Payload;
114113
for await (const event of events) {
115114
// Check the top-level field to determine which event this is.
116115
if (event.Records) {
117116
// handle Records event
118-
records = event.Records.Payload.toString();
117+
records = Buffer.from(event.Records.Payload).toString();
119118
records = JSON.parse("[" + records.slice(0, records.length - 1) + "]");
120119
}
121120
}

‎views/create-amazonS3-adapter.ejs

+6-7
Original file line numberDiff line numberDiff line change
@@ -168,13 +168,13 @@ module.exports = class <%- adapterName -%> {
168168
let item = null;
169169
try {
170170
const s3 = await this.storageHandler;
171-
const result = await s3.selectObjectContent(query).promise();
171+
const result = await s3.selectObjectContent(query);
172172
const events = result.Payload;
173173
for await (const event of events) {
174174
// Check the top-level field to determine which event this is.
175175
if (event.Records) {
176176
// handle Records event
177-
item = event.Records.Payload.toString();
177+
item = Buffer.from(event.Records.Payload).toString();
178178
item = JSON.parse(item.slice(0, item.length - 1));
179179
}
180180
}
@@ -210,13 +210,13 @@ module.exports = class <%- adapterName -%> {
210210
let num = null;
211211
try {
212212
const s3 = await this.storageHandler;
213-
const result = await s3.selectObjectContent(query).promise();
213+
const result = await s3.selectObjectContent(query);
214214
const events = result.Payload;
215215
for await (const event of events) {
216216
// Check the top-level field to determine which event this is.
217217
if (event.Records) {
218218
// handle Records event
219-
num = event.Records.Payload.toString();
219+
num = Buffer.from(event.Records.Payload).toString();
220220
num = JSON.parse(num.slice(0, num.length - 1)).num;
221221
}
222222
}
@@ -264,14 +264,13 @@ module.exports = class <%- adapterName -%> {
264264
.selectObjectContent({
265265
...params,
266266
Expression: query,
267-
})
268-
.promise();
267+
});
269268
const events = result.Payload;
270269
for await (const event of events) {
271270
// Check the top-level field to determine which event this is.
272271
if (event.Records) {
273272
// handle Records event
274-
records = event.Records.Payload.toString();
273+
records = Buffer.from(event.Records.Payload).toString();
275274
records = JSON.parse("[" + records.slice(0, records.length - 1) + "]");
276275
}
277276
}

‎views/create-models-amazonS3.ejs

+6-7
Original file line numberDiff line numberDiff line change
@@ -158,13 +158,13 @@ module.exports = class <%- nameLc -%> {
158158
let item = null;
159159
try {
160160
const s3 = await this.storageHandler;
161-
const result = await s3.selectObjectContent(query).promise();
161+
const result = await s3.selectObjectContent(query);
162162
const events = result.Payload;
163163
for await (const event of events) {
164164
// Check the top-level field to determine which event this is.
165165
if (event.Records) {
166166
// handle Records event
167-
item = event.Records.Payload.toString();
167+
item = Buffer.from(event.Records.Payload).toString();
168168
item = JSON.parse(item.slice(0, item.length - 1));
169169
}
170170
}
@@ -200,13 +200,13 @@ module.exports = class <%- nameLc -%> {
200200
let num = null;
201201
try {
202202
const s3 = await this.storageHandler;
203-
const result = await s3.selectObjectContent(query).promise();
203+
const result = await s3.selectObjectContent(query);
204204
const events = result.Payload;
205205
for await (const event of events) {
206206
// Check the top-level field to determine which event this is.
207207
if (event.Records) {
208208
// handle Records event
209-
num = event.Records.Payload.toString();
209+
num = Buffer.from(event.Records.Payload).toString();
210210
num = JSON.parse(num.slice(0, num.length - 1)).num;
211211
}
212212
}
@@ -263,14 +263,13 @@ module.exports = class <%- nameLc -%> {
263263
.selectObjectContent({
264264
...params,
265265
Expression: query,
266-
})
267-
.promise();
266+
});
268267
const events = result.Payload;
269268
for await (const event of events) {
270269
// Check the top-level field to determine which event this is.
271270
if (event.Records) {
272271
// handle Records event
273-
records = event.Records.Payload.toString();
272+
records = Buffer.from(event.Records.Payload).toString();
274273
records = JSON.parse("[" + records.slice(0, records.length - 1) + "]");
275274
}
276275
}

0 commit comments

Comments
 (0)