diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index a3d49105..429ae1e8 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -22,11 +22,11 @@ jobs: architecture: "arm64" - platform: aws language: nodejs - version: "16" + version: "24" architecture: "x64" - platform: aws language: nodejs - version: "16" + version: "24" architecture: "arm64" - platform: aws language: cpp diff --git a/benchmarks/000.microbenchmarks/010.sleep/nodejs/function.js b/benchmarks/000.microbenchmarks/010.sleep/nodejs/function.js index e775e609..e79e4077 100644 --- a/benchmarks/000.microbenchmarks/010.sleep/nodejs/function.js +++ b/benchmarks/000.microbenchmarks/010.sleep/nodejs/function.js @@ -4,5 +4,5 @@ const timer = ms => new Promise( res => setTimeout(res, ms)); exports.handler = async function(event) { var sleep = event.sleep; timer(sleep*1000); - return sleep; + return {result: sleep}; }; diff --git a/benchmarks/100.webapps/110.dynamic-html/nodejs/function.js b/benchmarks/100.webapps/110.dynamic-html/nodejs/function.js index 9e6e57eb..cdcaba01 100644 --- a/benchmarks/100.webapps/110.dynamic-html/nodejs/function.js +++ b/benchmarks/100.webapps/110.dynamic-html/nodejs/function.js @@ -23,7 +23,7 @@ exports.handler = async function(event) { fs.readFile(file, "utf-8", function(err, data) { if(err) reject(err); - resolve(Mustache.render(data, input)); + resolve({result: Mustache.render(data, input)}); } ); }); diff --git a/benchmarks/100.webapps/130.crud-api/nodejs/function.js b/benchmarks/100.webapps/130.crud-api/nodejs/function.js new file mode 100644 index 00000000..f1859c72 --- /dev/null +++ b/benchmarks/100.webapps/130.crud-api/nodejs/function.js @@ -0,0 +1,78 @@ +const nosql = require('./nosql'); + +const nosqlClient = nosql.nosql.get_instance(); +const nosqlTableName = "shopping_cart"; + +async function addProduct(cartId, productId, productName, price, quantity) { + await nosqlClient.insert( + nosqlTableName, + ["cart_id", cartId], + ["product_id", productId], + { price: price, quantity: quantity, name: productName } + ); +} + +async function getProducts(cartId, productId) { + return await nosqlClient.get( + nosqlTableName, + ["cart_id", cartId], + ["product_id", productId] + ); +} + +async function queryProducts(cartId) { + const res = await nosqlClient.query( + nosqlTableName, + ["cart_id", cartId], + "product_id" + ); + + const products = []; + let priceSum = 0; + let quantitySum = 0; + + for (const product of res) { + products.push(product.name); + priceSum += product.price * product.quantity; + quantitySum += product.quantity; + } + + const avgPrice = quantitySum > 0 ? priceSum / quantitySum : 0.0; + + return { + products: products, + total_cost: priceSum, + avg_price: avgPrice + }; +} + +exports.handler = async function(event) { + const results = []; + + for (const request of event.requests) { + const route = request.route; + const body = request.body; + let res; + + if (route === "PUT /cart") { + await addProduct( + body.cart, + body.product_id, + body.name, + body.price, + body.quantity + ); + res = {}; + } else if (route === "GET /cart/{id}") { + res = await getProducts(body.cart, request.path.id); + } else if (route === "GET /cart") { + res = await queryProducts(body.cart); + } else { + throw new Error(`Unknown request route: ${route}`); + } + + results.push(res); + } + + return results; +}; diff --git a/benchmarks/100.webapps/130.crud-api/nodejs/package.json b/benchmarks/100.webapps/130.crud-api/nodejs/package.json new file mode 100644 index 00000000..e00c83dd --- /dev/null +++ b/benchmarks/100.webapps/130.crud-api/nodejs/package.json @@ -0,0 +1,9 @@ +{ + "name": "crud-api", + "version": "1.0.0", + "description": "CRUD API benchmark", + "author": "", + "license": "", + "dependencies": { + } +} diff --git a/benchmarks/200.multimedia/210.thumbnailer/nodejs/function.js b/benchmarks/200.multimedia/210.thumbnailer/nodejs/function.js index 26ac7374..090bd7d2 100644 --- a/benchmarks/200.multimedia/210.thumbnailer/nodejs/function.js +++ b/benchmarks/200.multimedia/210.thumbnailer/nodejs/function.js @@ -25,5 +25,5 @@ exports.handler = async function(event) { } ); await promise; - return {bucket: output_prefix, key: uploadName} + return {result: {bucket: output_prefix, key: uploadName}} }; diff --git a/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.22 b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.22 new file mode 100644 index 00000000..aaa3f883 --- /dev/null +++ b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.22 @@ -0,0 +1,10 @@ +{ + "name": "", + "version": "1.0.0", + "description": "", + "author": "", + "license": "", + "dependencies": { + "sharp": "^0.33" + } +} diff --git a/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.24 b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.24 new file mode 100644 index 00000000..aaa3f883 --- /dev/null +++ b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.24 @@ -0,0 +1,10 @@ +{ + "name": "", + "version": "1.0.0", + "description": "", + "author": "", + "license": "", + "dependencies": { + "sharp": "^0.33" + } +} diff --git a/benchmarks/300.utilities/311.compression/nodejs/function.js b/benchmarks/300.utilities/311.compression/nodejs/function.js new file mode 100644 index 00000000..afa627c5 --- /dev/null +++ b/benchmarks/300.utilities/311.compression/nodejs/function.js @@ -0,0 +1,124 @@ +const fs = require('fs'); +const path = require('path'); +const archiver = require('archiver'); +const { v4: uuidv4 } = require('uuid'); +const storage = require('./storage'); + +let storage_handler = new storage.storage(); + +/** + * Calculate total size of a directory recursively + * @param {string} directory - Path to directory + * @returns {number} Total size in bytes + */ +function parseDirectory(directory) { + let size = 0; + + function walkDir(dir) { + const files = fs.readdirSync(dir); + for (const file of files) { + const filepath = path.join(dir, file); + const stat = fs.statSync(filepath); + if (stat.isDirectory()) { + walkDir(filepath); + } else { + size += stat.size; + } + } + } + + walkDir(directory); + return size; +} + +/** + * Create a zip archive from a directory using archiver + * @param {string} sourceDir - Directory to compress + * @param {string} outputPath - Path for the output archive file + * @returns {Promise} + */ +async function createZipArchive(sourceDir, outputPath) { + return new Promise((resolve, reject) => { + const output = fs.createWriteStream(outputPath); + const archive = archiver('zip', { + zlib: { level: 9 } // Maximum compression + }); + + output.on('close', () => { + resolve(); + }); + + archive.on('error', (err) => { + reject(err); + }); + + archive.pipe(output); + + // Add all files from the directory, excluding the archive itself + archive.glob('**/*', { + cwd: sourceDir, + ignore: [path.basename(outputPath)] + }); + + archive.finalize(); + }); +} + +exports.handler = async function(event) { + const bucket = event.bucket.bucket; + const input_prefix = event.bucket.input; + const output_prefix = event.bucket.output; + const key = event.object.key; + + // Create unique download path + const download_path = path.join('/tmp', `${key}-${uuidv4()}`); + fs.mkdirSync(download_path, { recursive: true }); + + // Download directory from storage + const s3_download_begin = Date.now(); + await storage_handler.downloadDirectory(bucket, path.join(input_prefix, key), download_path); + const s3_download_stop = Date.now(); + + // Calculate size of downloaded files + const size = parseDirectory(download_path); + + // Compress directory + const compress_begin = Date.now(); + const archive_name = `${key}.zip`; + const archive_path = path.join(download_path, archive_name); + await createZipArchive(download_path, archive_path); + const compress_end = Date.now(); + + // Get archive size + const archive_size = fs.statSync(archive_path).size; + + // Upload compressed archive + const s3_upload_begin = Date.now(); + const [key_name, uploadPromise] = storage_handler.upload( + bucket, + path.join(output_prefix, archive_name), + archive_path + ); + await uploadPromise; + const s3_upload_stop = Date.now(); + + // Calculate times in microseconds + const download_time = (s3_download_stop - s3_download_begin) * 1000; + const upload_time = (s3_upload_stop - s3_upload_begin) * 1000; + const process_time = (compress_end - compress_begin) * 1000; + + return { + result: { + bucket: bucket, + key: key_name + }, + measurement: { + download_time: download_time, + download_size: size, + upload_time: upload_time, + upload_size: archive_size, + compute_time: process_time + } + }; +}; + diff --git a/benchmarks/300.utilities/311.compression/nodejs/package.json b/benchmarks/300.utilities/311.compression/nodejs/package.json new file mode 100644 index 00000000..d90c53c6 --- /dev/null +++ b/benchmarks/300.utilities/311.compression/nodejs/package.json @@ -0,0 +1,9 @@ +{ + "name": "compression-benchmark", + "version": "1.0.0", + "description": "Compression benchmark for serverless platforms", + "dependencies": { + "uuid": "^10.0.0", + "archiver": "^7.0.0" + } +} diff --git a/benchmarks/wrappers/aws/nodejs/handler.js b/benchmarks/wrappers/aws/nodejs/handler.js index 7434432f..2b0c6ff5 100644 --- a/benchmarks/wrappers/aws/nodejs/handler.js +++ b/benchmarks/wrappers/aws/nodejs/handler.js @@ -35,7 +35,7 @@ exports.handler = async function(event, context) { end: end, compute_time: micro, results_time: 0, - result: {result: result}, + result: result, is_cold: is_cold, request_id: context.awsRequestId }, http_trigger) diff --git a/benchmarks/wrappers/aws/nodejs/nosql.js b/benchmarks/wrappers/aws/nodejs/nosql.js new file mode 100644 index 00000000..b1c1bba0 --- /dev/null +++ b/benchmarks/wrappers/aws/nodejs/nosql.js @@ -0,0 +1,104 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. +// +// This is pretty much a Node.js rewrite of our Python wrapper. + + + +const { DynamoDBDocument } = require("@aws-sdk/lib-dynamodb"); +const { DynamoDB } = require("@aws-sdk/client-dynamodb"); + +class nosql { + constructor() { + this.client = DynamoDBDocument.from(new DynamoDB()); + this._tables = {}; + } + + _get_table(table_name) { + if (!(table_name in this._tables)) { + const env_name = `NOSQL_STORAGE_TABLE_${table_name}`; + if (env_name in process.env) { + this._tables[table_name] = process.env[env_name]; + } else { + throw new Error( + `Couldn't find an environment variable ${env_name} for table ${table_name}` + ); + } + } + return this._tables[table_name]; + } + + async insert(table_name, primary_key, secondary_key, data) { + data[primary_key[0]] = primary_key[1]; + data[secondary_key[0]] = secondary_key[1]; + await this.client + .put({ TableName: this._get_table(table_name), Item: data }); + } + + async get(table_name, primary_key, secondary_key) { + const key = {}; + key[primary_key[0]] = primary_key[1]; + key[secondary_key[0]] = secondary_key[1]; + const res = await this.client + .get({ TableName: this._get_table(table_name), Key: key }); + return res.Item; + } + + async update(table_name, primary_key, secondary_key, updates) { + + const key = {}; + key[primary_key[0]] = primary_key[1]; + key[secondary_key[0]] = secondary_key[1]; + + const update_names = {}; + const update_values = {}; + const update_expression = ["SET"]; + for (const [key_name, value] of Object.entries(updates)) { + update_expression.push(`#${key_name}_name = :${key_name}_value,`); + update_names[`#${key_name}_name`] = key_name; + update_values[`:${key_name}_value`] = value; + } + // remove trailing comma from the last assignment + update_expression[update_expression.length - 1] = update_expression[ + update_expression.length - 1 + ].slice(0, -1); + + await this.client + .update({ + TableName: this._get_table(table_name), + Key: key, + UpdateExpression: update_expression.join(" "), + ExpressionAttributeNames: update_names, + ExpressionAttributeValues: update_values, + }); + } + + async query(table_name, primary_key, _) { + const key_name = primary_key[0]; + const res = await this.client + .query({ + TableName: this._get_table(table_name), + KeyConditionExpression: "#key_name = :keyvalue", + ExpressionAttributeNames: { "#key_name": key_name }, + ExpressionAttributeValues: { ":keyvalue": primary_key[1] }, + }); + return res.Items; + } + + async delete(table_name, primary_key, secondary_key) { + const key = {}; + key[primary_key[0]] = primary_key[1]; + key[secondary_key[0]] = secondary_key[1]; + await this.client + .delete({ TableName: this._get_table(table_name), Key: key }); + } + + static get_instance() { + // The instance is independent of the storage table, + // as we provide table name for each operation. + if (!nosql.instance) { + nosql.instance = new nosql(); + } + return nosql.instance; + } +} +exports.nosql = nosql; diff --git a/benchmarks/wrappers/aws/nodejs/storage.js b/benchmarks/wrappers/aws/nodejs/storage.js index 38de3d9b..1a8e7e32 100644 --- a/benchmarks/wrappers/aws/nodejs/storage.js +++ b/benchmarks/wrappers/aws/nodejs/storage.js @@ -1,16 +1,16 @@ // Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. -const aws = require('aws-sdk'), - fs = require('fs'), - path = require('path'), - uuid = require('uuid'), - util = require('util'), - stream = require('stream'); +const fs = require('fs'), path = require('path'), uuid = require('uuid'), util = require('util'), stream = require('stream'); + +const { pipeline } = require("stream/promises"); + +const { Upload } = require('@aws-sdk/lib-storage'); +const { S3, GetObjectCommand, ListObjectsV2Command} = require('@aws-sdk/client-s3'); class aws_storage { constructor() { - this.S3 = new aws.S3(); + this.S3 = new S3(); } unique_name(file) { @@ -23,13 +23,34 @@ class aws_storage { var upload_stream = fs.createReadStream(filepath); let uniqueName = this.unique_name(file); let params = {Bucket: bucket, Key: uniqueName, Body: upload_stream}; - var upload = this.S3.upload(params); - return [uniqueName, upload.promise()]; + var upload = new Upload({ + client: this.S3, + params + }); + return [uniqueName, upload.done()]; + }; + + async download(bucket, file, filepath) { + var file_stream = fs.createWriteStream(filepath); + const response = await this.S3.send(new GetObjectCommand({ Bucket: bucket, Key: file })); + await pipeline(response.Body, file_stream); }; - download(bucket, file, filepath) { - var file = fs.createWriteStream(filepath); - this.S3.getObject( {Bucket: bucket, Key: file} ).createReadStream().pipe(file); + async downloadDirectory(bucket, prefix, downloadPath) { + const response = await this.S3.send(new ListObjectsV2Command({ Bucket: bucket, Prefix: prefix })); + + if (!response.Contents) { + throw new Error(`No objects found in bucket '${bucket}' with prefix '${prefix}'`); + } + + const downloadPromises = response.Contents.map(obj => { + const fileName = obj.Key; + const pathToFile = path.dirname(fileName); + fs.mkdirSync(path.join(downloadPath, pathToFile), { recursive: true }); + return this.download(bucket, fileName, path.join(downloadPath, fileName)); + }); + + await Promise.all(downloadPromises); }; uploadStream(bucket, file) { @@ -37,15 +58,19 @@ class aws_storage { let uniqueName = this.unique_name(file); // putObject won't work correctly for streamed data (length has to be known before) // https://stackoverflow.com/questions/38442512/difference-between-upload-and-putobject-for-uploading-a-file-to-s3 - var upload = this.S3.upload( {Bucket: bucket, Key: uniqueName, Body: write_stream} ); - return [write_stream, upload.promise(), uniqueName]; + var upload = new Upload({ + client: this.S3, + params: {Bucket: bucket, Key: uniqueName, Body: write_stream} + }); + return [write_stream, upload.done(), uniqueName]; }; // We return a promise to match the API for other providers downloadStream(bucket, file) { - // AWS.Request -> read stream - let downloaded = this.S3.getObject( {Bucket: bucket, Key: file} ).createReadStream(); - return Promise.resolve(downloaded); + return this.S3.send(new GetObjectCommand({ + Bucket: bucket, + Key: file + })).then(response => response.Body); }; -}; +} exports.storage = aws_storage; diff --git a/benchmarks/wrappers/azure/nodejs/handler.js b/benchmarks/wrappers/azure/nodejs/handler.js index 1e6294e5..bd6c3428 100644 --- a/benchmarks/wrappers/azure/nodejs/handler.js +++ b/benchmarks/wrappers/azure/nodejs/handler.js @@ -1,12 +1,17 @@ // Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. - const path = require('path'), fs = require('fs'); +if('NOSQL_STORAGE_DATABASE' in process.env) { + const nosql = require('./nosql'); + nosql.nosql.get_instance( + process.env['NOSQL_STORAGE_DATABASE'], + process.env['NOSQL_STORAGE_URL'], + process.env['NOSQL_STORAGE_CREDS'] + ); +} + module.exports = async function(context, req) { - if('connection_string' in req.body) { - process.env['STORAGE_CONNECTION_STRING'] = req.body.connection_string - } var begin = Date.now()/1000; var start = process.hrtime(); var func = require('./function'); @@ -30,7 +35,7 @@ module.exports = async function(context, req) { end: end, compute_time: micro, results_time: 0, - result: {result: result}, + result: result, is_cold: is_cold, request_id: context.invocationId }, diff --git a/benchmarks/wrappers/azure/nodejs/nosql.js b/benchmarks/wrappers/azure/nodejs/nosql.js new file mode 100644 index 00000000..c3eaaf08 --- /dev/null +++ b/benchmarks/wrappers/azure/nodejs/nosql.js @@ -0,0 +1,90 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +const { CosmosClient } = require("@azure/cosmos"); + +class nosql { + constructor(url, credential, database) { + this._client = new CosmosClient({ endpoint: url, key: credential }); + this._db_client = this._client.database(database); + this._containers = {}; + } + + _get_table(table_name) { + if (!(table_name in this._containers)) { + this._containers[table_name] = this._db_client.container(table_name); + } + return this._containers[table_name]; + } + + async insert(table_name, primary_key, secondary_key, data) { + data[primary_key[0]] = primary_key[1]; + // secondary key must have that name in CosmosDB + data.id = secondary_key[1]; + + await this._get_table(table_name).items.upsert(data); + } + + async get(table_name, primary_key, secondary_key) { + const { resource } = await this._get_table(table_name) + .item(secondary_key[1], primary_key[1]) + .read(); + + if (!resource) { + return null; + } + + resource[secondary_key[0]] = secondary_key[1]; + + // remove Azure-specific fields + delete resource.id; + delete resource._etag; + delete resource._rid; + delete resource._self; + delete resource._ts; + delete resource._attachments; + + return resource; + } + + async update(table_name, primary_key, secondary_key, updates) { + const ops = []; + for (const [key, value] of Object.entries(updates)) { + ops.push({ op: "add", path: `/${key}`, value: value }); + } + await this._get_table(table_name).item(secondary_key[1], primary_key[1]).patch(ops); + } + + async query(table_name, primary_key, secondary_key_name) { + const query = { + query: `SELECT * FROM c WHERE c.${primary_key[0]} = @keyvalue`, + parameters: [{ name: "@keyvalue", value: primary_key[1] }], + }; + const { resources } = await this._get_table(table_name) + .items.query(query, { enableCrossPartitionQuery: false }) + .fetchAll(); + + // Emulate the kind key + for (const item of resources) { + item[secondary_key_name] = item.id; + } + + return resources; + } + + async delete(table_name, primary_key, secondary_key) { + await this._get_table(table_name).item(secondary_key[1], primary_key[1]).delete(); + } + + static get_instance(database = null, url = null, credential = null) { + if (!nosql.instance) { + if (!database || !url || !credential) { + throw new Error( + "NoSQL database, URL and credentials must be provided when creating an instance." + ); + } + nosql.instance = new nosql(url, credential, database); + } + return nosql.instance; + } +} +exports.nosql = nosql; diff --git a/benchmarks/wrappers/azure/nodejs/storage.js b/benchmarks/wrappers/azure/nodejs/storage.js index 9228d356..c0ff103f 100644 --- a/benchmarks/wrappers/azure/nodejs/storage.js +++ b/benchmarks/wrappers/azure/nodejs/storage.js @@ -1,6 +1,7 @@ // Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. const { BlobServiceClient } = require('@azure/storage-blob'), + fs = require('fs'), path = require('path'), uuid = require('uuid'), util = require('util'), @@ -29,8 +30,28 @@ class azure_storage { return [uniqueName, blockBlobClient.uploadFile(filepath)]; }; - download(bucket, file, filepath) { - // TODO: + async download(container, file, filepath) { + let containerClient = this.client.getContainerClient(container); + let blockBlobClient = containerClient.getBlockBlobClient(file); + await blockBlobClient.downloadToFile(filepath); + }; + + async downloadDirectory(container, prefix, downloadPath) { + let containerClient = this.client.getContainerClient(container); + const blobs = containerClient.listBlobsFlat({ prefix: prefix }); + + // We don't get an array of objects, but an async iterator + // Thus, intead of a map, we use await-for to iterate over + // the blobs and create an array of promises. + const downloadPromises = []; + for await (const blob of blobs) { + const fileName = blob.name; + const pathToFile = path.dirname(fileName); + fs.mkdirSync(path.join(downloadPath, pathToFile), { recursive: true }); + downloadPromises.push(this.download(container, fileName, path.join(downloadPath, fileName))); + } + + await Promise.all(downloadPromises); }; // We could provide additional API for just providing a byte buffer and uploading diff --git a/benchmarks/wrappers/gcp/nodejs/handler.js b/benchmarks/wrappers/gcp/nodejs/handler.js index 66392e86..de455097 100644 --- a/benchmarks/wrappers/gcp/nodejs/handler.js +++ b/benchmarks/wrappers/gcp/nodejs/handler.js @@ -2,6 +2,11 @@ const path = require('path'), fs = require('fs'); +if('NOSQL_STORAGE_DATABASE' in process.env) { + const nosql = require('./function/nosql'); + nosql.nosql.get_instance(process.env['NOSQL_STORAGE_DATABASE']); +} + exports.handler = async function(req, res) { var begin = Date.now()/1000; var start = process.hrtime(); @@ -25,7 +30,7 @@ exports.handler = async function(req, res) { end: end, compute_time: micro, results_time: 0, - result: {result: result}, + result: result, is_cold: is_cold, request_id: req.headers["function-execution-id"] }); diff --git a/benchmarks/wrappers/gcp/nodejs/nosql.js b/benchmarks/wrappers/gcp/nodejs/nosql.js new file mode 100644 index 00000000..7c17cef9 --- /dev/null +++ b/benchmarks/wrappers/gcp/nodejs/nosql.js @@ -0,0 +1,93 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +const { Datastore, KEY } = require("@google-cloud/datastore"); + +/* +This implementation is the Node.js reimplementation of the reference +implementation in Python. It is used for the NoSQL benchmarks. + +Each benchmark supports up to two keys - one for grouping items, +and for unique identification of each item. + +In Google Cloud Datastore, we determine different tables by using +its value for `kind` name. + +The primary key is assigned to the `kind` value. + +To implement sorting semantics, we use the ancestor relation: +the sorting key is used as the parent. +It is the assumption that all related items will have the same parent. +*/ + +class nosql { + constructor(database) { + this._client = new Datastore({ 'databaseId': database }); + } + + _get_entity_key(table_name, primary_key, secondary_key) { + return this._client.key([ + primary_key[0], primary_key[1], table_name, secondary_key[1] + ]); + } + + async insert(table_name, primary_key, secondary_key, data) { + const key = this._get_entity_key(table_name, primary_key, secondary_key); + await this._client.save({ key, data }); + } + + async update(table_name, primary_key, secondary_key, updates) { + // Just like in the Python version, we don't have a direct update. + // Instead, we fetch the existing data, update fields, and write it. + // Otherwise, we would also rewrite fields that we do not want to modify. + const key = this._get_entity_key(table_name, primary_key, secondary_key); + let [res] = await this._client.get(key); + if (!res) { + res = {}; + } + Object.assign(res, updates); + await this._client.save({ key, data: res }); + } + + async get(table_name, primary_key, secondary_key) { + const key = this._get_entity_key(table_name, primary_key, secondary_key); + const [res] = await this._client.get(key); + if (!res) { + return null; + } + + // Emulate the kind and main keys + res[secondary_key[0]] = secondary_key[1]; + res[primary_key[0]] = primary_key[1]; + return res; + } + + async query(table_name, primary_key, secondary_key_name) { + const ancestor = this._client.key([primary_key[0], primary_key[1]]); + const query = this._client.createQuery(table_name).hasAncestor(ancestor); + const [res] = await this._client.runQuery(query); + + // Emulate the kind key + for (const item of res) { + item[secondary_key_name] = item[Datastore.KEY].name; + } + + return res; + } + + async delete(table_name, primary_key, secondary_key) { + const key = this._get_entity_key(table_name, primary_key, secondary_key); + await this._client.delete(key); + } + + static get_instance(database = null) { + // There's one database we connect to, so we can preallocate storage instance. + if (!nosql.instance) { + if (!database) { + throw new Error("NoSQL database must be provided when creating an instance."); + } + nosql.instance = new nosql(database); + } + return nosql.instance; + } +} +exports.nosql = nosql; diff --git a/benchmarks/wrappers/gcp/nodejs/storage.js b/benchmarks/wrappers/gcp/nodejs/storage.js index d8b56c28..4465842f 100644 --- a/benchmarks/wrappers/gcp/nodejs/storage.js +++ b/benchmarks/wrappers/gcp/nodejs/storage.js @@ -28,7 +28,21 @@ class gcp_storage { download(container, file, filepath) { let bucket = this.storage.bucket(container); var file = bucket.file(file); - file.download({destination: filepath}); + return file.download({destination: filepath}); + }; + + async downloadDirectory(container, prefix, downloadPath) { + let bucket = this.storage.bucket(container); + const [files] = await bucket.getFiles({ prefix: prefix }); + + const downloadPromises = files.map(file => { + const fileName = file.name; + const pathToFile = path.dirname(fileName); + fs.mkdirSync(path.join(downloadPath, pathToFile), { recursive: true }); + return this.download(container, fileName, path.join(downloadPath, fileName)); + }); + + await Promise.all(downloadPromises); }; uploadStream(container, file) { diff --git a/benchmarks/wrappers/local/nodejs/nosql.js b/benchmarks/wrappers/local/nodejs/nosql.js new file mode 100644 index 00000000..7ae9cbbd --- /dev/null +++ b/benchmarks/wrappers/local/nodejs/nosql.js @@ -0,0 +1,113 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +const aws = require("aws-sdk"); + +class nosql { + constructor() { + if (process.env.NOSQL_STORAGE_TYPE !== "scylladb") { + throw new Error(`Unsupported NoSQL storage type: ${process.env.NOSQL_STORAGE_TYPE}!`); + } + this.client = new aws.DynamoDB.DocumentClient({ + region: "None", + accessKeyId: "None", + secretAccessKey: "None", + endpoint: `http://${process.env.NOSQL_STORAGE_ENDPOINT}`, + maxRetries: 0, + }); + this._tables = {}; + } + + _get_table(table_name) { + if (!(table_name in this._tables)) { + const env_name = `NOSQL_STORAGE_TABLE_${table_name}`; + if (env_name in process.env) { + this._tables[table_name] = process.env[env_name]; + } else { + throw new Error( + `Couldn't find an environment variable ${env_name} for table ${table_name}` + ); + } + } + return this._tables[table_name]; + } + + async insert(table_name, primary_key, secondary_key, data) { + data[primary_key[0]] = primary_key[1]; + data[secondary_key[0]] = secondary_key[1]; + await this.client + .put({ TableName: this._get_table(table_name), Item: data }) + .promise(); + } + + async get(table_name, primary_key, secondary_key) { + const key = {}; + key[primary_key[0]] = primary_key[1]; + key[secondary_key[0]] = secondary_key[1]; + const res = await this.client + .get({ TableName: this._get_table(table_name), Key: key }) + .promise(); + return res.Item; + } + + async update(table_name, primary_key, secondary_key, updates) { + if (Object.keys(updates).length === 0) { + return; + } + + const key = {}; + key[primary_key[0]] = primary_key[1]; + key[secondary_key[0]] = secondary_key[1]; + + const update_names = {}; + const update_values = {}; + const update_expression = ["SET"]; + for (const [key_name, value] of Object.entries(updates)) { + update_expression.push(`#${key_name}_name = :${key_name}_value,`); + update_names[`#${key_name}_name`] = key_name; + update_values[`:${key_name}_value`] = value; + } + update_expression[update_expression.length - 1] = update_expression[ + update_expression.length - 1 + ].slice(0, -1); + + await this.client + .update({ + TableName: this._get_table(table_name), + Key: key, + UpdateExpression: update_expression.join(" "), + ExpressionAttributeNames: update_names, + ExpressionAttributeValues: update_values, + }) + .promise(); + } + + async query(table_name, primary_key, _) { + const key_name = primary_key[0]; + const res = await this.client + .query({ + TableName: this._get_table(table_name), + KeyConditionExpression: "#key_name = :keyvalue", + ExpressionAttributeNames: { "#key_name": key_name }, + ExpressionAttributeValues: { ":keyvalue": primary_key[1] }, + }) + .promise(); + return res.Items; + } + + async delete(table_name, primary_key, secondary_key) { + const key = {}; + key[primary_key[0]] = primary_key[1]; + key[secondary_key[0]] = secondary_key[1]; + await this.client + .delete({ TableName: this._get_table(table_name), Key: key }) + .promise(); + } + + static get_instance() { + if (!nosql.instance) { + nosql.instance = new nosql(); + } + return nosql.instance; + } +} +exports.nosql = nosql; diff --git a/benchmarks/wrappers/openwhisk/nodejs/nosql.js b/benchmarks/wrappers/openwhisk/nodejs/nosql.js new file mode 100644 index 00000000..7ae9cbbd --- /dev/null +++ b/benchmarks/wrappers/openwhisk/nodejs/nosql.js @@ -0,0 +1,113 @@ +// Copyright 2020-2025 ETH Zurich and the SeBS authors. All rights reserved. + +const aws = require("aws-sdk"); + +class nosql { + constructor() { + if (process.env.NOSQL_STORAGE_TYPE !== "scylladb") { + throw new Error(`Unsupported NoSQL storage type: ${process.env.NOSQL_STORAGE_TYPE}!`); + } + this.client = new aws.DynamoDB.DocumentClient({ + region: "None", + accessKeyId: "None", + secretAccessKey: "None", + endpoint: `http://${process.env.NOSQL_STORAGE_ENDPOINT}`, + maxRetries: 0, + }); + this._tables = {}; + } + + _get_table(table_name) { + if (!(table_name in this._tables)) { + const env_name = `NOSQL_STORAGE_TABLE_${table_name}`; + if (env_name in process.env) { + this._tables[table_name] = process.env[env_name]; + } else { + throw new Error( + `Couldn't find an environment variable ${env_name} for table ${table_name}` + ); + } + } + return this._tables[table_name]; + } + + async insert(table_name, primary_key, secondary_key, data) { + data[primary_key[0]] = primary_key[1]; + data[secondary_key[0]] = secondary_key[1]; + await this.client + .put({ TableName: this._get_table(table_name), Item: data }) + .promise(); + } + + async get(table_name, primary_key, secondary_key) { + const key = {}; + key[primary_key[0]] = primary_key[1]; + key[secondary_key[0]] = secondary_key[1]; + const res = await this.client + .get({ TableName: this._get_table(table_name), Key: key }) + .promise(); + return res.Item; + } + + async update(table_name, primary_key, secondary_key, updates) { + if (Object.keys(updates).length === 0) { + return; + } + + const key = {}; + key[primary_key[0]] = primary_key[1]; + key[secondary_key[0]] = secondary_key[1]; + + const update_names = {}; + const update_values = {}; + const update_expression = ["SET"]; + for (const [key_name, value] of Object.entries(updates)) { + update_expression.push(`#${key_name}_name = :${key_name}_value,`); + update_names[`#${key_name}_name`] = key_name; + update_values[`:${key_name}_value`] = value; + } + update_expression[update_expression.length - 1] = update_expression[ + update_expression.length - 1 + ].slice(0, -1); + + await this.client + .update({ + TableName: this._get_table(table_name), + Key: key, + UpdateExpression: update_expression.join(" "), + ExpressionAttributeNames: update_names, + ExpressionAttributeValues: update_values, + }) + .promise(); + } + + async query(table_name, primary_key, _) { + const key_name = primary_key[0]; + const res = await this.client + .query({ + TableName: this._get_table(table_name), + KeyConditionExpression: "#key_name = :keyvalue", + ExpressionAttributeNames: { "#key_name": key_name }, + ExpressionAttributeValues: { ":keyvalue": primary_key[1] }, + }) + .promise(); + return res.Items; + } + + async delete(table_name, primary_key, secondary_key) { + const key = {}; + key[primary_key[0]] = primary_key[1]; + key[secondary_key[0]] = secondary_key[1]; + await this.client + .delete({ TableName: this._get_table(table_name), Key: key }) + .promise(); + } + + static get_instance() { + if (!nosql.instance) { + nosql.instance = new nosql(); + } + return nosql.instance; + } +} +exports.nosql = nosql; diff --git a/configs/nodejs.json b/configs/nodejs.json index 9abc0ce1..151a9c8e 100644 --- a/configs/nodejs.json +++ b/configs/nodejs.json @@ -8,7 +8,7 @@ "container_deployment": true, "runtime": { "language": "nodejs", - "version": "16" + "version": "20" }, "type": "invocation-overhead", "perf-cost": { diff --git a/configs/systems.json b/configs/systems.json index 347229bc..3c66ac5b 100644 --- a/configs/systems.json +++ b/configs/systems.json @@ -61,9 +61,10 @@ "username": "docker_user", "deployment": { "files": [ - "storage.js" + "storage.js", + "nosql.js" ], - "packages": [] + "packages": {} } } }, @@ -102,10 +103,14 @@ "nodejs": { "base_images": { "x64": { - "16": "amazon/aws-lambda-nodejs:16.2024.09.06.14" + "24": "amazon/aws-lambda-nodejs:24.2026.04.22.11-x86_64", + "22": "amazon/aws-lambda-nodejs:22.2026.04.22.11-x86_64", + "20": "amazon/aws-lambda-nodejs:20.2026.04.22.11-x86_64" }, "arm64": { - "16": "amazon/aws-lambda-nodejs:16.2024.09.06.13" + "24": "amazon/aws-lambda-nodejs:24.2026.04.22.11-arm64", + "22": "amazon/aws-lambda-nodejs:22.2026.04.22.11-arm64", + "20": "amazon/aws-lambda-nodejs:20.2026.04.22.11-arm64" } }, "images": [ @@ -114,7 +119,8 @@ "deployment": { "files": [ "handler.js", - "storage.js" + "storage.js", + "nosql.js" ], "packages": { "uuid": "3.4.0" @@ -216,9 +222,11 @@ "deployment": { "files": [ "handler.js", - "storage.js" + "storage.js", + "nosql.js" ], "packages": { + "@azure/cosmos": "^4.3.0", "@azure/storage-blob": "^12.0.0", "uuid": "3.4.0" } @@ -299,9 +307,11 @@ "deployment": { "files": [ "handler.js", - "storage.js" + "storage.js", + "nosql.js" ], "packages": { + "@google-cloud/datastore": "^9.1.0", "@google-cloud/storage": "^4.0.0", "uuid": "3.4.0" } @@ -388,9 +398,11 @@ "deployment": { "files": [ "index.js", - "storage.js" + "storage.js", + "nosql.js" ], "packages": { + "aws-sdk": "^2.1692.0", "minio": "7.0.16" } } diff --git a/dockerfiles/aws/nodejs/Dockerfile.build b/dockerfiles/aws/nodejs/Dockerfile.build index c5ef797e..13dabd1c 100755 --- a/dockerfiles/aws/nodejs/Dockerfile.build +++ b/dockerfiles/aws/nodejs/Dockerfile.build @@ -3,7 +3,7 @@ FROM ${BASE_IMAGE} ARG TARGETARCH # useradd, groupmod -RUN yum install -y shadow-utils cmake curl libcurl libcurl-devel +RUN dnf swap -y curl-minimal curl && dnf install -y shadow-utils ENV GOSU_VERSION 1.14 # https://github.com/tianon/gosu/releases/tag/1.14 # key https://keys.openpgp.org/search?q=tianon%40debian.org diff --git a/sebs/aws/s3.py b/sebs/aws/s3.py index bd70c6fd..180abdea 100644 --- a/sebs/aws/s3.py +++ b/sebs/aws/s3.py @@ -230,7 +230,13 @@ def download(self, bucket_name: str, key: str, filepath: str) -> None: filepath: Local path where the file should be saved """ self.logging.info("Download {}:{} to {}".format(bucket_name, key, filepath)) - self.client.download_file(Bucket=bucket_name, Key=key, Filename=filepath) + try: + self.client.download_file(Bucket=bucket_name, Key=key, Filename=filepath) + except self.client.exceptions.ClientError as error: + raise RuntimeError( + f"Failed to download {key} from bucket {bucket_name}, " + f"reason: {error.response['Error']['Message']}" + ) from None def exists_bucket(self, bucket_name: str) -> bool: """Check if an S3 bucket exists and is accessible. diff --git a/sebs/regression.py b/sebs/regression.py index cf242eae..91a41a3b 100644 --- a/sebs/regression.py +++ b/sebs/regression.py @@ -49,7 +49,14 @@ "504.dna-visualisation", # DNA visualization ] -benchmarks_nodejs = ["010.sleep", "110.dynamic-html", "120.uploader", "210.thumbnailer"] +benchmarks_nodejs = [ + "010.sleep", + "110.dynamic-html", + "120.uploader", + "130.crud-api", + "210.thumbnailer", + "311.compression", +] benchmarks_java = ["010.sleep", "110.dynamic-html"]