diff --git a/.npmrc b/.npmrc deleted file mode 100644 index 41583e36c..000000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -@jsr:registry=https://npm.jsr.io diff --git a/package-lock.json b/package-lock.json index c5e603529..083893967 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,6 @@ "version": "31.1.0", "license": "MIT", "dependencies": { - "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.4", "ansi-colors": "^4.1.3", "axios": "^1.13.2", "body-parser": "^1.20.3", @@ -30,6 +29,7 @@ "pg-query-stream": "^4.10.3", "pm2": "^6.0.14", "protobufjs": "^7.5.4", + "s3mini": "^0.9.1", "steam-user-odota": "^5.2.11", "stripe": "^9.16.0", "vdf-parser": "^1.2.1" @@ -115,12 +115,6 @@ "integrity": "sha512-idpUcNQ2co6T1oU/7/DG/ZRfipSSkTn9Ozw9f5vaXH7nzV3qhqZnhFVlHTzGGnRlzKlBwWOBzOdWi4Zeqg1c5A==", "license": "MIT" }, - "node_modules/@bradenmacdonald/s3-lite-client": { - "name": "@jsr/bradenmacdonald__s3-lite-client", - "version": "0.9.4", - "resolved": "https://npm.jsr.io/~/11/@jsr/bradenmacdonald__s3-lite-client/0.9.4.tgz", - "integrity": "sha512-Uar/IGGcnhLKppxjCMSd6r8P5UO7FDPsUaqs0WcoJiV1qfKGPQjiH7IxMUbnVoa0k6bH8HkriOvZf6TWOUAcvQ==" - }, "node_modules/@doctormckay/stdlib": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/@doctormckay/stdlib/-/stdlib-2.10.0.tgz", @@ -3315,6 +3309,15 @@ ], "license": "MIT" }, + "node_modules/s3mini": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/s3mini/-/s3mini-0.9.1.tgz", + "integrity": "sha512-aIsdMFbNEjA10KRRTWuEn52zX8kkB5ByrWOoyZg/WFrsMLqGbypTf0y2M7DdltoKkl+FYzl3g/SXJLvVZ54eXw==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", diff --git a/package.json b/package.json index 977ddc187..0a1af27bd 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,6 @@ "madge": "npx madge everything.ts --image deptree.svg" }, "dependencies": { - "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.4", "ansi-colors": "^4.1.3", "axios": "^1.13.2", "body-parser": "^1.20.3", @@ -46,6 +45,7 @@ "pg-query-stream": "^4.10.3", "pm2": "^6.0.14", "protobufjs": "^7.5.4", + "s3mini": "^0.9.1", "steam-user-odota": "^5.2.11", "stripe": "^9.16.0", "vdf-parser": "^1.2.1" diff --git a/svc/store/archive.ts b/svc/store/archive.ts index 241b62c58..dbdfcdfe7 100644 --- a/svc/store/archive.ts +++ b/svc/store/archive.ts @@ -1,11 +1,7 @@ import config from "../../config.ts"; import { gzipSync, gunzipSync } from "node:zlib"; import redis, { redisCount } from "./redis.ts"; -import { - S3Client, - type S3UploadedObjectInfo, - type S3Errors, -} from "@bradenmacdonald/s3-lite-client"; +import { S3mini } from 's3mini'; type ArchiveType = "match" | "player" | "blob"; class Archive { @@ -13,7 +9,7 @@ class Archive { private accessKeyId: string = ""; private secretAccessKey: string = ""; private bucket: string = ""; - private client: S3Client | null = null; + private client: S3mini | null = null; private type: ArchiveType | null = null; constructor(type: ArchiveType) { this.endpoint = config.ARCHIVE_S3_ENDPOINT; @@ -27,12 +23,11 @@ class Archive { } else if (type === "blob") { this.bucket = config.BLOB_ARCHIVE_S3_BUCKET; } - this.client = new S3Client({ - endPoint: this.endpoint, + this.client = new S3mini({ + endpoint: this.endpoint + '/' + this.bucket, region: "local", - bucket: this.bucket, - accessKey: this.accessKeyId, - secretKey: this.secretAccessKey, + accessKeyId: this.accessKeyId, + secretAccessKey: this.secretAccessKey, // put the bucket name in the path rather than the domain to avoid DNS issues with minio // forcePathStyle: true, }); @@ -66,15 +61,14 @@ class Archive { throw new Error("[ARCHIVE] s3 client not available"); } try { - const data = await this.client.getObject(key); - buffer = Buffer.from(await data.arrayBuffer()); - } catch (e: unknown) { - const error = e as S3Errors.ServerError; - if (error.statusCode === 404) { + const data = await this.client.getObjectArrayBuffer(key); + if (data == null) { // expected response if key not valid redisCount("archive_miss"); return null; } + buffer = Buffer.from(data); + } catch (e: unknown) { redisCount("archive_get_error"); throw e; } @@ -96,7 +90,7 @@ class Archive { key: string, blob: Buffer, noCache = false, - ): Promise => { + ): Promise => { if (!this.client) { throw new Error("[ARCHIVE] s3 client not available"); } @@ -106,7 +100,7 @@ class Archive { ); } const zip = gzipSync(blob); - let result; + let result: Response; try { // if (ifNotExists) { // // May not be implemented by some s3 providers @@ -114,13 +108,7 @@ class Archive { // } result = await this.client.putObject(key, zip); } catch (e: unknown) { - const error = e as S3Errors.ServerError; - console.log( - "[ARCHIVE] put error [key: %s] (%s): %s", - key, - error.code, - error.message, - ); + // const error = err as ErrorWithCode; // if (ifNotExists && e.Code === 'PreconditionFailed') { // // Expected error if ifNotExists was passed // return { message: 'already exists' }; diff --git a/test/test.ts b/test/test.ts index cff834752..f9944931a 100644 --- a/test/test.ts +++ b/test/test.ts @@ -23,9 +23,9 @@ import db, { upsertPlayer } from "../svc/store/db.ts"; import cassandra from "../svc/store/cassandra.ts"; import c from "ansi-colors"; import { suite, test, before, beforeEach, after } from "node:test"; -import { S3Client } from "@bradenmacdonald/s3-lite-client"; import { averageMedal } from "../svc/util/utility.ts"; import { addReliableJob } from "../svc/store/queue.ts"; +import { S3mini } from 's3mini'; const { RETRIEVER_HOST, POSTGRES_URL, CASSANDRA_URL } = config; const initPostgresHost = POSTGRES_URL.replace("/yasp_test", "/postgres"); @@ -133,26 +133,24 @@ async function initCassandra() { } async function initMinio() { - const client = new S3Client({ - endPoint: config.ARCHIVE_S3_ENDPOINT, + const client = new S3mini({ + endpoint: config.ARCHIVE_S3_ENDPOINT + '/' + config.BLOB_ARCHIVE_S3_BUCKET, region: "local", - accessKey: config.ARCHIVE_S3_KEY_ID, - secretKey: config.ARCHIVE_S3_KEY_SECRET, - bucket: config.BLOB_ARCHIVE_S3_BUCKET, + accessKeyId: config.ARCHIVE_S3_KEY_ID, + secretAccessKey: config.ARCHIVE_S3_KEY_SECRET, }); // Make a new test bucket with a new name // There's no good way to delete the test bucket automatically since we can't delete nonempty buckets console.log("create minio test bucket"); - await client.makeBucket(config.BLOB_ARCHIVE_S3_BUCKET); + await client.createBucket(); // Put a test blob await client.putObject("test", "test"); // Assert we can read without exception - const result = await client.getObject("test"); - assert.equal(await result.text(), "test"); - // Assert it rejects when reading invalid key - assert.rejects(async () => { - await client.getObject("invalid"); - }); + let result = await client.getObjectArrayBuffer("test"); + assert.equal(Buffer.from(result!).toString(), "test"); + // Assert it's null when reading invalid key + result = await client.getObjectArrayBuffer("invalid"); + assert.equal(result, null); } async function startServices() {