アップロードを完全API経由に変更し、MinIOを非公開化
- presigned URL / multipartアップロードを廃止し、ブラウザ→API→MinIOの中継に統一 - S3Storage.createUploadPlan は常にsingle(API内部エンドポイント)を返す - PUT /api/v1/uploads/content/:encodedKey をS3モードでも有効化しガード削除 - body parserをstream化し5GiBをメモリに載せないよう修正 - フロントのmultipart分岐を削除しsingle PUTに一本化 これでMinIOに公開ドメインが不要になり、Coolifyではapi/webのドメイン設定だけで完結する
This commit is contained in:
amania-jailbreak
+5
-11
@@ -1,6 +1,5 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { posix } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import Fastify, { type FastifyReply, type FastifyRequest } from "fastify";
|
||||
import cookie from "@fastify/cookie";
|
||||
import cors from "@fastify/cors";
|
||||
@@ -43,10 +42,6 @@ const appInput = z.object({
|
||||
appPermissions: z.object({ entitlements: z.array(z.string()).max(200), privacy: z.record(z.string(), z.string()).default({}) }).optional(),
|
||||
});
|
||||
|
||||
const multipartPartsInput = z.object({
|
||||
parts: z.array(z.object({ partNumber: z.number().int().positive(), etag: z.string().min(1) })).max(10000).optional(),
|
||||
}).default({});
|
||||
|
||||
function safePath(value: string) {
|
||||
const decoded = decodeURIComponent(value).replaceAll("\\", "/");
|
||||
const normalized = posix.normalize(decoded);
|
||||
@@ -79,7 +74,9 @@ export async function createServer() {
|
||||
const appConfig = loadConfig();
|
||||
const runtime = await createRuntime(appConfig);
|
||||
const app = Fastify({ logger: appConfig.NODE_ENV !== "test", bodyLimit: appConfig.MAX_UPLOAD_BYTES });
|
||||
app.addContentTypeParser(["application/zip", "application/octet-stream"], { parseAs: "buffer" }, (_request, payload, done) => done(null, payload));
|
||||
// Stream large upload bodies straight through to storage without buffering
|
||||
// them in memory, so a 5 GiB ADP doesn't exhaust the process.
|
||||
app.addContentTypeParser(["application/zip", "application/octet-stream"], (_request, payload, done) => done(null, payload));
|
||||
await app.register(cookie);
|
||||
// The control plane (web :3000) and API (:4000) are cross-origin, so allow
|
||||
// the write methods the dashboard uses; @fastify/cors otherwise defaults to
|
||||
@@ -171,19 +168,18 @@ export async function createServer() {
|
||||
const safeFilename = input.filename.replace(/[^a-zA-Z0-9._-]/g, "-");
|
||||
const objectKey = `uploads/${workspace.id}/${uploadId}/${safeFilename}`;
|
||||
const uploadPlan = await runtime.storage.createUploadPlan(objectKey, "application/zip", input.sizeBytes);
|
||||
const upload = await runtime.store.createUpload({ id: uploadId, workspaceId: workspace.id, sourceId: source.id, appId: input.appId, objectKey, multipartUploadId: uploadPlan.mode === "multipart" ? uploadPlan.uploadId : undefined, originalFilename: input.filename, expectedSize: input.sizeBytes });
|
||||
const upload = await runtime.store.createUpload({ id: uploadId, workspaceId: workspace.id, sourceId: source.id, appId: input.appId, objectKey, originalFilename: input.filename, expectedSize: input.sizeBytes });
|
||||
if (uploadPlan.mode === "single" && uploadPlan.uploadUrl.startsWith("/")) uploadPlan.uploadUrl = `${appConfig.PUBLIC_BASE_URL}${uploadPlan.uploadUrl}`;
|
||||
return reply.code(201).send({ upload, uploadPlan, mode: appConfig.STORAGE_MODE });
|
||||
});
|
||||
|
||||
app.put("/api/v1/uploads/content/:encodedKey", async (request, reply) => {
|
||||
if (appConfig.STORAGE_MODE !== "local") return reply.code(404).send({ error: "DIRECT_UPLOAD_USES_STORAGE_URL" });
|
||||
const key = decodeURIComponent((request.params as { encodedKey: string }).encodedKey);
|
||||
if (!key.startsWith("uploads/") || key.includes("..")) return reply.code(400).send({ error: "UPLOAD_KEY_INVALID" });
|
||||
const uploadId = key.split("/")[2];
|
||||
const upload = await runtime.store.getUpload(uploadId);
|
||||
if (!upload || upload.objectKey !== key) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" });
|
||||
const body = Buffer.isBuffer(request.body) ? Readable.from(request.body) : request.body as NodeJS.ReadableStream;
|
||||
const body = request.body as NodeJS.ReadableStream;
|
||||
const receivedSize = await runtime.storage.writeUploadFromStream(key, body);
|
||||
if (receivedSize !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_SIZE_MISMATCH", expectedSize: upload.expectedSize, receivedSize });
|
||||
await runtime.store.updateUpload(upload.id, { receivedSize, status: "uploaded" });
|
||||
@@ -197,8 +193,6 @@ export async function createServer() {
|
||||
const workspace = await runtime.store.getWorkspace(identity);
|
||||
const upload = await runtime.store.getUpload(uploadId, workspace.id);
|
||||
if (!upload) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" });
|
||||
const completeInput = multipartPartsInput.parse(request.body);
|
||||
if (upload.multipartUploadId) await runtime.storage.completeMultipartUpload(upload.objectKey, upload.multipartUploadId, completeInput.parts);
|
||||
const head = await runtime.storage.headObject(upload.objectKey);
|
||||
if (!head || head.sizeBytes !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_INCOMPLETE", expectedSize: upload.expectedSize, receivedSize: head?.sizeBytes || 0 });
|
||||
await runtime.store.updateUpload(upload.id, { receivedSize: head.sizeBytes, status: "queued" });
|
||||
|
||||
+7
-13
@@ -7,15 +7,12 @@ import { Readable } from "node:stream";
|
||||
import {
|
||||
CreateBucketCommand,
|
||||
CompleteMultipartUploadCommand,
|
||||
CreateMultipartUploadCommand,
|
||||
GetObjectCommand,
|
||||
HeadBucketCommand,
|
||||
HeadObjectCommand,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
UploadPartCommand,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
||||
import type { AppConfig } from "./config";
|
||||
|
||||
export interface StoredObject {
|
||||
@@ -25,8 +22,7 @@ export interface StoredObject {
|
||||
}
|
||||
|
||||
export type UploadPlan =
|
||||
| { mode: "single"; uploadUrl: string }
|
||||
| { mode: "multipart"; uploadId: string; partSizeBytes: number; parts: Array<{ partNumber: number; uploadUrl: string }> };
|
||||
{ mode: "single"; uploadUrl: string };
|
||||
|
||||
export interface StorageAdapter {
|
||||
createUploadPlan(key: string, contentType: string, sizeBytes: number): Promise<UploadPlan>;
|
||||
@@ -149,14 +145,12 @@ export class S3Storage implements StorageAdapter {
|
||||
}
|
||||
|
||||
|
||||
async createUploadPlan(key: string, contentType: string, sizeBytes: number) {
|
||||
const partSizeBytes = 16 * 1024 * 1024;
|
||||
const partCount = Math.ceil(sizeBytes / partSizeBytes);
|
||||
if (partCount > 10000) throw new Error("UPLOAD_PART_COUNT_LIMIT");
|
||||
const multipart = await this.client.send(new CreateMultipartUploadCommand({ Bucket: this.appConfig.S3_BUCKET, Key: key, ContentType: contentType }));
|
||||
if (!multipart.UploadId) throw new Error("MULTIPART_UPLOAD_ID_MISSING");
|
||||
const parts = await Promise.all(Array.from({ length: partCount }, async (_, index) => ({ partNumber: index + 1, uploadUrl: await getSignedUrl(this.client, new UploadPartCommand({ Bucket: this.appConfig.S3_BUCKET, Key: key, UploadId: multipart.UploadId, PartNumber: index + 1 }), { expiresIn: 900 }) })));
|
||||
return { mode: "multipart" as const, uploadId: multipart.UploadId, partSizeBytes, parts };
|
||||
// Uploads always flow through the API (browser -> API -> MinIO), so there is
|
||||
// no presigned/multipart plan returned to the client. The API exposes a
|
||||
// single PUT endpoint that streams the body straight into object storage,
|
||||
// which keeps MinIO fully private and removes the need for a public S3 URL.
|
||||
async createUploadPlan(key: string) {
|
||||
return { mode: "single" as const, uploadUrl: `/api/v1/uploads/content/${encodeURIComponent(key)}` };
|
||||
}
|
||||
|
||||
async completeMultipartUpload(key: string, uploadId: string | undefined, parts: Array<{ partNumber: number; etag: string }> | undefined) {
|
||||
|
||||
Reference in New Issue
Block a user