diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 45aa2c0..7b18549 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -1,6 +1,5 @@ import { randomUUID } from "node:crypto"; import { posix } from "node:path"; -import { Readable } from "node:stream"; import Fastify, { type FastifyReply, type FastifyRequest } from "fastify"; import cookie from "@fastify/cookie"; import cors from "@fastify/cors"; @@ -43,10 +42,6 @@ const appInput = z.object({ appPermissions: z.object({ entitlements: z.array(z.string()).max(200), privacy: z.record(z.string(), z.string()).default({}) }).optional(), }); -const multipartPartsInput = z.object({ - parts: z.array(z.object({ partNumber: z.number().int().positive(), etag: z.string().min(1) })).max(10000).optional(), -}).default({}); - function safePath(value: string) { const decoded = decodeURIComponent(value).replaceAll("\\", "/"); const normalized = posix.normalize(decoded); @@ -79,7 +74,9 @@ export async function createServer() { const appConfig = loadConfig(); const runtime = await createRuntime(appConfig); const app = Fastify({ logger: appConfig.NODE_ENV !== "test", bodyLimit: appConfig.MAX_UPLOAD_BYTES }); - app.addContentTypeParser(["application/zip", "application/octet-stream"], { parseAs: "buffer" }, (_request, payload, done) => done(null, payload)); + // Stream large upload bodies straight through to storage without buffering + // them in memory, so a 5 GiB ADP doesn't exhaust the process. + app.addContentTypeParser(["application/zip", "application/octet-stream"], (_request, payload, done) => done(null, payload)); await app.register(cookie); // The control plane (web :3000) and API (:4000) are cross-origin, so allow // the write methods the dashboard uses; @fastify/cors otherwise defaults to @@ -171,19 +168,18 @@ export async function createServer() { const safeFilename = input.filename.replace(/[^a-zA-Z0-9._-]/g, "-"); const objectKey = `uploads/${workspace.id}/${uploadId}/${safeFilename}`; const uploadPlan = await runtime.storage.createUploadPlan(objectKey, "application/zip", input.sizeBytes); - const upload = await runtime.store.createUpload({ id: uploadId, workspaceId: workspace.id, sourceId: source.id, appId: input.appId, objectKey, multipartUploadId: uploadPlan.mode === "multipart" ? uploadPlan.uploadId : undefined, originalFilename: input.filename, expectedSize: input.sizeBytes }); + const upload = await runtime.store.createUpload({ id: uploadId, workspaceId: workspace.id, sourceId: source.id, appId: input.appId, objectKey, originalFilename: input.filename, expectedSize: input.sizeBytes }); if (uploadPlan.mode === "single" && uploadPlan.uploadUrl.startsWith("/")) uploadPlan.uploadUrl = `${appConfig.PUBLIC_BASE_URL}${uploadPlan.uploadUrl}`; return reply.code(201).send({ upload, uploadPlan, mode: appConfig.STORAGE_MODE }); }); app.put("/api/v1/uploads/content/:encodedKey", async (request, reply) => { - if (appConfig.STORAGE_MODE !== "local") return reply.code(404).send({ error: "DIRECT_UPLOAD_USES_STORAGE_URL" }); const key = decodeURIComponent((request.params as { encodedKey: string }).encodedKey); if (!key.startsWith("uploads/") || key.includes("..")) return reply.code(400).send({ error: "UPLOAD_KEY_INVALID" }); const uploadId = key.split("/")[2]; const upload = await runtime.store.getUpload(uploadId); if (!upload || upload.objectKey !== key) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" }); - const body = Buffer.isBuffer(request.body) ? Readable.from(request.body) : request.body as NodeJS.ReadableStream; + const body = request.body as NodeJS.ReadableStream; const receivedSize = await runtime.storage.writeUploadFromStream(key, body); if (receivedSize !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_SIZE_MISMATCH", expectedSize: upload.expectedSize, receivedSize }); await runtime.store.updateUpload(upload.id, { receivedSize, status: "uploaded" }); @@ -197,8 +193,6 @@ export async function createServer() { const workspace = await runtime.store.getWorkspace(identity); const upload = await runtime.store.getUpload(uploadId, workspace.id); if (!upload) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" }); - const completeInput = multipartPartsInput.parse(request.body); - if (upload.multipartUploadId) await runtime.storage.completeMultipartUpload(upload.objectKey, upload.multipartUploadId, completeInput.parts); const head = await runtime.storage.headObject(upload.objectKey); if (!head || head.sizeBytes !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_INCOMPLETE", expectedSize: upload.expectedSize, receivedSize: head?.sizeBytes || 0 }); await runtime.store.updateUpload(upload.id, { receivedSize: head.sizeBytes, status: "queued" }); diff --git a/apps/api/src/storage.ts b/apps/api/src/storage.ts index 331e3ab..8048f1d 100644 --- a/apps/api/src/storage.ts +++ b/apps/api/src/storage.ts @@ -7,15 +7,12 @@ import { Readable } from "node:stream"; import { CreateBucketCommand, CompleteMultipartUploadCommand, - CreateMultipartUploadCommand, GetObjectCommand, HeadBucketCommand, HeadObjectCommand, PutObjectCommand, S3Client, - UploadPartCommand, } from "@aws-sdk/client-s3"; -import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; import type { AppConfig } from "./config"; export interface StoredObject { @@ -25,8 +22,7 @@ export interface StoredObject { } export type UploadPlan = - | { mode: "single"; uploadUrl: string } - | { mode: "multipart"; uploadId: string; partSizeBytes: number; parts: Array<{ partNumber: number; uploadUrl: string }> }; + { mode: "single"; uploadUrl: string }; export interface StorageAdapter { createUploadPlan(key: string, contentType: string, sizeBytes: number): Promise; @@ -149,14 +145,12 @@ export class S3Storage implements StorageAdapter { } - async createUploadPlan(key: string, contentType: string, sizeBytes: number) { - const partSizeBytes = 16 * 1024 * 1024; - const partCount = Math.ceil(sizeBytes / partSizeBytes); - if (partCount > 10000) throw new Error("UPLOAD_PART_COUNT_LIMIT"); - const multipart = await this.client.send(new CreateMultipartUploadCommand({ Bucket: this.appConfig.S3_BUCKET, Key: key, ContentType: contentType })); - if (!multipart.UploadId) throw new Error("MULTIPART_UPLOAD_ID_MISSING"); - const parts = await Promise.all(Array.from({ length: partCount }, async (_, index) => ({ partNumber: index + 1, uploadUrl: await getSignedUrl(this.client, new UploadPartCommand({ Bucket: this.appConfig.S3_BUCKET, Key: key, UploadId: multipart.UploadId, PartNumber: index + 1 }), { expiresIn: 900 }) }))); - return { mode: "multipart" as const, uploadId: multipart.UploadId, partSizeBytes, parts }; + // Uploads always flow through the API (browser -> API -> MinIO), so there is + // no presigned/multipart plan returned to the client. The API exposes a + // single PUT endpoint that streams the body straight into object storage, + // which keeps MinIO fully private and removes the need for a public S3 URL. + async createUploadPlan(key: string) { + return { mode: "single" as const, uploadUrl: `/api/v1/uploads/content/${encodeURIComponent(key)}` }; } async completeMultipartUpload(key: string, uploadId: string | undefined, parts: Array<{ partNumber: number; etag: string }> | undefined) { diff --git a/components/altdock/dashboard-provider.tsx b/components/altdock/dashboard-provider.tsx index ed77839..8503c2b 100644 --- a/components/altdock/dashboard-provider.tsx +++ b/components/altdock/dashboard-provider.tsx @@ -6,9 +6,7 @@ import { API_BASE, api, type DashboardData, type Notice, type Visibility } from type SourceInput = Pick & Partial>; -type UploadPlan = - | { mode: "single"; uploadUrl: string } - | { mode: "multipart"; uploadId: string; partSizeBytes: number; parts: Array<{ partNumber: number; uploadUrl: string }> }; +type UploadPlan = { mode: "single"; uploadUrl: string }; type AppPatch = Partial>; @@ -98,22 +96,12 @@ export function DashboardProvider({ children }: PropsWithChildren) { setNotice({ tone: "info", text: "ADPをアップロードし、Manifestとアセットを検証しています。" }); try { const init = await api<{ upload: { id: string }; uploadPlan: UploadPlan }>("/api/v1/uploads", { method: "POST", body: JSON.stringify({ sourceId, appId, filename: file.name, sizeBytes: file.size }) }); - const uploadedParts: Array<{ partNumber: number; etag: string }> = []; - if (init.uploadPlan.mode === "multipart") { - for (const part of init.uploadPlan.parts) { - const start = (part.partNumber - 1) * init.uploadPlan.partSizeBytes; - const response = await fetch(part.uploadUrl, { method: "PUT", body: file.slice(start, Math.min(file.size, start + init.uploadPlan.partSizeBytes)) }); - if (!response.ok) throw new Error(`ADPパート${part.partNumber}の転送に失敗しました。`); - const etag = response.headers.get("etag"); - if (!etag) throw new Error(`ADPパート${part.partNumber}のETagを取得できませんでした。`); - uploadedParts.push({ partNumber: part.partNumber, etag }); - } - } else { - const response = await fetch(init.uploadPlan.uploadUrl, { method: "PUT", headers: { "content-type": "application/zip" }, body: file }); - if (!response.ok) throw new Error("ADPファイルの転送に失敗しました。"); - } + // Uploads always go through the API, which streams to MinIO internally, + // so a single PUT is all that's needed (no presigned URLs, no multipart). + const response = await fetch(init.uploadPlan.uploadUrl, { method: "PUT", headers: { "content-type": "application/zip" }, body: file }); + if (!response.ok) throw new Error("ADPファイルの転送に失敗しました。"); - let complete = await api<{ release?: ReleaseRecord }>(`/api/v1/uploads/${init.upload.id}/complete`, { method: "POST", body: JSON.stringify({ parts: uploadedParts.length ? uploadedParts : undefined }) }); + let complete = await api<{ release?: ReleaseRecord }>(`/api/v1/uploads/${init.upload.id}/complete`, { method: "POST", body: "{}" }); if (!complete.release) { for (let attempt = 0; attempt < 20; attempt += 1) { await new Promise((resolve) => setTimeout(resolve, 1200));