アップロードを完全API経由に変更し、MinIOを非公開化

- presigned URL / multipartアップロードを廃止し、ブラウザ→API→MinIOの中継に統一
- S3Storage.createUploadPlan は常にsingle(API内部エンドポイント)を返す
- PUT /api/v1/uploads/content/:encodedKey をS3モードでも有効化しガード削除
- body parserをstream化し5GiBをメモリに載せないよう修正
- フロントのmultipart分岐を削除しsingle PUTに一本化

これでMinIOに公開ドメインが不要になり、Coolifyではapi/webのドメイン設定だけで完結する
This commit is contained in:
amania-jailbreak
2026-08-05 10:47:11 +09:00
parent 62c32ad0ad
commit 21bd070dc1
3 changed files with 18 additions and 42 deletions
+5 -11
View File
@@ -1,6 +1,5 @@
import { randomUUID } from "node:crypto"; import { randomUUID } from "node:crypto";
import { posix } from "node:path"; import { posix } from "node:path";
import { Readable } from "node:stream";
import Fastify, { type FastifyReply, type FastifyRequest } from "fastify"; import Fastify, { type FastifyReply, type FastifyRequest } from "fastify";
import cookie from "@fastify/cookie"; import cookie from "@fastify/cookie";
import cors from "@fastify/cors"; import cors from "@fastify/cors";
@@ -43,10 +42,6 @@ const appInput = z.object({
appPermissions: z.object({ entitlements: z.array(z.string()).max(200), privacy: z.record(z.string(), z.string()).default({}) }).optional(), appPermissions: z.object({ entitlements: z.array(z.string()).max(200), privacy: z.record(z.string(), z.string()).default({}) }).optional(),
}); });
const multipartPartsInput = z.object({
parts: z.array(z.object({ partNumber: z.number().int().positive(), etag: z.string().min(1) })).max(10000).optional(),
}).default({});
function safePath(value: string) { function safePath(value: string) {
const decoded = decodeURIComponent(value).replaceAll("\\", "/"); const decoded = decodeURIComponent(value).replaceAll("\\", "/");
const normalized = posix.normalize(decoded); const normalized = posix.normalize(decoded);
@@ -79,7 +74,9 @@ export async function createServer() {
const appConfig = loadConfig(); const appConfig = loadConfig();
const runtime = await createRuntime(appConfig); const runtime = await createRuntime(appConfig);
const app = Fastify({ logger: appConfig.NODE_ENV !== "test", bodyLimit: appConfig.MAX_UPLOAD_BYTES }); const app = Fastify({ logger: appConfig.NODE_ENV !== "test", bodyLimit: appConfig.MAX_UPLOAD_BYTES });
app.addContentTypeParser(["application/zip", "application/octet-stream"], { parseAs: "buffer" }, (_request, payload, done) => done(null, payload)); // Stream large upload bodies straight through to storage without buffering
// them in memory, so a 5 GiB ADP doesn't exhaust the process.
app.addContentTypeParser(["application/zip", "application/octet-stream"], (_request, payload, done) => done(null, payload));
await app.register(cookie); await app.register(cookie);
// The control plane (web :3000) and API (:4000) are cross-origin, so allow // The control plane (web :3000) and API (:4000) are cross-origin, so allow
// the write methods the dashboard uses; @fastify/cors otherwise defaults to // the write methods the dashboard uses; @fastify/cors otherwise defaults to
@@ -171,19 +168,18 @@ export async function createServer() {
const safeFilename = input.filename.replace(/[^a-zA-Z0-9._-]/g, "-"); const safeFilename = input.filename.replace(/[^a-zA-Z0-9._-]/g, "-");
const objectKey = `uploads/${workspace.id}/${uploadId}/${safeFilename}`; const objectKey = `uploads/${workspace.id}/${uploadId}/${safeFilename}`;
const uploadPlan = await runtime.storage.createUploadPlan(objectKey, "application/zip", input.sizeBytes); const uploadPlan = await runtime.storage.createUploadPlan(objectKey, "application/zip", input.sizeBytes);
const upload = await runtime.store.createUpload({ id: uploadId, workspaceId: workspace.id, sourceId: source.id, appId: input.appId, objectKey, multipartUploadId: uploadPlan.mode === "multipart" ? uploadPlan.uploadId : undefined, originalFilename: input.filename, expectedSize: input.sizeBytes }); const upload = await runtime.store.createUpload({ id: uploadId, workspaceId: workspace.id, sourceId: source.id, appId: input.appId, objectKey, originalFilename: input.filename, expectedSize: input.sizeBytes });
if (uploadPlan.mode === "single" && uploadPlan.uploadUrl.startsWith("/")) uploadPlan.uploadUrl = `${appConfig.PUBLIC_BASE_URL}${uploadPlan.uploadUrl}`; if (uploadPlan.mode === "single" && uploadPlan.uploadUrl.startsWith("/")) uploadPlan.uploadUrl = `${appConfig.PUBLIC_BASE_URL}${uploadPlan.uploadUrl}`;
return reply.code(201).send({ upload, uploadPlan, mode: appConfig.STORAGE_MODE }); return reply.code(201).send({ upload, uploadPlan, mode: appConfig.STORAGE_MODE });
}); });
app.put("/api/v1/uploads/content/:encodedKey", async (request, reply) => { app.put("/api/v1/uploads/content/:encodedKey", async (request, reply) => {
if (appConfig.STORAGE_MODE !== "local") return reply.code(404).send({ error: "DIRECT_UPLOAD_USES_STORAGE_URL" });
const key = decodeURIComponent((request.params as { encodedKey: string }).encodedKey); const key = decodeURIComponent((request.params as { encodedKey: string }).encodedKey);
if (!key.startsWith("uploads/") || key.includes("..")) return reply.code(400).send({ error: "UPLOAD_KEY_INVALID" }); if (!key.startsWith("uploads/") || key.includes("..")) return reply.code(400).send({ error: "UPLOAD_KEY_INVALID" });
const uploadId = key.split("/")[2]; const uploadId = key.split("/")[2];
const upload = await runtime.store.getUpload(uploadId); const upload = await runtime.store.getUpload(uploadId);
if (!upload || upload.objectKey !== key) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" }); if (!upload || upload.objectKey !== key) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" });
const body = Buffer.isBuffer(request.body) ? Readable.from(request.body) : request.body as NodeJS.ReadableStream; const body = request.body as NodeJS.ReadableStream;
const receivedSize = await runtime.storage.writeUploadFromStream(key, body); const receivedSize = await runtime.storage.writeUploadFromStream(key, body);
if (receivedSize !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_SIZE_MISMATCH", expectedSize: upload.expectedSize, receivedSize }); if (receivedSize !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_SIZE_MISMATCH", expectedSize: upload.expectedSize, receivedSize });
await runtime.store.updateUpload(upload.id, { receivedSize, status: "uploaded" }); await runtime.store.updateUpload(upload.id, { receivedSize, status: "uploaded" });
@@ -197,8 +193,6 @@ export async function createServer() {
const workspace = await runtime.store.getWorkspace(identity); const workspace = await runtime.store.getWorkspace(identity);
const upload = await runtime.store.getUpload(uploadId, workspace.id); const upload = await runtime.store.getUpload(uploadId, workspace.id);
if (!upload) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" }); if (!upload) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" });
const completeInput = multipartPartsInput.parse(request.body);
if (upload.multipartUploadId) await runtime.storage.completeMultipartUpload(upload.objectKey, upload.multipartUploadId, completeInput.parts);
const head = await runtime.storage.headObject(upload.objectKey); const head = await runtime.storage.headObject(upload.objectKey);
if (!head || head.sizeBytes !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_INCOMPLETE", expectedSize: upload.expectedSize, receivedSize: head?.sizeBytes || 0 }); if (!head || head.sizeBytes !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_INCOMPLETE", expectedSize: upload.expectedSize, receivedSize: head?.sizeBytes || 0 });
await runtime.store.updateUpload(upload.id, { receivedSize: head.sizeBytes, status: "queued" }); await runtime.store.updateUpload(upload.id, { receivedSize: head.sizeBytes, status: "queued" });
+7 -13
View File
@@ -7,15 +7,12 @@ import { Readable } from "node:stream";
import { import {
CreateBucketCommand, CreateBucketCommand,
CompleteMultipartUploadCommand, CompleteMultipartUploadCommand,
CreateMultipartUploadCommand,
GetObjectCommand, GetObjectCommand,
HeadBucketCommand, HeadBucketCommand,
HeadObjectCommand, HeadObjectCommand,
PutObjectCommand, PutObjectCommand,
S3Client, S3Client,
UploadPartCommand,
} from "@aws-sdk/client-s3"; } from "@aws-sdk/client-s3";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import type { AppConfig } from "./config"; import type { AppConfig } from "./config";
export interface StoredObject { export interface StoredObject {
@@ -25,8 +22,7 @@ export interface StoredObject {
} }
export type UploadPlan = export type UploadPlan =
| { mode: "single"; uploadUrl: string } { mode: "single"; uploadUrl: string };
| { mode: "multipart"; uploadId: string; partSizeBytes: number; parts: Array<{ partNumber: number; uploadUrl: string }> };
export interface StorageAdapter { export interface StorageAdapter {
createUploadPlan(key: string, contentType: string, sizeBytes: number): Promise<UploadPlan>; createUploadPlan(key: string, contentType: string, sizeBytes: number): Promise<UploadPlan>;
@@ -149,14 +145,12 @@ export class S3Storage implements StorageAdapter {
} }
async createUploadPlan(key: string, contentType: string, sizeBytes: number) { // Uploads always flow through the API (browser -> API -> MinIO), so there is
const partSizeBytes = 16 * 1024 * 1024; // no presigned/multipart plan returned to the client. The API exposes a
const partCount = Math.ceil(sizeBytes / partSizeBytes); // single PUT endpoint that streams the body straight into object storage,
if (partCount > 10000) throw new Error("UPLOAD_PART_COUNT_LIMIT"); // which keeps MinIO fully private and removes the need for a public S3 URL.
const multipart = await this.client.send(new CreateMultipartUploadCommand({ Bucket: this.appConfig.S3_BUCKET, Key: key, ContentType: contentType })); async createUploadPlan(key: string) {
if (!multipart.UploadId) throw new Error("MULTIPART_UPLOAD_ID_MISSING"); return { mode: "single" as const, uploadUrl: `/api/v1/uploads/content/${encodeURIComponent(key)}` };
const parts = await Promise.all(Array.from({ length: partCount }, async (_, index) => ({ partNumber: index + 1, uploadUrl: await getSignedUrl(this.client, new UploadPartCommand({ Bucket: this.appConfig.S3_BUCKET, Key: key, UploadId: multipart.UploadId, PartNumber: index + 1 }), { expiresIn: 900 }) })));
return { mode: "multipart" as const, uploadId: multipart.UploadId, partSizeBytes, parts };
} }
async completeMultipartUpload(key: string, uploadId: string | undefined, parts: Array<{ partNumber: number; etag: string }> | undefined) { async completeMultipartUpload(key: string, uploadId: string | undefined, parts: Array<{ partNumber: number; etag: string }> | undefined) {
+6 -18
View File
@@ -6,9 +6,7 @@ import { API_BASE, api, type DashboardData, type Notice, type Visibility } from
type SourceInput = Pick<SourceRecord, "name" | "subtitle" | "description" | "visibility"> & Partial<Pick<SourceRecord, "iconURL" | "headerURL" | "website" | "tintColor">>; type SourceInput = Pick<SourceRecord, "name" | "subtitle" | "description" | "visibility"> & Partial<Pick<SourceRecord, "iconURL" | "headerURL" | "website" | "tintColor">>;
type UploadPlan = type UploadPlan = { mode: "single"; uploadUrl: string };
| { mode: "single"; uploadUrl: string }
| { mode: "multipart"; uploadId: string; partSizeBytes: number; parts: Array<{ partNumber: number; uploadUrl: string }> };
type AppPatch = Partial<Pick<AppRecord, "name" | "developerName" | "subtitle" | "localizedDescription" | "iconURL" | "category" | "appPermissions">>; type AppPatch = Partial<Pick<AppRecord, "name" | "developerName" | "subtitle" | "localizedDescription" | "iconURL" | "category" | "appPermissions">>;
@@ -98,22 +96,12 @@ export function DashboardProvider({ children }: PropsWithChildren) {
setNotice({ tone: "info", text: "ADPをアップロードし、Manifestとアセットを検証しています。" }); setNotice({ tone: "info", text: "ADPをアップロードし、Manifestとアセットを検証しています。" });
try { try {
const init = await api<{ upload: { id: string }; uploadPlan: UploadPlan }>("/api/v1/uploads", { method: "POST", body: JSON.stringify({ sourceId, appId, filename: file.name, sizeBytes: file.size }) }); const init = await api<{ upload: { id: string }; uploadPlan: UploadPlan }>("/api/v1/uploads", { method: "POST", body: JSON.stringify({ sourceId, appId, filename: file.name, sizeBytes: file.size }) });
const uploadedParts: Array<{ partNumber: number; etag: string }> = []; // Uploads always go through the API, which streams to MinIO internally,
if (init.uploadPlan.mode === "multipart") { // so a single PUT is all that's needed (no presigned URLs, no multipart).
for (const part of init.uploadPlan.parts) { const response = await fetch(init.uploadPlan.uploadUrl, { method: "PUT", headers: { "content-type": "application/zip" }, body: file });
const start = (part.partNumber - 1) * init.uploadPlan.partSizeBytes; if (!response.ok) throw new Error("ADPファイルの転送に失敗しました。");
const response = await fetch(part.uploadUrl, { method: "PUT", body: file.slice(start, Math.min(file.size, start + init.uploadPlan.partSizeBytes)) });
if (!response.ok) throw new Error(`ADPパート${part.partNumber}の転送に失敗しました。`);
const etag = response.headers.get("etag");
if (!etag) throw new Error(`ADPパート${part.partNumber}のETagを取得できませんでした。`);
uploadedParts.push({ partNumber: part.partNumber, etag });
}
} else {
const response = await fetch(init.uploadPlan.uploadUrl, { method: "PUT", headers: { "content-type": "application/zip" }, body: file });
if (!response.ok) throw new Error("ADPファイルの転送に失敗しました。");
}
let complete = await api<{ release?: ReleaseRecord }>(`/api/v1/uploads/${init.upload.id}/complete`, { method: "POST", body: JSON.stringify({ parts: uploadedParts.length ? uploadedParts : undefined }) }); let complete = await api<{ release?: ReleaseRecord }>(`/api/v1/uploads/${init.upload.id}/complete`, { method: "POST", body: "{}" });
if (!complete.release) { if (!complete.release) {
for (let attempt = 0; attempt < 20; attempt += 1) { for (let attempt = 0; attempt < 20; attempt += 1) {
await new Promise((resolve) => setTimeout(resolve, 1200)); await new Promise((resolve) => setTimeout(resolve, 1200));