アップロードを完全API経由に変更し、MinIOを非公開化
- presigned URL / multipartアップロードを廃止し、ブラウザ→API→MinIOの中継に統一 - S3Storage.createUploadPlan は常にsingle(API内部エンドポイント)を返す - PUT /api/v1/uploads/content/:encodedKey をS3モードでも有効化しガード削除 - body parserをstream化し5GiBをメモリに載せないよう修正 - フロントのmultipart分岐を削除しsingle PUTに一本化 これでMinIOに公開ドメインが不要になり、Coolifyではapi/webのドメイン設定だけで完結する
This commit is contained in:
+5
-11
@@ -1,6 +1,5 @@
|
|||||||
import { randomUUID } from "node:crypto";
|
import { randomUUID } from "node:crypto";
|
||||||
import { posix } from "node:path";
|
import { posix } from "node:path";
|
||||||
import { Readable } from "node:stream";
|
|
||||||
import Fastify, { type FastifyReply, type FastifyRequest } from "fastify";
|
import Fastify, { type FastifyReply, type FastifyRequest } from "fastify";
|
||||||
import cookie from "@fastify/cookie";
|
import cookie from "@fastify/cookie";
|
||||||
import cors from "@fastify/cors";
|
import cors from "@fastify/cors";
|
||||||
@@ -43,10 +42,6 @@ const appInput = z.object({
|
|||||||
appPermissions: z.object({ entitlements: z.array(z.string()).max(200), privacy: z.record(z.string(), z.string()).default({}) }).optional(),
|
appPermissions: z.object({ entitlements: z.array(z.string()).max(200), privacy: z.record(z.string(), z.string()).default({}) }).optional(),
|
||||||
});
|
});
|
||||||
|
|
||||||
const multipartPartsInput = z.object({
|
|
||||||
parts: z.array(z.object({ partNumber: z.number().int().positive(), etag: z.string().min(1) })).max(10000).optional(),
|
|
||||||
}).default({});
|
|
||||||
|
|
||||||
function safePath(value: string) {
|
function safePath(value: string) {
|
||||||
const decoded = decodeURIComponent(value).replaceAll("\\", "/");
|
const decoded = decodeURIComponent(value).replaceAll("\\", "/");
|
||||||
const normalized = posix.normalize(decoded);
|
const normalized = posix.normalize(decoded);
|
||||||
@@ -79,7 +74,9 @@ export async function createServer() {
|
|||||||
const appConfig = loadConfig();
|
const appConfig = loadConfig();
|
||||||
const runtime = await createRuntime(appConfig);
|
const runtime = await createRuntime(appConfig);
|
||||||
const app = Fastify({ logger: appConfig.NODE_ENV !== "test", bodyLimit: appConfig.MAX_UPLOAD_BYTES });
|
const app = Fastify({ logger: appConfig.NODE_ENV !== "test", bodyLimit: appConfig.MAX_UPLOAD_BYTES });
|
||||||
app.addContentTypeParser(["application/zip", "application/octet-stream"], { parseAs: "buffer" }, (_request, payload, done) => done(null, payload));
|
// Stream large upload bodies straight through to storage without buffering
|
||||||
|
// them in memory, so a 5 GiB ADP doesn't exhaust the process.
|
||||||
|
app.addContentTypeParser(["application/zip", "application/octet-stream"], (_request, payload, done) => done(null, payload));
|
||||||
await app.register(cookie);
|
await app.register(cookie);
|
||||||
// The control plane (web :3000) and API (:4000) are cross-origin, so allow
|
// The control plane (web :3000) and API (:4000) are cross-origin, so allow
|
||||||
// the write methods the dashboard uses; @fastify/cors otherwise defaults to
|
// the write methods the dashboard uses; @fastify/cors otherwise defaults to
|
||||||
@@ -171,19 +168,18 @@ export async function createServer() {
|
|||||||
const safeFilename = input.filename.replace(/[^a-zA-Z0-9._-]/g, "-");
|
const safeFilename = input.filename.replace(/[^a-zA-Z0-9._-]/g, "-");
|
||||||
const objectKey = `uploads/${workspace.id}/${uploadId}/${safeFilename}`;
|
const objectKey = `uploads/${workspace.id}/${uploadId}/${safeFilename}`;
|
||||||
const uploadPlan = await runtime.storage.createUploadPlan(objectKey, "application/zip", input.sizeBytes);
|
const uploadPlan = await runtime.storage.createUploadPlan(objectKey, "application/zip", input.sizeBytes);
|
||||||
const upload = await runtime.store.createUpload({ id: uploadId, workspaceId: workspace.id, sourceId: source.id, appId: input.appId, objectKey, multipartUploadId: uploadPlan.mode === "multipart" ? uploadPlan.uploadId : undefined, originalFilename: input.filename, expectedSize: input.sizeBytes });
|
const upload = await runtime.store.createUpload({ id: uploadId, workspaceId: workspace.id, sourceId: source.id, appId: input.appId, objectKey, originalFilename: input.filename, expectedSize: input.sizeBytes });
|
||||||
if (uploadPlan.mode === "single" && uploadPlan.uploadUrl.startsWith("/")) uploadPlan.uploadUrl = `${appConfig.PUBLIC_BASE_URL}${uploadPlan.uploadUrl}`;
|
if (uploadPlan.mode === "single" && uploadPlan.uploadUrl.startsWith("/")) uploadPlan.uploadUrl = `${appConfig.PUBLIC_BASE_URL}${uploadPlan.uploadUrl}`;
|
||||||
return reply.code(201).send({ upload, uploadPlan, mode: appConfig.STORAGE_MODE });
|
return reply.code(201).send({ upload, uploadPlan, mode: appConfig.STORAGE_MODE });
|
||||||
});
|
});
|
||||||
|
|
||||||
app.put("/api/v1/uploads/content/:encodedKey", async (request, reply) => {
|
app.put("/api/v1/uploads/content/:encodedKey", async (request, reply) => {
|
||||||
if (appConfig.STORAGE_MODE !== "local") return reply.code(404).send({ error: "DIRECT_UPLOAD_USES_STORAGE_URL" });
|
|
||||||
const key = decodeURIComponent((request.params as { encodedKey: string }).encodedKey);
|
const key = decodeURIComponent((request.params as { encodedKey: string }).encodedKey);
|
||||||
if (!key.startsWith("uploads/") || key.includes("..")) return reply.code(400).send({ error: "UPLOAD_KEY_INVALID" });
|
if (!key.startsWith("uploads/") || key.includes("..")) return reply.code(400).send({ error: "UPLOAD_KEY_INVALID" });
|
||||||
const uploadId = key.split("/")[2];
|
const uploadId = key.split("/")[2];
|
||||||
const upload = await runtime.store.getUpload(uploadId);
|
const upload = await runtime.store.getUpload(uploadId);
|
||||||
if (!upload || upload.objectKey !== key) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" });
|
if (!upload || upload.objectKey !== key) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" });
|
||||||
const body = Buffer.isBuffer(request.body) ? Readable.from(request.body) : request.body as NodeJS.ReadableStream;
|
const body = request.body as NodeJS.ReadableStream;
|
||||||
const receivedSize = await runtime.storage.writeUploadFromStream(key, body);
|
const receivedSize = await runtime.storage.writeUploadFromStream(key, body);
|
||||||
if (receivedSize !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_SIZE_MISMATCH", expectedSize: upload.expectedSize, receivedSize });
|
if (receivedSize !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_SIZE_MISMATCH", expectedSize: upload.expectedSize, receivedSize });
|
||||||
await runtime.store.updateUpload(upload.id, { receivedSize, status: "uploaded" });
|
await runtime.store.updateUpload(upload.id, { receivedSize, status: "uploaded" });
|
||||||
@@ -197,8 +193,6 @@ export async function createServer() {
|
|||||||
const workspace = await runtime.store.getWorkspace(identity);
|
const workspace = await runtime.store.getWorkspace(identity);
|
||||||
const upload = await runtime.store.getUpload(uploadId, workspace.id);
|
const upload = await runtime.store.getUpload(uploadId, workspace.id);
|
||||||
if (!upload) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" });
|
if (!upload) return reply.code(404).send({ error: "UPLOAD_NOT_FOUND" });
|
||||||
const completeInput = multipartPartsInput.parse(request.body);
|
|
||||||
if (upload.multipartUploadId) await runtime.storage.completeMultipartUpload(upload.objectKey, upload.multipartUploadId, completeInput.parts);
|
|
||||||
const head = await runtime.storage.headObject(upload.objectKey);
|
const head = await runtime.storage.headObject(upload.objectKey);
|
||||||
if (!head || head.sizeBytes !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_INCOMPLETE", expectedSize: upload.expectedSize, receivedSize: head?.sizeBytes || 0 });
|
if (!head || head.sizeBytes !== upload.expectedSize) return reply.code(400).send({ error: "UPLOAD_INCOMPLETE", expectedSize: upload.expectedSize, receivedSize: head?.sizeBytes || 0 });
|
||||||
await runtime.store.updateUpload(upload.id, { receivedSize: head.sizeBytes, status: "queued" });
|
await runtime.store.updateUpload(upload.id, { receivedSize: head.sizeBytes, status: "queued" });
|
||||||
|
|||||||
+7
-13
@@ -7,15 +7,12 @@ import { Readable } from "node:stream";
|
|||||||
import {
|
import {
|
||||||
CreateBucketCommand,
|
CreateBucketCommand,
|
||||||
CompleteMultipartUploadCommand,
|
CompleteMultipartUploadCommand,
|
||||||
CreateMultipartUploadCommand,
|
|
||||||
GetObjectCommand,
|
GetObjectCommand,
|
||||||
HeadBucketCommand,
|
HeadBucketCommand,
|
||||||
HeadObjectCommand,
|
HeadObjectCommand,
|
||||||
PutObjectCommand,
|
PutObjectCommand,
|
||||||
S3Client,
|
S3Client,
|
||||||
UploadPartCommand,
|
|
||||||
} from "@aws-sdk/client-s3";
|
} from "@aws-sdk/client-s3";
|
||||||
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
|
|
||||||
import type { AppConfig } from "./config";
|
import type { AppConfig } from "./config";
|
||||||
|
|
||||||
export interface StoredObject {
|
export interface StoredObject {
|
||||||
@@ -25,8 +22,7 @@ export interface StoredObject {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export type UploadPlan =
|
export type UploadPlan =
|
||||||
| { mode: "single"; uploadUrl: string }
|
{ mode: "single"; uploadUrl: string };
|
||||||
| { mode: "multipart"; uploadId: string; partSizeBytes: number; parts: Array<{ partNumber: number; uploadUrl: string }> };
|
|
||||||
|
|
||||||
export interface StorageAdapter {
|
export interface StorageAdapter {
|
||||||
createUploadPlan(key: string, contentType: string, sizeBytes: number): Promise<UploadPlan>;
|
createUploadPlan(key: string, contentType: string, sizeBytes: number): Promise<UploadPlan>;
|
||||||
@@ -149,14 +145,12 @@ export class S3Storage implements StorageAdapter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
async createUploadPlan(key: string, contentType: string, sizeBytes: number) {
|
// Uploads always flow through the API (browser -> API -> MinIO), so there is
|
||||||
const partSizeBytes = 16 * 1024 * 1024;
|
// no presigned/multipart plan returned to the client. The API exposes a
|
||||||
const partCount = Math.ceil(sizeBytes / partSizeBytes);
|
// single PUT endpoint that streams the body straight into object storage,
|
||||||
if (partCount > 10000) throw new Error("UPLOAD_PART_COUNT_LIMIT");
|
// which keeps MinIO fully private and removes the need for a public S3 URL.
|
||||||
const multipart = await this.client.send(new CreateMultipartUploadCommand({ Bucket: this.appConfig.S3_BUCKET, Key: key, ContentType: contentType }));
|
async createUploadPlan(key: string) {
|
||||||
if (!multipart.UploadId) throw new Error("MULTIPART_UPLOAD_ID_MISSING");
|
return { mode: "single" as const, uploadUrl: `/api/v1/uploads/content/${encodeURIComponent(key)}` };
|
||||||
const parts = await Promise.all(Array.from({ length: partCount }, async (_, index) => ({ partNumber: index + 1, uploadUrl: await getSignedUrl(this.client, new UploadPartCommand({ Bucket: this.appConfig.S3_BUCKET, Key: key, UploadId: multipart.UploadId, PartNumber: index + 1 }), { expiresIn: 900 }) })));
|
|
||||||
return { mode: "multipart" as const, uploadId: multipart.UploadId, partSizeBytes, parts };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async completeMultipartUpload(key: string, uploadId: string | undefined, parts: Array<{ partNumber: number; etag: string }> | undefined) {
|
async completeMultipartUpload(key: string, uploadId: string | undefined, parts: Array<{ partNumber: number; etag: string }> | undefined) {
|
||||||
|
|||||||
@@ -6,9 +6,7 @@ import { API_BASE, api, type DashboardData, type Notice, type Visibility } from
|
|||||||
|
|
||||||
type SourceInput = Pick<SourceRecord, "name" | "subtitle" | "description" | "visibility"> & Partial<Pick<SourceRecord, "iconURL" | "headerURL" | "website" | "tintColor">>;
|
type SourceInput = Pick<SourceRecord, "name" | "subtitle" | "description" | "visibility"> & Partial<Pick<SourceRecord, "iconURL" | "headerURL" | "website" | "tintColor">>;
|
||||||
|
|
||||||
type UploadPlan =
|
type UploadPlan = { mode: "single"; uploadUrl: string };
|
||||||
| { mode: "single"; uploadUrl: string }
|
|
||||||
| { mode: "multipart"; uploadId: string; partSizeBytes: number; parts: Array<{ partNumber: number; uploadUrl: string }> };
|
|
||||||
|
|
||||||
type AppPatch = Partial<Pick<AppRecord, "name" | "developerName" | "subtitle" | "localizedDescription" | "iconURL" | "category" | "appPermissions">>;
|
type AppPatch = Partial<Pick<AppRecord, "name" | "developerName" | "subtitle" | "localizedDescription" | "iconURL" | "category" | "appPermissions">>;
|
||||||
|
|
||||||
@@ -98,22 +96,12 @@ export function DashboardProvider({ children }: PropsWithChildren) {
|
|||||||
setNotice({ tone: "info", text: "ADPをアップロードし、Manifestとアセットを検証しています。" });
|
setNotice({ tone: "info", text: "ADPをアップロードし、Manifestとアセットを検証しています。" });
|
||||||
try {
|
try {
|
||||||
const init = await api<{ upload: { id: string }; uploadPlan: UploadPlan }>("/api/v1/uploads", { method: "POST", body: JSON.stringify({ sourceId, appId, filename: file.name, sizeBytes: file.size }) });
|
const init = await api<{ upload: { id: string }; uploadPlan: UploadPlan }>("/api/v1/uploads", { method: "POST", body: JSON.stringify({ sourceId, appId, filename: file.name, sizeBytes: file.size }) });
|
||||||
const uploadedParts: Array<{ partNumber: number; etag: string }> = [];
|
// Uploads always go through the API, which streams to MinIO internally,
|
||||||
if (init.uploadPlan.mode === "multipart") {
|
// so a single PUT is all that's needed (no presigned URLs, no multipart).
|
||||||
for (const part of init.uploadPlan.parts) {
|
|
||||||
const start = (part.partNumber - 1) * init.uploadPlan.partSizeBytes;
|
|
||||||
const response = await fetch(part.uploadUrl, { method: "PUT", body: file.slice(start, Math.min(file.size, start + init.uploadPlan.partSizeBytes)) });
|
|
||||||
if (!response.ok) throw new Error(`ADPパート${part.partNumber}の転送に失敗しました。`);
|
|
||||||
const etag = response.headers.get("etag");
|
|
||||||
if (!etag) throw new Error(`ADPパート${part.partNumber}のETagを取得できませんでした。`);
|
|
||||||
uploadedParts.push({ partNumber: part.partNumber, etag });
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
const response = await fetch(init.uploadPlan.uploadUrl, { method: "PUT", headers: { "content-type": "application/zip" }, body: file });
|
const response = await fetch(init.uploadPlan.uploadUrl, { method: "PUT", headers: { "content-type": "application/zip" }, body: file });
|
||||||
if (!response.ok) throw new Error("ADPファイルの転送に失敗しました。");
|
if (!response.ok) throw new Error("ADPファイルの転送に失敗しました。");
|
||||||
}
|
|
||||||
|
|
||||||
let complete = await api<{ release?: ReleaseRecord }>(`/api/v1/uploads/${init.upload.id}/complete`, { method: "POST", body: JSON.stringify({ parts: uploadedParts.length ? uploadedParts : undefined }) });
|
let complete = await api<{ release?: ReleaseRecord }>(`/api/v1/uploads/${init.upload.id}/complete`, { method: "POST", body: "{}" });
|
||||||
if (!complete.release) {
|
if (!complete.release) {
|
||||||
for (let attempt = 0; attempt < 20; attempt += 1) {
|
for (let attempt = 0; attempt < 20; attempt += 1) {
|
||||||
await new Promise((resolve) => setTimeout(resolve, 1200));
|
await new Promise((resolve) => setTimeout(resolve, 1200));
|
||||||
|
|||||||
Reference in New Issue
Block a user