songtianlun/prmbr-image-mksaas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(storage) replace with s3mini sdk & fix upload issue in cloudflare worker

Browse Source
This commit is contained in:
javayhu 2025-06-20 02:03:22 +08:00
parent 196f72ff68
commit 7c101d595e
11 changed files with 137 additions and 409 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
env.example
View File

@ -56,7 +56,6 @@ STORAGE_BUCKET_NAME=""
STORAGE_ACCESS_KEY_ID=""
STORAGE_SECRET_ACCESS_KEY=""
STORAGE_ENDPOINT=""
STORAGE_FORCE_PATH_STYLE="false"
STORAGE_PUBLIC_URL=""
# -----------------------------------------------------------------------------

1
package.json
View File

@ -116,6 +116,7 @@
"react-use-measure": "^2.1.7",
"recharts": "^2.15.1",
"resend": "^4.4.1",
"s3mini": "^0.2.0",
"shiki": "^2.4.2",
"sonner": "^2.0.0",
"stripe": "^17.6.0",

9
pnpm-lock.yaml generated
View File

@ -287,6 +287,9 @@ importers:
resend:
specifier: ^4.4.1
version: 4.4.1(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
s3mini:
specifier: ^0.2.0
version: 0.2.0
shiki:
specifier: ^2.4.2
version: 2.5.0
@ -6021,6 +6024,10 @@ packages:
rxjs@7.8.1:
resolution: {integrity: sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==}
s3mini@0.2.0:
resolution: {integrity: sha512-wEiPibnyGN8B/1vJiOQGZbaF/t5A356RjAH4ch75UZi5e04LM9qaRh3A5KXk2Eq/qdU7I/2qXtigk1ifFl6yEA==}
engines: {node: '>=20'}
safe-buffer@5.2.1:
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
@ -12724,6 +12731,8 @@ snapshots:
dependencies:
tslib: 2.8.1
s3mini@0.2.0: {}
safe-buffer@5.2.1: {}
scheduler@0.25.0: {}

79
src/app/api/storage/file-url/route.ts
View File

@ -1,79 +0,0 @@
import { StorageError } from '@/storage/types';
import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { type NextRequest, NextResponse } from 'next/server';
export async function POST(request: NextRequest) {
try {
const body = await request.json();
const { key } = body as { key: string };
if (!key) {
return NextResponse.json(
{ error: 'File key is required' },
{ status: 400 }
);
}
const bucket = process.env.STORAGE_BUCKET_NAME;
const region = process.env.STORAGE_REGION;
const endpoint = process.env.STORAGE_ENDPOINT;
const publicUrl = process.env.STORAGE_PUBLIC_URL;
if (!bucket || !region) {
return NextResponse.json(
{ error: 'Storage configuration is incomplete' },
{ status: 500 }
);
}
let url: string;
// If a public URL is configured, use it
if (publicUrl) {
url = `${publicUrl.replace(/\/$/, '')}/${key}`;
} else {
// Otherwise, generate a pre-signed URL
const clientOptions: any = {
region,
credentials: {
accessKeyId: process.env.STORAGE_ACCESS_KEY_ID || '',
secretAccessKey: process.env.STORAGE_SECRET_ACCESS_KEY || '',
},
};
// Add custom endpoint for S3-compatible services like Cloudflare R2
if (endpoint) {
clientOptions.endpoint = endpoint;
// For services like R2 that don't use path-style URLs
if (process.env.STORAGE_FORCE_PATH_STYLE === 'false') {
clientOptions.forcePathStyle = false;
} else {
clientOptions.forcePathStyle = true;
}
}
const s3 = new S3Client(clientOptions);
const command = new GetObjectCommand({
Bucket: bucket,
Key: key,
});
url = await getSignedUrl(s3, command, { expiresIn: 3600 * 24 * 7 }); // 7 days
}
return NextResponse.json({ url, key });
} catch (error) {
console.error('Error getting file URL:', error);
if (error instanceof StorageError) {
return NextResponse.json({ error: error.message }, { status: 500 });
}
return NextResponse.json(
{ error: 'Something went wrong while getting the file URL' },
{ status: 500 }
);
}
}

62
src/app/api/storage/presigned-url/route.ts
View File

@ -1,62 +0,0 @@
import { randomUUID } from 'crypto';
import { getPresignedUploadUrl } from '@/storage';
import { StorageError } from '@/storage/types';
import { type NextRequest, NextResponse } from 'next/server';
export async function POST(request: NextRequest) {
try {
const body = await request.json();
const { filename, contentType, folder } = body as {
filename: string;
contentType: string;
folder: string;
};
if (!filename) {
return NextResponse.json(
{ error: 'Filename is required' },
{ status: 400 }
);
}
if (!contentType) {
return NextResponse.json(
{ error: 'Content type is required' },
{ status: 400 }
);
}
// Validate content type (optional, based on your requirements)
const allowedTypes = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
if (!allowedTypes.includes(contentType)) {
return NextResponse.json(
{ error: 'File type not supported' },
{ status: 400 }
);
}
// Generate a unique filename to prevent collisions
const extension = filename.split('.').pop() || '';
const uniqueFilename = `${randomUUID()}${extension ? `.${extension}` : ''}`;
// Get pre-signed URL
const result = await getPresignedUploadUrl(
uniqueFilename,
contentType,
folder || undefined
);
return NextResponse.json(result);
} catch (error) {
console.error('Error generating pre-signed URL:', error);
if (error instanceof StorageError) {
return NextResponse.json({ error: error.message }, { status: 500 });
}
return NextResponse.json(
{ error: 'Something went wrong while generating pre-signed URL' },
{ status: 500 }
);
}
}

2
src/components/settings/profile/update-avatar-card.tsx
View File

@ -13,7 +13,7 @@ import {
} from '@/components/ui/card';
import { authClient } from '@/lib/auth-client';
import { cn } from '@/lib/utils';
import { uploadFileFromBrowser } from '@/storage';
import { uploadFileFromBrowser } from '@/storage/client';
import { User2Icon } from 'lucide-react';
import { useTranslations } from 'next-intl';
import { useEffect, useState } from 'react';

63
src/storage/README.md
View File

@ -1,18 +1,18 @@
# Storage Module
This module provides a unified interface for storing and retrieving files using various cloud storage providers. Currently, it supports Amazon S3 and compatible services like Cloudflare R2.
This module provides a unified interface for storing and retrieving files using various cloud storage providers. Currently, it supports Amazon S3 and compatible services like Cloudflare R2 using the `s3mini` library for better Cloudflare Workers compatibility.
## Features
- Upload files to cloud storage
- Generate pre-signed URLs for direct browser-to-storage uploads
- Delete files from storage
- Client-side upload helpers for both small and large files
- Client-side upload helpers through API endpoints
- Cloudflare Workers compatible using s3mini
## Basic Usage
```typescript
import { uploadFile, deleteFile, getPresignedUploadUrl } from '@/storage';
import { uploadFile, deleteFile } from '@/storage';
// Upload a file
const { url, key } = await uploadFile(
@ -24,13 +24,6 @@ const { url, key } = await uploadFile(
// Delete a file
await deleteFile(key);
// Generate a pre-signed URL for direct upload
const { url, key } = await getPresignedUploadUrl(
'filename.jpg',
'image/jpeg',
'uploads/images'
);
```
## Client-Side Upload
@ -40,15 +33,15 @@ For client-side uploads, use the `uploadFileFromBrowser` function:
```typescript
'use client';
import { uploadFileFromBrowser } from '@/storage';
import { uploadFileFromBrowser } from '@/storage/client';
// In your component
async function handleFileUpload(event) {
const file = event.target.files[0];
try {
// This will automatically use the most appropriate upload method
// based on the file size
// All uploads go through the direct upload API endpoint
// since s3mini doesn't support presigned URLs
const { url, key } = await uploadFileFromBrowser(file, 'uploads/images');
console.log('File uploaded:', url);
} catch (error) {
@ -78,15 +71,16 @@ export const websiteConfig = {
```
# Required
STORAGE_REGION=us-east-1
STORAGE_REGION=auto
STORAGE_ACCESS_KEY_ID=your-access-key
STORAGE_SECRET_ACCESS_KEY=your-secret-key
STORAGE_BUCKET_NAME=your-bucket-name
STORAGE_ENDPOINT=https://custom-endpoint.com
STORAGE_PUBLIC_URL=https://cdn.example.com
STORAGE_FORCE_PATH_STYLE=true
```
**Note**: When using s3mini, the `STORAGE_ENDPOINT` is required and the bucket name will be included in the endpoint URL.
## Advanced Usage
### Using the Storage Provider Directly
@ -125,11 +119,6 @@ class CustomStorageProvider implements StorageProvider {
// Your implementation
}
async getPresignedUploadUrl(params: PresignedUploadUrlParams): Promise<PresignedUploadUrlResult> {
// Your implementation
return { url: 'https://example.com/upload', key: 'file.jpg' };
}
getProviderName(): string {
return 'CustomProvider';
}
@ -144,14 +133,32 @@ const result = await customProvider.uploadFile({
});
```
## Important Limitations
### s3mini Limitations
Since this implementation uses `s3mini` for Cloudflare Workers compatibility, there are some limitations compared to the full AWS SDK:
- **No Presigned URLs**: s3mini doesn't support presigned URLs. All browser uploads must go through your API server.
- **Endpoint Configuration**: The bucket name is included in the endpoint URL configuration.
- **Manual URL Construction**: File URLs are constructed manually rather than using AWS SDK's getSignedUrl.
## API Reference
### Main Functions
### Server-Side Functions
For server-side usage (in API routes, server actions, etc.):
- `uploadFile(file, filename, contentType, folder?)`: Upload a file to storage
- `deleteFile(key)`: Delete a file from storage
- `getPresignedUploadUrl(filename, contentType, folder?, expiresIn?)`: Generate a pre-signed URL
- `uploadFileFromBrowser(file, folder?)`: Upload a file from the browser
### Client-Side Functions
For client-side usage (in React components with 'use client'):
- `uploadFileFromBrowser(file, folder?)`: Upload a file from the browser (via API)
**Note**: Import client-side functions from `@/storage/client` to avoid Node.js module conflicts in the browser.
### Provider Interface
@ -159,17 +166,15 @@ The `StorageProvider` interface defines the following methods:
- `uploadFile(params)`: Upload a file to storage
- `deleteFile(key)`: Delete a file from storage
- `getPresignedUploadUrl(params)`: Generate a pre-signed URL
- `getProviderName()`: Get the provider name
### Configuration
The `StorageConfig` interface defines the configuration options:
- `region`: Storage region (e.g., 'us-east-1')
- `endpoint?`: Custom endpoint URL for S3-compatible services
- `region`: Storage region (e.g., 'auto' for Cloudflare R2)
- `endpoint?`: Custom endpoint URL for S3-compatible services (required for s3mini)
- `accessKeyId`: Access key ID for authentication
- `secretAccessKey`: Secret access key for authentication
- `bucketName`: Storage bucket name
- `publicUrl?`: Public URL for accessing files
- `forcePathStyle?`: Whether to use path-style URLs

45
src/storage/client.ts Normal file
View File

@ -0,0 +1,45 @@
import type { UploadFileResult } from './types';
const API_STORAGE_UPLOAD = '/api/storage/upload';
/**
* Uploads a file from the browser to the storage provider
* This function is meant to be used in client components
*
* Note: Since s3mini doesn't support presigned URLs, all uploads
* go through the direct upload API endpoint regardless of file size.
*
* @param file - The file object from an input element
* @param folder - Optional folder path to store the file in
* @returns Promise with the URL of the uploaded file
*/
export const uploadFileFromBrowser = async (
file: File,
folder?: string
): Promise<UploadFileResult> => {
try {
// With s3mini, we use direct upload for all file sizes
// since presigned URLs are not supported
const formData = new FormData();
formData.append('file', file);
formData.append('folder', folder || '');
const response = await fetch(API_STORAGE_UPLOAD, {
method: 'POST',
body: formData,
});
if (!response.ok) {
const error = (await response.json()) as { message: string };
throw new Error(error.message || 'Failed to upload file');
}
return await response.json();
} catch (error) {
const message =
error instanceof Error
? error.message
: 'Unknown error occurred during file upload';
throw new Error(message);
}
};

121
src/storage/index.ts
View File

@ -3,10 +3,6 @@ import { storageConfig } from './config/storage-config';
import { S3Provider } from './provider/s3';
import type { StorageConfig, StorageProvider, UploadFileResult } from './types';
const API_STORAGE_UPLOAD = '/api/storage/upload';
const API_STORAGE_PRESIGNED_URL = '/api/storage/presigned-url';
const API_STORAGE_FILE_URL = '/api/storage/file-url';
/**
* Default storage configuration
*/
@ -75,120 +71,3 @@ export const deleteFile = async (key: string): Promise<void> => {
const provider = getStorageProvider();
return provider.deleteFile(key);
};
/**
* Generates a pre-signed URL for direct browser uploads
*
* @param filename - Filename with extension
* @param contentType - MIME type of the file
* @param folder - Optional folder path to store the file in
* @param expiresIn - Expiration time in seconds (default: 3600)
* @returns Promise with the pre-signed URL and the storage key
*/
export const getPresignedUploadUrl = async (
filename: string,
contentType: string,
folder?: string,
expiresIn = 3600
): Promise<UploadFileResult> => {
const provider = getStorageProvider();
return provider.getPresignedUploadUrl({
filename,
contentType,
folder,
expiresIn,
});
};
/**
* Uploads a file from the browser to the storage provider
* This function is meant to be used in client components
*
* @param file - The file object from an input element
* @param folder - Optional folder path to store the file in
* @returns Promise with the URL of the uploaded file
*/
export const uploadFileFromBrowser = async (
file: File,
folder?: string
): Promise<UploadFileResult> => {
try {
// For small files (< 10MB), use direct upload
if (file.size < 10 * 1024 * 1024) {
const formData = new FormData();
formData.append('file', file);
formData.append('folder', folder || '');
const response = await fetch(API_STORAGE_UPLOAD, {
method: 'POST',
body: formData,
});
if (!response.ok) {
const error = (await response.json()) as { message: string };
throw new Error(error.message || 'Failed to upload file');
}
return await response.json();
}
// For larger files, use pre-signed URL
// First, get a pre-signed URL
const presignedUrlResponse = await fetch(API_STORAGE_PRESIGNED_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
filename: file.name,
contentType: file.type,
folder: folder || '',
}),
});
if (!presignedUrlResponse.ok) {
const error = (await presignedUrlResponse.json()) as { message: string };
throw new Error(error.message || 'Failed to get pre-signed URL');
}
const { url, key } = (await presignedUrlResponse.json()) as {
url: string;
key: string;
};
// Then upload directly to the storage provider
const uploadResponse = await fetch(url, {
method: 'PUT',
headers: {
'Content-Type': file.type,
},
body: file,
});
if (!uploadResponse.ok) {
throw new Error('Failed to upload file using pre-signed URL');
}
// Get the public URL
const fileUrlResponse = await fetch(API_STORAGE_FILE_URL, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ key }),
});
if (!fileUrlResponse.ok) {
const error = (await fileUrlResponse.json()) as { message: string };
throw new Error(error.message || 'Failed to get file URL');
}
return await fileUrlResponse.json();
} catch (error) {
const message =
error instanceof Error
? error.message
: 'Unknown error occurred during file upload';
throw new Error(message);
}
};

146
src/storage/provider/s3.ts
View File

@ -1,14 +1,8 @@
import { randomUUID } from 'crypto';
import {
GetObjectCommand,
PutObjectCommand,
S3Client,
} from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { s3mini } from 's3mini';
import { storageConfig } from '../config/storage-config';
import {
ConfigurationError,
type PresignedUploadUrlParams,
type StorageConfig,
StorageError,
type StorageProvider,
@ -18,19 +12,19 @@ import {
} from '../types';
/**
* Amazon S3 storage provider implementation
* Amazon S3 storage provider implementation using s3mini
*
* docs:
* https://mksaas.com/docs/storage
*
* This provider works with Amazon S3 and compatible services like Cloudflare R2
* https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html
* https://www.npmjs.com/package/@aws-sdk/client-s3
* using s3mini for better Cloudflare Workers compatibility
* https://github.com/good-lly/s3mini
* https://developers.cloudflare.com/r2/
*/
export class S3Provider implements StorageProvider {
private config: StorageConfig;
private s3Client: S3Client | null = null;
private s3Client: s3mini | null = null;
constructor(config: StorageConfig = storageConfig) {
this.config = config;
@ -46,33 +40,41 @@ export class S3Provider implements StorageProvider {
/**
* Get the S3 client instance
*/
private getS3Client(): S3Client {
private getS3Client(): s3mini {
if (this.s3Client) {
return this.s3Client;
}
const { region, endpoint, accessKeyId, secretAccessKey, forcePathStyle } =
const { region, endpoint, accessKeyId, secretAccessKey, bucketName } =
this.config;
if (!region) {
throw new ConfigurationError('Storage region is not configured');
}
const clientOptions: any = {
region,
credentials: {
accessKeyId,
secretAccessKey,
},
};
// Add custom endpoint for S3-compatible services like Cloudflare R2
if (endpoint) {
clientOptions.endpoint = endpoint;
clientOptions.forcePathStyle = forcePathStyle !== false;
if (!accessKeyId || !secretAccessKey) {
throw new ConfigurationError('Storage credentials are not configured');
}
this.s3Client = new S3Client(clientOptions);
if (!endpoint) {
throw new ConfigurationError('Storage endpoint is required for s3mini');
}
if (!bucketName) {
throw new ConfigurationError('Storage bucket name is not configured');
}
// s3mini client configuration
// The bucket name needs to be included in the endpoint URL for s3mini
const endpointWithBucket = `${endpoint.replace(/\/$/, '')}/${bucketName}`;
this.s3Client = new s3mini({
accessKeyId,
secretAccessKey,
endpoint: endpointWithBucket,
region,
});
return this.s3Client;
}
@ -94,30 +96,23 @@ export class S3Provider implements StorageProvider {
const s3 = this.getS3Client();
const { bucketName } = this.config;
if (!bucketName) {
throw new ConfigurationError('Storage bucket name is not configured');
}
const uniqueFilename = this.generateUniqueFilename(filename);
const key = folder ? `${folder}/${uniqueFilename}` : uniqueFilename;
// Convert Blob to Buffer if needed
let fileBuffer: Buffer;
let fileContent: Buffer | string;
if (file instanceof Blob) {
fileBuffer = Buffer.from(await file.arrayBuffer());
fileContent = Buffer.from(await file.arrayBuffer());
} else {
fileBuffer = file;
fileContent = file;
}
// Upload the file
const command = new PutObjectCommand({
Bucket: bucketName,
Key: key,
Body: fileBuffer,
ContentType: contentType,
});
// Upload the file using s3mini
const response = await s3.putObject(key, fileContent, contentType);
await s3.send(command);
if (!response.ok) {
throw new UploadError(`Failed to upload file: ${response.statusText}`);
}
// Generate the URL
const { publicUrl } = this.config;
@ -128,13 +123,11 @@ export class S3Provider implements StorageProvider {
url = `${publicUrl.replace(/\/$/, '')}/${key}`;
console.log('uploadFile, public url', url);
} else {
// Generate a pre-signed URL if no public URL is provided
const getCommand = new GetObjectCommand({
Bucket: bucketName,
Key: key,
});
url = await getSignedUrl(s3, getCommand, { expiresIn: 3600 * 24 * 7 }); // 7 days
console.log('uploadFile, signed url', url);
// For s3mini, we construct the URL manually
// Since bucket is included in endpoint, we just append the key
const baseUrl = this.config.endpoint?.replace(/\/$/, '') || '';
url = `${baseUrl}/${key}`;
console.log('uploadFile, constructed url', url);
}
return { url, key };
@ -159,23 +152,14 @@ export class S3Provider implements StorageProvider {
public async deleteFile(key: string): Promise<void> {
try {
const s3 = this.getS3Client();
const { bucketName } = this.config;
if (!bucketName) {
throw new ConfigurationError('Storage bucket name is not configured');
const wasDeleted = await s3.deleteObject(key);
if (!wasDeleted) {
console.warn(
`File with key ${key} was not found or could not be deleted`
);
}
const command = {
Bucket: bucketName,
Key: key,
};
await s3.send(
new PutObjectCommand({
...command,
Body: '',
})
);
} catch (error) {
const message =
error instanceof Error
@ -185,40 +169,4 @@ export class S3Provider implements StorageProvider {
throw new StorageError(message);
}
}
/**
* Generate a pre-signed URL for direct browser uploads
*/
public async getPresignedUploadUrl(
params: PresignedUploadUrlParams
): Promise<UploadFileResult> {
try {
const { filename, contentType, folder, expiresIn = 3600 } = params;
const s3 = this.getS3Client();
const { bucketName } = this.config;
if (!bucketName) {
throw new ConfigurationError('Storage bucket name is not configured');
}
const uniqueFilename = this.generateUniqueFilename(filename);
const key = folder ? `${folder}/${uniqueFilename}` : uniqueFilename;
const command = new PutObjectCommand({
Bucket: bucketName,
Key: key,
ContentType: contentType,
});
const url = await getSignedUrl(s3, command, { expiresIn });
return { url, key };
} catch (error) {
const message =
error instanceof Error
? error.message
: 'Unknown error occurred while generating presigned URL';
console.error('getPresignedUploadUrl, error', message);
throw new StorageError(message);
}
}
}

17
src/storage/types.ts
View File

@ -53,16 +53,6 @@ export interface UploadFileResult {
key: string;
}
/**
* Presigned upload URL parameters
*/
export interface PresignedUploadUrlParams {
filename: string;
contentType: string;
folder?: string;
expiresIn?: number;
}
/**
* Storage provider interface
*/
@ -77,13 +67,6 @@ export interface StorageProvider {
*/
deleteFile(key: string): Promise<void>;
/**
* Generate a pre-signed URL for client-side uploads
*/
getPresignedUploadUrl(
params: PresignedUploadUrlParams
): Promise<UploadFileResult>;
/**
* Get the provider's name
*/