Update server.js

server.js

@@ -3,69 +3,68 @@
 import express from 'express';
 import cors from 'cors';
 import multer from 'multer';
-import { commit } from '@huggingface/hub'; // Use 'commit' for batch operations
+import { commit } from '@huggingface/hub'; 
 import path from 'path';
 import { fileURLToPath } from 'url';
 
-// Fix __dirname for ES Modules
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
 const app = express();
-// Hugging Face Spaces requires port 7860
 const port = process.env.PORT || 3001;
 
-// Middleware
 app.use(cors());
-app.use(express.json());
+// Increase body limit for large batch JSON/Form data
+app.use(express.json({ limit: '500mb' }));
+app.use(express.urlencoded({ limit: '500mb', extended: true }));
 
-// Configure Multer for memory storage
-// No limit on file count here, but we limit payload size via express/nginx usually
+// Configure Multer with higher limits
 const storage = multer.memoryStorage();
-const upload = multer({ storage: storage });
+const upload = multer({ 
+    storage: storage,
+    limits: {
+        fileSize: 500 * 1024 * 1024, // 500MB per file max
+        fieldSize: 500 * 1024 * 1024, // 500MB per field
+    }
+});
 
-// SERVER CONFIGURATION
 const SERVER_CONFIG = {
   TOKEN: process.env.HF_TOKEN || '',
   REPO: process.env.HF_REPO || 'TwanAPI/DataTwan',
   TYPE: 'dataset'
 };
 
-// --- API ROUTES ---
-
-// Batch Upload Endpoint
-// Accepts multiple files in the 'files' field
 app.post('/api/upload', upload.array('files'), async (req, res) => {
   try {
     const files = req.files;
-    // req.body.paths can be a string (if 1 file) or array (if multiple)
-    // We normalize it to an array
-    const paths = [].concat(req.body.paths || []);
+    let paths = req.body.paths;
+
+    // Normalize paths to array if it's a single string
+    if (!Array.isArray(paths)) {
+        paths = [paths].filter(Boolean);
+    }
 
     if (!files || files.length === 0) {
       return res.status(400).json({ error: 'No files provided' });
     }
 
     if (files.length !== paths.length) {
-      return res.status(400).json({ error: 'Mismatch between file count and path count' });
+      return res.status(400).json({ error: `Mismatch: ${files.length} files vs ${paths.length} paths` });
     }
 
-    // Security Check
     if (!SERVER_CONFIG.TOKEN) {
-       console.error('[SERVER] Upload Failed: HF_TOKEN missing');
        return res.status(500).json({ error: 'Server misconfiguration: HF_TOKEN secret is missing.' });
     }
 
     console.log(`[SERVER] Processing batch of ${files.length} files...`);
 
-    // Prepare operations for Hugging Face 'commit'
     const operations = files.map((file, index) => ({
       operation: 'addOrUpdate',
       path: paths[index],
       content: new Blob([file.buffer])
     }));
 
-    // Execute single commit for all files
+    // Using commit is atomic and faster for datasets than individual LFS uploads for small-medium files
     const response = await commit({
       credentials: {
         accessToken: SERVER_CONFIG.TOKEN,
@@ -75,16 +74,14 @@ app.post('/api/upload', upload.array('files'), async (req, res) => {
         name: SERVER_CONFIG.REPO
       },
       operations: operations,
-      title: `Upload batch of ${files.length} files via DataTwan`
+      title: `Batch upload of ${files.length} files`
     });
 
-    const commitHash = response.oid; // 'commit' returns { oid: string, ... }
+    const commitHash = response.oid;
     const urlPrefix = "https://huggingface.co/datasets";
-    
-    // Generate public URLs for response
     const urls = paths.map(p => `${urlPrefix}/${SERVER_CONFIG.REPO}/blob/${commitHash}/${p}`);
 
-    console.log(`[SERVER] Batch Success: ${files.length} files committed (Hash: ${commitHash})`);
+    console.log(`[SERVER] Batch Committed: ${commitHash}`);
 
     res.json({ 
       success: true, 
@@ -93,7 +90,7 @@ app.post('/api/upload', upload.array('files'), async (req, res) => {
     });
 
   } catch (error) {
-    console.error('[SERVER] Batch Upload Error:', error);
+    console.error('[SERVER] Error:', error);
     res.status(500).json({ 
       success: false, 
       error: error.message || 'Internal Server Error' 
@@ -101,18 +98,12 @@ app.post('/api/upload', upload.array('files'), async (req, res) => {
   }
 });
 
-// --- SERVE FRONTEND (Production) ---
-
 app.use(express.static(path.join(__dirname, 'dist')));
 
 app.get('*', (req, res) => {
   res.sendFile(path.join(__dirname, 'dist', 'index.html'));
 });
 
-// Start Server
 app.listen(port, () => {
-  console.log(`--------------------------------------------------`);
-  console.log(`✅ Server running on port ${port}`);
-  console.log(`   Target Repo: ${SERVER_CONFIG.REPO}`);
-  console.log(`--------------------------------------------------`);
+  console.log(`✅ Server running on port ${port} | Repo: ${SERVER_CONFIG.REPO}`);
 });