| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <title>Dataset Architect</title> |
| <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet"> |
| <style> |
| body { background-color: #f4f6f8; font-size: 0.9rem; } |
| .box { background: white; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.05); padding: 20px; margin-bottom: 20px; } |
| .source-col-row { cursor: pointer; } |
| .source-col-row:hover { background-color: #e9ecef; } |
| .source-col-row.active { background-color: #cfe2ff; font-weight: bold; } |
| pre { background: #212529; color: #adbac7; padding: 15px; border-radius: 6px; font-size: 0.8em; max-height: 250px; overflow: auto; } |
| |
| .builder-row { border: 1px solid #dee2e6; border-radius: 6px; background: #fff; padding: 10px; margin-bottom: 10px; position: relative; } |
| .builder-row.mode-list { border-left: 4px solid #198754; } |
| .builder-row.mode-python { border-left: 4px solid #fd7e14; } |
| .builder-row.mode-simple { border-left: 4px solid #0d6efd; } |
| |
| .mode-badge { position: absolute; top: -10px; right: 10px; font-size: 0.7em; padding: 2px 6px; background: #fff; border: 1px solid #ccc; border-radius: 4px; text-transform: uppercase; font-weight: bold; color: #666; } |
| |
| .logic-group { background: #f8f9fa; padding: 8px; border-radius: 4px; margin-top: 5px; border: 1px dashed #ccc; } |
| </style> |
| </head> |
| <body> |
|
|
| <div class="container-fluid p-4"> |
| <div class="d-flex justify-content-between align-items-center mb-4"> |
| <h3>🏗️ Hugging Face Dataset Architect</h3> |
| <div><input type="password" id="token" class="form-control form-control-sm" placeholder="HF Token" style="width: 200px;"></div> |
| </div> |
|
|
| <div class="box"> |
| <div class="row g-2"> |
| <div class="col-md-4"> |
| <div class="input-group"> |
| <input type="text" id="dataset_id" class="form-control" placeholder="Source ID (e.g. the_stack)"> |
| <button class="btn btn-primary" onclick="loadMetadata()">Connect</button> |
| </div> |
| </div> |
| <div class="col-md-2"><select id="config_select" class="form-select" onchange="updateSplits()" disabled><option>Config...</option></select></div> |
| <div class="col-md-2"><select id="split_select" class="form-select" disabled><option>Split...</option></select></div> |
| <div class="col-md-4"><button id="inspect_btn" class="btn btn-success w-100" onclick="startInspection()" disabled>Analyze Schema</button></div> |
| </div> |
| </div> |
|
|
| <div class="row" id="workspace" style="display:none;"> |
| <div class="col-md-4"> |
| <div class="box h-100"> |
| <h6>Source Columns</h6> |
| <div class="mb-3"><input type="text" id="source_filter" class="form-control form-control-sm" placeholder="Filter rows: len(text) > 100"></div> |
| <ul class="list-group list-group-flush mb-3" id="source_col_list" style="max-height: 300px; overflow-y: auto;"></ul> |
| <h6>Data Preview</h6> |
| <div id="data_preview_box"><pre class="text-muted">Select a column.</pre></div> |
| </div> |
| </div> |
|
|
| <div class="col-md-8"> |
| <div class="box h-100"> |
| <div class="d-flex justify-content-between mb-3"> |
| <h6>New Schema Definition</h6> |
| <button class="btn btn-sm btn-outline-primary" onclick="addBuilderRow()">+ Add Column</button> |
| </div> |
| |
| <div id="builder_container"></div> |
|
|
| <hr class="my-4"> |
| |
| <div class="step-header">Metadata & Publish</div> |
| |
| <div class="row g-2 mb-3"> |
| <div class="col-md-8"> |
| <label class="small fw-bold text-muted">Target Repository Name</label> |
| <input type="text" id="target_id" class="form-control" placeholder="username/my-new-dataset"> |
| </div> |
| <div class="col-md-4"> |
| <label class="small fw-bold text-muted">License</label> |
| <input type="text" id="license_input" class="form-control" value="apache-2.0" placeholder="mit, cc-by-4.0..."> |
| </div> |
| </div> |
|
|
| <div class="row g-2 align-items-end"> |
| <div class="col-md-4"> |
| <label class="small fw-bold text-muted">Max Rows (Test)</label> |
| <input type="number" id="max_rows" class="form-control" placeholder="All"> |
| </div> |
| <div class="col-md-8 d-flex gap-2"> |
| <button class="btn btn-outline-dark w-50" onclick="runPreview()">Preview Data</button> |
| <button class="btn btn-success w-50" onclick="executeJob()">🚀 Push to Hub + Card</button> |
| </div> |
| </div> |
| |
| <div id="job_status" class="alert mt-3" style="display:none;"></div> |
| <div id="final_preview" class="mt-3" style="display:none;"></div> |
| </div> |
| </div> |
| </div> |
|
|
| <script> |
| let schemaData = {}; |
| let sampleRows = []; |
| |
| function addBuilderRow() { |
| const container = document.getElementById('builder_container'); |
| const id = 'row_' + Date.now(); |
| |
| let sourceOpts = '<option value="">Select Source...</option>'; |
| for(let col in schemaData) { |
| let label = col + (schemaData[col].type === 'List' ? ' [List]' : ''); |
| sourceOpts += `<option value="${col}">${label}</option>`; |
| } |
| |
| const div = document.createElement('div'); |
| div.className = "builder-row mode-simple"; |
| div.id = id; |
| |
| div.innerHTML = ` |
| <span class="mode-badge">Simple Path</span> |
| <div class="row g-2 align-items-center mb-2"> |
| <div class="col-md-3"> |
| <input type="text" class="form-control form-control-sm target-name" placeholder="New Col Name"> |
| </div> |
| <div class="col-md-3"> |
| <select class="form-select form-select-sm mode-select" onchange="changeMode('${id}', this.value)"> |
| <option value="simple">Simple Path</option> |
| <option value="list_search">List Filter (Get X where Y=Z)</option> |
| <option value="python">Custom Python</option> |
| <option value="requests">Request</option> |
| </select> |
| </div> |
| <div class="col-md-5 text-end"> |
| <button class="btn btn-sm btn-link text-danger text-decoration-none" onclick="this.closest('.builder-row').remove()">Remove</button> |
| </div> |
| </div> |
| |
| <div class="inputs-area"> |
| <div class="input-group input-group-sm"> |
| <span class="input-group-text">Source</span> |
| <select class="form-select source-col">${sourceOpts}</select> |
| <input type="text" class="form-control source-extra" placeholder="Optional .dot.notation"> |
| </div> |
| </div> |
| `; |
| container.appendChild(div); |
| } |
| |
| function changeMode(rowId, mode) { |
| const row = document.getElementById(rowId); |
| const area = row.querySelector('.inputs-area'); |
| const badge = row.querySelector('.mode-badge'); |
| |
| row.className = `builder-row mode-${mode}`; |
| |
| let sourceOpts = '<option value="">Select Col...</option>'; |
| for(let col in schemaData) { |
| sourceOpts += `<option value="${col}">${col}</option>`; |
| } |
| |
| if(mode === 'simple') { |
| badge.innerText = "Simple Path"; |
| area.innerHTML = ` |
| <div class="input-group input-group-sm"> |
| <span class="input-group-text">Source</span> |
| <select class="form-select source-col">${sourceOpts}</select> |
| <input type="text" class="form-control source-extra" placeholder="Optional .dot.notation (e.g. meta.url)"> |
| </div>`; |
| } |
| else if (mode === 'list_search') { |
| badge.innerText = "List Logic"; |
| area.innerHTML = ` |
| <div class="logic-group"> |
| <div class="d-flex gap-2 align-items-center mb-1"> |
| <span class="fw-bold small">FROM</span> |
| <select class="form-select form-select-sm source-col" style="width:150px">${sourceOpts}</select> |
| <span class="fw-bold small">FIND ITEM WHERE</span> |
| </div> |
| <div class="d-flex gap-2 align-items-center mb-1"> |
| <input type="text" class="form-control form-control-sm filter-key" placeholder="Key (e.g. role)"> |
| <span class="fw-bold small">=</span> |
| <input type="text" class="form-control form-control-sm filter-val" placeholder="Value (e.g. user)"> |
| </div> |
| <div class="d-flex gap-2 align-items-center"> |
| <span class="fw-bold small">EXTRACT</span> |
| <input type="text" class="form-control form-control-sm target-key" placeholder="Key (e.g. content or content.analysis)"> |
| </div> |
| </div>`; |
| } |
| else if (mode === 'python') { |
| badge.innerText = "Python"; |
| area.innerHTML = ` |
| <div class="input-group input-group-sm"> |
| <span class="input-group-text">val = </span> |
| <input type="text" class="form-control python-expr" placeholder="row['text'].upper() or json.loads(row['meta'])['id']"> |
| </div>`; |
| } |
| else if (mode === 'requests') { |
| badge.innerText = "Request"; |
| area.innerHTML = ` |
| <div class="input-group input-group-sm"> |
| <span class="input-group-text">url = </span> |
| <input type="text" class="form-control request-url" placeholder="row['text'].upper() or json.loads(row['meta'])['id']"> |
| <span class="input-group-text">payload = </span> |
| <input type="text" class="form-control request-payload" placeholder="row['text'].upper() or json.loads(row['meta'])['id']"> |
| </div>`; |
| } |
| } |
| |
| function getRecipe() { |
| const rows = document.querySelectorAll('.builder-row'); |
| let columns = []; |
| |
| rows.forEach(r => { |
| const name = r.querySelector('.target-name').value.trim(); |
| const mode = r.querySelector('.mode-select').value; |
| |
| |
| if(!name) { |
| console.warn('Skipping row with empty target name'); |
| return; |
| } |
| |
| if(mode === 'simple') { |
| let src = r.querySelector('.source-col').value; |
| let extra = r.querySelector('.source-extra').value.trim(); |
| |
| |
| if(!src) { |
| console.error(`Row "${name}" has no source column selected`); |
| alert(`Error: Column "${name}" has no source selected!`); |
| return; |
| } |
| |
| |
| if(extra) { |
| |
| src = extra.startsWith('.') ? src + extra : src + '.' + extra; |
| } |
| |
| console.log(`Simple path: ${name} <- ${src}`); |
| columns.push({ type: 'simple', name: name, source: src }); |
| } |
| else if(mode === 'list_search') { |
| const srcCol = r.querySelector('.source-col').value; |
| const filterKey = r.querySelector('.filter-key').value.trim(); |
| const filterVal = r.querySelector('.filter-val').value.trim(); |
| const targetKey = r.querySelector('.target-key').value.trim(); |
| |
| if(!srcCol || !filterKey || !filterVal || !targetKey) { |
| console.error(`List search column "${name}" is incomplete`); |
| alert(`Error: Column "${name}" list search is incomplete!`); |
| return; |
| } |
| |
| columns.push({ |
| type: 'list_search', |
| name: name, |
| source: srcCol, |
| filter_key: filterKey, |
| filter_val: filterVal, |
| target_key: targetKey |
| }); |
| } |
| else if(mode === 'python') { |
| const expr = r.querySelector('.python-expr').value.trim(); |
| |
| if(!expr) { |
| console.error(`Python column "${name}" has no expression`); |
| alert(`Error: Column "${name}" has no Python expression!`); |
| return; |
| } |
| |
| columns.push({ |
| type: 'python', |
| name: name, |
| expression: expr |
| }); |
| } |
| else if(mode === 'requests') { |
| const rurl = r.querySelector('.request-url').value.trim(); |
| const rpay = r.querySelector('.request-payload').value.trim(); |
| |
| if(!rurl || !rpay) { |
| console.error(`Python column "${name}" has no expression`); |
| alert(`Error: Column "${name}" has no Python expression!`); |
| return; |
| } |
| |
| columns.push({ |
| type: 'request', |
| name: name, |
| rurl: rurl, |
| rpay: rpay |
| }); |
| } |
| }); |
| |
| const recipe = { |
| filter_rule: document.getElementById('source_filter').value.trim(), |
| columns: columns |
| }; |
| |
| console.log('Generated recipe:', JSON.stringify(recipe, null, 2)); |
| return recipe; |
| } |
| |
| async function loadMetadata() { |
| const btn = document.querySelector('button[onclick="loadMetadata()"]'); |
| const tokenVal = document.getElementById('token').value; |
| const datasetVal = document.getElementById('dataset_id').value; |
| if(!datasetVal) { alert("Please enter a Dataset ID"); return; } |
| btn.disabled = true; btn.innerText = "..."; |
| |
| try { |
| const res = await fetch('/analyze_metadata', { |
| method: 'POST', headers: {'Content-Type': 'application/json'}, |
| body: JSON.stringify({ |
| token: tokenVal, |
| dataset_id: datasetVal |
| }) |
| }); |
| |
| const data = await res.json(); |
| |
| if(data.status === 'success') { |
| const confSel = document.getElementById('config_select'); |
| confSel.innerHTML = ''; |
| |
| if (Array.isArray(data.configs) && data.configs.length > 0) { |
| data.configs.forEach(c => { |
| confSel.innerHTML += `<option value="${c}">${c}</option>`; |
| }); |
| } else { |
| confSel.innerHTML = `<option value="default">default</option>`; |
| } |
| confSel.disabled = false; |
| |
| populateSplits(data.splits); |
| |
| const licInput = document.getElementById('license_input'); |
| if(data.license_detected && licInput) { |
| licInput.value = data.license_detected; |
| } |
| |
| document.getElementById('inspect_btn').disabled = false; |
| } else { |
| alert('Server Error: ' + data.message); |
| } |
| } catch(e) { |
| console.error(e); |
| alert("Network Error: " + e); |
| } |
| |
| btn.disabled = false; btn.innerText = "Connect"; |
| } |
| |
| function populateSplits(splits) { |
| const sel = document.getElementById('split_select'); |
| sel.innerHTML = ''; |
| |
| if (Array.isArray(splits) && splits.length > 0) { |
| splits.forEach(s => sel.innerHTML += `<option value="${s}">${s}</option>`); |
| } else { |
| sel.innerHTML = `<option value="train">train (fallback)</option>`; |
| } |
| sel.disabled = false; |
| } |
| |
| async function updateSplits() { |
| const conf = document.getElementById('config_select').value; |
| const ds = document.getElementById('dataset_id').value; |
| const token = document.getElementById('token').value; |
| try { |
| const res = await fetch('/get_splits', { |
| method: 'POST', headers: {'Content-Type': 'application/json'}, |
| body: JSON.stringify({dataset_id: ds, config: conf, token: token}) |
| }); |
| const data = await res.json(); |
| |
| if(data.status === 'success') { |
| populateSplits(data.splits); |
| } else { |
| console.warn("Could not fetch specific splits, using defaults."); |
| populateSplits(['train', 'test', 'validation']); |
| } |
| } catch(e) { |
| console.error(e); |
| populateSplits(['train', 'test', 'validation']); |
| } |
| } |
| |
| async function startInspection() { |
| const btn = document.getElementById('inspect_btn'); |
| btn.innerText = "Scanning..."; btn.disabled = true; |
| |
| try { |
| const res = await fetch('/inspect_rows', { |
| method: 'POST', headers: {'Content-Type': 'application/json'}, |
| body: JSON.stringify({ |
| token: document.getElementById('token').value, |
| dataset_id: document.getElementById('dataset_id').value, |
| config: document.getElementById('config_select').value, |
| split: document.getElementById('split_select').value |
| }) |
| }); |
| const data = await res.json(); |
| |
| if(data.status === 'success') { |
| document.getElementById('workspace').style.display = 'flex'; |
| sampleRows = data.samples; |
| schemaData = data.schema; |
| |
| const list = document.getElementById('source_col_list'); |
| list.innerHTML = ''; |
| for(let col in schemaData) { |
| let badge = schemaData[col].type === 'List' ? '<span class="badge bg-warning text-dark float-end">List</span>' : ''; |
| list.innerHTML += `<li class="list-group-item source-col-row" onclick="previewCol('${col}')">${col} ${badge}</li>`; |
| } |
| } else { |
| alert('Error: ' + data.message); |
| } |
| } catch(e) { |
| console.error(e); |
| alert('Network error: ' + e); |
| } |
| |
| btn.innerText = "Analyze Schema"; |
| btn.disabled = false; |
| } |
| |
| function previewCol(col) { |
| let html = ''; |
| sampleRows.forEach(r => { |
| let v = r[col]; |
| if(typeof v === 'object') v = JSON.stringify(v, null, 2); |
| html += v + "\n" + "-".repeat(20) + "\n"; |
| }); |
| document.getElementById('data_preview_box').innerHTML = `<pre>${html}</pre>`; |
| } |
| |
| async function runPreview() { |
| const box = document.getElementById('final_preview'); |
| box.style.display='block'; |
| box.innerHTML = "Generating preview..."; |
| |
| const recipe = getRecipe(); |
| |
| |
| if(!recipe.columns || recipe.columns.length === 0) { |
| box.innerHTML = '<div class="alert alert-warning">No columns defined! Please add at least one column.</div>'; |
| return; |
| } |
| |
| console.log('Sending preview request with recipe:', recipe); |
| |
| try { |
| const res = await fetch('/preview', { |
| method: 'POST', |
| headers: {'Content-Type': 'application/json'}, |
| body: JSON.stringify({ |
| token: document.getElementById('token').value, |
| dataset_id: document.getElementById('dataset_id').value, |
| config: document.getElementById('config_select').value, |
| split: document.getElementById('split_select').value, |
| recipe: recipe |
| }) |
| }); |
| |
| const data = await res.json(); |
| console.log('Preview response:', data); |
| |
| if(data.status === 'success') { |
| box.innerHTML = `<pre>${JSON.stringify(data.rows, null, 2)}</pre>`; |
| } else { |
| box.innerHTML = `<div class="alert alert-danger">Error: ${data.message}</div>`; |
| } |
| } catch(e) { |
| console.error('Preview error:', e); |
| box.innerHTML = `<div class="alert alert-danger">Network error: ${e}</div>`; |
| } |
| } |
| |
| async function executeJob() { |
| const recipe = getRecipe(); |
| |
| if(!recipe.columns || recipe.columns.length === 0) { |
| alert('No columns defined!'); |
| return; |
| } |
| |
| const targetId = document.getElementById('target_id').value.trim(); |
| if(!targetId) { |
| alert('Please enter a target repository name!'); |
| return; |
| } |
| |
| const res = await fetch('/execute', { |
| method: 'POST', |
| headers: {'Content-Type': 'application/json'}, |
| body: JSON.stringify({ |
| token: document.getElementById('token').value, |
| dataset_id: document.getElementById('dataset_id').value, |
| config: document.getElementById('config_select').value, |
| split: document.getElementById('split_select').value, |
| target_id: targetId, |
| recipe: recipe, |
| max_rows: document.getElementById('max_rows').value, |
| license: document.getElementById('license_input').value |
| }) |
| }); |
| |
| const data = await res.json(); |
| |
| if(data.status === 'started') { |
| const statusBox = document.getElementById('job_status'); |
| statusBox.style.display = 'block'; |
| statusBox.className = 'alert alert-info mt-3'; |
| statusBox.innerText = "Job started, processing..."; |
| |
| const interval = setInterval(async () => { |
| const s = await fetch(`/status/${data.job_id}`).then(r=>r.json()); |
| if(s.status === 'completed') { |
| clearInterval(interval); |
| statusBox.className = 'alert alert-success mt-3'; |
| statusBox.innerText = `✅ Done! Processed ${s.result.rows_processed} rows.`; |
| } else if(s.status === 'failed') { |
| clearInterval(interval); |
| statusBox.className = 'alert alert-danger mt-3'; |
| statusBox.innerText = `❌ Job failed: ${s.error}`; |
| } |
| }, 2000); |
| } else { |
| alert('Failed to start job: ' + (data.message || 'Unknown error')); |
| } |
| } |
| </script> |
| </body> |
| </html> |
