update dnotebook

notebook.ipynb

@@ -25,7 +25,8 @@
     "from sklearn.model_selection import train_test_split\n",
     "from sklearn.metrics import accuracy_score\n",
     "import numpy as np\n",
-    "from tqdm import tqdm"
+    "from tqdm import tqdm\n",
+    "import pickle"
    ]
   },
   {
@@ -34,29 +35,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "class ESC50Dataset(Dataset):\n",
-    "    def __init__(self, audio_dir, metadata_path, sr=16000, max_length=160000):\n",
-    "        self.audio_dir = audio_dir\n",
-    "        self.sr = sr\n",
-    "        self.max_length = max_length\n",
-    "        self.metadata = pd.read_csv(metadata_path)\n",
-    "    \n",
-    "    def __len__(self):\n",
-    "        return len(self.metadata)\n",
-    "    \n",
-    "    def __getitem__(self, idx):\n",
-    "        row = self.metadata.iloc[idx]\n",
-    "        filename = row['filename']\n",
-    "        label = row['target']\n",
-    "        \n",
-    "        audio_path = os.path.join(self.audio_dir, filename)\n",
-    "        audio, sr = librosa.load(audio_path, sr=self.sr)\n",
-    "        \n",
-    "        audio_tensor = torch.tensor(audio).float()\n",
-    "        label_tensor = torch.tensor(label).long()\n",
-    "        \n",
-    "        return audio_tensor, label_tensor\n",
-    "\n",
     "def download_esc50():\n",
     "    import urllib.request\n",
     "    import zipfile\n",
@@ -79,43 +57,131 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Download dataset\n",
+    "def extract_features():\n",
+    "    \"\"\"Extract and save features for all ESC-50 audio files\"\"\"\n",
+    "    \n",
+    "    if os.path.exists('esc50_features.pkl'):\n",
+    "        print(\"Features already extracted, loading from file...\")\n",
+    "        with open('esc50_features.pkl', 'rb') as f:\n",
+    "            return pickle.load(f)\n",
+    "    \n",
+    "    # Load model\n",
+    "    model = AutoModel.from_pretrained(\"mispeech/dashengtokenizer\", trust_remote_code=True)\n",
+    "    model.eval()\n",
+    "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
+    "    model.to(device)\n",
+    "    \n",
+    "    # Load metadata\n",
+    "    metadata_path = 'ESC-50/meta/esc50.csv'\n",
+    "    df = pd.read_csv(metadata_path)\n",
+    "    \n",
+    "    features_list = []\n",
+    "    labels_list = []\n",
+    "    \n",
+    "    print(\"Extracting features...\")\n",
+    "    for idx, row in tqdm(df.iterrows(), total=len(df)):\n",
+    "        filename = row['filename']\n",
+    "        label = row['target']\n",
+    "        \n",
+    "        audio_path = os.path.join('ESC-50/audio', filename)\n",
+    "        \n",
+    "        try:\n",
+    "            # Load and preprocess audio\n",
+    "            audio, sr = librosa.load(audio_path, sr=16000)\n",
+    "            audio_tensor = torch.tensor(audio).float().unsqueeze(0).to(device)\n",
+    "            \n",
+    "            # Extract features\n",
+    "            with torch.no_grad(), torch.autocast(device_type='cuda'):\n",
+    "                features = model.encode(audio_tensor)\n",
+    "                if isinstance(features, dict):\n",
+    "                    for key in ['last_hidden_state', 'embeddings', 'audio']:\n",
+    "                        if key in features:\n",
+    "                            features = features[key]\n",
+    "                            break\n",
+    "                    else:\n",
+    "                        features = list(features.values())[0]\n",
+    "                \n",
+    "                # Global average pooling\n",
+    "                if features.dim() > 2:\n",
+    "                    features = features.mean(dim=1)\n",
+    "                \n",
+    "                features = features.squeeze().cpu().numpy()\n",
+    "            \n",
+    "            features_list.append(features)\n",
+    "            labels_list.append(label)\n",
+    "            \n",
+    "        except Exception as e:\n",
+    "            print(f\"Error processing {filename}: {e}\")\n",
+    "    \n",
+    "    # Save features\n",
+    "    features_data = {\n",
+    "        'features': np.array(features_list),\n",
+    "        'labels': np.array(labels_list),\n",
+    "        'embedding_dim': features_list[0].shape[0]\n",
+    "    }\n",
+    "    \n",
+    "    with open('esc50_features.pkl', 'wb') as f:\n",
+    "        pickle.dump(features_data, f)\n",
+    "    \n",
+    "    print(f\"Features extracted: {len(features_list)} samples, embedding dim: {features_data['embedding_dim']}\")\n",
+    "    return features_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Download dataset and extract features\n",
     "download_esc50()\n",
+    "features_data = extract_features()\n",
     "\n",
-    "# Load model\n",
-    "model = AutoModel.from_pretrained(\"mispeech/dashengtokenizer\", trust_remote_code=True)\n",
+    "X = features_data['features']\n",
+    "y = features_data['labels']\n",
+    "embedding_dim = features_data['embedding_dim']\n",
+    "\n",
+    "print(f\"Features shape: {X.shape}, Labels shape: {y.shape}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Convert to PyTorch tensors\n",
+    "X_tensor = torch.tensor(X, dtype=torch.float32)\n",
+    "y_tensor = torch.tensor(y, dtype=torch.long)\n",
     "\n",
-    "# Get embedding dimension\n",
-    "embedding_dim = 1280\n",
-    "print(f\"Model embedding dimension: {embedding_dim}\")\n",
+    "# Split into train/val\n",
+    "train_idx, val_idx = train_test_split(range(len(X_tensor)), test_size=0.2, random_state=42)\n",
     "\n",
-    "# Freeze model\n",
-    "for param in model.parameters():\n",
-    "    param.requires_grad = False\n",
+    "X_train = X_tensor[train_idx]\n",
+    "y_train = y_tensor[train_idx]\n",
+    "X_val = X_tensor[val_idx]\n",
+    "y_val = y_tensor[val_idx]\n",
     "\n",
+    "print(f\"Train: {X_train.shape}, Val: {X_val.shape}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "# Single linear layer\n",
     "classifier = nn.Linear(embedding_dim, 50)  # 50 ESC-50 classes\n",
     "\n",
     "# Setup\n",
     "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
-    "model.to(device)\n",
     "classifier.to(device)\n",
     "print(f\"Using device: {device}\")\n",
-    "# Create datasets\n",
-    "audio_dir = 'ESC-50/audio'\n",
-    "metadata_path = 'ESC-50/meta/esc50.csv'\n",
-    "\n",
-    "dataset = ESC50Dataset(audio_dir, metadata_path)\n",
     "\n",
-    "# Split into train/val\n",
-    "train_idx, val_idx = train_test_split(range(len(dataset)), test_size=0.2, random_state=42)\n",
-    "train_dataset = torch.utils.data.Subset(dataset, train_idx)\n",
-    "val_dataset = torch.utils.data.Subset(dataset, val_idx)\n",
-    "\n",
-    "train_loader = DataLoader(train_dataset, batch_size=4, shuffle=True, num_workers=2)\n",
-    "val_loader = DataLoader(val_dataset, batch_size=4, shuffle=False, num_workers=2)\n",
-    "\n",
-    "print(f\"Train samples: {len(train_dataset)}, Val samples: {len(val_dataset)}\")"
+    "# Training setup\n",
+    "optimizer = torch.optim.Adam(classifier.parameters(), lr=1e-3)\n",
+    "criterion = nn.CrossEntropyLoss()"
    ]
   },
   {
@@ -124,94 +190,49 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Training setup\n",
-    "optimizer = torch.optim.Adam(classifier.parameters(), lr=1e-3)\n",
-    "criterion = nn.CrossEntropyLoss()\n",
+    "# Training loop (much faster since features are pre-extracted)\n",
+    "batch_size = 32\n",
     "\n",
-    "# Training loop\n",
     "for epoch in range(10):\n",
-    "    model.eval()\n",
     "    classifier.train()\n",
     "    \n",
     "    # Training\n",
     "    train_loss = 0\n",
     "    train_preds = []\n",
     "    train_labels = []\n",
-    "\n",
-    "    pbar = tqdm(train_loader, desc=f'Epoch {epoch+1}/10 Training')\n",
-    "    for batch_audio, batch_labels in pbar:\n",
-    "        batch_audio = batch_audio.to(device)\n",
-    "        batch_labels = batch_labels.to(device)\n",
-    "\n",
-    "        # Forward through frozen model\n",
-    "        with torch.no_grad(), torch.autocast(device_type='cuda'):\n",
-    "            features = model.encode(batch_audio)\n",
-    "            if isinstance(features, dict):\n",
-    "                for key in ['last_hidden_state', 'embeddings', 'audio']:\n",
-    "                    if key in features:\n",
-    "                        features = features[key]\n",
-    "                        break\n",
-    "                else:\n",
-    "                    features = list(features.values())[0]\n",
-    "\n",
-    "            # Global average pooling if needed\n",
-    "            if features.dim() > 2:\n",
-    "                features = features.mean(dim=1)\n",
-    "\n",
-    "        # Classifier\n",
-    "        logits = classifier(features)\n",
+    "    \n",
+    "    # Mini-batch training\n",
+    "    for i in range(0, len(X_train), batch_size):\n",
+    "        batch_features = X_train[i:i+batch_size].to(device)\n",
+    "        batch_labels = y_train[i:i+batch_size].to(device)\n",
+    "        \n",
+    "        # Forward pass\n",
+    "        logits = classifier(batch_features)\n",
     "        loss = criterion(logits, batch_labels)\n",
-    "\n",
-    "        # Backward\n",
+    "        \n",
+    "        # Backward pass\n",
     "        optimizer.zero_grad()\n",
     "        loss.backward()\n",
     "        optimizer.step()\n",
-    "\n",
+    "        \n",
     "        train_loss += loss.item()\n",
     "        preds = torch.argmax(logits, dim=1)\n",
     "        train_preds.extend(preds.cpu().numpy())\n",
     "        train_labels.extend(batch_labels.cpu().numpy())\n",
-    "\n",
-    "        # Update progress bar\n",
-    "        pbar.set_postfix({'loss': f'{loss.item():.4f}'})\n",
-    "\n",
+    "    \n",
     "    train_acc = accuracy_score(train_labels, train_preds)\n",
     "    \n",
     "    # Validation\n",
     "    classifier.eval()\n",
-    "    val_preds = []\n",
-    "    val_labels = []\n",
-    "\n",
-    "    with torch.no_grad(),torch.autocast(device_type='cuda'):\n",
-    "        pbar_val = tqdm(val_loader, desc=f'Epoch {epoch+1}/10 Validation')\n",
-    "        for batch_audio, batch_labels in pbar_val:\n",
-    "            batch_audio = batch_audio.to(device)\n",
-    "            batch_labels = batch_labels.to(device)\n",
-    "\n",
-    "            features = model(batch_audio)\n",
-    "            if isinstance(features, dict):\n",
-    "                for key in ['last_hidden_state', 'embeddings', 'audio']:\n",
-    "                    if key in features:\n",
-    "                        features = features[key]\n",
-    "                        break\n",
-    "                else:\n",
-    "                    features = list(features.values())[0]\n",
-    "\n",
-    "            if features.dim() > 2:\n",
-    "                features = features.mean(dim=1)\n",
-    "\n",
-    "            logits = classifier(features)\n",
-    "            preds = torch.argmax(logits, dim=1)\n",
-    "            val_preds.extend(preds.cpu().numpy())\n",
-    "            val_labels.extend(batch_labels.cpu().numpy())\n",
-    "\n",
-    "            # Update validation progress bar\n",
-    "            batch_acc = (preds == batch_labels).float().mean().item()\n",
-    "            pbar_val.set_postfix({'batch_acc': f'{batch_acc:.4f}'})\n",
-    "\n",
-    "    val_acc = accuracy_score(val_labels, val_preds)\n",
+    "    with torch.no_grad():\n",
+    "        val_features = X_val.to(device)\n",
+    "        val_labels = y_val.cpu().numpy()\n",
+    "        \n",
+    "        val_logits = classifier(val_features)\n",
+    "        val_preds = torch.argmax(val_logits, dim=1).cpu().numpy()\n",
+    "        val_acc = accuracy_score(val_labels, val_preds)\n",
     "    \n",
-    "    print(f\"Epoch {epoch+1}/10 - Train Loss: {train_loss/len(train_loader):.4f} - Train Acc: {train_acc:.4f} - Val Acc: {val_acc:.4f}\")"
+    "    print(f\"Epoch {epoch+1}/10 - Train Loss: {train_loss/len(range(0, len(X_train), batch_size)):.4f} - Train Acc: {train_acc:.4f} - Val Acc: {val_acc:.4f}\")"
    ]
   }
  ],