{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "89ef2fd8-f832-4c29-b47a-c4c5bef42610",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "import torch.nn.functional as F\n",
    "import torch.nn.init as init\n",
    "\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from tqdm import tqdm\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "153e50f0-ff3f-47ea-94eb-28f11c031d51",
   "metadata": {},
   "source": [
    "# Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "73830b0e-7386-40f4-ab67-2bfe0beb1e5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the image transformations\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize((28, 28)),  # Resizing to 136x136 pixels\n",
    "    transforms.Grayscale(num_output_channels=1),  # Convert to grayscale\n",
    "    transforms.ToTensor(),  # Convert to tensor\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f68342a4-9977-4c01-94d6-6bcc63f61f3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the training and validation datasets\n",
    "train_dataset = datasets.ImageFolder(root='dataset/train', transform=transform)\n",
    "val_dataset = datasets.ImageFolder(root='dataset/validation', transform=transform)\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "# Create DataLoaders\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84ba899d-2698-40ad-a6a9-d8b45942e8b8",
   "metadata": {},
   "source": [
    "# FCN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "118e2b0d-d2b8-4c66-af21-5da5b1499140",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLP(\n",
       "  (fc1): Linear(in_features=784, out_features=512, bias=True)\n",
       "  (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
       "  (fc3): Linear(in_features=256, out_features=5, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MLP, self).__init__()\n",
    "        self.fc1 = nn.Linear(28 * 28, 512)  # Adjust input size to match resized images\n",
    "        self.fc2 = nn.Linear(512, 256)\n",
    "        self.fc3 = nn.Linear(256, 5)  # Adjust output size to match the number of classes\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 28 * 28)  # Adjust input size to match resized images\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return F.log_softmax(x, dim=1)\n",
    "\n",
    "def weight_init(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        init.kaiming_uniform_(m.weight.data)\n",
    "\n",
    "# Determine if GPU (CUDA) is available, else default to CPU\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "\n",
    "model = MLP().to(device)\n",
    "model.apply(weight_init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cebf981c-89a5-4c6a-a57f-2147347d215d",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.01\n",
    "\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=0.9)\n",
    "\n",
    "loss_fn = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79285511-db0a-48cf-881a-291acb4678e0",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bb8052fa-e00c-4622-b813-2ead799266d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/20, Train Loss: 0.0494, Train Accuracy: 27.04%, Val Loss: 0.0506, Val Accuracy: 29.83%\n",
      "Epoch: 2/20, Train Loss: 0.0475, Train Accuracy: 33.05%, Val Loss: 0.0492, Val Accuracy: 30.17%\n",
      "Epoch: 3/20, Train Loss: 0.0470, Train Accuracy: 33.81%, Val Loss: 0.0507, Val Accuracy: 29.02%\n",
      "Epoch: 4/20, Train Loss: 0.0463, Train Accuracy: 35.95%, Val Loss: 0.0500, Val Accuracy: 30.40%\n",
      "Epoch: 5/20, Train Loss: 0.0456, Train Accuracy: 37.80%, Val Loss: 0.0494, Val Accuracy: 28.67%\n",
      "Epoch: 6/20, Train Loss: 0.0452, Train Accuracy: 38.43%, Val Loss: 0.0506, Val Accuracy: 27.05%\n",
      "Epoch: 7/20, Train Loss: 0.0449, Train Accuracy: 38.52%, Val Loss: 0.0502, Val Accuracy: 31.10%\n",
      "Epoch: 8/20, Train Loss: 0.0440, Train Accuracy: 39.68%, Val Loss: 0.0515, Val Accuracy: 27.28%\n",
      "Epoch: 9/20, Train Loss: 0.0436, Train Accuracy: 40.40%, Val Loss: 0.0504, Val Accuracy: 31.91%\n",
      "Epoch: 10/20, Train Loss: 0.0432, Train Accuracy: 41.58%, Val Loss: 0.0504, Val Accuracy: 30.29%\n",
      "Epoch: 11/20, Train Loss: 0.0427, Train Accuracy: 43.29%, Val Loss: 0.0507, Val Accuracy: 29.71%\n",
      "Epoch: 12/20, Train Loss: 0.0427, Train Accuracy: 43.29%, Val Loss: 0.0502, Val Accuracy: 30.06%\n",
      "Epoch: 13/20, Train Loss: 0.0419, Train Accuracy: 43.96%, Val Loss: 0.0519, Val Accuracy: 28.67%\n",
      "Epoch: 14/20, Train Loss: 0.0411, Train Accuracy: 46.21%, Val Loss: 0.0543, Val Accuracy: 28.21%\n",
      "Epoch: 15/20, Train Loss: 0.0417, Train Accuracy: 45.03%, Val Loss: 0.0504, Val Accuracy: 32.02%\n",
      "Epoch: 16/20, Train Loss: 0.0405, Train Accuracy: 47.25%, Val Loss: 0.0517, Val Accuracy: 30.52%\n",
      "Epoch: 17/20, Train Loss: 0.0409, Train Accuracy: 46.47%, Val Loss: 0.0585, Val Accuracy: 28.55%\n",
      "Epoch: 18/20, Train Loss: 0.0426, Train Accuracy: 43.93%, Val Loss: 0.0511, Val Accuracy: 32.25%\n",
      "Epoch: 19/20, Train Loss: 0.0393, Train Accuracy: 48.93%, Val Loss: 0.0521, Val Accuracy: 32.02%\n",
      "Epoch: 20/20, Train Loss: 0.0396, Train Accuracy: 48.35%, Val Loss: 0.0575, Val Accuracy: 26.71%\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 20\n",
    "\n",
    "# Training loop\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    total_train_loss = 0\n",
    "    total_train_correct = 0\n",
    "\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        # Move the data and target labels to the chosen device (GPU/CPU)\n",
    "        data, target = data.to(device), target.to(device)\n",
    "\n",
    "        # Zero out any gradients from the previous iteration\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "\n",
    "        # Compute the loss between the predictions and actual labels\n",
    "        loss = loss_fn(output, target)\n",
    "        loss.backward()\n",
    "\n",
    "        # Update the model parameters\n",
    "        optimizer.step()\n",
    "\n",
    "        # Accumulate total loss and correct predictions for this epoch\n",
    "        total_train_loss += loss.item()\n",
    "        _, predicted = torch.max(output.data, 1)\n",
    "        total_train_correct += (predicted == target).sum().item()\n",
    "\n",
    "    # Compute average training loss and training accuracy for this epoch\n",
    "    avg_train_loss = total_train_loss / len(train_loader.dataset)\n",
    "    train_accuracy = 100. * total_train_correct / len(train_loader.dataset)\n",
    "\n",
    "    # Switch the model to evaluation mode. Disables features like dropout.\n",
    "    model.eval()\n",
    "    total_val_loss = 0\n",
    "    total_val_correct = 0\n",
    "\n",
    "    # No gradient computation is needed in validation. Improves performance.\n",
    "    with torch.no_grad():\n",
    "        # Process each batch of data from the validation set\n",
    "        for data, target in val_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model(data)\n",
    "            loss = loss_fn(output, target)\n",
    "            \n",
    "            # Accumulate total validation loss and correct predictions\n",
    "            total_val_loss += loss.item()\n",
    "            _, predicted = torch.max(output.data, 1)\n",
    "            total_val_correct += (predicted == target).sum().item()\n",
    "\n",
    "    # Compute average validation loss and validation accuracy for this epoch\n",
    "    avg_val_loss = total_val_loss / len(val_loader.dataset)\n",
    "    val_accuracy = 100. * total_val_correct / len(val_loader.dataset)\n",
    "\n",
    "    # Print training and validation results for this epoch\n",
    "    print(f\"Epoch: {epoch+1}/{num_epochs}, Train Loss: {avg_train_loss:.4f}, Train Accuracy: {train_accuracy:.2f}%, Val Loss: {avg_val_loss:.4f}, Val Accuracy: {val_accuracy:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "565f018d-4e8e-4e3e-9324-2efd38923eab",
   "metadata": {},
   "source": [
    "# Improve FCN Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3e8b347-f8e1-4791-abbf-2213dd0364c7",
   "metadata": {},
   "source": [
    "### 1. update pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b5659715-1f8d-4f7b-aacc-d872ab369fdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the image transformations\n",
    "transform2 = transforms.Compose([\n",
    "    transforms.Resize((64, 64)),  # Resizing to 128x128 pixels\n",
    "    transforms.Grayscale(num_output_channels=1),  # Convert to grayscale\n",
    "    transforms.ToTensor(),  # Convert to tensor\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "24ce6db9-a04f-4f5d-9e02-4c2f6f34a438",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the training and validation datasets\n",
    "train_dataset = datasets.ImageFolder(root='dataset/train', transform=transform2)\n",
    "val_dataset = datasets.ImageFolder(root='dataset/validation', transform=transform2)\n",
    "\n",
    "batch_size = 32\n",
    "\n",
    "# Create DataLoaders\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "651a963d-e504-4d82-be2f-57e7dd35ff54",
   "metadata": {},
   "source": [
    "### 2. update learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "75b972c8-d916-4144-aba0-f0750eecade9",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.001"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ada2fde-236a-4bf7-a696-f4bd2fab1273",
   "metadata": {},
   "source": [
    "### 3. update model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "88cb9e6a-065e-4800-a571-400459dda747",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLP2(\n",
       "  (fc1): Linear(in_features=4096, out_features=512, bias=True)\n",
       "  (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
       "  (fc3): Linear(in_features=256, out_features=5, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class MLP2(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MLP2, self).__init__()\n",
    "        self.fc1 = nn.Linear(64 * 64, 512)  # Adjust input size to match resized images\n",
    "        self.fc2 = nn.Linear(512, 256)\n",
    "        self.fc3 = nn.Linear(256, 5)  # Adjust output size to match the number of classes\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 64 * 64)  # Adjust input size to match resized images\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return F.log_softmax(x, dim=1)\n",
    "\n",
    "def weight_init(m):\n",
    "    if isinstance(m, nn.Linear):\n",
    "        init.kaiming_uniform_(m.weight.data)\n",
    "\n",
    "# Determine if GPU (CUDA) is available, else default to CPU\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "\n",
    "model2 = MLP2().to(device)\n",
    "model2.apply(weight_init)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "258fd16f-75ce-40d2-a0a9-34e112b70168",
   "metadata": {},
   "source": [
    "### 4. update optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8906d1e6-35bd-416e-a402-0d586c21af03",
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer2 = torch.optim.Adam(model2.parameters(), lr=lr)\n",
    "loss_fn2 = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9fa0ffa-ae2c-4fef-b69d-78af3370c237",
   "metadata": {},
   "source": [
    "# Train w/ Update"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c0198cc7-008a-4f8b-881a-a71705409c24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/20, Train Loss: 0.0497, Train Accuracy: 27.82%, Val Loss: 0.0525, Val Accuracy: 27.40%\n",
      "Epoch: 2/20, Train Loss: 0.0478, Train Accuracy: 32.16%, Val Loss: 0.0494, Val Accuracy: 29.36%\n",
      "Epoch: 3/20, Train Loss: 0.0473, Train Accuracy: 32.88%, Val Loss: 0.0497, Val Accuracy: 28.32%\n",
      "Epoch: 4/20, Train Loss: 0.0467, Train Accuracy: 33.57%, Val Loss: 0.0506, Val Accuracy: 29.02%\n",
      "Epoch: 5/20, Train Loss: 0.0462, Train Accuracy: 34.73%, Val Loss: 0.0501, Val Accuracy: 31.79%\n",
      "Epoch: 6/20, Train Loss: 0.0459, Train Accuracy: 36.09%, Val Loss: 0.0496, Val Accuracy: 30.40%\n",
      "Epoch: 7/20, Train Loss: 0.0462, Train Accuracy: 36.35%, Val Loss: 0.0512, Val Accuracy: 30.06%\n",
      "Epoch: 8/20, Train Loss: 0.0450, Train Accuracy: 37.16%, Val Loss: 0.0517, Val Accuracy: 27.86%\n",
      "Epoch: 9/20, Train Loss: 0.0448, Train Accuracy: 39.53%, Val Loss: 0.0497, Val Accuracy: 31.79%\n",
      "Epoch: 10/20, Train Loss: 0.0438, Train Accuracy: 40.60%, Val Loss: 0.0509, Val Accuracy: 28.32%\n",
      "Epoch: 11/20, Train Loss: 0.0432, Train Accuracy: 41.96%, Val Loss: 0.0549, Val Accuracy: 26.13%\n",
      "Epoch: 12/20, Train Loss: 0.0424, Train Accuracy: 44.74%, Val Loss: 0.0511, Val Accuracy: 28.55%\n",
      "Epoch: 13/20, Train Loss: 0.0421, Train Accuracy: 45.63%, Val Loss: 0.0517, Val Accuracy: 27.63%\n",
      "Epoch: 14/20, Train Loss: 0.0424, Train Accuracy: 44.16%, Val Loss: 0.0509, Val Accuracy: 31.68%\n",
      "Epoch: 15/20, Train Loss: 0.0405, Train Accuracy: 47.14%, Val Loss: 0.0549, Val Accuracy: 28.55%\n",
      "Epoch: 16/20, Train Loss: 0.0399, Train Accuracy: 47.86%, Val Loss: 0.0522, Val Accuracy: 29.25%\n",
      "Epoch: 17/20, Train Loss: 0.0400, Train Accuracy: 48.61%, Val Loss: 0.0527, Val Accuracy: 29.83%\n",
      "Epoch: 18/20, Train Loss: 0.0391, Train Accuracy: 49.54%, Val Loss: 0.0503, Val Accuracy: 30.17%\n",
      "Epoch: 19/20, Train Loss: 0.0378, Train Accuracy: 52.40%, Val Loss: 0.0573, Val Accuracy: 28.55%\n",
      "Epoch: 20/20, Train Loss: 0.0379, Train Accuracy: 52.86%, Val Loss: 0.0520, Val Accuracy: 29.02%\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 20\n",
    "\n",
    "# Training loop\n",
    "for epoch in range(num_epochs):\n",
    "    model2.train()\n",
    "    total_train_loss = 0\n",
    "    total_train_correct = 0\n",
    "\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        # Move the data and target labels to the chosen device (GPU/CPU)\n",
    "        data, target = data.to(device), target.to(device)\n",
    "\n",
    "        # Zero out any gradients from the previous iteration\n",
    "        optimizer2.zero_grad()\n",
    "        output = model2(data)\n",
    "\n",
    "        # Compute the loss between the predictions and actual labels\n",
    "        loss = loss_fn2(output, target)\n",
    "        loss.backward()\n",
    "\n",
    "        # Update the model parameters\n",
    "        optimizer2.step()\n",
    "\n",
    "        # Accumulate total loss and correct predictions for this epoch\n",
    "        total_train_loss += loss.item()\n",
    "        _, predicted = torch.max(output.data, 1)\n",
    "        total_train_correct += (predicted == target).sum().item()\n",
    "\n",
    "    # Compute average training loss and training accuracy for this epoch\n",
    "    avg_train_loss = total_train_loss / len(train_loader.dataset)\n",
    "    train_accuracy = 100. * total_train_correct / len(train_loader.dataset)\n",
    "\n",
    "    # Switch the model to evaluation mode. Disables features like dropout.\n",
    "    model2.eval()\n",
    "    total_val_loss = 0\n",
    "    total_val_correct = 0\n",
    "\n",
    "    # No gradient computation is needed in validation. Improves performance.\n",
    "    with torch.no_grad():\n",
    "        # Process each batch of data from the validation set\n",
    "        for data, target in val_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = model2(data)\n",
    "            loss = loss_fn2(output, target)\n",
    "            \n",
    "            # Accumulate total validation loss and correct predictions\n",
    "            total_val_loss += loss.item()\n",
    "            _, predicted = torch.max(output.data, 1)\n",
    "            total_val_correct += (predicted == target).sum().item()\n",
    "\n",
    "    # Compute average validation loss and validation accuracy for this epoch\n",
    "    avg_val_loss = total_val_loss / len(val_loader.dataset)\n",
    "    val_accuracy = 100. * total_val_correct / len(val_loader.dataset)\n",
    "\n",
    "    # Print training and validation results for this epoch\n",
    "    print(f\"Epoch: {epoch+1}/{num_epochs}, Train Loss: {avg_train_loss:.4f}, Train Accuracy: {train_accuracy:.2f}%, Val Loss: {avg_val_loss:.4f}, Val Accuracy: {val_accuracy:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a66d8328-dc37-4d85-a344-80dd846f5919",
   "metadata": {},
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "88cd98d3-b42f-4512-9277-589b303313d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the training and validation datasets\n",
    "train_dataset = datasets.ImageFolder(root='dataset/train', transform=transform)\n",
    "val_dataset = datasets.ImageFolder(root='dataset/validation', transform=transform)\n",
    "\n",
    "# Flatten the images and prepare the data for SVM\n",
    "X_train = torch.stack([img for img, _ in train_dataset]).view(len(train_dataset), -1).numpy()\n",
    "y_train = torch.tensor([label for _, label in train_dataset]).numpy()\n",
    "X_val = torch.stack([img for img, _ in val_dataset]).view(len(val_dataset), -1).numpy()\n",
    "y_val = torch.tensor([label for _, label in val_dataset]).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "16399848-bdb0-4182-874d-e7185f2927cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CascadeSVM:\n",
    "    def __init__(self, kernel='linear'):\n",
    "        # Initialize SVM kernel type (default is linear) and a list to store trained classifiers\n",
    "        self.kernel = kernel\n",
    "        self.classifiers = []\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        unique_classes = np.unique(y)\n",
    "        remaining_data = X.copy()\n",
    "        remaining_labels = y.copy()\n",
    "        \n",
    "        # Train individual SVMs for each unique class \n",
    "        for cls in tqdm(unique_classes[:-1], desc=\"Training SVMs\"):\n",
    "            \n",
    "            # Convert multi-class labels to binary: 1 for the current class and 0 for all others\n",
    "            binary_labels = np.where(remaining_labels == cls, 1, 0)\n",
    "            \n",
    "            # Initialize an SVM with the specified kernel\n",
    "            svm_model = SVC(kernel=self.kernel)\n",
    "\n",
    "            # Train the SVM on the remaining data using the binary labels\n",
    "            svm_model.fit(remaining_data, binary_labels)\n",
    "            \n",
    "            # Store the trained SVM classifier\n",
    "            self.classifiers.append(svm_model)\n",
    "            \n",
    "            # Remove samples of the current class from the dataset\n",
    "            non_cls_indices = np.where(remaining_labels != cls)\n",
    "            remaining_data = remaining_data[non_cls_indices]\n",
    "            remaining_labels = remaining_labels[non_cls_indices]\n",
    "\n",
    "    def predict(self, X):\n",
    "        predictions = []\n",
    "\n",
    "        for sample in X:\n",
    "            sample_class = None\n",
    "\n",
    "            # Iterate over all trained SVM classifiers\n",
    "            for idx, classifier in enumerate(self.classifiers):\n",
    "                \n",
    "                # Predict using the current SVM classifier\n",
    "                prediction = classifier.predict([sample])\n",
    "\n",
    "                # If the SVM classifies the sample as '1', assign its index as the class label\n",
    "                if prediction == 1:\n",
    "                    sample_class = idx\n",
    "                    break  # Stop checking further classifiers for this sample\n",
    "\n",
    "            # If the sample was not classified by any SVM, assign it to the last class\n",
    "            if sample_class is None:  \n",
    "                sample_class = len(self.classifiers)\n",
    "\n",
    "            predictions.append(sample_class)\n",
    "\n",
    "        # Convert predictions list to a numpy array and return\n",
    "        return np.array(predictions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e1dffaea-e918-4fbc-8862-014ac30a2dc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Training SVMs: 100%|██████████████████████████████| 4/4 [00:12<00:00,  3.23s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation Accuracy: 25.90%\n"
     ]
    }
   ],
   "source": [
    "# Create a cascading SVM and train it\n",
    "cascade_svm = CascadeSVM()\n",
    "cascade_svm.fit(X_train, y_train)\n",
    "\n",
    "# Predict on validation set\n",
    "y_val_pred = cascade_svm.predict(X_val)\n",
    "\n",
    "# Calculate accuracy\n",
    "accuracy = accuracy_score(y_val, y_val_pred)\n",
    "print(f\"Validation Accuracy: {accuracy * 100:.2f}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "952d5c02-3b53-4ec4-9068-f8ad453e4d9f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "prac",
   "language": "python",
   "name": "prac"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}