Repository: crawlab-team/webspot (language: python)

Issue:
feat: added fields display
[AB#75](https://dev.azure.com/crawlab-team/24e63d12-e9d5-4201-93d7-9ee8fcfce033/_workitems/edit/75)



Discussion / hints:

Fix patch:
diff --git a/.dockerignore b/.dockerignore
index 480ec21..90eb601 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -5,3 +5,4 @@ data/
 .ipynb_checkpoints/
 .idea/
 .models/
+dev/
diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..66143b1
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,1 @@
+WEBSPOT_DATABASE_URL=
diff --git a/.github/workflows/dockerpublish.yml b/.github/workflows/dockerpublish.yml
index ec8f4eb..c176780 100644
--- a/.github/workflows/dockerpublish.yml
+++ b/.github/workflows/dockerpublish.yml
@@ -2,7 +2,9 @@ name: "Docker Image CI: websplot"
 
 on:
   push:
-    branches: [ main ]
+    branches:
+      - main
+      - develop
 
 env:
   IMAGE_NAME: crawlabteam/webspot
diff --git a/.github/workflows/main_webspot.yml b/.github/workflows/main_webspot.yml
index 4a5398a..c8039f9 100644
--- a/.github/workflows/main_webspot.yml
+++ b/.github/workflows/main_webspot.yml
@@ -7,6 +7,7 @@ on:
   push:
     branches:
       - main
+      - develop
   workflow_dispatch:
 
 jobs:
diff --git a/.gitignore b/.gitignore
index ebac091..5d7e369 100644
--- a/.gitignore
+++ b/.gitignore
@@ -139,3 +139,7 @@ node_modules/
 
 *.db
 *.db-journal
+
+dev/data/
+
+*.lock
diff --git a/Dockerfile b/Dockerfile
index 33186d5..64f2b9f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,13 @@
+FROM golang:1.19 AS build
+
+WORKDIR /go/src/app
+COPY ./webspot_rod .
+
+ENV GO111MODULE on
+
+RUN go mod tidy \
+  && go install -v ./...
+
 FROM python:3.10.9
 
 # Working directory
@@ -7,6 +17,36 @@ WORKDIR /app
 RUN echo `uname -a`
 RUN echo `python --version`
 
+# Install supervisor
+RUN apt-get update && apt-get install -y supervisor
+
+# Install dependencies
+RUN apt-get -qq install --no-install-recommends -y \
+    # chromium dependencies
+    libnss3 \
+    libxss1 \
+    libasound2 \
+    libxtst6 \
+    libgtk-3-0 \
+    libgbm1 \
+    ca-certificates \
+    # fonts
+    fonts-liberation fonts-noto-color-emoji fonts-noto-cjk
+
+# Start and enable SSH
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends dialog \
+    && apt-get install -y --no-install-recommends openssh-server \
+    && echo "root:Docker!" | chpasswd
+COPY webspot_rod/conf/sshd_config /etc/ssh/
+
+# Expose SSH port
+EXPOSE 8000 2222
+
+# Copy webspot_rod
+COPY --from=build /go/bin/webspot_rod /go/bin/webspot_rod
+COPY webspot_rod/conf/supervisord.conf /etc/supervisor/supervisord.conf
+
 # Install requirements
 COPY ./requirements.txt /app
 RUN pip install --upgrade pip
@@ -22,5 +62,7 @@ ADD . /app
 ENV PORT 80
 EXPOSE 80
 
-ENTRYPOINT ["python", "main.py"]
-CMD ["web"]
+# Change mode of entrypoint.sh
+RUN chmod u+x ./entrypoint.sh
+
+CMD ["sh", "entrypoint.sh"]
diff --git a/README.md b/README.md
index 5b86034..f7522d9 100644
--- a/README.md
+++ b/README.md
@@ -32,6 +32,22 @@ Then you can access the web UI at http://localhost:80.
 webspot web --help
 ```
 
+## Architecture
+
+### Overview
+
+The overall process of how Webspot detects meaningful elements from HTML or web pages is shown in the following figure.
+
+```mermaid
+graph LR
+    hr[HtmlRequester]
+    gl[GraphLoader]
+    d[Detector]
+    r[Results]
+
+    hr --"html + json"--> gl --"graph"--> d --"output"--> r
+```
+
 ## Development
 
 Development with Webspot is easy. You can follow the following guidance to get started.
diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
new file mode 100644
index 0000000..46963ee
--- /dev/null
+++ b/dev/docker-compose.yml
@@ -0,0 +1,12 @@
+version: '3.3'
+services:
+  mysql:
+    image: mysql:8
+    container_name: mysql
+    ports:
+      - "3306:3306"
+    environment:
+      MYSQL_DATABASE: "webspot"
+      MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+    volumes:
+      - ./data/mysql:/var/lib/mysql
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000..50562c8
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,13 @@
+version: '3.3'
+services:
+  webspot:
+    image: crawlabteam/webspot:develop
+    container_name: webspot
+    environment:
+      WEBSPOT_DATABASE_URL: mongodb://mongo:27017/webspot
+    ports:
+      - '9999:80'
+
+  mongo:
+    image: mongo:4.2
+    container_name: webspot_mongo
diff --git a/entrypoint.sh b/entrypoint.sh
new file mode 100644
index 0000000..879caf6
--- /dev/null
+++ b/entrypoint.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+set -e
+
+# Start ssh
+service ssh start
+
+# Start supervisor
+supervisord -c /etc/supervisor/supervisord.conf
diff --git a/main.py b/main.py
index c5b2915..22f5b99 100644
--- a/main.py
+++ b/main.py
@@ -1,8 +1,13 @@
 from argparse import ArgumentParser
 
+from dotenv import load_dotenv
+
 from webspot.cmd.crawl import cmd_crawl
 from webspot.cmd.request import cmd_request
 from webspot.cmd.web import cmd_web
+from webspot.constants.html_request_method import HTML_REQUEST_METHOD_REQUEST
+
+load_dotenv()
 
 parser = ArgumentParser()
 
@@ -27,7 +32,7 @@
 
 request_parser = subparsers.add_parser('request')
 request_parser.add_argument('--url', '-U', help='url to request', required=True)
-request_parser.add_argument('--method', '-M', help='request method', default='rod')
+request_parser.add_argument('--method', '-M', help='request method', default=HTML_REQUEST_METHOD_REQUEST)
 request_parser.set_defaults(func=cmd_request)
 
 if __name__ == '__main__':
diff --git a/notebooks/pagination_detector.ipynb b/notebooks/pagination_detector.ipynb
new file mode 100644
index 0000000..9aaabee
--- /dev/null
+++ b/notebooks/pagination_detector.ipynb
@@ -0,0 +1,781 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "12cd9398",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import init"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "fc3ef28f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from urllib.parse import urlparse\n",
+    "import numpy as np\n",
+    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
+    "from sklearn.metrics.pairwise import cosine_similarity"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "38a0c675",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from webspot.request.html_requester import HtmlRequester\n",
+    "from webspot.graph.graph_loader import GraphLoader\n",
+    "from webspot.detect.detectors.pagination import PaginationDetector\n",
+    "from webspot.detect.detectors.plain_list import PlainListDetector\n",
+    "from webspot.detect.utils.transform_html_links import transform_url"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "f1b35f81",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/marvzhang/projects/crawlab-team/webspot/venv/lib/python3.10/site-packages/dgl/heterograph.py:72: DGLWarning: Recommend creating graphs by `dgl.graph(data)` instead of `dgl.DGLGraph(data)`.\n",
+      "  dgl_warning('Recommend creating graphs by `dgl.graph(data)`'\n",
+      "/Users/marvzhang/projects/crawlab-team/webspot/venv/lib/python3.10/site-packages/sklearn/preprocessing/_label.py:116: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
+      "  y = column_or_1d(y, warn=True)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# url = 'https://quotes.toscrape.com'\n",
+    "# url = 'https://books.toscrape.com'\n",
+    "url = 'https://cuiqingcai.com/archives/'\n",
+    "# url = 'https://cuiqingcai.com/archives/page/2/'\n",
+    "\n",
+    "html_requester = HtmlRequester(url=url)\n",
+    "html_requester.run()\n",
+    "\n",
+    "graph_loader = GraphLoader(html=html_requester.html_, json_data=html_requester.json_data)\n",
+    "graph_loader.run()\n",
+    "\n",
+    "pagination_detector = PaginationDetector(html_requester=html_requester, graph_loader=graph_loader)\n",
+    "plain_list_detector = PlainListDetector(html_requester=html_requester, graph_loader=graph_loader)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "c7d45271",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([ 12,  25,  27,  29,  31,  33,  35,  37,  67,  74,  81,  88,  95,\n",
+       "       102, 109, 116, 123, 130, 137, 144, 151, 158, 165, 172, 179, 186,\n",
+       "       193, 200, 207, 214, 221, 228, 235, 242, 249, 256, 263, 270, 277,\n",
+       "       284, 291, 298, 305, 312, 319, 326, 333, 340, 344, 346, 347, 368,\n",
+       "       372, 376, 393, 400, 403, 406, 409, 412, 415, 418, 421, 424, 427,\n",
+       "       430, 433, 436, 439, 442, 445, 448, 451, 454, 457, 460, 463, 466,\n",
+       "       469, 472, 475, 478, 487, 492, 497, 502, 507, 512, 517, 522, 527,\n",
+       "       532, 537, 542, 547, 552, 557, 562, 567, 572, 577, 582, 587, 592,\n",
+       "       597, 602, 607, 612, 617, 621, 622, 623, 624, 625, 626, 627, 628,\n",
+       "       629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641,\n",
+       "       642, 643, 644, 645, 646, 647, 648, 649, 650, 651, 652, 653, 654,\n",
+       "       655, 656, 657, 658, 659, 660, 661, 662, 663, 664, 665, 666, 667,\n",
+       "       668, 669, 670, 671, 672, 673, 674, 675, 676, 677, 678, 679, 680,\n",
+       "       681, 682, 683, 684, 685, 686, 687, 688, 689, 690, 691, 692, 693,\n",
+       "       694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 704, 705, 706,\n",
+       "       707, 708, 709, 710, 711, 712, 713, 714, 715, 716, 717, 718, 719,\n",
+       "       720, 730, 732])"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "feature_tag_a_idx = np.argwhere(np.array(graph_loader.nodes_features_enc.feature_names_) == 'tag=a')[0]\n",
+    "link_nodes_idx = np.argwhere(graph_loader.nodes_features_tensor[:, feature_tag_a_idx].T[0] == 1)[0].detach().numpy()\n",
+    "link_nodes_idx"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "3d125255",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'id': 57,\n",
+       " 'parent_id': 56,\n",
+       " 'features': [('tag', 'a'),\n",
+       "  ('href', '/'),\n",
+       "  ('class', 'brand'),\n",
+       "  ('rel', 'start')],\n",
+       " 'text': None}"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "nodes_a = graph_loader.nodes[link_nodes_idx]\n",
+    "\n",
+    "nodes_a[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "id": "89270d88",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['https://cuiqingcai.com/archives/',\n",
+       " 'https://cuiqingcai.com/tags/',\n",
+       " 'https://cuiqingcai.com/categories/',\n",
+       " 'https://cuiqingcai.com/about/',\n",
+       " 'https://cuiqingcai.com/message/',\n",
+       " 'https://cuiqingcai.com/36065.html',\n",
+       " 'https://cuiqingcai.com/36064.html',\n",
+       " 'https://cuiqingcai.com/36063.html',\n",
+       " 'https://cuiqingcai.com/36062.html',\n",
+       " 'https://cuiqingcai.com/36061.html',\n",
+       " 'https://cuiqingcai.com/36060.html',\n",
+       " 'https://cuiqingcai.com/36059.html',\n",
+       " 'https://cuiqingcai.com/36058.html',\n",
+       " 'https://cuiqingcai.com/36057.html',\n",
+       " 'https://cuiqingcai.com/36056.html',\n",
+       " 'https://cuiqingcai.com/17777.html',\n",
+       " 'https://cuiqingcai.com/2022114.html',\n",
+       " 'https://cuiqingcai.com/2022113.html',\n",
+       " 'https://cuiqingcai.com/2022111.html',\n",
+       " 'https://cuiqingcai.com/2022112.html',\n",
+       " 'https://cuiqingcai.com/2022104.html',\n",
+       " 'https://cuiqingcai.com/2022103.html',\n",
+       " 'https://cuiqingcai.com/2022102.html',\n",
+       " 'https://cuiqingcai.com/202293.html',\n",
+       " 'https://cuiqingcai.com/2022101.html',\n",
+       " 'https://cuiqingcai.com/36054.html',\n",
+       " 'https://cuiqingcai.com/202292.html',\n",
+       " 'https://cuiqingcai.com/202291.html',\n",
+       " 'https://cuiqingcai.com/36055.html',\n",
+       " 'https://cuiqingcai.com/36052.html',\n",
+       " 'https://cuiqingcai.com/36053.html',\n",
+       " 'https://cuiqingcai.com/202282.html',\n",
+       " 'https://cuiqingcai.com/202272.html',\n",
+       " 'https://cuiqingcai.com/202281.html',\n",
+       " 'https://cuiqingcai.com/202261.html',\n",
+       " 'https://cuiqingcai.com/202246.html',\n",
+       " 'https://cuiqingcai.com/202271.html',\n",
+       " 'https://cuiqingcai.com/202262.html',\n",
+       " 'https://cuiqingcai.com/202253.html',\n",
+       " 'https://cuiqingcai.com/202251.html',\n",
+       " 'https://cuiqingcai.com/202242.html',\n",
+       " 'https://cuiqingcai.com/202232.html',\n",
+       " 'https://cuiqingcai.com/202252.html',\n",
+       " 'https://cuiqingcai.com/202241.html',\n",
+       " 'https://cuiqingcai.com/202231.html',\n",
+       " 'https://cuiqingcai.com/archives/page/2/',\n",
+       " 'https://cuiqingcai.com/archives/page/17/',\n",
+       " 'https://cuiqingcai.com/archives/page/2/',\n",
+       " 'https://cuiqingcai.com/archives/',\n",
+       " 'https://cuiqingcai.com/categories/',\n",
+       " 'https://cuiqingcai.com/tags/',\n",
+       " 'https://cuiqingcai.com/categories/C-C/',\n",
+       " 'https://cuiqingcai.com/categories/HTML/',\n",
+       " 'https://cuiqingcai.com/categories/Java/',\n",
+       " 'https://cuiqingcai.com/categories/JavaScript/',\n",
+       " 'https://cuiqingcai.com/categories/Linux/',\n",
+       " 'https://cuiqingcai.com/categories/Markdown/',\n",
+       " 'https://cuiqingcai.com/categories/Net/',\n",
+       " 'https://cuiqingcai.com/categories/Other/',\n",
+       " 'https://cuiqingcai.com/categories/PHP/',\n",
+       " 'https://cuiqingcai.com/categories/Paper/',\n",
+       " 'https://cuiqingcai.com/categories/Python/',\n",
+       " 'https://cuiqingcai.com/categories/TypeScript/',\n",
+       " 'https://cuiqingcai.com/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/',\n",
+       " 'https://cuiqingcai.com/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/',\n",
+       " 'https://cuiqingcai.com/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/',\n",
+       " 'https://cuiqingcai.com/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/',\n",
+       " 'https://cuiqingcai.com/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/',\n",
+       " 'https://cuiqingcai.com/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/',\n",
+       " 'https://cuiqingcai.com/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/',\n",
+       " 'https://cuiqingcai.com/categories/%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0/',\n",
+       " 'https://cuiqingcai.com/categories/%E7%88%AC%E8%99%AB/',\n",
+       " 'https://cuiqingcai.com/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/',\n",
+       " 'https://cuiqingcai.com/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/',\n",
+       " 'https://cuiqingcai.com/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/',\n",
+       " 'https://cuiqingcai.com/tags/2022/',\n",
+       " 'https://cuiqingcai.com/tags/2048/',\n",
+       " 'https://cuiqingcai.com/tags/ADSL/',\n",
+       " 'https://cuiqingcai.com/tags/API/',\n",
+       " 'https://cuiqingcai.com/tags/Ajax/',\n",
+       " 'https://cuiqingcai.com/tags/Bootstrap/',\n",
+       " 'https://cuiqingcai.com/tags/Bug/',\n",
+       " 'https://cuiqingcai.com/tags/CDN/',\n",
+       " 'https://cuiqingcai.com/tags/CQC/',\n",
+       " 'https://cuiqingcai.com/tags/CSS/',\n",
+       " 'https://cuiqingcai.com/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/',\n",
+       " 'https://cuiqingcai.com/tags/CV/',\n",
+       " 'https://cuiqingcai.com/tags/ChatGPT/',\n",
+       " 'https://cuiqingcai.com/tags/Cookie/',\n",
+       " 'https://cuiqingcai.com/tags/Django/',\n",
+       " 'https://cuiqingcai.com/tags/Eclipse/',\n",
+       " 'https://cuiqingcai.com/tags/Elasticsearch/',\n",
+       " 'https://cuiqingcai.com/tags/FTP/',\n",
+       " 'https://cuiqingcai.com/tags/Git/',\n",
+       " 'https://cuiqingcai.com/tags/GitHub/',\n",
+       " 'https://cuiqingcai.com/tags/HTML5/',\n",
+       " 'https://cuiqingcai.com/tags/HTTP/',\n",
+       " 'https://cuiqingcai.com/tags/Hexo/',\n",
+       " 'https://cuiqingcai.com/tags/Hook/',\n",
+       " 'https://cuiqingcai.com/tags/IT/',\n",
+       " 'https://cuiqingcai.com/tags/JSON/',\n",
+       " 'https://cuiqingcai.com/tags/JSP/',\n",
+       " 'https://cuiqingcai.com/tags/JavaScript/',\n",
+       " 'https://cuiqingcai.com/tags/K8s/',\n",
+       " 'https://cuiqingcai.com/tags/LOGO/',\n",
+       " 'https://cuiqingcai.com/tags/Linux/',\n",
+       " 'https://cuiqingcai.com/tags/MIUI/',\n",
+       " 'https://cuiqingcai.com/tags/Markdown/',\n",
+       " 'https://cuiqingcai.com/tags/MongoDB/',\n",
+       " 'https://cuiqingcai.com/tags/MySQL/',\n",
+       " 'https://cuiqingcai.com/tags/Mysql/',\n",
+       " 'https://cuiqingcai.com/tags/NBA/',\n",
+       " 'https://cuiqingcai.com/tags/OCR/',\n",
+       " 'https://cuiqingcai.com/tags/OpenCV/',\n",
+       " 'https://cuiqingcai.com/tags/PHP/',\n",
+       " 'https://cuiqingcai.com/tags/PPT/',\n",
+       " 'https://cuiqingcai.com/tags/PS/',\n",
+       " 'https://cuiqingcai.com/tags/Pathlib/',\n",
+       " 'https://cuiqingcai.com/tags/PhantomJS/',\n",
+       " 'https://cuiqingcai.com/tags/Playwright/',\n",
+       " 'https://cuiqingcai.com/tags/Python/',\n",
+       " 'https://cuiqingcai.com/tags/Python-%E7%88%AC%E8%99%AB/',\n",
+       " 'https://cuiqingcai.com/tags/Python3/',\n",
+       " 'https://cuiqingcai.com/tags/Python3%E7%88%AC%E8%99%AB%E6%95%99%E7%A8%8B/',\n",
+       " 'https://cuiqingcai.com/tags/Pythonic/',\n",
+       " 'https://cuiqingcai.com/tags/Python%E7%88%AC%E8%99%AB/',\n",
+       " 'https://cuiqingcai.com/tags/Python%E7%88%AC%E8%99%AB%E4%B9%A6/',\n",
+       " 'https://cuiqingcai.com/tags/Python%E7%88%AC%E8%99%AB%E6%95%99%E7%A8%8B/',\n",
+       " 'https://cuiqingcai.com/tags/QQ/',\n",
+       " 'https://cuiqingcai.com/tags/RabbitMQ/',\n",
+       " 'https://cuiqingcai.com/tags/ReCAPTCHA/',\n",
+       " 'https://cuiqingcai.com/tags/Redis/',\n",
+       " 'https://cuiqingcai.com/tags/SAE/',\n",
+       " 'https://cuiqingcai.com/tags/SSH/',\n",
+       " 'https://cuiqingcai.com/tags/SVG/',\n",
+       " 'https://cuiqingcai.com/tags/Scrapy-redis/',\n",
+       " 'https://cuiqingcai.com/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/',\n",
+       " 'https://cuiqingcai.com/tags/Selenium/',\n",
+       " 'https://cuiqingcai.com/tags/Session/',\n",
+       " 'https://cuiqingcai.com/tags/Shell/',\n",
+       " 'https://cuiqingcai.com/tags/TKE/',\n",
+       " 'https://cuiqingcai.com/tags/TXT/',\n",
+       " 'https://cuiqingcai.com/tags/Terminal/',\n",
+       " 'https://cuiqingcai.com/tags/Ubuntu/',\n",
+       " 'https://cuiqingcai.com/tags/VS-Code/',\n",
+       " 'https://cuiqingcai.com/tags/Vs-Code/',\n",
+       " 'https://cuiqingcai.com/tags/Vue/',\n",
+       " 'https://cuiqingcai.com/tags/Web/',\n",
+       " 'https://cuiqingcai.com/tags/Webpack/',\n",
+       " 'https://cuiqingcai.com/tags/Web%E7%BD%91%E9%A1%B5/',\n",
+       " 'https://cuiqingcai.com/tags/Windows/',\n",
+       " 'https://cuiqingcai.com/tags/Winpcap/',\n",
+       " 'https://cuiqingcai.com/tags/WordPress/',\n",
+       " 'https://cuiqingcai.com/tags/XPath/',\n",
+       " 'https://cuiqingcai.com/tags/Youtube/',\n",
+       " 'https://cuiqingcai.com/tags/aiohttp/',\n",
+       " 'https://cuiqingcai.com/tags/android/',\n",
+       " 'https://cuiqingcai.com/tags/ansible/',\n",
+       " 'https://cuiqingcai.com/tags/cocos2d-x/',\n",
+       " 'https://cuiqingcai.com/tags/dummy-change/',\n",
+       " 'https://cuiqingcai.com/tags/e6/',\n",
+       " 'https://cuiqingcai.com/tags/fitvids/',\n",
+       " 'https://cuiqingcai.com/tags/git/',\n",
+       " 'https://cuiqingcai.com/tags/json/',\n",
+       " 'https://cuiqingcai.com/tags/js%E9%80%86%E5%90%91/',\n",
+       " 'https://cuiqingcai.com/tags/kubernetes/',\n",
+       " 'https://cuiqingcai.com/tags/log/',\n",
+       " 'https://cuiqingcai.com/tags/log4j/',\n",
+       " 'https://cuiqingcai.com/tags/logging/',\n",
+       " 'https://cuiqingcai.com/tags/matlab/',\n",
+       " 'https://cuiqingcai.com/tags/parsel/',\n",
+       " 'https://cuiqingcai.com/tags/pymongo/',\n",
+       " 'https://cuiqingcai.com/tags/python/',\n",
+       " 'https://cuiqingcai.com/tags/pytube/',\n",
+       " 'https://cuiqingcai.com/tags/pywin32/',\n",
+       " 'https://cuiqingcai.com/sitemap.xml',\n",
+       " 'https://cuiqingcai.com/sitemap.html']"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "hrefs = [\n",
+    "    transform_url(html_requester.url, n.attrs.get('href'))\n",
+    "    for n in pagination_detector.link_nodes\n",
+    "    if n.attrs.get('href') is not None \n",
+    "    and urlparse(html_requester.url).netloc in transform_url(html_requester.url, n.attrs.get('href'))\n",
+    "]\n",
+    "\n",
+    "hrefs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "id": "1e2a2b38",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[['archives'],\n",
+       " ['tags'],\n",
+       " ['categories'],\n",
+       " ['about'],\n",
+       " ['message'],\n",
+       " ['36065.html'],\n",
+       " ['36064.html'],\n",
+       " ['36063.html'],\n",
+       " ['36062.html'],\n",
+       " ['36061.html'],\n",
+       " ['36060.html'],\n",
+       " ['36059.html'],\n",
+       " ['36058.html'],\n",
+       " ['36057.html'],\n",
+       " ['36056.html'],\n",
+       " ['17777.html'],\n",
+       " ['2022114.html'],\n",
+       " ['2022113.html'],\n",
+       " ['2022111.html'],\n",
+       " ['2022112.html'],\n",
+       " ['2022104.html'],\n",
+       " ['2022103.html'],\n",
+       " ['2022102.html'],\n",
+       " ['202293.html'],\n",
+       " ['2022101.html'],\n",
+       " ['36054.html'],\n",
+       " ['202292.html'],\n",
+       " ['202291.html'],\n",
+       " ['36055.html'],\n",
+       " ['36052.html'],\n",
+       " ['36053.html'],\n",
+       " ['202282.html'],\n",
+       " ['202272.html'],\n",
+       " ['202281.html'],\n",
+       " ['202261.html'],\n",
+       " ['202246.html'],\n",
+       " ['202271.html'],\n",
+       " ['202262.html'],\n",
+       " ['202253.html'],\n",
+       " ['202251.html'],\n",
+       " ['202242.html'],\n",
+       " ['202232.html'],\n",
+       " ['202252.html'],\n",
+       " ['202241.html'],\n",
+       " ['202231.html'],\n",
+       " ['archives', 'page', '2'],\n",
+       " ['archives', 'page', '17'],\n",
+       " ['archives', 'page', '2'],\n",
+       " ['archives'],\n",
+       " ['categories'],\n",
+       " ['tags'],\n",
+       " ['categories', 'C-C'],\n",
+       " ['categories', 'HTML'],\n",
+       " ['categories', 'Java'],\n",
+       " ['categories', 'JavaScript'],\n",
+       " ['categories', 'Linux'],\n",
+       " ['categories', 'Markdown'],\n",
+       " ['categories', 'Net'],\n",
+       " ['categories', 'Other'],\n",
+       " ['categories', 'PHP'],\n",
+       " ['categories', 'Paper'],\n",
+       " ['categories', 'Python'],\n",
+       " ['categories', 'TypeScript'],\n",
+       " ['categories', '%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA'],\n",
+       " ['categories', '%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0'],\n",
+       " ['categories', '%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95'],\n",
+       " ['categories', '%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94'],\n",
+       " ['categories', '%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE'],\n",
+       " ['categories', '%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88'],\n",
+       " ['categories', '%E6%9C%AA%E5%88%86%E7%B1%BB'],\n",
+       " ['categories', '%E6%B7%B1%E5%BA%A6%E5%AD%A6%E4%B9%A0'],\n",
+       " ['categories', '%E7%88%AC%E8%99%AB'],\n",
+       " ['categories', '%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0'],\n",
+       " ['categories', '%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA'],\n",
+       " ['categories', '%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90'],\n",
+       " ['tags', '2022'],\n",
+       " ['tags', '2048'],\n",
+       " ['tags', 'ADSL'],\n",
+       " ['tags', 'API'],\n",
+       " ['tags', 'Ajax'],\n",
+       " ['tags', 'Bootstrap'],\n",
+       " ['tags', 'Bug'],\n",
+       " ['tags', 'CDN'],\n",
+       " ['tags', 'CQC'],\n",
+       " ['tags', 'CSS'],\n",
+       " ['tags', 'CSS-%E5%8F%8D%E7%88%AC%E8%99%AB'],\n",
+       " ['tags', 'CV'],\n",
+       " ['tags', 'ChatGPT'],\n",
+       " ['tags', 'Cookie'],\n",
+       " ['tags', 'Django'],\n",
+       " ['tags', 'Eclipse'],\n",
+       " ['tags', 'Elasticsearch'],\n",
+       " ['tags', 'FTP'],\n",
+       " ['tags', 'Git'],\n",
+       " ['tags', 'GitHub'],\n",
+       " ['tags', 'HTML5'],\n",
+       " ['tags', 'HTTP'],\n",
+       " ['tags', 'Hexo'],\n",
+       " ['tags', 'Hook'],\n",
+       " ['tags', 'IT'],\n",
+       " ['tags', 'JSON'],\n",
+       " ['tags', 'JSP'],\n",
+       " ['tags', 'JavaScript'],\n",
+       " ['tags', 'K8s'],\n",
+       " ['tags', 'LOGO'],\n",
+       " ['tags', 'Linux'],\n",
+       " ['tags', 'MIUI'],\n",
+       " ['tags', 'Markdown'],\n",
+       " ['tags', 'MongoDB'],\n",
+       " ['tags', 'MySQL'],\n",
+       " ['tags', 'Mysql'],\n",
+       " ['tags', 'NBA'],\n",
+       " ['tags', 'OCR'],\n",
+       " ['tags', 'OpenCV'],\n",
+       " ['tags', 'PHP'],\n",
+       " ['tags', 'PPT'],\n",
+       " ['tags', 'PS'],\n",
+       " ['tags', 'Pathlib'],\n",
+       " ['tags', 'PhantomJS'],\n",
+       " ['tags', 'Playwright'],\n",
+       " ['tags', 'Python'],\n",
+       " ['tags', 'Python-%E7%88%AC%E8%99%AB'],\n",
+       " ['tags', 'Python3'],\n",
+       " ['tags', 'Python3%E7%88%AC%E8%99%AB%E6%95%99%E7%A8%8B'],\n",
+       " ['tags', 'Pythonic'],\n",
+       " ['tags', 'Python%E7%88%AC%E8%99%AB'],\n",
+       " ['tags', 'Python%E7%88%AC%E8%99%AB%E4%B9%A6'],\n",
+       " ['tags', 'Python%E7%88%AC%E8%99%AB%E6%95%99%E7%A8%8B'],\n",
+       " ['tags', 'QQ'],\n",
+       " ['tags', 'RabbitMQ'],\n",
+       " ['tags', 'ReCAPTCHA'],\n",
+       " ['tags', 'Redis'],\n",
+       " ['tags', 'SAE'],\n",
+       " ['tags', 'SSH'],\n",
+       " ['tags', 'SVG'],\n",
+       " ['tags', 'Scrapy-redis'],\n",
+       " ['tags', 'Scrapy%E5%88%86%E5%B8%83%E5%BC%8F'],\n",
+       " ['tags', 'Selenium'],\n",
+       " ['tags', 'Session'],\n",
+       " ['tags', 'Shell'],\n",
+       " ['tags', 'TKE'],\n",
+       " ['tags', 'TXT'],\n",
+       " ['tags', 'Terminal'],\n",
+       " ['tags', 'Ubuntu'],\n",
+       " ['tags', 'VS-Code'],\n",
+       " ['tags', 'Vs-Code'],\n",
+       " ['tags', 'Vue'],\n",
+       " ['tags', 'Web'],\n",
+       " ['tags', 'Webpack'],\n",
+       " ['tags', 'Web%E7%BD%91%E9%A1%B5'],\n",
+       " ['tags', 'Windows'],\n",
+       " ['tags', 'Winpcap'],\n",
+       " ['tags', 'WordPress'],\n",
+       " ['tags', 'XPath'],\n",
+       " ['tags', 'Youtube'],\n",
+       " ['tags', 'aiohttp'],\n",
+       " ['tags', 'android'],\n",
+       " ['tags', 'ansible'],\n",
+       " ['tags', 'cocos2d-x'],\n",
+       " ['tags', 'dummy-change'],\n",
+       " ['tags', 'e6'],\n",
+       " ['tags', 'fitvids'],\n",
+       " ['tags', 'git'],\n",
+       " ['tags', 'json'],\n",
+       " ['tags', 'js%E9%80%86%E5%90%91'],\n",
+       " ['tags', 'kubernetes'],\n",
+       " ['tags', 'log'],\n",
+       " ['tags', 'log4j'],\n",
+       " ['tags', 'logging'],\n",
+       " ['tags', 'matlab'],\n",
+       " ['tags', 'parsel'],\n",
+       " ['tags', 'pymongo'],\n",
+       " ['tags', 'python'],\n",
+       " ['tags', 'pytube'],\n",
+       " ['tags', 'pywin32'],\n",
+       " ['sitemap.xml'],\n",
+       " ['sitemap.html']]"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "hrefs_frag = [[f for f in urlparse(href).path.split('/') if f != ''] for href in hrefs if href.startswith('http')]\n",
+    "hrefs_frag"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "b86d614b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'https://cuiqingcai.com/archives/'"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "html_requester.url"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "a79d157d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['archives']"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "current_fragments = pagination_detector.current_url_path_fragments\n",
+    "current_fragments"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "919cb36b",
+   "metadata": {},
+   "source": [
+    "# create tf-idf vectorizer\n",
+    "vectorizer = TfidfVectorizer()"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "0280587f",
+   "metadata": {},
+   "source": [
+    "# join the words in each list into a single string\n",
+    "corpus = [' '.join(fragments) for fragments in fragments_list]\n",
+    "\n",
+    "# create tf-idf vectorizer\n",
+    "vectorizer = TfidfVectorizer()\n",
+    "\n",
+    "# fit the vectorizer on the collection of words\n",
+    "matrix = vectorizer.fit_transform(corpus)\n",
+    "matrix"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "02522599",
+   "metadata": {},
+   "source": [
+    "current_fragments_vec = vectorizer.transform([' '.join(current_fragments)])\n",
+    "current_fragments_vec.todense()"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "f2d60717",
+   "metadata": {},
+   "source": [
+    "similarity_scores = cosine_similarity(matrix, current_fragments_vec)\n",
+    "similarity_scores"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "8c7dae66",
+   "metadata": {},
+   "source": [
+    "# get the index of the most similar word list\n",
+    "most_similar_index = similarity_scores.argmax()\n",
+    "most_similar_index"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "5682cf26",
+   "metadata": {},
+   "source": [
+    "# print the most similar word list\n",
+    "print(fragments_list[most_similar_index])"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "de45938c",
+   "metadata": {},
+   "source": [
+    "similarity_scores.argsort(axis=0)[::-1]"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "id": "68f302a5",
+   "metadata": {},
+   "source": [
+    "fragments_list[47]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "55bafc67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pagination_detector.run()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 54,
+   "id": "fd479118",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
+       "       0])"
+      ]
+     },
+     "execution_count": 54,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "nodes_texts = np.array([len(set(n.text.lower().split(' ')).intersection(pagination_detector.keywords)) > 0 if n.text else False for n in pagination_detector.internal_link_nodes])\n",
+    "nodes_texts * 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "id": "6fae406c",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([ 0, 45, 46, 47, 48])"
+      ]
+     },
+     "execution_count": 58,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "scores_idx = np.argwhere(pagination_detector.scores >= 1.).T[0]\n",
+    "scores_idx"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 63,
+   "id": "dd95c34f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "3"
+      ]
+     },
+     "execution_count": 63,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "np.argmax(pagination_detector.scores[scores_idx])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6ea38a9c",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/requirements.txt b/requirements.txt
index 2af8d64..e29a7f9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,3 +8,9 @@ uvicorn==0.20.0
 SQLAlchemy==2.0.2
 PyMySQL==1.0.2
 alembic==1.9.3
+python-dotenv==0.21.1
+mongoengine==0.26.0
+pytest==7.2.1
+pytest-async==0.1.1
+retrying==1.3.4
+httpx==0.23.3
diff --git a/webspot/cmd/request.py b/webspot/cmd/request.py
index 608a869..c12e35b 100644
--- a/webspot/cmd/request.py
+++ b/webspot/cmd/request.py
@@ -1,5 +1,7 @@
-from webspot.request.get_html import get_html
+from webspot.request.html_requester import HtmlRequester
 
 
 def cmd_request(args):
-    get_html(args.url, request_method=args.method, save=True)
+    html_requester = HtmlRequester(url=args.url, request_method=args.method)
+    html_requester.run()
+    print(html_requester.html)
diff --git a/webspot/constants/detector.py b/webspot/constants/detector.py
new file mode 100644
index 0000000..916199e
--- /dev/null
+++ b/webspot/constants/detector.py
@@ -0,0 +1,7 @@
+DETECTOR_PLAIN_LIST = 'plain_list'
+DETECTOR_PAGINATION = 'pagination'
+
+ALL_DETECTORS = [
+    DETECTOR_PLAIN_LIST,
+    DETECTOR_PAGINATION,
+]
diff --git a/webspot/constants/html_request_method.py b/webspot/constants/html_request_method.py
new file mode 100644
index 0000000..df29b7e
--- /dev/null
+++ b/webspot/constants/html_request_method.py
@@ -0,0 +1,2 @@
+HTML_REQUEST_METHOD_ROD = 'rod'
+HTML_REQUEST_METHOD_REQUEST = 'request'
diff --git a/webspot/constants/request_status.py b/webspot/constants/request_status.py
new file mode 100644
index 0000000..d798879
--- /dev/null
+++ b/webspot/constants/request_status.py
@@ -0,0 +1,3 @@
+REQUEST_STATUS_PENDING = 'pending'
+REQUEST_STATUS_SUCCESS = 'success'
+REQUEST_STATUS_ERROR = 'error'
diff --git a/webspot/db/connect.py b/webspot/db/connect.py
new file mode 100644
index 0000000..884c434
--- /dev/null
+++ b/webspot/db/connect.py
@@ -0,0 +1,20 @@
+import os
+
+from mongoengine import connect as mongo_connect
+
+from webspot.utils.test import is_running_unit_tests
+
+DEFAULT_DATABASE_URL = 'mongodb://localhost:27017/webspot'
+
+
+def connect():
+    if is_running_unit_tests():
+        mongo_connect("mockdb", host="mongomock://localhost")
+        return
+
+    conn_str = get_connection_str()
+    mongo_connect(host=conn_str)
+
+
+def get_connection_str():
+    return os.environ.get('WEBSPOT_DATABASE_URL', DEFAULT_DATABASE_URL)
diff --git a/webspot/db/engine.py b/webspot/db/engine.py
deleted file mode 100644
index 6e5d7b7..0000000
--- a/webspot/db/engine.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import os
-from typing import Union
-
-from sqlalchemy import create_engine, Engine
-
-from webspot.models.base import Base
-
-_engine: Union[Engine, None] = None
-
-
-def get_engine():
-    global _engine
-    if _engine is None:
-        _engine = create_engine(get_connection_str(), echo=True)
-        _create_tables(_engine)
-    return _engine
-
-
-def _create_tables(engine: Engine = None):
-    if engine is None:
-        engine = get_engine()
-    Base.metadata.create_all(engine)
-
-
-def get_connection_str():
-    if os.environ.get('WEBSPOT_DATABASE_URL'):
-        return os.environ.get('WEBSPOT_DATABASE_URL')
-    return 'sqlite:///webspot.db'
diff --git a/webspot/db/session.py b/webspot/db/session.py
deleted file mode 100644
index 6fe222c..0000000
--- a/webspot/db/session.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from typing import Union
-
-from sqlalchemy.orm import Session
-
-from webspot.db.engine import get_engine
-
-_session: Union[Session, None] = None
-
-
-def get_session():
-    global _session
-    if _session is None or _session.is_active is False:
-        _session = Session(bind=get_engine())
-    return _session
diff --git a/webspot/detect/detectors/base.py b/webspot/detect/detectors/base.py
new file mode 100644
index 0000000..e9a3392
--- /dev/null
+++ b/webspot/detect/detectors/base.py
@@ -0,0 +1,41 @@
+from abc import abstractmethod
+from typing import List
+
+import numpy as np
+
+from webspot.detect.models.result import Result
+from webspot.graph.graph_loader import GraphLoader
+from webspot.request.html_requester import HtmlRequester
+
+
+class BaseDetector(object):
+    def __init__(
+        self,
+        graph_loader: GraphLoader,
+        html_requester: HtmlRequester,
+    ):
+        # graph loader
+        self.graph_loader = graph_loader
+
+        # html requester
+        self.html_requester = html_requester
+
+        # results
+        self.results: List[Result] = []
+
+    def get_nodes_idx_by_feature(self, feature_key: str, feature_value: str):
+        feature = f'{feature_key}={feature_value}'
+        feature_tag_a_idx = np.argwhere(np.array(self.graph_loader.nodes_features_enc.feature_names_) == feature)[0]
+        return np.argwhere(self.graph_loader.nodes_features_tensor[:, feature_tag_a_idx].T[0] == 1)[0].detach().numpy()
+
+    def get_nodes_by_feature(self, feature_key: str, feature_value: str):
+        nodes_idx = self.get_nodes_idx_by_feature(feature_key, feature_value)
+        return [n for n in self.graph_loader.nodes[nodes_idx]]
+
+    @abstractmethod
+    def highlight_html(self, html: str, **kwargs) -> str:
+        pass
+
+    @abstractmethod
+    def run(self):
+        pass
diff --git a/webspot/detect/detectors/pagination.py b/webspot/detect/detectors/pagination.py
new file mode 100644
index 0000000..17dc48f
--- /dev/null
+++ b/webspot/detect/detectors/pagination.py
@@ -0,0 +1,261 @@
+from typing import Optional, List
+from urllib.parse import urlparse
+
+import numpy as np
+from bs4 import BeautifulSoup
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.preprocessing import normalize
+
+from webspot.constants.detector import DETECTOR_PAGINATION
+from webspot.detect.detectors.base import BaseDetector
+from webspot.detect.models.result import Result
+from webspot.detect.models.selector import Selector
+from webspot.detect.utils.highlight_html import add_class, add_label
+from webspot.detect.utils.transform_html_links import transform_url
+from webspot.detect.utils.url import get_url_domain
+from webspot.graph.graph_loader import GraphLoader
+from webspot.request.html_requester import HtmlRequester
+
+
+class PaginationDetector(BaseDetector):
+    def __init__(
+        self,
+        score_threshold: float = 1.,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+
+        # settings
+        self.keywords = [
+            'next',
+            '下一页',
+        ]
+        self.escape_feature_names = [
+            'src',
+            'href',
+        ]
+        self.url_sep = '/'
+        self.numeric_mask = '#NUMERIC#'
+        self.score_threshold = score_threshold
+
+        # vectorizer
+        self.vec = TfidfVectorizer()
+
+        # data
+        self.internal_link_nodes = None
+        self.scores_url_path_fragments = None
+        self.scores_feature_next = None
+        self.scores_text = None
+        self._scores = None
+        self._scores_idx = None
+        self._scores_idx_max = None
+
+    def highlight_html(self, html: str, **kwargs) -> str:
+        soup = BeautifulSoup(html, 'html.parser')
+
+        if len(self.results) == 0:
+            return html
+
+        result = self.results[0]
+        next_selector = result.selectors.get('next')
+        next_el = soup.select_one(next_selector.selector)
+
+        add_class(next_el, ['webspot-highlight-container', 'webspot-highlight-node-color__red'])
+        add_label(next_el, soup, f'Pagination', 'primary')
+
+        return str(soup)
+
+    @property
+    def root_url(self):
+        return self.html_requester.url
+
+    def _get_masked_fragment(self, fragment: str):
+        if fragment.isdigit():
+            return self.numeric_mask
+        return fragment
+
+    def _get_url_path_fragment(self, url: str):
+        return [
+            self._get_masked_fragment(f.lower())
+            for f in urlparse(url).path.split(self.url_sep)
+            if f != ''
+        ]
+
+    @property
+    def current_url_path_fragments(self):
+        return self._get_url_path_fragment(self.root_url)
+
+    @property
+    def link_nodes(self):
+        return [n for n in self.get_nodes_by_feature(feature_key='tag', feature_value='a')
+                if n.attrs.get('href') is not None]
+
+    def _get_internal_link_nodes(self):
+        self.internal_link_nodes = [
+            n
+            for n in self.link_nodes
+            if get_url_domain(transform_url(self.root_url, n.attrs.get('href'))) == get_url_domain(self.root_url)
+        ]
+
+    @property
+    def internal_link_nodes_ids(self):
+        return [n.id for n in self.internal_link_nodes]
+
+    @property
+    def internal_link_nodes_idx(self):
+        nodes_ids = self.graph_loader.nodes_ids_tensor.detach().numpy()
+        internal_link_nodes_ids_enc = self.graph_loader.node_ids_enc.transform(self.internal_link_nodes_ids)
+        return np.argwhere(np.isin(nodes_ids, internal_link_nodes_ids_enc))
+
+    def _calculate_scores_url_path_fragments(self):
+        hrefs = [transform_url(self.root_url, n.attrs.get('href')) for n in self.internal_link_nodes]
+
+        fragments_list = [self._get_url_path_fragment(href) for href in hrefs]
+
+        # join the fragments of list into a single string
+        corpus = [' '.join(fragments) for fragments in fragments_list]
+
+        # fit the vectorizer on the collection
+        fragments_list_mat = self.vec.fit_transform(corpus)
+
+        # transform the current url path fragments
+        current_fragments_vec = self.vec.transform([' '.join(self.current_url_path_fragments)])
+
+        # calculate the cosine similarity
+        self.scores_url_path_fragments = cosine_similarity(fragments_list_mat, current_fragments_vec)
+
+    def _calculate_scores_feature_next(self):
+        feature_names = self.graph_loader.nodes_features_enc.feature_names_
+
+        features_hits = np.zeros(len(feature_names))
+
+        for i, f in enumerate(feature_names):
+            k = f.split('=')[0]
+            if k in self.escape_feature_names:
+                continue
+
+            v = '='.join(f.split('=')[1:])
+            for keyword in self.keywords:
+                if keyword in v.lower():
+                    features_hits[i] = 1
+                    break
+
+        features_hit_idx = np.argwhere(features_hits == 1).T[0]
+
+        internal_nodes_hit_features = self.graph_loader.nodes_features_tensor[
+            self.internal_link_nodes_idx,
+            features_hit_idx,
+        ].detach().numpy()
+
+        normalized_feat = np.array([internal_nodes_hit_features.sum(axis=1)]).T
+
+        self.scores_feature_next = np.log(normalized_feat + 1)
+
+    def _calculate_scores_text(self):
+        texts_hits = np.array([[
+            len(set(n.text.lower().split(' ')).intersection(self.keywords)) > 0 if n.text else False
+            for n in self.internal_link_nodes
+        ]]) * 1
+        texts_hits = texts_hits.T
+        self.scores_text = np.log(texts_hits + 1)
+
+    def _pre_process(self):
+        self._get_internal_link_nodes()
+
+    def _train(self) -> np.array:
+        self._calculate_scores_url_path_fragments()
+        self._calculate_scores_feature_next()
+        self._calculate_scores_text()
+
+        self._scores = normalize(
+            np.concatenate(
+                (
+                    self.scores_url_path_fragments,
+                    self.scores_feature_next,
+                    self.scores_text,
+                ),
+                axis=1,
+            ),
+            norm='l2',
+            axis=1,
+        )
+        self.scores = self._scores.sum(axis=1)
+
+    def _filter(self):
+        self._scores_idx = np.argwhere(self.scores >= self.score_threshold).T[0]
+
+    def _sort(self):
+        if len(self._scores_idx) == 0:
+            return
+
+        _idx = np.argmax(self.scores[self._scores_idx])
+        self._scores_idx_max = self._scores_idx[_idx]
+
+    def _extract(self):
+        if self._scores_idx_max is None:
+            return
+
+        node = self.internal_link_nodes[self._scores_idx_max]
+        next_selector = Selector(name='next', selector=self.graph_loader.get_node_css_selector_path(node))
+        score = self.scores[self._scores_idx_max]
+        score_url_path_fragments = self._scores[self._scores_idx_max][0]
+        score_feature_next = self._scores[self._scores_idx_max][1]
+        score_text = self._scores[self._scores_idx_max][2]
+
+        self.results.append(Result(
+            name='Next',
+            score=score,
+            scores={
+                'url_path_fragments': score_url_path_fragments,
+                'feature_next': score_feature_next,
+                'text': score_text,
+            },
+            selectors={
+                'next': next_selector,
+            },
+            detector=DETECTOR_PAGINATION,
+        ))
+
+    def run(self):
+        self._pre_process()
+
+        self._train()
+
+        self._filter()
+
+        self._sort()
+
+        self._extract()
+
+        return self.results
+
+
+if __name__ == '__main__':
+    urls = [
+        # 'https://www.baidu.com/s?wd=crawlab',
+        # 'https://github.com/search?q=spider',
+        # 'https://www.bing.com/search?q=crawlab&form=QBLHCN&sp=-1&lq=0&pq=&sc=0-0&qs=n&sk=&cvid=BB20795C2CB64FCFBC5A0B69070A11AA&ghsh=0&ghacc=0&ghpl=',
+        # 'https://www.bing.com/search?q=crawlab&sp=-1&lq=0&pq=&sc=0-0&qs=n&sk=&cvid=BB20795C2CB64FCFBC5A0B69070A11AA&ghsh=0&ghacc=0&ghpl=&first=11&FORM=PORE',
+        # 'https://github.com/trending',
+        # 'https://github.com/crawlab-team/crawlab/actions',
+        'https://quotes.toscrape.com',
+        # 'https://quotes.toscrape.com/page/2/',
+        # 'https://books.toscrape.com',
+        # 'https://cuiqingcai.com/archives/',
+        # 'https://cuiqingcai.com/archives/page/2/',
+    ]
+
+    for url in urls:
+        html_requester = HtmlRequester(url=url, request_method='request')
+        html_requester.run()
+
+        graph_loader = GraphLoader(html=html_requester.html_, json_data=html_requester.json_data)
+        graph_loader.run()
+
+        pagination_detector = PaginationDetector(html_requester=html_requester, graph_loader=graph_loader)
+        pagination_detector.run()
+        print(url)
+        print(pagination_detector.results)
+        print('\n\n')
diff --git a/webspot/detect/detectors/plain_list.py b/webspot/detect/detectors/plain_list.py
index 666ac60..ee3a050 100644
--- a/webspot/detect/detectors/plain_list.py
+++ b/webspot/detect/detectors/plain_list.py
@@ -1,117 +1,117 @@
 import base64
 import json
-import logging
-import os.path
 import time
-import traceback
-from typing import List, Set
+from typing import List, Set, Dict
 from urllib.parse import urljoin
 
-import html_to_json_enhanced
 import numpy as np
 import pandas as pd
 from bs4 import BeautifulSoup
-from scipy.stats import entropy, norm
+from scipy.sparse import csr_matrix
+from scipy.stats import entropy
 from sklearn.cluster import DBSCAN
 from sklearn.preprocessing import normalize
 
+from webspot.constants.detector import DETECTOR_PLAIN_LIST
 from webspot.constants.field_extract_rule_type import FIELD_EXTRACT_RULE_TYPE_TEXT, FIELD_EXTRACT_RULE_TYPE_LINK_URL, \
     FIELD_EXTRACT_RULE_TYPE_IMAGE_URL
-from webspot.detect.utils.math import sigmoid, log_positive
-from webspot.detect.utils.node import get_node_inner_text
-from webspot.graph.graph_loader import GraphLoader
-from webspot.detect.utils.highlight_html import highlight_html
-from webspot.detect.models.field import Field
+from webspot.detect.detectors.base import BaseDetector
+from webspot.detect.models.selector import Selector
+from webspot.detect.utils.math import log_positive
+from webspot.detect.utils.highlight_html import add_class, add_label
 from webspot.detect.models.list_result import ListResult
-from webspot.detect.utils.transform_html_links import transform_html_links
-from webspot.request.get_html import get_html
+from webspot.graph.graph_loader import GraphLoader
+from webspot.graph.models.node import Node
+from webspot.logging import get_logger
+from webspot.request.html_requester import HtmlRequester
 
+logger = get_logger('webspot.detect.detectors.plain_list')
 
-class PlainListDetector(object):
+
+class PlainListDetector(BaseDetector):
     def __init__(
         self,
-        url: str = None,
-        json_data: str = None,
-        json_path: str = None,
-        html_path: str = None,
-        save_path: str = None,
         dbscan_eps: float = 0.5,
         dbscan_min_samples: int = 3,
         dbscan_metric: str = 'euclidean',
+        dbscan_n_jobs: int = -1,
         entropy_threshold: float = 1e-3,
         score_threshold: float = 1.,
-        embed_walk_length: int = 5,
-        item_nodes_samples: int = 5,
+        min_item_nodes: int = 5,
         node2vec_ratio: float = 1.,
-        request_method: str = 'request',
-        request_rod_url: str = 'http://localhost:7777/request',
-        request_rod_duration: int = 3,
-        text_length_discount: float = 1e-2,
+        result_name_prefix: str = 'List',
+        text_length_discount: float = 0.01,
+        max_text_length: float = 1024,
+        max_item_count: int = 10,
+        max_feature_count: int = 10,
+        *args,
+        **kwargs,
     ):
+        super().__init__(*args, **kwargs)
+
         # settings
-        self.url = url
-        self.json_data = json_data
-        self.json_path = json_path
-        self.html_path = html_path
         self.entropy_threshold = entropy_threshold
         self.score_threshold = score_threshold
-        self.item_nodes_samples = item_nodes_samples
+        self.min_item_nodes = min_item_nodes
         self.node2vec_ratio = node2vec_ratio
-        self.request_method = request_method
-        self.request_rod_url = request_rod_url
-        self.request_rod_duration = request_rod_duration
+        self.result_name_prefix = result_name_prefix
         self.text_length_discount = text_length_discount
+        self.max_text_length = max_text_length
+        self.max_item_count = max_item_count
+        self.max_feature_count = max_feature_count
 
-        # validate
-        if not self.url:
-            if self.json_path:
-                assert self.html_path, 'html_path is required if json_path is provided and url is not provided'
-            else:
-                assert self.json_data, 'json_data is required if json_path is not provided and url is not provided'
-
-        # save path
-        self.save_path = save_path
-        if not self.save_path:
-            if self.url:
-                self.save_path = os.path.abspath(
-                    os.path.join(os.path.dirname(__file__), '..', '..', 'data', 'detect', 'plain_list',
-                                 self.url.replace('/', '_').replace(':', '_').replace('.', '_') + '.json'))
-            elif self.json_path:
-                self.save_path = os.path.abspath(
-                    os.path.join(os.path.dirname(__file__), '..', '..', 'data', 'detect', 'plain_list',
-                                 self.json_path.split('/')[-1]))
-
-        # data
-        self._html = None
-
-        # request html page if url exists
-        if self.url:
-            self._request()
-        else:
-            with open(self.html_path, 'r') as f:
-                self._html = f.read()
-
-        # graph loader
-        self.graph_loader = GraphLoader(json_data=self.json_data, json_path=self.json_path,
-                                        embed_walk_length=embed_walk_length)
-        self.graph_loader.run()
-
-        # dbscan clustering model
+        # dbscan model
         self.dbscan = DBSCAN(
             metric=dbscan_metric,
             eps=dbscan_eps,
             min_samples=dbscan_min_samples,
+            n_jobs=dbscan_n_jobs,
         )
 
         # data
         self.results: List[ListResult] = []
 
+    @property
+    def _html(self):
+        return self.graph_loader.html
+
+    @property
+    def url(self):
+        return self.html_requester.url
+
+    def highlight_html(self, html: str, **kwargs) -> str:
+        soup = BeautifulSoup(html, 'html.parser')
+        for i, result in enumerate(self.results):
+            # list
+            list_selector = result.selectors.get('list')
+            list_el = soup.select_one(list_selector.selector)
+            if not list_el:
+                continue
+            add_class(list_el, ['webspot-highlight-container', 'webspot-highlight-node-color__blue'])
+            add_label(list_el, soup, f'List {i + 1}', 'primary')
+
+            # items
+            items_selector = result.selectors.get('items')
+            item_els = list_el.select(items_selector.selector)
+            for j, item_el in enumerate(item_els):
+                add_class(item_el, ['webspot-highlight-container', 'webspot-highlight-node-color__orange'])
+                # _add_label(item_el, soup, f'Item {j + 1}', 'warning')
+
+                # fields
+                for k, field in enumerate(result.fields):
+                    try:
+                        field_els = item_el.select(field.selector)
+                    except Exception as e:
+                        # logging.warning(e)
+                        field_els = []
+                    for field_el in field_els:
+                        add_class(field_el, ['webspot-highlight-container', 'webspot-highlight-node-color__green'])
+                        # _add_label(field_el, soup, f'Field {k + 1}', 'success')
+        return str(soup)
+
     @property
     def html(self):
-        html = self._html
-        html = transform_html_links(html, self.url)
-        html = highlight_html(html, self.results)
-        return html
+        return self.highlight_html(self._html, )
 
     @property
     def html_base64(self):
@@ -121,15 +121,6 @@ def html_base64(self):
     def results_base64(self):
         return base64.b64encode(json.dumps(self.results).encode('utf-8')).decode('utf-8')
 
-    def _request(self):
-        self._html = get_html(
-            url=self.url,
-            request_method=self.request_method,
-            request_rod_url=self.request_rod_url,
-            request_rod_duration=self.request_rod_duration,
-        )
-        self.json_data = html_to_json_enhanced.convert(self._html, with_id=True)
-
     def _get_nodes_features_tags_attrs(self, nodes_idx: np.ndarray = None):
         """
         nodes features (tags + attributes)
@@ -163,13 +154,13 @@ def _get_nodes_features_node2vec(self, nodes_idx: np.ndarray = None):
             axis=1,
         )
 
-    def _get_nodes_features(self, nodes_idx: np.ndarray = None):
+    def _get_nodes_features(self, nodes_idx: np.ndarray = None, to_sparse: bool = False):
         """
         nodes features (tags + attributes + node2vec)
         """
         x1 = self._get_nodes_features_tags_attrs(nodes_idx)
         x2 = self._get_nodes_features_node2vec(nodes_idx) * self.node2vec_ratio
-        return normalize(
+        x = normalize(
             np.concatenate(
                 (x1, x2),
                 axis=1,
@@ -177,10 +168,16 @@ def _get_nodes_features(self, nodes_idx: np.ndarray = None):
             norm='l2',
             axis=1,
         )
+        logger.debug(f'nodes features size: {x.shape}')
+
+        if to_sparse:
+            return csr_matrix(x)
+        else:
+            return x
 
-    def _extract_fields_by_id(self, list_id: int) -> List[Field]:
+    def _extract_fields_by_id(self, list_id: int, item_nodes: List[Node]) -> List[Selector]:
         # fields
-        fields: List[Field] = []
+        fields: List[Selector] = []
 
         # extract rules set (css selector path, type, attribute)
         fields_extract_rules_set: Set[(str, str)] = set()
@@ -189,9 +186,13 @@ def _extract_fields_by_id(self, list_id: int) -> List[Field]:
         list_child_nodes = self.graph_loader.get_node_children_by_id(list_id)
 
         # iterate list child nodes
-        for i, c in enumerate(list_child_nodes):
+        i = 0
+        for c in list_child_nodes:
+            if c.tag != item_nodes[0].tag:
+                continue
+
             # stop if item nodes samples reached
-            if i == self.item_nodes_samples:
+            if i == self.min_item_nodes:
                 break
 
             # item child nodes
@@ -217,18 +218,20 @@ def _extract_fields_by_id(self, list_id: int) -> List[Field]:
                                                                                     numbered=False)
                     fields_extract_rules_set.add((extract_rule_css, FIELD_EXTRACT_RULE_TYPE_IMAGE_URL, 'src'))
 
+            i += 1
+
         for i, item in enumerate(fields_extract_rules_set):
             extract_rule_css, type_, attribute = item
             name = f'Field_{type_}_{i + 1}'
-            fields.append(Field({
-                'name': name,
-                'selector': extract_rule_css,
-                'type': type_,
-                'attribute': attribute,
-            }))
+            fields.append(Selector(
+                name=name,
+                selector=extract_rule_css,
+                type=type_,
+                attribute=attribute,
+            ))
         return fields
 
-    def _extract_data(self, soup: BeautifulSoup, items_selector_full: str, fields: List[Field]):
+    def _extract_data(self, soup: BeautifulSoup, items_selector_full: str, fields: List[Selector]):
         data = []
         for item_el in soup.select(items_selector_full):
             row = {}
@@ -250,130 +253,199 @@ def _extract_data(self, soup: BeautifulSoup, items_selector_full: str, fields: L
         return data
 
     def _train(self):
-        self.dbscan.fit(self._get_nodes_features())
+        self.dbscan.fit(self._get_nodes_features(to_sparse=True))
 
-    def _pre_filter(self):
+    def _pre_filter(self) -> (List[Node], List[List[Node]]):
         df_nodes = pd.DataFrame({
             'id': [n.id for n in self.graph_loader.nodes_],
             'parent_id': [n.parent_id for n in self.graph_loader.nodes_],
             'label': self.dbscan.labels_,
         })
-        df_nodes = df_nodes[df_nodes.label != -1]
+        df_nodes_filtered = df_nodes[df_nodes.label != -1]
+        logger.debug('nodes before pre-filter: %s' % df_nodes.shape[0])
+        logger.debug('nodes after pre-filter: %s' % df_nodes_filtered.shape[0])
 
-        df_labels = df_nodes \
+        df_labels = df_nodes_filtered \
             .groupby('label')[['parent_id']] \
             .agg(lambda s: entropy(s.value_counts())) \
             .rename(columns={'parent_id': 'entropy'}) \
             .sort_values(by='entropy', ascending=True)
 
-        threshold_mask = df_labels.entropy < self.entropy_threshold
-        df_labels_filtered = df_labels[threshold_mask]
+        # threshold_mask = df_labels.entropy < self.entropy_threshold
+        # df_labels_filtered = df_labels[threshold_mask]
 
-        for label in df_labels_filtered.index:
-            # item node ids
-            item_nodes_ids = df_nodes[df_nodes.label == label].id.values
+        item_nodes_list = []
+        list_node_list = []
+        for label in df_labels.index:
+            # item nodes (filtered by label only)
+            df_nodes_filtered_by_label = df_nodes_filtered[df_nodes_filtered.label == label]
 
-            # item nodes
-            item_nodes = self.graph_loader.get_nodes_by_ids(item_nodes_ids)
+            # iterate parent id
+            for parent_id in df_nodes_filtered_by_label.parent_id.unique().tolist():
+                # item nodes (further filtered parent id)
+                df_nodes_filtered_by_label_parent_id = df_nodes_filtered_by_label[
+                    df_nodes_filtered.parent_id == parent_id]
 
-            # list node
-            list_node = self.graph_loader.get_node_by_id(item_nodes[0].parent_id)
-
-            # entropy
-            entropy_ = df_labels.loc[label].entropy
-
-            # add to result
-            self.results.append(
-                ListResult({
-                    'nodes': {
-                        'list': list_node,
-                        'items': item_nodes,
-                    },
-                    'stats': {
-                        'entropy': entropy_,
-                    },
-                }),
-            )
+                # skip if item nodes count is less than required
+                if df_nodes_filtered_by_label_parent_id.shape[0] < self.min_item_nodes:
+                    continue
 
-    def _filter(self):
-        for i, result in enumerate(self.results):
-            nodes_ids = np.array([n.id for n in result.item_nodes])
-            nodes_idx = np.argwhere(np.isin(self.graph_loader.nodes_ids, nodes_ids))
+                # item node ids
+                item_nodes_ids = df_nodes_filtered_by_label_parent_id.id.values
+
+                # item nodes
+                item_nodes = self.graph_loader.get_nodes_by_ids(item_nodes_ids)
+                item_nodes_list.append(item_nodes)
+
+                # list node
+                list_node = self.graph_loader.get_node_by_id(item_nodes[0].parent_id)
+                list_node_list.append(list_node)
+
+        return list_node_list, item_nodes_list
 
-            score = 0
-            scores = {}
+    def _filter(
+        self,
+        list_node_list: List[Node],
+        item_nodes_list: List[List[Node]],
+    ) -> (List[Node], List[List[Node]], List[float], List[Dict[str, float]]):
+        score_list = []
+        scores_list = []
+        idx: List[int] = []
+        for i, list_node, item_nodes in zip(range(len(list_node_list)), list_node_list, item_nodes_list):
+            nodes_ids = np.random.choice(np.array([n.id for n in item_nodes]), size=10)
             try:
-                nodes_features_vec = self._get_nodes_features(nodes_idx)
-                nodes_features_vec_norm = normalize(nodes_features_vec.sum(axis=0).reshape(1, -1), norm='l1')
+                score_text_richness = 0
+                score_complexity = 0
+                for node_id in nodes_ids:
+                    child_nodes_idx = self.graph_loader.get_node_children_idx_recursive_by_id(node_id)
+                    if len(child_nodes_idx) == 0:
+                        continue
+                    nodes_text_length_vec = self.graph_loader.nodes_text_length_vec[child_nodes_idx]
+                    nodes_text_length_vec_non_zero = nodes_text_length_vec[nodes_text_length_vec > 0]
+                    _score_text_richness = log_positive(
+                        min(nodes_text_length_vec_non_zero.sum(), self.max_text_length) * self.text_length_discount)
+                    _score_complexity = log_positive(min(len(nodes_text_length_vec_non_zero), self.max_feature_count))
+                    if _score_text_richness > score_text_richness:
+                        score_text_richness = _score_text_richness
+                    if _score_complexity > score_complexity:
+                        score_complexity = _score_complexity
+                score_item_count = log_positive(min(len(item_nodes), self.max_item_count))
+
+                logger.debug(f'score_text_richness: {score_text_richness}')
+                logger.debug(f'score_complexity: {score_complexity}')
+                logger.debug(f'score_item_count: {score_item_count}')
 
                 # score
-                score_text_richness = log_positive(
-                    np.array([(len(get_node_inner_text(self.graph_loader, n)) * self.text_length_discount)
-                              for n in result.item_nodes]).sum())
-                score_complexity = float(entropy(nodes_features_vec_norm, axis=1)[0])
-                score_item_count = log_positive(len(result.item_nodes))
-                score = score_text_richness * score_complexity * score_item_count
-                scores = {
+                score = score_text_richness + score_complexity + score_item_count
+                logger.debug(f'score: {score}')
+
+                # skip score less than threshold
+                if score < self.score_threshold:
+                    continue
+
+                # skip zero sub-score
+                if score_text_richness == 0 or score_complexity == 0 or score_item_count == 0:
+                    continue
+
+                # add to scores list
+                scores_list.append({
                     'text_richness': score_text_richness,
                     'complexity': score_complexity,
                     'item_count': score_item_count,
-                }
+                })
+
+                # add to score list
+                score_list.append(score)
+
+                # add to idx
+                idx.append(i)
 
             except ValueError as e:
                 # logging.warning(f'ValueError: {e}')
                 pass
 
-            self.results[i]['stats']['score'] = float(score)
-            self.results[i]['stats']['scores'] = scores
-
-        self.results = [r for r in self.results if r.stats.score > self.score_threshold]
+        res_list_node_list = [list_node_list[i] for i in idx]
+        res_item_nodes_list = [item_nodes_list[i] for i in idx]
 
-    def _sort(self):
-        self.results = sorted(self.results, key=lambda x: x.stats.score, reverse=True)
+        return res_list_node_list, res_item_nodes_list, score_list, scores_list
 
-        # assign name
-        for i, result in enumerate(self.results):
-            self.results[i]['name'] = f'List {i + 1}'
-
-    def _extract(self):
+    def _extract(
+        self,
+        list_node_list: List[Node],
+        item_nodes_list: List[List[Node]],
+        score_list: List[float],
+        scores_list: List[Dict[str, float]],
+    ) -> List[ListResult]:
+        # soup
         soup = BeautifulSoup(self._html, features='lxml')
 
-        for i, result in enumerate(self.results):
+        # results
+        results = []
+
+        # iterate inputs
+        for i, list_node, item_nodes, score, scores in zip(range(len(list_node_list)),
+                                                           list_node_list,
+                                                           item_nodes_list,
+                                                           score_list,
+                                                           scores_list):
             # list node
-            list_node = result.list_node
+            list_node = list_node
 
             # item nodes
-            item_nodes = result.item_nodes
+            item_nodes = item_nodes
 
-            # list node extract rule (css)
-            list_node_extract_rule_css = self.graph_loader.get_node_css_selector_path(list_node)
+            # list selector
+            list_selector = Selector(
+                name='list',
+                selector=self.graph_loader.get_node_css_selector_path(list_node),
+                type='css',
+            )
 
             # items node extract rule (css)
-            items_node_extract_rule_css = self.graph_loader.get_node_css_selector_repr(item_nodes[0], False)
+            items_selector = Selector(
+                name='items',
+                selector=self.graph_loader.get_node_css_selector_repr(item_nodes[0], False),
+                type='css',
+            )
 
             # items node full extract rule (css)
-            items_node_extract_rule_css_full = f'{list_node_extract_rule_css} > {items_node_extract_rule_css}'
+            full_items_selector = Selector(
+                name='items',
+                selector=f'{list_selector.selector} > {items_selector.selector}',
+                type='css',
+            )
 
             # fields node extract rule
-            fields_node_extract_rule_css = self._extract_fields_by_id(list_node.id)
+            fields_selectors = self._extract_fields_by_id(list_id=list_node.id, item_nodes=item_nodes)
 
             # data
-            data = self._extract_data(soup, items_node_extract_rule_css_full, fields_node_extract_rule_css)
+            data = self._extract_data(soup, full_items_selector.selector, fields_selectors)
+
+            # result
+            result = ListResult(
+                selectors={
+                    'list': list_selector,
+                    'items': items_selector,
+                    'full_items': full_items_selector,
+                },
+                score=score,
+                scores=scores,
+                fields=fields_selectors,
+                data=data,
+                detector=DETECTOR_PLAIN_LIST,
+            )
+            results.append(result)
+
+        return results
 
-            self.results[i]['extract_rules_css'] = {
-                'list': list_node_extract_rule_css,
-                'items': items_node_extract_rule_css,
-                'items_full': items_node_extract_rule_css_full,
-                'fields': fields_node_extract_rule_css,
-                'data': data,
-            }
+    def _sort(self, results: List[ListResult]) -> List[ListResult]:
+        results = sorted(results, key=lambda x: x.score, reverse=True)
 
-    def _save(self):
-        if not os.path.exists(os.path.dirname(self.save_path)):
-            os.makedirs(os.path.dirname(self.save_path))
+        # assign name
+        for i, result in enumerate(results):
+            results[i].name = f'{self.result_name_prefix} {i + 1}'
 
-        with open(self.save_path, 'w') as f:
-            f.write(json.dumps(self.results, indent=2))
+        return results
 
     def run(self):
         tic = time.time()
@@ -382,44 +454,39 @@ def run(self):
         self._train()
 
         # pre-filter nodes
-        self._pre_filter()
+        res = self._pre_filter()
 
         # filter nodes
-        self._filter()
-
-        # sort results
-        self._sort()
+        res = self._filter(*res)
 
         # extract fields into results
-        self._extract()
+        results = self._extract(*res)
 
-        # save results
-        self._save()
+        # sort results
+        self.results = self._sort(results)
 
         toc = time.time()
-        logging.info(f'PlainListExtractor: {toc - tic:.2f}s')
+        logger.debug(f'PlainListExtractor: {toc - tic:.2f}s')
 
         return self.results
 
 
+def run_plain_list_detector(url: str, method: str = None) -> PlainListDetector:
+    html_requester = HtmlRequester(url=url, request_method=method)
+    html_requester.run()
+
+    graph_loader = GraphLoader(html=html_requester.html_, json_data=html_requester.json_data)
+    graph_loader.run()
+
+    plain_list_detector = PlainListDetector(html_requester=html_requester, graph_loader=graph_loader)
+    plain_list_detector.run()
+
+    return plain_list_detector
+
+
 if __name__ == '__main__':
-    data_path = '/Users/marvzhang/projects/tikazyq/auto-html/data/quotes.toscrape.com/json/http___quotes_toscrape_com_.json'
-    # data_path = '/Users/marvzhang/projects/tikazyq/auto-html/data/github.com/json/https___github_com_trending.json'
-    # data_path = '/Users/marvzhang/projects/tikazyq/auto-html/data/news.ycombinator.com/json/https___news_ycombinator_com.json'
-    # data_path = '/Users/marvzhang/projects/tikazyq/auto-html/data/www.ruanyifeng.com/json/http___www_ruanyifeng_com_blog_.json'
-    # data_path = '/Users/marvzhang/projects/tikazyq/auto-html/data/www.producthunt.com/json/https___www_producthunt_com.json'
-    # data_path = '/Users/marvzhang/projects/tikazyq/auto-html/data/edition.cnn.com/json/https___edition_cnn_com.json'
-    # data_path = '/Users/marvzhang/projects/tikazyq/auto-html/data/shixian.com/json/https___shixian_com_jobs_part-time.json'
-    # data_path = '/Users/marvzhang/projects/tikazyq/auto-html/data/github.com/json/https___github_com_search?q=crawlab.json'
-    # detector = PlainListDetector(json_path=data_path)
-
-    # url = 'https://quotes.toscrape.com/'
-    # url = 'https://cuiqingcai.com/'
-    url = 'https://cuiqingcai.com/archives/'
-    detector = PlainListDetector(url=url)
-
-    detector.run()
-
-    for result in detector.results:
-        print(result.extract_rules)
-        # n = detector.graph_loader.get_node_by_id(result.list_node.id)
+    urls = [
+        'https://cuiqingcai.com'
+    ]
+    for url in urls:
+        run_plain_list_detector(url)
diff --git a/webspot/detect/detectors/plain_table.py b/webspot/detect/detectors/plain_table.py
new file mode 100644
index 0000000..1f89734
--- /dev/null
+++ b/webspot/detect/detectors/plain_table.py
@@ -0,0 +1,30 @@
+from bs4 import BeautifulSoup
+import numpy as np
+
+from webspot.detect.detectors.base import BaseDetector
+
+
+class PlainTableDetector(BaseDetector):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # data
+        self._table_nodes = None
+
+    def highlight_html(self, html: str, **kwargs) -> str:
+        pass
+
+    @property
+    def table_nodes(self):
+        return self.get_nodes_by_feature(feature_key='tag', feature_value='table')
+
+    def _pre_process(self):
+        self._table_nodes = self.table_nodes
+
+    def _train(self):
+        pass
+
+    def run(self):
+        self._pre_process()
+
+        self._train()
diff --git a/webspot/detect/models/field.py b/webspot/detect/models/field.py
deleted file mode 100644
index 09151e4..0000000
--- a/webspot/detect/models/field.py
+++ /dev/null
@@ -1,16 +0,0 @@
-class Field(dict):
-    @property
-    def name(self):
-        return self.get('name')
-
-    @property
-    def selector(self):
-        return self.get('selector')
-
-    @property
-    def type(self):
-        return self.get('type')
-
-    @property
-    def attribute(self):
-        return self.get('attribute')
diff --git a/webspot/detect/models/list_result.py b/webspot/detect/models/list_result.py
index d96d2f3..4b630cc 100644
--- a/webspot/detect/models/list_result.py
+++ b/webspot/detect/models/list_result.py
@@ -1,31 +1,9 @@
 from typing import List
 
-from webspot.detect.models.stats import Stats
-from webspot.graph.models.node import Node
+from webspot.detect.models.result import Result
+from webspot.detect.models.selector import Selector
 
 
-class ListResult(dict):
-    @property
-    def name(self) -> str:
-        return self.get('name')
-
-    @property
-    def item_nodes(self) -> List[Node]:
-        return self.get('nodes').get('items')
-
-    @property
-    def list_node(self) -> Node:
-        return self.get('nodes').get('list')
-
-    @property
-    def stats(self) -> Stats:
-        return Stats(self.get('stats') or {})
-
-    @property
-    def extract_rules(self):
-        return {
-            'list': self.get('extract_rules_css').get('list'),
-            'items': self.get('extract_rules_css').get('items'),
-            'fields': self.get('extract_rules_css').get('fields'),
-            'data': self.get('extract_rules_css').get('data'),
-        }
+class ListResult(Result):
+    fields: List[Selector]
+    data: List[dict]
diff --git a/webspot/detect/models/result.py b/webspot/detect/models/result.py
new file mode 100644
index 0000000..0cf3daa
--- /dev/null
+++ b/webspot/detect/models/result.py
@@ -0,0 +1,13 @@
+from typing import Dict, Any, Optional
+
+from pydantic import BaseModel
+
+from webspot.detect.models.selector import Selector
+
+
+class Result(BaseModel):
+    name: Optional[str]
+    selectors: Optional[Dict[str, Selector]]
+    score: Optional[float]
+    scores: Optional[Dict[str, Optional[float]]]
+    detector: Optional[str]
diff --git a/webspot/detect/models/selector.py b/webspot/detect/models/selector.py
new file mode 100644
index 0000000..82cb00e
--- /dev/null
+++ b/webspot/detect/models/selector.py
@@ -0,0 +1,10 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class Selector(BaseModel):
+    name: str
+    selector: str
+    type: Optional[str]
+    attribute: Optional[str]
diff --git a/webspot/detect/models/table_result.py b/webspot/detect/models/table_result.py
new file mode 100644
index 0000000..9ca754f
--- /dev/null
+++ b/webspot/detect/models/table_result.py
@@ -0,0 +1,8 @@
+from typing import Optional, List
+
+from webspot.detect.models.result import Result
+
+
+class TableResult(Result):
+    columns: Optional[List[str]]
+    data: Optional[List[dict]]
diff --git a/webspot/detect/utils/highlight_html.py b/webspot/detect/utils/highlight_html.py
index c2decc8..25afea3 100644
--- a/webspot/detect/utils/highlight_html.py
+++ b/webspot/detect/utils/highlight_html.py
@@ -13,40 +13,11 @@ def get_embed_css() -> str:
         return f.read()
 
 
-def highlight_html(html, results: List[ListResult]) -> str:
+def embed_highlight_css(html: str) -> str:
     soup = BeautifulSoup(html, 'html.parser')
-    for i, result in enumerate(results):
-        # list
-        list_rule = result.extract_rules.get('list')
-        list_el = soup.select_one(list_rule)
-        if not list_el:
-            continue
-        _add_class(list_el, ['webspot-highlight-container', 'webspot-highlight-list-node'])
-        _add_label(list_el, soup, f'List {i + 1}', 'primary')
-
-        # items
-        items_rule = result.extract_rules.get('items')
-        item_els = list_el.select(items_rule)
-        for j, item_el in enumerate(item_els):
-            _add_class(item_el, ['webspot-highlight-container', 'webspot-highlight-item-node'])
-            # _add_label(item_el, soup, f'Item {j + 1}', 'warning')
-
-            # fields
-            for k, field in enumerate(result.extract_rules.get('fields')):
-                try:
-                    field_els = item_el.select(field.get('selector'))
-                except Exception as e:
-                    # logging.warning(e)
-                    field_els = []
-                for field_el in field_els:
-                    _add_class(field_el, ['webspot-highlight-container', 'webspot-highlight-item-field-node'])
-                    # _add_label(field_el, soup, f'Field {k + 1}', 'success')
-
-    # add embed css
     style_el = soup.new_tag('style')
     style_el.append(get_embed_css())
     soup.select_one('head').append(style_el)
-
     return str(soup)
 
 
@@ -67,4 +38,11 @@ def _add_label(el: Tag, soup: BeautifulSoup, label: str, type_: str = None):
     _add_class(label_el, ['webspot-highlight-label'])
     if type_:
         _add_class(label_el, [f'webspot-highlight-label-{type_}'])
-    pass
+
+
+def add_class(el: Tag, classes: List[str]):
+    return _add_class(el, classes)
+
+
+def add_label(el: Tag, soup: BeautifulSoup, label: str, type_: str = None):
+    return _add_label(el, soup, label, type_)
diff --git a/webspot/detect/utils/node.py b/webspot/detect/utils/node.py
deleted file mode 100644
index e93d68c..0000000
--- a/webspot/detect/utils/node.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from webspot.graph.graph_loader import GraphLoader
-from webspot.graph.models.node import Node
-
-
-def get_node_inner_text(g: GraphLoader, n: Node) -> str:
-    texts = []
-    for c in g.get_node_children_recursive_by_id(n.id):
-        if c.text:
-            texts.append(c.text)
-    return ' '.join(texts)
diff --git a/webspot/detect/utils/transform_html_links.py b/webspot/detect/utils/transform_html_links.py
index 5aece35..e6e3e9d 100644
--- a/webspot/detect/utils/transform_html_links.py
+++ b/webspot/detect/utils/transform_html_links.py
@@ -26,14 +26,14 @@ def transform_html_links(html: str, url: str) -> str:
 
 
 def _transform(el: Tag, key: str, root_url: str):
-    if el.get(key) and not re.search(re.escape(el[key]), r'^(https?:)?//'):
-        el[key] = urljoin(root_url, el[key])
+    if el.get(key) and _is_relative_url_path(el[key]):
+        el[key] = transform_url(root_url, el[key])
 
 
 def _transform_a(el: Tag, root_url: str, target_blank: bool = True):
     key = 'href'
-    if el.get(key) and el[key].startswith('/'):
-        url = urljoin(root_url, el[key])
+    if el.get(key) and _is_relative_url_path(el[key]):
+        url = transform_url(root_url, el[key])
         el[key] = f'/?url={url}'
         if target_blank:
             el['target'] = '_blank'
@@ -46,3 +46,13 @@ def _is_valid_asset_url(url: str) -> bool:
         if keyword in url.lower():
             return True
     return False
+
+
+def _is_relative_url_path(url: str) -> bool:
+    return not re.search(re.escape(url), r'^(https?:)?//')
+
+
+def transform_url(root_url: str, url: str) -> str:
+    if not _is_relative_url_path(url):
+        return url
+    return urljoin(root_url, url)
diff --git a/webspot/detect/utils/url.py b/webspot/detect/utils/url.py
new file mode 100644
index 0000000..5fe2ecd
--- /dev/null
+++ b/webspot/detect/utils/url.py
@@ -0,0 +1,11 @@
+from urllib.parse import urlparse
+
+
+def get_url_domain(url: str) -> str:
+    """Get the domain from a url."""
+    return urlparse(url).netloc
+
+
+def get_url_path(url: str) -> str:
+    """Get the path from a url."""
+    return urlparse(url).path
diff --git a/webspot/graph/graph_loader.py b/webspot/graph/graph_loader.py
index 53e5775..c0368c4 100644
--- a/webspot/graph/graph_loader.py
+++ b/webspot/graph/graph_loader.py
@@ -1,35 +1,57 @@
 import json
-from typing import List, Tuple, Dict, Union
+from typing import List, Tuple, Dict, Optional
 
 import dgl
+import html_to_json_enhanced
 import networkx as nx
 import numpy as np
 import torch
+from bs4 import BeautifulSoup
 from html_to_json_enhanced import iterate
+from networkx import DiGraph, dfs_successors
 from sklearn.feature_extraction import DictVectorizer
 from sklearn.preprocessing import LabelEncoder
 from dgl import DGLGraph
 
 from webspot.graph.models.node import Node
+from webspot.logging import get_logger
+
+logger = get_logger('webspot.graph.graph_loader')
+
+SVG_TAG_NAMES = ['circle', 'clipPath', 'defs', 'ellipse', 'g', 'image', 'line', 'linearGradient',
+                 'mask', 'path', 'pattern', 'polygon', 'polyline', 'radialGradient', 'rect',
+                 'stop', 'svg', 'text', 'tspan']
+ESCAPED_TAG_NAMES = [
+    'script',
+    'link',
+    'meta',
+    *SVG_TAG_NAMES,
+]
 
 
 class GraphLoader(object):
-    def __init__(self, json_data: str = None, json_path: str = None, file_pattern: str = None, body_only: bool = True,
-                 embed_walk_length: int = 3):
+    def __init__(
+        self,
+        html: str,
+        json_data: dict,
+        body_only: bool = True,
+        embed_walk_length: int = 8,
+        dfs_depth: int = 8,
+    ):
         # settings
-        self.json_data = json_data
-        self.json_path = json_path
-        self.file_pattern = file_pattern or r'\.json$'
         self.body_only = body_only
         self.embed_walk_length = embed_walk_length
         self.available_feature_keys = [
             'tag',
             'id',
             'class',
+            'style',
         ]
-        assert self.json_data or self.json_path, 'json_data or json_path must be provided'
+        self.dfs_depth = dfs_depth
 
         # data
+        self.html = html
+        self.json_data = json_data
         self.root_node_json: dict = {}
         self.nodes_: List[Node] = []
         self.nodes_ids: List[int] = []
@@ -37,9 +59,13 @@ def __init__(self, json_data: str = None, json_path: str = None, file_pattern: s
         self.edge_nodes: List[Node] = []
 
         # internals
-        self._nodes_dict = {}
+        self._nodes_dict = None
         self._css_selector_repr_cache = {}
-        self._unique_node_feature_id_dict: Union[dict, None] = None
+        self._unique_node_feature_id_dict: Optional[dict] = None
+        self._nodes_texts: Optional[List[str]] = None
+        self._nodes_ids_idx_dict = None
+        self._soup: Optional[BeautifulSoup] = None
+        self._g_nx: Optional[DiGraph] = None
 
         # encoders
         self.nodes_features_enc = DictVectorizer()
@@ -51,9 +77,14 @@ def __init__(self, json_data: str = None, json_path: str = None, file_pattern: s
         self.edges_source_tensor = torch.LongTensor()
         self.edges_target_tensor = torch.LongTensor()
         self.nodes_embedded_tensor = torch.LongTensor()
+        self.nodes_text_length_tensor = torch.LongTensor()
+
+        # vectors
+        self.nodes_text_length_vec: Optional[np.ndarray] = None
+        self.nodes_full_text_length_vec: Optional[np.ndarray] = None
 
         # graph
-        self.g_dgl = DGLGraph()
+        self.g_dgl: Optional[DGLGraph] = None
 
     @property
     def nodes_dict(self):
@@ -63,24 +94,18 @@ def nodes_dict(self):
         return self._nodes_dict
 
     @property
-    def g_nx(self) -> nx.Graph:
-        g_nx = self.g_dgl.to_networkx()
-        return g_nx
-
-    def load_graph_data(self):
-        # get list data and root node
-        if self.json_data is not None:
-            list_data, self.root_node_json = self.get_nodes_json_data(self.json_data)
-        else:
-            list_data, self.root_node_json = self.get_nodes_json_data_by_filename(self.json_path)
-
-        # load graph data
-        self._load_graph_data(list_data)
+    def nodes_ids_idx_dict(self):
+        if self._nodes_ids_idx_dict:
+            return self._nodes_ids_idx_dict
+        self._nodes_ids_idx_dict = {n.id: i for i, n in enumerate(self.nodes_)}
+        return self._nodes_ids_idx_dict
 
-        # encode node ids
-        self.nodes_ids = self.node_ids_enc.fit_transform(
-            np.array([node.id for node in self.nodes_]).reshape(-1, 1)
-        )
+    @property
+    def g_nx(self) -> nx.DiGraph:
+        if self._g_nx:
+            return self._g_nx
+        self._g_nx = DiGraph(self.g_dgl.to_networkx())
+        return self._g_nx
 
     def get_nodes_json_data_by_filename(self, filename: str) -> (List[dict], dict):
         with open(filename) as f:
@@ -106,11 +131,18 @@ def _load_graph_data(self, nodes_json_data: List[dict]):
             # node
             node = self._get_node(node_json_data)
 
+            # skip escaped tags
+            if node.tag in ESCAPED_TAG_NAMES:
+                continue
+
             # add to all nodes
             self.nodes_.append(node)
 
             # add to all node features
-            self.nodes_features.append(node.features_dict)
+            # self.nodes_features.append(node.features_dict)
+            self.nodes_features.append({k: v for k, v in node.features_dict.items()
+                                        if k.split('=')[0] in self.available_feature_keys
+                                        })
 
     def _get_node(self, node_json_data: dict) -> Node:
         node_id = node_json_data.get('_id')
@@ -144,7 +176,7 @@ def _get_node_features(node_json_data: dict) -> List[Tuple[str, str]]:
         return features
 
     @staticmethod
-    def _get_node_text(node_json_data: dict) -> Union[str, None]:
+    def _get_node_text(node_json_data: dict) -> Optional[str]:
         if node_json_data.get('_text') is not None:
             return node_json_data.get('_text')
 
@@ -154,6 +186,19 @@ def _get_node_text(node_json_data: dict) -> Union[str, None]:
         else:
             return
 
+    def load_graph_data(self):
+        # get list data and root node
+        self.json_data = html_to_json_enhanced.convert_html.convert(self.html, with_id=True)
+        list_data, self.root_node_json = self.get_nodes_json_data(self.json_data)
+
+        # load graph data
+        self._load_graph_data(list_data)
+
+        # encode node ids
+        self.nodes_ids = self.node_ids_enc.fit_transform(
+            np.array([node.id for node in self.nodes_]).reshape(-1, 1)
+        )
+
     def load_tensors(self):
         # nodes tensor
         encoded_nodes = self.node_ids_enc.transform([n.id for n in self.nodes_])
@@ -185,31 +230,52 @@ def load_embeddings(self):
             walk_length=self.embed_walk_length,
         )
 
+    def load_soup(self):
+        self._soup = BeautifulSoup(self.html, 'html.parser')
+
+    def load_texts(self):
+        self._nodes_texts = [n.text for n in self.nodes_]
+        self.nodes_text_length_vec = np.array([len(t or []) for t in self._nodes_texts])
+
     def run(self):
         self.load_graph_data()
         self.load_tensors()
         self.load_dgl_graph()
         self.load_embeddings()
+        self.load_soup()
+        self.load_texts()
 
     @property
     def nodes(self):
         return np.array(self.nodes_)
 
-    def get_node_by_id(self, id: int) -> Union[Node, None]:
+    def get_node_by_id(self, id: int) -> Optional[Node]:
         return self.nodes_dict.get(id)
 
     def get_nodes_by_ids(self, ids: List[int]) -> List[Node]:
         return [self.get_node_by_id(id) for id in ids]
 
-    def get_node_children_recursive(self, n: Node) -> List[Node]:
-        return self.get_node_children_recursive_by_id(n.id)
-
     def get_node_children_recursive_by_id(self, id: int) -> List[Node]:
         idx = self.node_ids_enc.transform([id])[0]
-        tree_data = nx.tree_data(self.g_nx, idx)
-        tree_graph = nx.tree_graph(tree_data)
-        child_ids = self.node_ids_enc.inverse_transform([nid for nid in tree_graph.nodes() if nid != idx])
-        return [self.get_node_by_id(cid) for cid in child_ids]
+        child_nodes_idx = self.get_node_children_idx_recursive_by_id(id)
+        child_ids = self.node_ids_enc.inverse_transform([nid for nid in child_nodes_idx if nid != idx])
+        return self.get_nodes_by_ids(child_ids)
+
+    def get_node_children_idx_recursive_by_id(self, id: int) -> np.ndarray:
+        idx = self.node_ids_enc.transform([id])[0]
+        return self.get_node_children_idx_recursive_by_idx(idx)
+
+    def get_node_children_idx_recursive_by_idx(self, idx: int) -> np.ndarray:
+        successors = dfs_successors(G=self.g_nx, source=idx, depth_limit=self.dfs_depth)
+        child_nodes_idx = [item for sublist in successors.values() for item in sublist]
+        return np.array(list(child_nodes_idx))
+
+    def get_node_text_length(self, n: Node, max_length: int = 1024) -> int:
+        selector = self.get_node_css_selector_path(n)
+        el = self._soup.select_one(selector)
+        if el is None:
+            return 0
+        return min(len(el.text), max_length)
 
     def get_node_children_by_id(self, id: int) -> List[Node]:
         return [n for n in self.nodes_ if n.parent_id == id]
@@ -273,6 +339,11 @@ def _get_node_previous_siblings_with_classes(self, node: Node) -> List[Node]:
                 and n.feature_tag == node.feature_tag
                 and node_feature_classes_set.issubset(set(n.feature_classes))]
 
+    def _is_node_last_child(self, node: Node) -> bool:
+        parent = self.get_node_by_id(node.parent_id)
+        children = self.get_node_children_by_id(parent.id)
+        return node.id == children[-1].id
+
     def get_node_css_selector_repr(self, node: Node, numbered: bool = True) -> str:
         if numbered:
             return self._get_node_css_selector_repr(node, numbered)
@@ -294,7 +365,9 @@ def _get_node_css_selector_repr(self, node: Node, numbered: bool = True) -> str:
                 previous_siblings = self._get_node_previous_siblings_with_classes(node)
                 length = len(previous_siblings) + 1
                 if length > 1:
-                    return f'{node.feature_tag}.{".".join(node.feature_classes)}:nth-child({length})'
+                    if self._is_node_last_child(node):
+                        return f'{node.feature_tag}.{".".join(node.feature_classes)}:last-child'
+                    return f'{node.feature_tag}.{".".join(node.feature_classes)}:nth-of-type({length})'
             return f'{node.feature_tag}.{".".join(node.feature_classes)}'
 
         # tag
@@ -302,7 +375,9 @@ def _get_node_css_selector_repr(self, node: Node, numbered: bool = True) -> str:
             if numbered:
                 length = len(self._get_node_previous_siblings(node)) + 1
                 if length > 1:
-                    return f'{node.feature_tag}:nth-child({length})'
+                    if self._is_node_last_child(node):
+                        return f'{node.feature_tag}:last-child'
+                    return f'{node.feature_tag}:nth-of-type({length})'
             return f'{node.feature_tag}'
 
     @staticmethod
@@ -350,9 +425,3 @@ def get_node_css_selector_path(self, node: Node, root_id: int = None, numbered:
             node = parent
 
         return ' > '.join(path)
-
-
-if __name__ == '__main__':
-    data_path = '/Users/marvzhang/projects/tikazyq/auto-html/data/quotes.toscrape.com/json/http___quotes_toscrape_com_.json'
-    loader = GraphLoader(json_path=data_path)
-    loader.run()
diff --git a/webspot/graph/models/node.py b/webspot/graph/models/node.py
index a8592f4..fd74ca3 100644
--- a/webspot/graph/models/node.py
+++ b/webspot/graph/models/node.py
@@ -1,4 +1,5 @@
-from typing import Tuple, List, Dict, Union
+import re
+from typing import Tuple, List, Dict, Optional
 
 
 class Node(dict):
@@ -11,22 +12,22 @@ def parent_id(self):
         return self.get('parent_id')
 
     @property
-    def features(self) -> List[Tuple[str, str]]:
-        features = []
+    def features(self) -> Tuple[Tuple[str, str]]:
+        features: List[Tuple[str, str]] = []
         for k, v in self.get('features'):
             # skip pseudo-classes
             if k == 'class':
                 if ':' in v:
                     continue
             features.append((k, v))
-        return features
+        return tuple(features)
 
     @property
     def features_dict(self) -> Dict[str, int]:
         return {f'{k}={v}': 1 for k, v in self.features}
 
     @property
-    def feature_tag(self) -> Union[str, None]:
+    def feature_tag(self) -> Optional[str]:
         for k, v in self.features:
             if k == 'tag':
                 return v
@@ -37,11 +38,14 @@ def feature_classes(self) -> List[str]:
         classes = []
         for k, v in self.features:
             if k == 'class':
+                # css parser cannot parse class starting with a digit
+                if re.search(r'^\d', v) is not None:
+                    continue
                 classes.append(v)
         return classes
 
     @property
-    def feature_id(self) -> Union[str, None]:
+    def feature_id(self) -> Optional[str]:
         for k, v in self.features:
             if k == 'id':
                 return v
diff --git a/webspot/logging/__init__.py b/webspot/logging/__init__.py
new file mode 100644
index 0000000..f941d82
--- /dev/null
+++ b/webspot/logging/__init__.py
@@ -0,0 +1,30 @@
+import logging
+import os
+import sys
+
+_loggers = {}
+
+log_level_name = os.environ.get('WEBSPOT_LOG_LEVEL', 'info')
+log_level = logging.getLevelName(log_level_name.upper())
+
+
+def get_logger(name: str, stream=sys.stdout):
+    # check if logger already exists
+    if _loggers.get(name) is not None:
+        return _loggers.get(name)
+
+    # logger
+    logger = logging.getLogger(name)
+    logger.setLevel(log_level)
+
+    # handler
+    handler = logging.StreamHandler(stream)
+    handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+
+    # add handler
+    logger.addHandler(handler)
+
+    # store logger
+    _loggers[name] = logger
+
+    return logger
diff --git a/webspot/models/base.py b/webspot/models/base.py
index 45597b0..c677576 100644
--- a/webspot/models/base.py
+++ b/webspot/models/base.py
@@ -1,9 +1,42 @@
+import json
+from abc import abstractmethod
 from datetime import datetime
+from typing import Optional
 
-from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+from mongoengine import Document, StringField, DateTimeField, pre_save
+from pydantic import BaseModel, Field
 
+from webspot.utils.mongo import encode_mongo_document
 
-class Base(DeclarativeBase):
-    __table_args__ = {'sqlite_autoincrement': True}
 
-    created_at: Mapped[str] = mapped_column(default=lambda _: datetime.now().timestamp())
+class BaseOut(BaseModel):
+    id: str
+    created_at: datetime
+    created_by: Optional[str]
+    updated_at: Optional[datetime]
+    updated_by: Optional[str]
+
+
+class Base(Document):
+    created_at = DateTimeField(default=datetime.utcnow)
+    created_by = StringField()
+    updated_at = DateTimeField(default=datetime.utcnow)
+    updated_by = StringField()
+
+    meta = {
+        'abstract': True,
+        'allow_inheritance': True,
+    }
+
+    @classmethod
+    def pre_save(cls, sender, document, **kwargs):
+        document.updated_at = datetime.utcnow()
+
+    def to_dict(self, *args, **kwargs):
+        use_db_field = kwargs.pop("use_db_field", True)
+        doc_json = encode_mongo_document(self.to_mongo(use_db_field))
+        return json.loads(doc_json)
+
+    @abstractmethod
+    def to_out(self) -> BaseOut:
+        return BaseOut(**self.to_dict())
diff --git a/webspot/models/page.py b/webspot/models/page.py
deleted file mode 100644
index c5e71dc..0000000
--- a/webspot/models/page.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from sqlalchemy.orm import Mapped, mapped_column
-
-from webspot.models.base import Base
-
-
-class PageModel(Base):
-    __tablename__ = 'pages'
-
-    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
-    url: Mapped[str] = mapped_column(index=True, unique=True)
diff --git a/webspot/models/request.py b/webspot/models/request.py
index f053426..844128e 100644
--- a/webspot/models/request.py
+++ b/webspot/models/request.py
@@ -1,13 +1,61 @@
-from sqlalchemy.orm import mapped_column, Mapped
+from typing import List, Optional, Dict, Union
 
-from webspot.models.base import Base
+from mongoengine import StringField, BooleanField, ListField, IntField, DictField
 
+from webspot.constants.detector import DETECTOR_PLAIN_LIST, DETECTOR_PAGINATION
+from webspot.constants.html_request_method import HTML_REQUEST_METHOD_REQUEST
+from webspot.constants.request_status import REQUEST_STATUS_PENDING
+from webspot.detect.models.list_result import ListResult
+from webspot.detect.models.result import Result
+from webspot.models.base import Base, BaseOut
 
-class RequestModel(Base):
-    __tablename__ = 'requests'
 
-    id: Mapped[int] = mapped_column(primary_key=True)
-    page_id: Mapped[int] = mapped_column(index=True)
-    html: Mapped[str] = mapped_column(nullable=True)
-    results: Mapped[str] = mapped_column(nullable=True)
-    valid: Mapped[bool] = mapped_column(nullable=True)
+class RequestOut(BaseOut):
+    url: str
+    method: Optional[str]
+    duration: Optional[int]
+    status: Optional[str]
+    results: Optional[Dict[str, List[Union[ListResult, Result]]]]
+    valid: Optional[bool]
+    error: Optional[str]
+    no_async: Optional[bool]
+    detectors: List[str]
+    execution_time: Optional[dict]
+
+
+class Request(Base):
+    meta = {'allow_inheritance': True}
+
+    url = StringField()
+    method = StringField(default=HTML_REQUEST_METHOD_REQUEST)
+    duration = IntField()
+    status = StringField(default=REQUEST_STATUS_PENDING)
+    html = StringField()
+    html_highlighted = StringField()
+    results = DictField(default={})
+    valid = BooleanField(default=True)
+    error = StringField()
+    no_async = BooleanField(default=False)
+    detectors = ListField(default=[DETECTOR_PLAIN_LIST, DETECTOR_PAGINATION])
+    execution_time = DictField(default={})
+
+    def to_out(self) -> BaseOut:
+        return RequestOut(
+            id=str(self.id),
+            created_at=self.created_at,
+            created_by=self.created_by,
+            updated_at=self.updated_at,
+            updated_by=self.updated_by,
+            url=self.url,
+            method=self.method,
+            duration=self.duration,
+            status=self.status,
+            html=self.html,
+            html_highlighted=self.html_highlighted,
+            results=self.results,
+            valid=self.valid,
+            error=self.error,
+            no_async=self.no_async,
+            detectors=self.detectors,
+            execution_time=self.execution_time,
+        )
diff --git a/webspot/models/user.py b/webspot/models/user.py
new file mode 100644
index 0000000..2bed439
--- /dev/null
+++ b/webspot/models/user.py
@@ -0,0 +1,10 @@
+from sqlalchemy.orm import mapped_column, Mapped
+
+from webspot.models.base import Base
+
+
+class UserModel(Base):
+    __tablename__ = 'users'
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    name: Mapped[str] = mapped_column()
diff --git a/webspot/request/get_html.py b/webspot/request/get_html.py
deleted file mode 100644
index d68ede7..0000000
--- a/webspot/request/get_html.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import json
-import os
-from urllib.parse import urlparse
-
-import html_to_json_enhanced
-import requests
-
-DEFAULT_REQUEST_ROD_URL = 'http://localhost:7777/request'
-DEFAULT_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'data'))
-
-
-def get_html(url: str, request_method: str, request_rod_url: str = DEFAULT_REQUEST_ROD_URL,
-             request_rod_duration: int = 1, save: bool = False) -> str:
-    if request_method == 'rod':
-        res = requests.post(
-            request_rod_url,
-            data=json.dumps({'url': url, 'duration': request_rod_duration}),
-            headers={'Content-Type': 'application/json'},
-        )
-        _html = json.loads(res.content.decode('utf-8')).get('html')
-    elif request_method == 'request':
-        res = requests.get(url)
-        _html = res.content.decode('utf-8')
-    else:
-        raise Exception(f'Invalid request method: {request_method}')
-
-    if save:
-        # domain
-        domain = urlparse(url).netloc
-
-        # create data html directory if not exists
-        data_html_dir = os.path.join(DEFAULT_DATA_DIR, domain, 'html')
-        if not os.path.exists(data_html_dir):
-            os.makedirs(data_html_dir)
-
-        # create data json directory if not exists
-        data_json_dir = os.path.join(DEFAULT_DATA_DIR, domain, 'json')
-        if not os.path.exists(data_json_dir):
-            os.makedirs(data_json_dir)
-
-        # file name prefix
-        filename_prefix = url.replace('/', '_').replace(':', '_').replace('.', '_')
-
-        # save response html
-        data_html_dir = os.path.join(DEFAULT_DATA_DIR, domain, 'html')
-        filename = os.path.join(data_html_dir, f'{filename_prefix}.html')
-        with open(filename, 'w') as f:
-            f.write(_html)
-
-        # save response json
-        filename = os.path.join(data_json_dir, f'{filename_prefix}.json')
-        with open(filename, 'w') as f:
-            json_data = html_to_json_enhanced.convert(_html, with_id=True)
-            f.write(json.dumps(json_data))
-
-    return _html
diff --git a/webspot/request/html_requester.py b/webspot/request/html_requester.py
new file mode 100644
index 0000000..e0df28a
--- /dev/null
+++ b/webspot/request/html_requester.py
@@ -0,0 +1,140 @@
+import json
+import logging
+import os
+from typing import Optional
+from urllib.parse import urlparse
+
+import html_to_json_enhanced
+import httpx
+from httpx import Timeout
+from requests import Response
+from retrying import retry
+
+from webspot.constants.html_request_method import HTML_REQUEST_METHOD_REQUEST, HTML_REQUEST_METHOD_ROD
+from webspot.detect.utils.transform_html_links import transform_html_links
+
+DEFAULT_REQUEST_ROD_URL = 'http://localhost:7777/request'
+DEFAULT_REQUEST_ROD_DURATION = 10
+DEFAULT_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'data'))
+
+
+class HtmlRequester(object):
+    def __init__(
+        self,
+        url: str = None,
+        html_path: str = None,
+        request_method: str = HTML_REQUEST_METHOD_REQUEST,
+        request_rod_url: str = DEFAULT_REQUEST_ROD_URL,
+        request_rod_duration: int = DEFAULT_REQUEST_ROD_DURATION,
+        save: bool = False,
+    ):
+        # settings
+        self.url = url
+        self.html_path = html_path
+        self.request_method = request_method
+        self.request_rod_url = request_rod_url
+        self.request_rod_duration = request_rod_duration
+        self.save = save
+        self.encodings = ['utf-8', 'gbk', 'iso-8859-1', 'cp1252']
+
+        # data
+        self.html_: Optional[str] = None
+        self.html_response: Optional[Response] = None
+        self.json_data: Optional[dict] = None
+
+        # logger
+        self.logger = logging.getLogger('webspot.request.html_requester')
+
+    def _decode_response_content(self, res: Response) -> str:
+        content = ''
+        for encoding in self.encodings:
+            try:
+                content = res.content.decode(encoding)
+                break
+            except UnicodeDecodeError:
+                pass
+        return content
+
+    @retry(stop_max_attempt_number=3, wait_fixed=100)
+    def _request_html(self):
+        url = self.url
+        request_method = self.request_method
+        request_rod_url = self.request_rod_url
+        request_rod_duration = self.request_rod_duration
+
+        # print info
+        self.logger.info(f'Requesting {url} [method="{request_method}", duration={self.request_rod_duration}]')
+
+        if request_method == HTML_REQUEST_METHOD_ROD:
+            # request rod (headless browser)
+            res = httpx.post(
+                request_rod_url,
+                json={'url': url, 'duration': request_rod_duration},
+                headers={'Content-Type': 'application/json'},
+            )
+            if res.status_code != 200:
+                raise Exception(f'Invalid response from request rod: {res.status_code}')
+            self.html_response = res
+            self.html_ = json.loads(self._decode_response_content(res)).get('html')
+        elif request_method == HTML_REQUEST_METHOD_REQUEST:
+            # plain request
+            res = httpx.get(url, timeout=Timeout(timeout=self.request_rod_duration))
+            if res.status_code != 200:
+                raise Exception(f'Invalid response from request: {res.status_code}')
+            self.html_response = res
+            self.html_ = self._decode_response_content(res)
+        else:
+            raise Exception(f'Invalid request method: {request_method}')
+
+        # convert html to json data
+        self.json_data = html_to_json_enhanced.convert(self.html_, with_id=True)
+
+        if self.save:
+            # domain
+            domain = urlparse(url).netloc
+
+            # create data html directory if not exists
+            data_html_dir = os.path.join(DEFAULT_DATA_DIR, domain, 'html')
+            if not os.path.exists(data_html_dir):
+                os.makedirs(data_html_dir)
+
+            # create data json directory if not exists
+            data_json_dir = os.path.join(DEFAULT_DATA_DIR, domain, 'json')
+            if not os.path.exists(data_json_dir):
+                os.makedirs(data_json_dir)
+
+            # file name prefix
+            filename_prefix = url.replace('/', '_').replace(':', '_').replace('.', '_')
+
+            # save response html
+            data_html_dir = os.path.join(DEFAULT_DATA_DIR, domain, 'html')
+            filename = os.path.join(data_html_dir, f'{filename_prefix}.html')
+            with open(filename, 'w') as f:
+                f.write(self.html_)
+
+            # save response json
+            filename = os.path.join(data_json_dir, f'{filename_prefix}.json')
+            with open(filename, 'w') as f:
+                f.write(json.dumps(self.json_data))
+
+    def _load_html(self):
+        # print info
+        self.logger.info(f'Loading html from {self.html_path}...')
+
+        with open(self.html_path, 'r') as f:
+            self.html_ = f.read()
+
+    def run(self):
+        # request html page if url exists
+        if self.url:
+            self._request_html()
+        elif self.html_path:
+            self._load_html()
+        else:
+            raise Exception('No url or html path provided')
+
+        assert self.html_ is not None, 'No html obtained!'
+
+    @property
+    def html(self):
+        return transform_html_links(self.html_, self.url)
diff --git a/webspot/utils/mongo.py b/webspot/utils/mongo.py
new file mode 100644
index 0000000..d79b83a
--- /dev/null
+++ b/webspot/utils/mongo.py
@@ -0,0 +1,19 @@
+import datetime
+import json
+from bson import ObjectId
+from mongoengine.base import BaseDocument
+
+
+class MongoEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, BaseDocument):
+            return str(obj.id)
+        elif isinstance(obj, ObjectId):
+            return str(obj)
+        elif isinstance(obj, datetime.datetime):
+            return obj.strftime('%Y-%m-%d %H:%M:%S')
+        return json.JSONEncoder.default(self, obj)
+
+
+def encode_mongo_document(obj):
+    return MongoEncoder().encode(obj)
diff --git a/webspot/utils/time.py b/webspot/utils/time.py
index 9ef5140..dd1d5e9 100644
--- a/webspot/utils/time.py
+++ b/webspot/utils/time.py
@@ -1,15 +1,25 @@
+import os
 from functools import wraps
 import time
 
+from webspot.logging import get_logger
+
+logger = get_logger('webspot.utils.time')
+
+is_debug = os.environ.get('WEBSPOT_DEBUG', 'false').lower() == 'true'
+
 
 def timeit(func):
     @wraps(func)
     def timeit_wrapper(*args, **kwargs):
-        start_time = time.perf_counter()
-        result = func(*args, **kwargs)
-        end_time = time.perf_counter()
-        total_time = (end_time - start_time) * 1e3
-        print(f'[{total_time:.0f}ms] {func.__name__}{args} {kwargs}')
-        return result
+        if is_debug:
+            start_time = time.perf_counter()
+            result = func(*args, **kwargs)
+            end_time = time.perf_counter()
+            total_time = (end_time - start_time) * 1e3
+            logger.debug(f'[{total_time:.0f}ms] {func.__name__}{args} {kwargs}')
+            return result
+        else:
+            return func(*args, **kwargs)
 
     return timeit_wrapper
diff --git a/webspot/web/app.py b/webspot/web/app.py
index e4a86e0..c8c6211 100644
--- a/webspot/web/app.py
+++ b/webspot/web/app.py
@@ -2,9 +2,9 @@
 
 from fastapi import FastAPI
 from fastapi.templating import Jinja2Templates
-from starlette.responses import FileResponse
 from starlette.staticfiles import StaticFiles
 
+from webspot.db.connect import connect as db_connect
 from webspot.web.routes import init_routes
 
 # root path
@@ -13,11 +13,14 @@
 # app instance
 app = FastAPI()
 
+# connect db
+db_connect()
+
 # static files
 app.mount('/static', StaticFiles(directory=os.path.join(root_path, 'static')), name='static')
 
 # set no cache for static files
-if os.environ.get('WEBSERVER_NO_CACHE', 'false').lower() == 'true':
+if os.environ.get('WEBSPOT_WEB_NO_CACHE', 'false').lower() == 'true':
     __import__('webspot.web.middlewares.no_cache')
 
 # templates
diff --git a/webspot/web/index.html b/webspot/web/index.html
index e6293c0..9b95242 100644
--- a/webspot/web/index.html
+++ b/webspot/web/index.html
@@ -4,9 +4,11 @@
   <link href="//unpkg.com/element-plus/dist/index.css" rel="stylesheet"/>
   <link href="//unpkg.com/font-awesome@4.7.0/css/font-awesome.min.css" rel="stylesheet">
   <script src="https://unpkg.com/vue@3/dist/vue.global.js"></script>
+  <script src="https://unpkg.com/vuex@4.1.0/dist//vuex.global.prod.js"></script>
   <script src="//unpkg.com/element-plus"></script>
   <script src="//unpkg.com/@element-plus/icons-vue"></script>
   <script src="https://unpkg.com/axios/dist/axios.min.js"></script>
+  <script src="https://unpkg.com/js-base64@3.7.5/base64.js"></script>
   <link href="/static/img/favicon.svg" rel="icon">
   <link href="/static/css/style.css" rel="stylesheet">
 </head>
@@ -14,9 +16,11 @@
 <div id="app"></div>
 <script type="module">
   import App from './static/js/app.js';
+  import store from './static/js/store/index.js';
 
   const {createApp} = Vue;
   const app = createApp(App);
+  app.use(store);
   app.use(ElementPlus);
   for (const [key, component] of Object.entries(ElementPlusIconsVue)) {
     app.component(key, component);
diff --git a/webspot/web/logging/__init__.py b/webspot/web/logging/__init__.py
new file mode 100644
index 0000000..f7e77c6
--- /dev/null
+++ b/webspot/web/logging/__init__.py
@@ -0,0 +1,3 @@
+from webspot.logging import get_logger
+
+logger = get_logger('webspot.web')
diff --git a/webspot/web/models/__init__.py b/webspot/web/models/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/webspot/web/models/payload/__init__.py b/webspot/web/models/payload/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/webspot/web/models/payload/request.py b/webspot/web/models/payload/request.py
new file mode 100644
index 0000000..9897efc
--- /dev/null
+++ b/webspot/web/models/payload/request.py
@@ -0,0 +1,33 @@
+from typing import Optional, List
+
+from pydantic import BaseModel, Field
+
+from webspot.constants.detector import DETECTOR_PLAIN_LIST, DETECTOR_PAGINATION
+
+
+class RequestPayload(BaseModel):
+    url: str = Field(
+        ...,
+        title='URL',
+        description='The URL to request.',
+    )
+    method: Optional[str] = Field(
+        default='request',
+        title='Request Method',
+        description='The method to use to request the page. (request, rod)',
+    )
+    no_async: Optional[bool] = Field(
+        default=False,
+        title='No Async',
+        description='If true, the request will be run synchronously, and return the result immediately.',
+    )
+    detectors: Optional[List[str]] = Field(
+        default=[DETECTOR_PLAIN_LIST, DETECTOR_PAGINATION],
+        title='Detectors',
+        description='The detectors to run on the page.',
+    )
+    duration: Optional[int] = Field(
+        default=3,
+        title='Duration (seconds)',
+        description='The duration to request the page (method "rod" only)',
+    )
diff --git a/webspot/web/routes/__init__.py b/webspot/web/routes/__init__.py
index 5dec54b..643e143 100644
--- a/webspot/web/routes/__init__.py
+++ b/webspot/web/routes/__init__.py
@@ -1,10 +1,8 @@
 import os
 
-MODULE_PREFIX = 'webspot.web.routes.'
+MODULE_PREFIX = 'webspot.web.routes'
 
 
 def init_routes():
-    for module in os.listdir(os.path.dirname(__file__)):
-        if module == '__init__.py' or module[-3:] != '.py':
-            continue
-        __import__(f'{MODULE_PREFIX}{module[:-3]}', locals(), globals())
+    __import__(f'{MODULE_PREFIX}.index')
+    __import__(f'{MODULE_PREFIX}.api.request')
diff --git a/webspot/web/routes/api/__init__.py b/webspot/web/routes/api/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/webspot/web/routes/api/request.py b/webspot/web/routes/api/request.py
new file mode 100644
index 0000000..1a47eca
--- /dev/null
+++ b/webspot/web/routes/api/request.py
@@ -0,0 +1,154 @@
+import threading
+import traceback
+from datetime import datetime
+from typing import List
+
+from fastapi import Body
+from starlette.responses import HTMLResponse
+
+from webspot.constants.detector import DETECTOR_PLAIN_LIST, DETECTOR_PAGINATION
+from webspot.constants.request_status import REQUEST_STATUS_SUCCESS, REQUEST_STATUS_ERROR
+from webspot.detect.detectors.pagination import PaginationDetector
+from webspot.detect.detectors.plain_list import PlainListDetector
+from webspot.detect.utils.highlight_html import embed_highlight_css
+from webspot.graph.graph_loader import GraphLoader
+from webspot.models.request import Request, RequestOut
+from webspot.request.html_requester import HtmlRequester
+from webspot.web.app import app
+from webspot.web.logging import logger
+from webspot.web.models.payload.request import RequestPayload
+
+
+@app.get('/api/requests')
+async def requests() -> List[RequestOut]:
+    """Get all requests."""
+    docs = Request.objects().order_by('-_id')
+    return [d.to_out() for d in docs]
+
+
+@app.get('/api/requests/{id}')
+async def request(id: str) -> RequestOut:
+    """Get a request."""
+    d = Request.objects(pk=id).first()
+    return d.to_out()
+
+
+@app.get('/api/requests/{id}/html')
+async def request_html(id: str) -> HTMLResponse:
+    """Get a request."""
+    d = Request.objects(pk=id).first()
+    return HTMLResponse(content=embed_highlight_css(d.html_highlighted))
+
+
+@app.put('/api/requests/{id}')
+async def request(id: str) -> RequestOut:
+    """Update a request."""
+    d = Request.objects(pk=id).first()
+    _d = await request.json()
+    d.update(_d)
+    return d.to_out()
+
+
+@app.post('/api/requests')
+async def request(payload: RequestPayload = Body(
+    example={
+        'url': 'https://quotes.toscrape.com',
+        'method': 'request',
+        'no_async': False,
+        'detectors': ['plain_list', 'pagination'],
+    }
+)) -> RequestOut:
+    """Create a request. This is used to generate a new request to detect a web page."""
+    d = Request(
+        url=payload.url,
+        method=payload.method,
+        no_async=payload.no_async,
+        detectors=payload.detectors,
+    )
+    d.save()
+
+    if d.no_async:
+        # run request (sync)
+        d = _run_request(d)
+    else:
+        # run request (async)
+        t = threading.Thread(target=_run_request, args=[d])
+        t.start()
+
+    return d.to_out()
+
+
+def _run_request(d: Request):
+    execution_time = {
+        'html_requester': None,
+        'graph_loader': None,
+        'detectors': {},
+    }
+    try:
+        # html requester
+        tic = datetime.now()
+        html_requester = HtmlRequester(
+            url=d.url,
+            request_method=d.method,
+            request_rod_duration=d.duration,
+        )
+        html_requester.run()
+        execution_time['html_requester'] = round((datetime.now() - tic).total_seconds() * 1000)
+
+        # graph loader
+        tic = datetime.now()
+        graph_loader = GraphLoader(
+            html=html_requester.html_,
+            json_data=html_requester.json_data,
+        )
+        graph_loader.run()
+        execution_time['graph_loader'] = round((datetime.now() - tic).total_seconds() * 1000)
+
+        # run detectors
+        html = html_requester.html
+        for detector_name in d.detectors:
+            # start time
+            tic = datetime.now()
+
+            # detector class
+            if detector_name == DETECTOR_PLAIN_LIST:
+                detector_cls = PlainListDetector
+            elif detector_name == DETECTOR_PAGINATION:
+                detector_cls = PaginationDetector
+            else:
+                raise Exception(f'Invalid detector: {detector_name}')
+
+            # run detector
+            detector = detector_cls(
+                graph_loader=graph_loader,
+                html_requester=html_requester,
+            )
+            detector.run()
+
+            # highlight html
+            html = detector.highlight_html(html)
+
+            # add to results
+            d.results[detector_name] = [r.dict() for r in detector.results]
+
+            # execution time
+            execution_time['detectors'][detector_name] = round((datetime.now() - tic).total_seconds() * 1000)
+
+        # update request
+        d.status = REQUEST_STATUS_SUCCESS
+        d.html = html_requester.html_
+        d.html_highlighted = html
+        d.execution_time = execution_time
+        d.save()
+
+    except Exception as e:
+        err_lines = traceback.format_exception(type(e), e, e.__traceback__)
+        err = ''.join(err_lines)
+        logger.error(err)
+
+        # update request
+        d.status = REQUEST_STATUS_ERROR
+        d.error = err
+        d.save()
+
+    return d
diff --git a/webspot/web/routes/detect.py b/webspot/web/routes/detect.py
deleted file mode 100644
index be150f5..0000000
--- a/webspot/web/routes/detect.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import asyncio
-import logging
-import os
-import traceback
-
-from fastapi import Request, HTTPException
-
-from webspot.detect.detectors.plain_list import PlainListDetector
-from webspot.web.app import app, templates
-from webspot.web.utils.db import save_page_request
-
-
-@app.post('/detect')
-async def detect(payload: dict):
-    url = payload.get('url')
-    if url is None or url == '':
-        raise HTTPException(status_code=400, detail='url is required')
-    method = payload.get('method') or os.environ.get('WEBSPOT_REQUEST_METHOD') or 'request'
-
-    try:
-        # create and run a detector
-        detector = PlainListDetector(
-            url=url,
-            request_method=method,
-        )
-        detector.run()
-    except Exception as e:
-        err_lines = traceback.format_exception(type(e), e, e.__traceback__)
-        err = ''.join(err_lines)
-        logging.error(err)
-        return {
-            'url': url,
-            'error': err,
-        }
-
-    # save to db
-    asyncio.ensure_future(save_page_request(detector, url))
-
-    return {
-        'url': url,
-        'html': detector.html_base64,
-        'results': detector.results_base64,
-    }
diff --git a/webspot/web/routes/root.py b/webspot/web/routes/index.py
similarity index 87%
rename from webspot/web/routes/root.py
rename to webspot/web/routes/index.py
index 588a439..3708b1b 100644
--- a/webspot/web/routes/root.py
+++ b/webspot/web/routes/index.py
@@ -5,7 +5,7 @@
 
 
 @app.get('/', response_class=HTMLResponse)
-async def root(request: Request):
+async def index(request: Request):
     return templates.TemplateResponse('index.html', {
         'request': request,
     })
diff --git a/webspot/web/routes/request.py b/webspot/web/routes/request.py
deleted file mode 100644
index c56d4eb..0000000
--- a/webspot/web/routes/request.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from sqlalchemy import select
-from fastapi import Request
-
-from webspot.db.session import get_session
-from webspot.models.page import PageModel
-from webspot.models.request import RequestModel
-from webspot.web.app import app
-
-
-@app.get('/requests')
-async def requests(request: Request):
-    session = get_session()
-    stmt = select(RequestModel, PageModel) \
-        .join(PageModel, PageModel.id == RequestModel.page_id)
-    res = session.execute(stmt)
-    return {
-        'requests': [{
-            'id': r.RequestModel.id,
-            'page_id': r.PageModel.id,
-            'page_url': r.PageModel.url,
-        } for r in res],
-    }
diff --git a/webspot/web/static/css/highlight.css b/webspot/web/static/css/highlight.css
index 874e0d1..02bd788 100644
--- a/webspot/web/static/css/highlight.css
+++ b/webspot/web/static/css/highlight.css
@@ -1,7 +1,10 @@
 :root {
-  --color-red: #F56C6C;
   --color-green: #67C23A;
   --color-blue: #409EFF;
+  --color-cyan: #5CB3FF;
+  --color-purple: #B37FEB;
+  --color-pink: #F78FB3;
+  --color-red: #F56C6C;
   --color-orange: #E6A23C;
   --color-light-grey: #EEEEEE;
   --color-grey: #AAAAAA;
@@ -28,39 +31,115 @@
   opacity: 0.15;
 }
 
-.webspot-highlight-list-node {
+.webspot-highlight-node {
   position: relative;
   display: block;
 }
 
-.webspot-highlight-list-node::before {
-  z-index: 777;
+.webspot-highlight-node-color__blue::before {
   border: 1px solid var(--color-blue);
 }
 
-.webspot-highlight-list-node:hover::before {
+.webspot-highlight-node-color__blue:hover::before {
   background: var(--color-blue);
 }
 
-.webspot-highlight-item-node::before {
-  z-index: 888;
+.webspot-highlight-node-color__orange::before {
   border: 1px solid var(--color-orange);
 }
 
-.webspot-highlight-item-node:hover::before {
+.webspot-highlight-node-color__orange:hover::before {
   background: var(--color-orange);
 }
 
-.webspot-highlight-item-field-node::before {
-  z-index: 999;
+.webspot-highlight-node-color__green::before {
   border: 1px solid var(--color-green);
 }
 
-.webspot-highlight-item-field-node:hover::before {
-  border: 1px solid var(--color-green);
+.webspot-highlight-node-color__green:hover::before {
   background: var(--color-green);
 }
 
+.webspot-highlight-node-color__cyan::before {
+  border: 1px solid var(--color-cyan);
+}
+
+.webspot-highlight-node-color__cyan:hover::before {
+  background: var(--color-cyan);
+}
+
+.webspot-highlight-node-color__purple::before {
+  border: 1px solid var(--color-purple);
+}
+
+.webspot-highlight-node-color__purple:hover::before {
+  background: var(--color-purple);
+}
+
+.webspot-highlight-node-color__pink::before {
+  border: 1px solid var(--color-pink);
+}
+
+.webspot-highlight-node-color__pink:hover::before {
+  background: var(--color-pink);
+}
+
+.webspot-highlight-node-color__red::before {
+  border: 1px solid var(--color-red);
+}
+
+.webspot-highlight-node-color__red:hover::before {
+  background: var(--color-red);
+}
+
+.webspot-highlight-node-color__grey::before {
+  border: 1px solid var(--color-grey);
+}
+
+.webspot-highlight-node-color__grey:hover::before {
+  background: var(--color-grey);
+}
+
+.webspot-highlight-node-level__1::before {
+  z-index: 100;
+}
+
+.webspot-highlight-node-level__2::before {
+  z-index: 200;
+}
+
+.webspot-highlight-node-level__3::before {
+  z-index: 300;
+}
+
+.webspot-highlight-node-level__4::before {
+  z-index: 400;
+}
+
+.webspot-highlight-node-level__5::before {
+  z-index: 500;
+}
+
+.webspot-highlight-node-level__6::before {
+  z-index: 600;
+}
+
+.webspot-highlight-node-level__7::before {
+  z-index: 700;
+}
+
+.webspot-highlight-node-level__8::before {
+  z-index: 800;
+}
+
+.webspot-highlight-node-level__9::before {
+  z-index: 900;
+}
+
+.webspot-highlight-node-level__10::before {
+  z-index: 1000;
+}
+
 .webspot-highlight-label {
   position: absolute;
   overflow: visible;
diff --git a/webspot/web/static/css/style.css b/webspot/web/static/css/style.css
index 1b575c1..44da54a 100644
--- a/webspot/web/static/css/style.css
+++ b/webspot/web/static/css/style.css
@@ -30,14 +30,23 @@ body {
 }
 
 .top-navbar .logo-wrapper {
-  margin-right: 10px;
+  margin-right: 18px;
   height: 36px;
+  padding: 0 24px;
+  border-right: var(--color-light-grey) 1px solid;
 }
 
 .top-navbar .logo-wrapper img {
   height: 100%;
 }
 
+.top-navbar .new-request-wrapper {
+  padding: 0 18px 0 0;
+}
+
+.top-navbar .config-wrapper {
+}
+
 .main-container {
   position: absolute;
   top: 48px;
@@ -48,9 +57,27 @@ body {
   height: calc(100vh - 48px);
 }
 
+.request-history {
+  max-width: 240px;
+  overflow: auto;
+}
+
+/*.request-history .el-menu {*/
+/*  border-right: none;*/
+/*}*/
+
 .nav-sidebar {
   max-width: 240px;
   overflow: hidden;
+  border-right: 1px solid var(--color-light-grey);
+}
+
+.nav-sidebar .el-tree .el-tree-node {
+  min-height: 56px;
+}
+
+.nav-sidebar .el-tree .el-tree-node__content {
+  min-height: 56px;
 }
 
 .preview-container {
@@ -63,3 +90,32 @@ body {
 iframe {
   border: none;
 }
+
+.el-menu-item:focus {
+  background: inherit;
+}
+
+.el-menu-item.is-active {
+  background: var(--el-menu-hover-bg-color);
+}
+
+.error-message-container {
+  width: calc(100% - 240px);
+  height: 100%;
+}
+
+.error-message-container .el-result__extra {
+  width: 100%;
+  max-height: 640px;
+}
+
+.error-message-container .el-result__extra .error-message {
+  height: 100%;
+  width: 100%;
+  overflow: auto;
+  text-align: left;
+  color: var(--color-red);
+  border: 1px solid var(--color-light-grey);
+  padding: 10px;
+  background: #ffffff;
+}
diff --git a/webspot/web/static/js/app.js b/webspot/web/static/js/app.js
index fdac76e..4946b68 100644
--- a/webspot/web/static/js/app.js
+++ b/webspot/web/static/js/app.js
@@ -1,91 +1,113 @@
-const {ref, computed, watch, onBeforeMount} = Vue;
-import TopNavbar from './components/top-navbar.js';
-import NavSidebar from './components/nav-sidebar.js';
-import PreviewContainer from './components/preview-container.js';
+const {ref, computed, onBeforeMount} = Vue;
+const {useStore} = Vuex;
+
+import RequestHistory from './components/nav/request-history-sidebar.js';
+import TopNavbar from './components/nav/top-navbar.js';
+import NavSidebar from './components/nav/results-sidebar.js';
+import PreviewContainer from './components/layout/preview-container.js';
+import ResultDialog from './components/dialog/result-dialog.js';
 
 export default {
   components: {
+    RequestHistory,
     TopNavbar,
     NavSidebar,
     PreviewContainer,
+    ResultDialog,
   },
-  setup(props) {
-    const url = ref('');
-    const results = ref('');
-    const html = ref('');
-    const error = ref('');
-
-    const htmlString = computed(() => {
-      return atob(props.html);
+  setup() {
+    const store = useStore();
+
+    onBeforeMount(() => {
+      store.dispatch(`getRequests`);
+
+      setInterval(() => {
+        store.dispatch(`getRequests`);
+      }, 5 * 1e3);
     });
 
-    const onUrlChange = (value) => {
-      url.value = value;
+    const isEmpty = computed(() => store.getters['isEmpty']);
+
+    const activeRequestStatus = computed(() => store.getters['activeRequestStatus']);
+
+    const activeRequestFormattedError = computed(() => store.getters['activeRequestFormattedError']);
+
+    const defaultRequestForm = {
+      url: '',
+      no_async: false,
     };
+    const requestForm = ref({...defaultRequestForm});
 
+    const isLoading = ref(false);
     const onSubmit = async () => {
-      const res = await axios.post(`/detect`, {url: url.value});
-      results.value = res.data.results;
-      html.value = res.data.html;
-      error.value = res.data.error;
+      isLoading.value = true;
+      await store.dispatch('postRequest', {...requestForm.value});
+      isLoading.value = false;
+      requestForm.value = {...defaultRequestForm};
     };
 
-    onBeforeMount(async () => {
-      const urlParam = new URL(window.location.href).searchParams.get('url');
-      if (urlParam) {
-        url.value = urlParam;
-        await onSubmit();
-      }
-    });
-
-    const formattedError = computed(() => {
-      if (!props.error) return '';
-      return props.error.split('\n').join('<br>');
-    });
-
     return {
-      url,
-      results,
-      html,
-      error,
-      htmlString,
-      onUrlChange,
+      isEmpty,
+      activeRequestStatus,
+      activeRequestFormattedError,
+      requestForm,
+      isLoading,
       onSubmit,
-      formattedError,
     };
   },
   template: `
-<top-navbar :url="url" @url-change="onUrlChange" @submit="onSubmit"/>
+<top-navbar/>
+
 <div class="main-container">
-  <template v-if="url">
+  <request-history/>
+
+  <!--is-empty-->
+  <template v-if="isEmpty">
+    <el-empty style="width: 100%; padding-bottom: 15%;" description="Please enter the URL to start">
+      <div class="url-wrapper" style="display: flex; width: 640px;">
+        <el-input v-model="requestForm.url" placeholder="Please enter the URL" @keyup.enter="onSubmit"/>
+        <el-button type="primary" @click="onSubmit" style="margin-left: 5px" :loading="isLoading">Submit</el-button>
+      </div>
+    </el-empty>
+  </template>
+  <!--./is-empty-->
+
+  <template v-else>
     <!--error-->
-    <template v-if="error">
-      <el-result icon="error" style="width: 100%" title="An error occurred" sub-title="Please try again">
+    <template v-if="activeRequestStatus === 'error'">
+      <el-result class="error-message-container" icon="error" style="" title="An error occurred" sub-title="Please try again">
         <template #extra>
-          <pre v-html="formattedError" style="text-align: left; color: var(--color-red); border: 1px solid var(--color-light-grey); padding: 10px; background: #ffffff"/>
+          <pre class="error-message" v-html="activeRequestFormattedError"/>
         </template>
       </el-result>
     </template>
     <!--./error-->
 
+    <!--skeleton-->
+    <template v-else-if="activeRequestStatus === 'pending'">
+      <el-skeleton style="padding-top: 2%; flex: 1" animated :throttle="500">
+        <template #template>
+          <div style="display: flex;">
+            <el-skeleton style="width: 240px; margin-left: 2%;" :rows="25" />
+            <el-skeleton style="flex: 1; margin: 0 10%;" :rows="25" />
+          </div>
+        </template>
+      </el-skeleton>
+    </template>
+    <!--./skeleton-->
+
     <!--results-->
-    <template v-else-if="results && html">
-      <nav-sidebar :results="results" @submit="onSubmit" @change="onUrlChange"/>
-      <preview-container :html="html"/>
+    <template v-else-if="activeRequestStatus === 'success'">
+      <nav-sidebar/>
+      <preview-container/>
     </template>
     <!--./results-->
-  </template>
 
-  <!--no-url-->
-  <template v-else>
-    <el-empty style="width: 100%" description="Please enter the URL to start" style="padding-bottom: 15%">
-      <div class="url-wrapper" style="display: flex; width: 640px;">
-        <el-input v-model="url" placeholder="Please enter the URL" @keyup.enter="onSubmit"/>
-        <el-button type="primary" @click="onSubmit" style="margin-left: 5px">Submit</el-button>
-      </div>
-    </el-empty>
+    <!--result dialog-->
+    <result-dialog/>
+    <!--./result dialog-->
+
   </template>
-  <!--./no-url-->
 </div>
 `
 };
diff --git a/webspot/web/static/js/components/dialog/result-dialog.js b/webspot/web/static/js/components/dialog/result-dialog.js
new file mode 100644
index 0000000..0fe1c98
--- /dev/null
+++ b/webspot/web/static/js/components/dialog/result-dialog.js
@@ -0,0 +1,54 @@
+const {ref, computed} = Vue;
+const {useStore} = Vuex;
+
+import PlainListForm from '../form/plain-list-form.js';
+import PaginationForm from '../form/pagination-form.js';
+
+export default {
+  name: 'ResultDialog',
+  components: {
+    PlainListForm,
+    PaginationForm,
+  },
+  setup() {
+    const store = useStore();
+
+    const activeTabName = ref('overview');
+
+
+    const dialogVisible = computed(() => store.state.activeResultDialogVisible);
+
+    const detectorName = computed(() => store.getters['activeResultDetector']);
+
+    const resultName = computed(() => store.getters['activeResultName']);
+
+    const onDialogClose = () => {
+      store.commit('resetActiveResult');
+      store.commit('resetActiveResultTabName');
+      store.commit('setActiveResultDialogVisible', false);
+    };
+
+    return {
+      dialogVisible,
+      detectorName,
+      resultName,
+      onDialogClose,
+      activeTabName,
+    };
+  },
+  template: `
+<el-dialog
+  :title="resultName"
+   v-model="dialogVisible"
+   @close="onDialogClose"
+   width="80%"
+>
+  <plain-list-form
+    v-if="detectorName === 'plain_list'"
+  />
+  <pagination-form
+    v-else-if="detectorName === 'pagination'"
+  />
+</el-dialog>
+`
+};
diff --git a/webspot/web/static/js/components/form/base-form.js b/webspot/web/static/js/components/form/base-form.js
new file mode 100644
index 0000000..f43604b
--- /dev/null
+++ b/webspot/web/static/js/components/form/base-form.js
@@ -0,0 +1,45 @@
+const {ref, computed, watch} = Vue;
+const {useStore} = Vuex;
+const {ElMessage} = ElementPlus;
+
+export const useBaseForm = () => {
+  const store = useStore();
+
+  const activeTabName = ref(store.state.activeResultTabName);
+  watch(() => store.state.activeResultTabName, () => {
+    activeTabName.value = store.state.activeResultTabName;
+  });
+  const onTabChange = (value) => {
+    store.commit('setActiveResultTabName', value);
+  };
+
+  const result = computed(() => store.state.activeResult);
+
+  const selectorType = ref('css');
+
+  const onCopy = async (text) => {
+    await navigator.clipboard.writeText(text);
+    await ElMessage({message: 'Copied to clipboard', duration: 1000});
+  };
+
+
+  return {
+    activeTabName,
+    onTabChange,
+    result,
+    selectorType,
+    onCopy,
+  };
+};
+
+export default {
+  name: 'BaseForm',
+  setup() {
+    return useBaseForm();
+  },
+  template: `
+<el-tabs v-model="activeTabName" @tab-change="onTabChange">
+  <slot></slot>
+</el-tabs>
+`
+};
diff --git a/webspot/web/static/js/components/form/pagination-form.js b/webspot/web/static/js/components/form/pagination-form.js
new file mode 100644
index 0000000..8ffc1b9
--- /dev/null
+++ b/webspot/web/static/js/components/form/pagination-form.js
@@ -0,0 +1,59 @@
+import BaseForm, {useBaseForm} from './base-form.js';
+
+export default {
+  name: 'PaginationForm',
+  components: {
+    BaseForm,
+  },
+  setup() {
+    return {
+      ...useBaseForm(),
+    };
+  },
+  template: `
+<base-form>
+  <el-tab-pane label="Overview" name="overview">
+    <el-form v-if="result" label-width="180px">
+      <el-form-item label="Score">
+        <el-tooltip content="Score (Overall)">
+          <el-tag type="primary" style="margin-right: 5px">
+            <i class="fa fa-star"></i>
+            {{ result.score.toFixed(2) }}
+          </el-tag>
+        </el-tooltip>
+        <el-tooltip content="Sub-Score (URL Path Fragments)">
+          <el-tag type="info" style="margin-right: 5px">
+            <i class="fa fa-at"></i>
+            {{ result.scores.url_path_fragments.toFixed(2) }}
+          </el-tag>
+        </el-tooltip>
+        <el-tooltip content="Sub-Score (Feature Next)">
+          <el-tag type="info" style="margin-right: 5px">
+            <i class="fa fa-caret-right"></i>
+            {{ result.scores.feature_next.toFixed(2) }}
+          </el-tag>
+        </el-tooltip>
+        <el-tooltip content="Sub-Score (Text)">
+          <el-tag type="info" style="margin-right: 5px">
+            <i class="fa fa-file-text"></i>
+            {{ result.scores.text.toFixed(2) }}
+          </el-tag>
+        </el-tooltip>
+      </el-form-item>
+      <el-form-item label="Selector Type">
+        <el-radio-group v-model="selectorType">
+          <el-radio label="css">CSS</el-radio>
+          <el-radio label="xpath" disabled>XPath</el-radio>
+        </el-radio-group>
+      </el-form-item>
+      <el-form-item label="Next">
+        <el-tag type="primary">{{ result.selectors.next.selector }}</el-tag>
+        <el-icon style="margin-left:5px;cursor:pointer;" @click="() => onCopy(result.selectors.next.selector)">
+          <i class="fa fa-paste"></i>
+        </el-icon>
+      </el-form-item>
+    </el-form>
+  </el-tab-pane>
+</base-form>
+`
+};
diff --git a/webspot/web/static/js/components/form/plain-list-form.js b/webspot/web/static/js/components/form/plain-list-form.js
new file mode 100644
index 0000000..1b95d2a
--- /dev/null
+++ b/webspot/web/static/js/components/form/plain-list-form.js
@@ -0,0 +1,171 @@
+const {computed} = Vue;
+
+import BaseForm, {useBaseForm} from './base-form.js';
+
+export default {
+  name: 'PlainListForm',
+  components: {
+    BaseForm,
+  },
+  setup() {
+    const {
+      result,
+    } = useBaseForm();
+
+    const columns = computed(() => {
+      if (!result.value) return [];
+      return result.value.fields.map(f => {
+        const {data} = result.value;
+        // get all the data for each column
+        const arr = data.map(d => d[f.name]).filter(d => d !== undefined);
+        // add the label of each column
+        arr.push(f.name);
+        f.width = Math.min(getMaxLength(arr) + 60, 320);
+        return {
+          ...f,
+          key: f.name,
+          label: f.name,
+          width: f.width,
+        };
+      });
+    });
+
+    const getMaxLength = (arr) => {
+      return arr.reduce((acc, item) => {
+        if (item) {
+          let calcLen = getTextWidth(item);
+          if (acc < calcLen) {
+            acc = calcLen;
+          }
+        }
+        return acc;
+      }, 0);
+    };
+
+    // use the span tag to wrap the content and then calculate the width of the span
+    const getTextWidth = (str) => {
+      let width = 0;
+      let html = document.createElement('span');
+      html.innerText = str;
+      html.className = 'getTextWidth';
+      document.querySelector('body').appendChild(html);
+      width = document.querySelector('.getTextWidth').offsetWidth;
+      document.querySelector('.getTextWidth').remove();
+      return width;
+    };
+
+    return {
+      ...useBaseForm(),
+      columns,
+      getMaxLength,
+      getTextWidth,
+    };
+  },
+  template: `
+<base-form>
+  <el-tab-pane label="Overview" name="overview">
+    <el-form v-if="result" label-width="180px">
+      <el-form-item label="Score">
+        <el-tooltip content="Score (Overall)">
+          <el-tag type="primary" style="margin-right: 5px">
+            <i class="fa fa-star"></i>
+            {{ result.score.toFixed(2) }}
+          </el-tag>
+        </el-tooltip>
+        <el-tooltip content="Sub-Score (Text Richness)">
+          <el-tag type="info" style="margin-right: 5px">
+            <i class="fa fa-font"></i>
+            {{ result.scores.text_richness.toFixed(2) }}
+          </el-tag>
+        </el-tooltip>
+        <el-tooltip content="Sub-Score (Complexity)">
+          <el-tag type="info" style="margin-right: 5px">
+            <i class="fa fa-code-fork"></i>
+            {{ result.scores.complexity.toFixed(2) }}
+          </el-tag>
+        </el-tooltip>
+        <el-tooltip content="Sub-Score (Item Count)">
+          <el-tag type="info" style="margin-right: 5px">
+            <i class="fa fa-list"></i>
+            {{ result.scores.item_count.toFixed(2) }}
+          </el-tag>
+        </el-tooltip>
+      </el-form-item>
+      <el-form-item label="Selector Type">
+        <el-radio-group v-model="selectorType">
+          <el-radio label="css">CSS</el-radio>
+          <el-radio label="xpath" disabled>XPath</el-radio>
+        </el-radio-group>
+      </el-form-item>
+      <el-form-item label="List">
+        <el-tag type="primary">{{ result.selectors.list.selector }}</el-tag>
+        <el-icon style="margin-left:5px;cursor:pointer;" @click="() => onCopy(result.selectors.list.selector)">
+          <i class="fa fa-paste"></i>
+        </el-icon>
+      </el-form-item>
+      <el-form-item label="Items">
+        <el-tag type="primary">{{ result.selectors.items.selector }}</el-tag>
+        <el-icon style="margin-left:5px;cursor:pointer;" @click="() => onCopy(result.selectors.items.selector)">
+          <i class="fa fa-paste"></i>
+        </el-icon>
+      </el-form-item>
+      <el-form-item label="Items (Full Path)">
+        <el-tag type="primary">{{ result.selectors.full_items.selector }}</el-tag>
+        <el-icon style="margin-left:5px;cursor:pointer;" @click="() => onCopy(result.selectors.full_items.selector)">
+          <i class="fa fa-paste"></i>
+        </el-icon>
+      </el-form-item>
+    </el-form>
+  </el-tab-pane>
+
+  <el-tab-pane label="Fields" name="fields">
+    <el-table v-if="result" :data="result.fields" :cell-style="{padding: '5px 10px'}" border>
+      <el-table-column prop="name" label="Name" :width="200"/>
+      <el-table-column prop="selector" label="Selector" :width="auto">
+        <template #default="scope">
+          <el-tag type="primary">{{ scope.row.selector }}</el-tag>
+          <el-icon style="margin-left:5px;cursor:pointer;" @click="() => onCopy(scope.row.selector)">
+            <i class="fa fa-paste"></i>
+          </el-icon>
+        </template>
+      </el-table-column>
+    </el-table>
+  </el-tab-pane>
+
+  <el-tab-pane label="Data" name="data">
+    <el-table v-if="result" :data="result.data" :cell-style="{padding: '5px 10px'}" border>
+      <el-table-column
+        v-for="column in columns"
+        :key="column.key"
+        :prop="column.key"
+        :label="column.label"
+        :width="column.width"
+      >
+        <template #header="scope">
+          <span>{{ scope.column.label }}</span>
+          <el-tooltip :content="result.fields[scope.column.no].selector">
+            <el-icon style="margin-left: 5px; cursor: pointer;" @click="() => onCopy(result.fields[scope.column.no].selector)">
+              <i class="fa fa-info-circle"></i>
+            </el-icon>
+          </el-tooltip>
+        </template>
+        <template #default="scope">
+          <template v-if="column.type === 'text'">
+            {{ scope.row[column.key] }}
+          </template>
+          <template v-else-if="column.type === 'link_url'">
+            <a :href="scope.row[column.key]" target="_blank">{{ scope.row[column.key] }}</a>
+          </template>
+          <template v-else-if="column.type === 'image_url'">
+            <img :src="scope.row[column.key]" style="max-width: 100px; max-height: 100px;" />
+          </template>
+          <template v-else>
+            {{ scope.row[column.key] }}
+          </template>
+        </template>
+      </el-table-column>
+    </el-table>
+  </el-tab-pane>
+</base-form>
+`
+};
diff --git a/webspot/web/static/js/components/layout/preview-container.js b/webspot/web/static/js/components/layout/preview-container.js
new file mode 100644
index 0000000..a794255
--- /dev/null
+++ b/webspot/web/static/js/components/layout/preview-container.js
@@ -0,0 +1,35 @@
+const {ref, computed, watch, onMounted} = Vue;
+const {useStore} = Vuex;
+
+export default {
+  name: 'PreviewContainer',
+  setup() {
+    const store = useStore();
+
+    const activeRequestId = computed(() => store.state.activeRequestId);
+
+    // const activeRequestHtmlHighlighted = computed(() => convertToBase64(store.getters.activeRequestHtmlHighlighted));
+    const activeRequestHtmlSrc = computed(() => `/api/requests/${activeRequestId.value}/html`);
+
+    const isLoading = ref(false);
+
+    const invokeLoading = () => {
+      isLoading.value = true;
+      setTimeout(() => {
+        isLoading.value = false;
+      }, 1000);
+    };
+
+    watch(() => activeRequestId.value, invokeLoading);
+    onMounted(invokeLoading);
+
+    return {
+      activeRequestHtmlSrc,
+      isLoading,
+    };
+  },
+  template: `<div class="preview-container">
+  <el-skeleton v-if="isLoading" style="height: 100%; width: 100%; padding: 2%" :rows="25"/>
+  <iframe v-else id="iframe" width="100%" height="100%" :src="activeRequestHtmlSrc"></iframe>
+</div>`
+};
diff --git a/webspot/web/static/js/components/list-dialog.js b/webspot/web/static/js/components/list-dialog.js
deleted file mode 100644
index 845ca4c..0000000
--- a/webspot/web/static/js/components/list-dialog.js
+++ /dev/null
@@ -1,131 +0,0 @@
-const {ref, computed, watch} = Vue;
-const {ElMessage} = ElementPlus;
-
-export default {
-  name: 'ListDialog',
-  props: {
-    visible: Boolean,
-    result: {
-      type: Object,
-      default: () => {
-        return {};
-      },
-    },
-  },
-  setup(props, {emit}) {
-    const activeTabName = ref('overview');
-    const columns = computed(() => {
-      if (!props.result) return [];
-      return props.result.extract_rules_css.fields.map(f => {
-        return {
-          key: f.name,
-          label: f.name,
-          ...f,
-        };
-      });
-    });
-    const dialogVisible = ref(false);
-    watch(() => props.visible, () => {
-      dialogVisible.value = props.visible;
-    });
-    const onDialogClose = () => {
-      emit('close');
-    };
-    const selectorType = ref('css');
-
-    const onCopy = async (text) => {
-      await navigator.clipboard.writeText(text);
-      await ElMessage({message: 'Copied to clipboard', duration: 1000});
-    };
-
-    return {
-      dialogVisible,
-      onDialogClose,
-      activeTabName,
-      columns,
-      selectorType,
-      onCopy,
-    };
-  },
-  template: `
-<el-dialog :title="result.name" v-model="dialogVisible" @close="onDialogClose" width="80%">
-  <el-tabs v-model="activeTabName">
-    <el-tab-pane label="Overview" name="overview">
-      <el-form v-if="result" label-width="180px">
-        <el-form-item label="Score">
-          <el-tooltip content="Score (Overall)">
-            <el-tag type="primary" style="margin-right: 5px">
-              <i class="fa fa-star"></i>
-              {{ result.stats.score.toFixed(2) }}
-            </el-tag>
-          </el-tooltip>
-          <el-tooltip content="Sub-Score (Text Richness)">
-            <el-tag type="info" style="margin-right: 5px">
-              <i class="fa fa-font"></i>
-              {{ result.stats.scores.text_richness.toFixed(2) }}
-            </el-tag>
-          </el-tooltip>
-          <el-tooltip content="Sub-Score (Complexity)">
-            <el-tag type="info" style="margin-right: 5px">
-              <i class="fa fa-code-fork"></i>
-              {{ result.stats.scores.complexity.toFixed(2) }}
-            </el-tag>
-          </el-tooltip>
-          <el-tooltip content="Sub-Score (Item Count)">
-            <el-tag type="info" style="margin-right: 5px">
-              <i class="fa fa-list"></i>
-              {{ result.stats.scores.item_count.toFixed(2) }}
-            </el-tag>
-          </el-tooltip>
-        </el-form-item>
-        <el-form-item label="Selector Type">
-          <el-radio-group v-model="selectorType">
-            <el-radio label="css">CSS</el-radio>
-            <el-radio label="xpath" disabled>XPath</el-radio>
-          </el-radio-group>
-        </el-form-item>
-        <el-form-item label="List">
-          <el-tag type="primary">{{ result.extract_rules_css.list }}</el-tag>
-          <el-icon style="margin-left:5px;cursor:pointer;" @click="() => onCopy(result.extract_rules_css.list)">
-            <i class="fa fa-paste"></i>
-          </el-icon>
-        </el-form-item>
-        <el-form-item label="Items">
-          <el-tag type="primary">{{ result.extract_rules_css.items }}</el-tag>
-          <el-icon style="margin-left:5px;cursor:pointer;" @click="() => onCopy(result.extract_rules_css.items)">
-            <i class="fa fa-paste"></i>
-          </el-icon>
-        </el-form-item>
-        <el-form-item label="Items (Full Path)">
-          <el-tag type="primary">{{ result.extract_rules_css.items_full }}</el-tag>
-          <el-icon style="margin-left:5px;cursor:pointer;" @click="() => onCopy(result.extract_rules_css.items_full)">
-            <i class="fa fa-paste"></i>
-          </el-icon>
-        </el-form-item>
-      </el-form>
-    </el-tab-pane>
-
-    <el-tab-pane label="Data" name="data">
-      <el-table v-if="result" :data="result.extract_rules_css.data" :cell-style="{padding: '5px 10px'}" border>
-        <el-table-column v-for="column in columns" :key="column.key" :prop="column.key" :label="column.label">
-          <template #default="scope">
-            <template v-if="column.type === 'text'">
-              {{ scope.row[column.key] }}
-            </template>
-            <template v-else-if="column.type === 'link_url'">
-              <a :href="scope.row[column.key]" target="_blank">{{ scope.row[column.key] }}</a>
-            </template>
-            <template v-else-if="column.type === 'image_url'">
-              <img :src="scope.row[column.key]" style="max-width: 100px; max-height: 100px;" />
-            </template>
-            <template v-else>
-              {{ scope.row[column.key] }}
-            </template>
-          </template>
-        </el-table-column>
-      </el-table>
-    </el-tab-pane>
-  </el-tabs>
-</el-dialog>
-`
-};
diff --git a/webspot/web/static/js/components/nav-sidebar.js b/webspot/web/static/js/components/nav-sidebar.js
deleted file mode 100644
index 0e6b469..0000000
--- a/webspot/web/static/js/components/nav-sidebar.js
+++ /dev/null
@@ -1,74 +0,0 @@
-import ListDialog from './list-dialog.js';
-
-const {ref, computed, onBeforeMount} = Vue;
-
-export default {
-  name: 'NavSidebar',
-  components: {
-    ListDialog,
-  },
-  props: {
-    results: String,
-  },
-  setup(props, {emit}) {
-    const resultsArray = computed(() => {
-      const resultsDecoded = window.atob(props.results);
-      return JSON.parse(resultsDecoded);
-    });
-
-    const isCollapse = ref(false);
-    const onToggle = () => {
-      isCollapse.value = !isCollapse.value;
-    };
-
-    const onClickList = (result) => {
-      activeResult.value = result;
-      dialogVisible.value = true;
-    };
-
-    const dialogVisible = ref(false);
-    const onDialogClose = () => {
-      dialogVisible.value = false;
-    };
-
-    const activeResult = ref({});
-
-    onBeforeMount(() => {
-      console.debug(resultsArray.value);
-    });
-
-    return {
-      resultsArray,
-      isCollapse,
-      onToggle,
-      onClickList,
-      dialogVisible,
-      onDialogClose,
-      activeResult,
-    };
-  },
-  template: `<div class="nav-sidebar" :style="{flexBasis: isCollapse ? 'auto' : '240px'}">
-  <el-menu :collapse="isCollapse" style="height: 100%">
-    <el-menu-item v-for="(result, $index) in resultsArray" :key="$index" :index="$index" @click="() => onClickList(result)">
-      <el-icon>
-        <i class="fa fa-circle-o"></i>
-      </el-icon>
-      <span>{{ result.name }}</span>
-      <el-tag type="primary" style="margin-left: 10px;">
-        <span class="score">{{ result.stats.score.toFixed(2) }}</span>
-        <span class="count"> ({{ result.nodes.items.length }})</span>
-      </el-tag>
-    </el-menu-item>
-    <el-menu-item style="border-top: 1px solid #CCCCCC; position: absolute; bottom: 0; width: 100%" @click="onToggle">
-      <el-icon>
-        <DArrowRight v-if="isCollapse"/>
-        <DArrowLeft v-else/>
-      </el-icon>
-      <span>{{ isCollapse ? 'Expand' : 'Collapse' }}</span>
-    </el-menu-item>
-  </el-menu>
-
-  <list-dialog :visible="dialogVisible" :result="activeResult" @close="onDialogClose"/>
-</div>
-`
-};
diff --git a/webspot/web/static/js/components/nav/request-history-sidebar.js b/webspot/web/static/js/components/nav/request-history-sidebar.js
new file mode 100644
index 0000000..7bf8c45
--- /dev/null
+++ b/webspot/web/static/js/components/nav/request-history-sidebar.js
@@ -0,0 +1,61 @@
+const {ref, computed} = Vue;
+const {useStore} = Vuex;
+const {ElMessageBox} = ElementPlus;
+
+export default {
+  setup() {
+    const store = useStore();
+
+    const requests = computed(() => store.state.requests);
+
+    const isCollapsed = ref(false);
+
+    const getRequestDataByStatus = (request) => {
+      switch (request.status) {
+        case 'pending':
+          return {
+            icon: 'fa fa-spinner fa-spin',
+            color: 'var(--color-orange)',
+          };
+        case 'success':
+          return {
+            icon: 'fa fa-check',
+            color: 'var(--color-green)',
+          };
+        case 'error':
+          return {
+            icon: 'fa fa-times',
+            color: 'var(--color-red)',
+          };
+      }
+    };
+
+    const activeRequestId = computed(() => store.state.activeRequestId);
+
+    const onClickRequest = (request) => {
+      store.commit('setActiveRequestId', request.id);
+    };
+
+    return {
+      requests,
+      isCollapsed,
+      getRequestDataByStatus,
+      activeRequestId,
+      onClickRequest,
+    };
+  },
+  template: `
+<div class="request-history" style="flex: 0 0 240px">
+  <el-menu :default-active="activeRequestId" v-model="" style="height: 100%">
+    <el-menu-item v-for="(request, $index) in requests" :key="$index" :index="request.id" @click="() => onClickRequest(request)">
+      <el-icon size="12">
+        <i :class="getRequestDataByStatus(request).icon" :style="{color: getRequestDataByStatus(request).color}"></i>
+      </el-icon>
+      <el-tooltip :content="'URL: ' + request.url" placement="right">
+        <span>{{ request.url }}</span>
+      </el-tooltip>
+    </el-menu-item>
+  </el-menu>
+</div>
+`
+};
diff --git a/webspot/web/static/js/components/nav/results-sidebar.js b/webspot/web/static/js/components/nav/results-sidebar.js
new file mode 100644
index 0000000..e4668c8
--- /dev/null
+++ b/webspot/web/static/js/components/nav/results-sidebar.js
@@ -0,0 +1,99 @@
+import PlainListDialog from '../form/plain-list-form.js';
+import PaginationDialog from '../form/pagination-form.js';
+
+const {ref, computed} = Vue;
+const {useStore} = Vuex;
+
+export default {
+  name: 'NavSidebar',
+  components: {
+    PlainListDialog,
+    PaginationDialog,
+  },
+  setup() {
+    const store = useStore();
+
+    const activeRequestResults = computed(() => store.getters['activeRequestResults']);
+
+    const isCollapsed = ref(false);
+    const onToggle = () => {
+      isCollapsed.value = !isCollapsed.value;
+    };
+
+    const onClickNode = (result) => {
+      if (result.children) return;
+      store.commit('setActiveResult', result);
+      store.commit('setActiveResultDialogVisible', true);
+    };
+
+    const resultsTree = computed(() => {
+      const res = Object.keys(activeRequestResults.value).map((key) => {
+        const value = activeRequestResults.value[key];
+        if (Array.isArray(value)) {
+          const label = key;
+          const children = value.map((item) => {
+            return {
+              label: item.name,
+              ...item,
+            };
+          });
+          return {
+            label,
+            children,
+          };
+        } else {
+          return {
+            label: key,
+            ...value,
+          };
+        }
+      });
+      return res;
+    });
+
+    const getIcon = (data) => {
+      if (data.children) {
+        switch (data.label) {
+          case 'pagination':
+            return 'fa fa-compass';
+          case 'plain_list':
+            return 'fa fa-list';
+        }
+      } else {
+        return 'fa fa-circle-o';
+      }
+    };
+
+    return {
+      activeRequestResults,
+      isCollapsed,
+      onToggle,
+      onClickNode,
+      resultsTree,
+      getIcon,
+    };
+  },
+  template: `<div class="nav-sidebar" :style="{flexBasis: isCollapsed ? 'auto' : '240px'}">
+  <h2 style="padding: 0 12px; height: 56px; margin: 0; display: flex; align-items: center">Detected Results</h2>
+  <el-tree :data="resultsTree" :props="{class:'item-node'}" default-expand-all @node-click="onClickNode">
+    <template v-slot="{node, data}">
+      <div
+       style="height: 100%; display: flex; align-items: center"
+        :style="{fontSize: data.children ? '16px' : '14px', fontWeight: data.children ? 'bolder' : 'normal'}"
+      >
+        <el-icon size="16px">
+          <i :class="getIcon(data)"></i>
+        </el-icon>
+        <span style="margin-left: 8px">
+          {{ node.label }}
+        </span>
+        <span v-if="data.children" style="margin-left: 3px">({{ data.children.length }})</span>
+        <el-tag v-if="data.score !== undefined" type="primary" style="margin-left: 10px;">
+          <span class="score">{{ data.score.toFixed(2) }}</span>
+        </el-tag>
+      </div>
+    </template>
+  </el-tree>
+</div>
+`
+};
diff --git a/webspot/web/static/js/components/nav/top-navbar.js b/webspot/web/static/js/components/nav/top-navbar.js
new file mode 100644
index 0000000..ab5c24c
--- /dev/null
+++ b/webspot/web/static/js/components/nav/top-navbar.js
@@ -0,0 +1,63 @@
+const {ref, watch, onBeforeMount, h} = Vue;
+const {useStore} = Vuex;
+const {ElMessageBox} = ElementPlus;
+
+export default {
+  name: 'TopNavbar',
+  setup() {
+    const store = useStore();
+
+    const requestForm = ref({});
+
+    const setRequestForm = () => {
+      requestForm.value = {...store.getters['requestForm']};
+    };
+
+    onBeforeMount(setRequestForm);
+    watch(() => JSON.stringify(store.getters['requestForm']), setRequestForm);
+
+    const updateRequestForm = () => {
+      store.commit('setRequestForm', {...requestForm.value});
+    };
+
+    const onClickNewRequest = async () => {
+      const res = await ElMessageBox.prompt('Please enter the URL to request', 'Request', {
+        type: 'info',
+        icon: h('i', {class: 'fa fa-paper-plane', style: 'font-size: 16px'}),
+        customStyle: {
+          maxWidth: '640px',
+        },
+        inputPlaceholder: 'Request URL',
+        confirmButtonText: 'Submit',
+      });
+      await store.dispatch('postRequest', {url: res.value});
+    };
+
+    return {
+      requestForm,
+      updateRequestForm,
+      onClickNewRequest,
+    };
+  },
+  template: `<div class="top-navbar">
+  <div class="logo-wrapper">
+    <img src="/static/img/logo.svg">
+  </div>
+  <div class="new-request-wrapper">
+    <el-button type="primary" icon="plus" @click="onClickNewRequest">
+      New Request
+    </el-button>
+  </div>
+  <div class="config-wrapper">
+    <el-tooltip content="Request Method" placement="bottom">
+      <el-icon size="12px" style="margin-right: 12px">
+        <i class="fa fa-paper-plane"></i>
+      </el-icon>
+    </el-tooltip>
+    <el-select v-model="requestForm.method" @change="updateRequestForm" style="width: 150px">
+      <el-option label="Request" value="request"/>
+      <el-option label="Rod (Browser)" value="rod"/>
+    </el-select>
+  </div>
+</div>`
+};
diff --git a/webspot/web/static/js/components/preview-container.js b/webspot/web/static/js/components/preview-container.js
deleted file mode 100644
index a162e9f..0000000
--- a/webspot/web/static/js/components/preview-container.js
+++ /dev/null
@@ -1,9 +0,0 @@
-export default {
-  name: 'PreviewContainer',
-  props: {
-    html: String,
-  },
-  template: `<div class="preview-container">
-  <iframe id="iframe" width="100%" height="100%" :src="'data:text/html;base64,' + html"></iframe>
-</div>`
-};
diff --git a/webspot/web/static/js/components/top-navbar.js b/webspot/web/static/js/components/top-navbar.js
deleted file mode 100644
index c505ce7..0000000
--- a/webspot/web/static/js/components/top-navbar.js
+++ /dev/null
@@ -1,33 +0,0 @@
-const {ref, watch} = Vue;
-
-export default {
-  name: 'TopNavbar',
-  props: {
-    url: String,
-  },
-  setup(props, {emit}) {
-    const internalUrl = ref(props.url);
-    watch(() => props.url, () => {
-      if (props.url) internalUrl.value = props.url;
-    });
-    const onUrlChange = () => emit('url-change', internalUrl.value);
-    watch(() => internalUrl.value, onUrlChange);
-    const onSubmit = () => {
-      emit('submit');
-    };
-    return {
-      internalUrl,
-      onUrlChange,
-      onSubmit,
-    };
-  },
-  template: `<div class="top-navbar">
-  <div class="logo-wrapper">
-    <img src="/static/img/logo.svg">
-  </div>
-  <div class="url-wrapper" style="display: flex; flex: 0 0 360px;">
-    <el-input id="url" type="text" placeholder="Please enter the URL" v-model="internalUrl" @keyup.enter="onSubmit"/>
-    <el-button type="primary" @click="onSubmit" style="margin-left: 5px">Submit</el-button>
-  </div>
-</div>`
-};
diff --git a/webspot/web/static/js/store/index.js b/webspot/web/static/js/store/index.js
new file mode 100644
index 0000000..31f2a13
--- /dev/null
+++ b/webspot/web/static/js/store/index.js
@@ -0,0 +1,135 @@
+const {h} = Vue;
+const {createStore} = Vuex;
+const {ElNotification} = ElementPlus;
+
+const defaultRequestForm = {
+  method: 'request',
+  no_async: false,
+};
+
+const store = createStore({
+  state() {
+    return {
+      requests: [],
+      activeRequestId: undefined,
+      requestForm: undefined,
+      activeResult: undefined,
+      activeResultTabName: 'overview',
+      activeResultDialogVisible: false,
+    };
+  },
+  getters: {
+    isEmpty(state) {
+      return state.requests.length === 0;
+    },
+    activeRequest(state) {
+      return state.requests.find((request) => request.id === state.activeRequestId);
+    },
+    activeRequestStatus(state, getters) {
+      return getters.activeRequest ? getters.activeRequest.status : '';
+    },
+    activeRequestResults(state, getters) {
+      return getters.activeRequest ? getters.activeRequest.results : [];
+    },
+    activeRequestHtml(state, getters) {
+      return getters.activeRequest ? getters.activeRequest.html : '';
+    },
+    activeRequestHtmlHighlighted(state, getters) {
+      return getters.activeRequest ? getters.activeRequest.html_highlighted : '';
+    },
+    activeRequestFormattedError(state, getters) {
+      if (!getters.activeRequest) return '';
+      if (!getters.activeRequest.error) return '';
+      return getters.activeRequest.error.split('\n').join('<br>');
+    },
+    requestForm(state) {
+      if (state.requestForm) {
+        return {...state.requestForm};
+      }
+      if (localStorage.getItem('requestForm')) {
+        try {
+          return JSON.parse(localStorage.getItem('requestForm'));
+        } catch {
+        }
+      }
+      localStorage.setItem('requestForm', JSON.stringify(defaultRequestForm));
+      state.requestForm = {...defaultRequestForm};
+      return {...defaultRequestForm};
+    },
+    activeResultDetector(state) {
+      return state.activeResult ? state.activeResult.detector : '';
+    },
+    activeResultName(state) {
+      return state.activeResult ? state.activeResult.name : '';
+    },
+  },
+  mutations: {
+    setRequests(state, requests) {
+      state.requests = requests;
+    },
+    setActiveRequestId(state, id) {
+      state.activeRequestId = id;
+    },
+    setRequestForm(state, requestForm) {
+      state.requestForm = {
+        ...requestForm,
+      };
+      localStorage.setItem('requestForm', JSON.stringify({
+        ...requestForm,
+      }));
+    },
+    resetRequestForm(state) {
+      localStorage.setItem('requestForm', JSON.stringify(defaultRequestForm));
+      return {...defaultRequestForm};
+    },
+    setActiveResult(state, result) {
+      state.activeResult = result;
+    },
+    resetActiveResult(state) {
+      state.activeResult = undefined;
+    },
+    setActiveResultTabName(state, tabName) {
+      state.activeResultTabName = tabName;
+    },
+    resetActiveResultTabName(state) {
+      state.activeResultTabName = 'overview';
+    },
+    setActiveResultDialogVisible(state, visible) {
+      state.activeResultDialogVisible = visible;
+    }
+  },
+  actions: {
+    async getRequests({commit, state}) {
+      const res = await axios.get(`/api/requests`);
+      if (JSON.stringify(res.data) === JSON.stringify(state.requests)) {
+        return;
+      }
+      commit('setRequests', res.data);
+
+      // Set active request id if it's not set yet
+      if (state.activeRequestId === undefined && res.data.length > 0) {
+        commit('setActiveRequestId', res.data[0].id);
+      }
+    },
+    async postRequest({getters, commit, dispatch}, request) {
+      const res = await axios.post(`/api/requests`, {
+        ...getters.requestForm,
+        ...request,
+      });
+      await dispatch('getRequests');
+      await ElNotification({
+        title: 'Triggered request',
+        message: h('span', [
+          h('label', {}, ['URL: ']),
+          h('a', {href: `${request.url}`, target: '_blank'}, [request.url]),
+        ]),
+        type: 'info',
+      });
+
+      // Set result id as active
+      commit('setActiveRequestId', res.data.id);
+    },
+  }
+});
+
+export default store;
diff --git a/webspot/web/static/js/utils/base64.js b/webspot/web/static/js/utils/base64.js
new file mode 100644
index 0000000..4fb2bb6
--- /dev/null
+++ b/webspot/web/static/js/utils/base64.js
@@ -0,0 +1,5 @@
+const {btoa, encode} = Base64;
+
+export const convertToBase64 = (htmlContent) => {
+  return encode(htmlContent);
+};
diff --git a/webspot/web/utils/db.py b/webspot/web/utils/db.py
deleted file mode 100644
index 380e4b1..0000000
--- a/webspot/web/utils/db.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import sqlalchemy
-from sqlalchemy.exc import NoResultFound
-
-from webspot.db.session import get_session
-from webspot.detect.detectors.plain_list import PlainListDetector
-from webspot.models.page import PageModel
-from webspot.models.request import RequestModel
-
-
-async def save_page_request(detector: PlainListDetector, url: str):
-    session = get_session()
-    p = session.query(PageModel).filter(PageModel.url == url).one()
-    if p is None:
-        p = PageModel(url=url)
-        session.add(p)
-    session.execute(sqlalchemy.insert(RequestModel).values(
-        page_id=p.id,
-        html=detector.html_base64,
-        results=detector.results_base64,
-    ))
-    session.commit()
diff --git a/webspot_rod/conf/sshd_config b/webspot_rod/conf/sshd_config
new file mode 100644
index 0000000..27e95ec
--- /dev/null
+++ b/webspot_rod/conf/sshd_config
@@ -0,0 +1,12 @@
+Port 			2222
+ListenAddress 		0.0.0.0
+LoginGraceTime 		180
+X11Forwarding 		yes
+Ciphers aes128-cbc,3des-cbc,aes256-cbc,aes128-ctr,aes192-ctr,aes256-ctr
+MACs hmac-sha1,hmac-sha1-96
+StrictModes 		yes
+SyslogFacility 		DAEMON
+PasswordAuthentication 	yes
+PermitEmptyPasswords 	no
+PermitRootLogin 	yes
+Subsystem sftp internal-sftp
diff --git a/webspot_rod/conf/supervisord.conf b/webspot_rod/conf/supervisord.conf
new file mode 100644
index 0000000..a3df881
--- /dev/null
+++ b/webspot_rod/conf/supervisord.conf
@@ -0,0 +1,17 @@
+[supervisord]
+nodaemon=true
+
+[program:webspot_web]
+command=python main.py web
+autostart=true
+autorestart=true
+stdout_logfile=/var/log/webspot_web.stdout.log
+stderr_logfile=/var/log/webspot_web.stderr.log
+
+
+[program:webspot_rod]
+command=/go/bin/webspot_rod api
+autostart=true
+autorestart=true
+stdout_logfile=/var/log/webspot_rod.stdout.log
+stderr_logfile=/var/log/webspot_rod.stderr.log


Test patch:
diff --git a/webspot/test/detect/__init__.py b/webspot/test/detect/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/webspot/test/detect/test_plain_list.py b/webspot/test/detect/test_plain_list.py
new file mode 100644
index 0000000..a79c7bb
--- /dev/null
+++ b/webspot/test/detect/test_plain_list.py
@@ -0,0 +1,103 @@
+import itertools
+import unittest
+
+import pytest
+
+from webspot.detect.detectors.plain_list import PlainListDetector, run_plain_list_detector
+from webspot.graph.graph_loader import GraphLoader
+from webspot.logging import get_logger
+from webspot.request.html_requester import HtmlRequester
+
+logger = get_logger('webspot.test.detect.test_plain_list')
+
+test_cases = [
+    {
+        'url': 'https://quotes.toscrape.com',
+        'result': {
+            'selectors': {
+                'list': 'body > div.container > div.row:last-child > div.col-md-8'
+            }
+        }
+    },
+    {
+        'url': 'https://books.toscrape.com',
+        'result': {
+            'selectors': {
+                'list': 'section > div:last-child > ol.row'
+            }
+        },
+    },
+    {
+        'url': 'https://github.com/trending',
+        'result': {
+            'selectors': {
+                'list': 'main > div.position-relative.container-lg.p-responsive.pt-6 > div.Box > div:last-child'
+            }
+        },
+    },
+    {
+        'url': 'https://github.com/search?q=spider',
+        'result': {
+            'selectors': {
+                'list': '.pt-md-0 > div.px-2 > ul.repo-list'
+            }
+        }
+    },
+    {
+        'url': 'http://bang.dangdang.com/books/newhotsales',
+        'result': {
+            'selectors': {
+                'list': '.bang_list_box > ul.bang_list.clearfix.bang_list_mode'
+            }
+        }
+    },
+    {
+        'url': 'https://cuiqingcai.com/archives/',
+        'result': {
+            'selectors': {
+                'list': '.post-block > div.posts-collapse'
+            }
+        }
+    },
+    # {
+    #     'url': 'https://github.com/crawlab-team/crawlab/actions',
+    #     'result': {
+    #         'selectors': {
+    #             'list': '#partial-actions-workflow-runs'
+    #         }
+    #     }
+    # },
+    # {
+    #     'url': 'https://cuiqingcai.com',
+    #     'result': {
+    #         'selectors': {
+    #             'list': '.content-wrap > div.content.index.posts-expand'
+    #         }
+    #     }
+    # },
+]
+
+
+# test_cases = [test_cases[-1]]
+
+
+@pytest.mark.parametrize('test_case', test_cases)
+def test_plain_list(test_case):
+    url = test_case.get('url')
+    method = test_case.get('method') or 'request'
+    target_index = test_case.get('target_index') or 0
+
+    logger.info(f'url: {url}')
+    logger.info(f'method: {method}')
+
+    plain_list_detector = run_plain_list_detector(url, method)
+
+    results = plain_list_detector.results
+    assert len(results) > 0, 'results should be more than 0'
+
+    test_case_result = test_case.get('result')
+    test_case_list_selector = test_case_result.get('selectors').get('list')
+    target_result = results[target_index]
+    assert target_result.selectors.get('list').selector == test_case_list_selector, 'selectors should be equal'
+    assert len(target_result.fields) > 0, 'target fields should be more than 0'
+    assert len(target_result.data) > 0, 'target data should be more than 0'
diff --git a/webspot/test/logging/__init__.py b/webspot/test/logging/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/webspot/test/logging/test_get_logger.py b/webspot/test/logging/test_get_logger.py
new file mode 100644
index 0000000..547949c
--- /dev/null
+++ b/webspot/test/logging/test_get_logger.py
@@ -0,0 +1,24 @@
+import unittest
+from io import StringIO
+from unittest.mock import patch
+
+from webspot.logging import get_logger
+
+
+class TestGetLogger(unittest.TestCase):
+    # @patch('sys.stdout', new_callable=StringIO)
+    @patch('sys.stderr', new_callable=StringIO)
+    def test_get_logger(self, stderr):
+        logger = get_logger('web.utils.time')
+        text = 'it works'
+        logger.info(text)
+        # print(stdout.getvalue())
+        # a = stdout.getvalue()
+        # print(stderr.getvalue())
+        # b = stderr.getvalue()
+        self.assertTrue(text in stderr.getvalue())
+        # self.assertTrue(text in stdout.getvalue())
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/webspot/test/web/routes/__init__.py b/webspot/test/web/routes/__init__.py
index e69de29..f07a754 100644
--- a/webspot/test/web/routes/__init__.py
+++ b/webspot/test/web/routes/__init__.py
@@ -0,0 +1,5 @@
+from fastapi.testclient import TestClient
+
+from webspot.web.app import app
+
+client = TestClient(app)
diff --git a/webspot/test/web/routes/api/__init__.py b/webspot/test/web/routes/api/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/webspot/test/web/routes/api/test_request.py b/webspot/test/web/routes/api/test_request.py
new file mode 100644
index 0000000..c41350a
--- /dev/null
+++ b/webspot/test/web/routes/api/test_request.py
@@ -0,0 +1,41 @@
+import itertools
+from unittest.mock import patch, MagicMock
+
+import pytest
+
+from webspot.constants.request_status import REQUEST_STATUS_SUCCESS, REQUEST_STATUS_PENDING
+from webspot.test.web.routes import client
+
+params_url = [
+    'https://quotes.toscrape.com',
+    'https://books.toscrape.com',
+]
+params_method = [
+    'request',
+    # 'rod',
+]
+params_no_async = [True, False]
+
+
+@pytest.mark.parametrize('url, method, no_async', list(itertools.product(params_url, params_method, params_no_async)))
+@patch('webspot.models.request.Request')
+def test_post(mocked_request_model, url, method, no_async):
+    # mock the model
+    mock_instance = MagicMock()
+    mock_instance.save.return_value = None
+    mocked_request_model.return_value = mock_instance
+
+    # post
+    res = client.post('/api/requests', json={'url': url, 'method': method, 'no_async': no_async})
+
+    # assert
+    assert res.status_code == 200
+    data = res.json()
+    assert data is not None
+    assert data.get('url') == url
+    assert data.get('method') == method
+    assert data.get('no_async') == no_async
+    if no_async:
+        assert data.get('status') == REQUEST_STATUS_PENDING
+    else:
+        assert data.get('status') == REQUEST_STATUS_SUCCESS
diff --git a/webspot/test/web/routes/test_detect.py b/webspot/test/web/routes/test_detect.py
deleted file mode 100644
index d04e373..0000000
--- a/webspot/test/web/routes/test_detect.py
+++ /dev/null
@@ -1,26 +0,0 @@
-import unittest
-
-import requests
-
-
-class TestWebRoutesDetect(unittest.TestCase):
-    def test_detect(self):
-        res = requests.post('http://localhost:80/detect', json={'url': 'https://quotes.toscrape.com'})
-        assert res.status_code == 200
-        data = res.json()
-        assert data is not None
-        assert data.get('url') == 'https://quotes.toscrape.com'
-        assert data.get('html') is not None and len(data.get('html')) > 0
-        assert data.get('results') is not None and len(data.get('results')) > 0
-
-    def test_detect_error(self):
-        res = requests.post('http://localhost:80/detect', json={'url': 'https://quotes.toscrape.com', 'method': 'xxx'})
-        assert res.status_code == 200
-        data = res.json()
-        assert data is not None
-        assert data.get('url') == 'https://quotes.toscrape.com'
-        assert data.get('error') is not None
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/webspot/utils/test.py b/webspot/utils/test.py
new file mode 100644
index 0000000..e30956d
--- /dev/null
+++ b/webspot/utils/test.py
@@ -0,0 +1,5 @@
+import sys
+
+
+def is_running_unit_tests():
+    return 'unittest' in sys.modules or 'pytest' in sys.modules