Spaces:
Runtime error
Runtime error
mohitmayank
commited on
Commit
•
5469918
1
Parent(s):
80eb70b
initial version
Browse files- .DS_Store +0 -0
- IPC_Semantic_Search.ipynb +0 -0
- README.md +16 -13
- app.py +42 -0
- devganscrap/.DS_Store +0 -0
- devganscrap/devganscrap/__init__.py +0 -0
- devganscrap/devganscrap/__pycache__/__init__.cpython-38.pyc +0 -0
- devganscrap/devganscrap/__pycache__/settings.cpython-38.pyc +0 -0
- devganscrap/devganscrap/items.py +12 -0
- devganscrap/devganscrap/middlewares.py +103 -0
- devganscrap/devganscrap/pipelines.py +13 -0
- devganscrap/devganscrap/settings.py +88 -0
- devganscrap/devganscrap/spiders/__init__.py +4 -0
- devganscrap/devganscrap/spiders/__pycache__/__init__.cpython-38.pyc +0 -0
- devganscrap/devganscrap/spiders/__pycache__/devgan.cpython-38.pyc +0 -0
- devganscrap/devganscrap/spiders/devgan.py +35 -0
- devganscrap/scrapy.cfg +11 -0
- devganscrap/sections_desc.csv +0 -0
- requirements.txt +4 -0
.DS_Store
ADDED
Binary file (6.15 kB). View file
|
|
IPC_Semantic_Search.ipynb
ADDED
The diff for this file is too large to render.
See raw diff
|
|
README.md
CHANGED
@@ -1,13 +1,16 @@
|
|
1 |
-
|
2 |
-
|
3 |
-
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
|
|
|
|
|
|
|
1 |
+
# IPC Semantic Search
|
2 |
+
|
3 |
+
**What:**
|
4 |
+
- The Indian Penal Code (IPC) is the official criminal code of India. [Wikipedia](https://en.wikipedia.org/wiki/Indian_Penal_Code)
|
5 |
+
- The intention behind this project is to provide an interfact for a common layman to search IPC sections.
|
6 |
+
|
7 |
+
**Steps:**
|
8 |
+
- Scrap the `devgan.in` website to get the different sections in IPC
|
9 |
+
- Use `LegalBERT` to get the embeddings of the descriptions of the IPC sections
|
10 |
+
- Perform Cosine Similarity to get the nearest matching section wrt query.
|
11 |
+
|
12 |
+
**Code:**
|
13 |
+
- `devganscrap`: Scrapy code to crawl and extract IPC sections and their description from the `devgan.in`
|
14 |
+
|
15 |
+
**Data:**
|
16 |
+
- `devagnscrp/sections_desc.csv`: IPC sections and descriptions *(Data Credits: http://devgan.in)*
|
app.py
ADDED
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# import
|
2 |
+
import pandas as pd
|
3 |
+
import streamlit as st
|
4 |
+
from txtai.embeddings import Embeddings
|
5 |
+
|
6 |
+
# set config
|
7 |
+
st.set_page_config(layout="wide", page_title="⚖️ Law Finder - IPC")
|
8 |
+
|
9 |
+
# load the summarization model (cache for faster loading)
|
10 |
+
@st.cache(allow_output_mutation=True)
|
11 |
+
def load_model_embeddings_data():
|
12 |
+
embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2"})
|
13 |
+
# embeddings.load("embedding")
|
14 |
+
df = pd.read_csv("devganscrap/sections_desc.csv")
|
15 |
+
embeddings.index([(uid, str(text), None) for uid, text in enumerate(df['description'].tolist())])
|
16 |
+
return embeddings, df
|
17 |
+
|
18 |
+
# loading the model
|
19 |
+
embeddings, df = load_model_embeddings_data()
|
20 |
+
|
21 |
+
# APP
|
22 |
+
# set title and subtitle
|
23 |
+
st.title("⚖️ Law Finder - IPC")
|
24 |
+
st.markdown("Search the [Indian Penal Code](https://en.wikipedia.org/wiki/Indian_Penal_Code) Sections with simple english.")
|
25 |
+
st.markdown("The data scraping procedure is explained in detail on [my website](http://mohitmayank.com/a_lazy_data_science_guide/python/scraping_websites/)")
|
26 |
+
st.markdown("The complete code is on [Github](https://github.com/imohitmayank/ipc_semantic_search)")
|
27 |
+
|
28 |
+
# create the input text box
|
29 |
+
query = st.text_area("Input your search phrase here!", "animal cruelty")
|
30 |
+
button = st.button("Find sections...")
|
31 |
+
|
32 |
+
# if button is clicked
|
33 |
+
with st.spinner("Finding the most similar sections...."):
|
34 |
+
if button:
|
35 |
+
# find and display the sections
|
36 |
+
st.markdown("**Sections:**")
|
37 |
+
results = []
|
38 |
+
for id, score in embeddings.search(query, limit=5):
|
39 |
+
st.write({
|
40 |
+
'section': df.loc[id, 'section'],
|
41 |
+
'description': df.loc[id, 'description']
|
42 |
+
})
|
devganscrap/.DS_Store
ADDED
Binary file (6.15 kB). View file
|
|
devganscrap/devganscrap/__init__.py
ADDED
File without changes
|
devganscrap/devganscrap/__pycache__/__init__.cpython-38.pyc
ADDED
Binary file (174 Bytes). View file
|
|
devganscrap/devganscrap/__pycache__/settings.cpython-38.pyc
ADDED
Binary file (290 Bytes). View file
|
|
devganscrap/devganscrap/items.py
ADDED
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Define here the models for your scraped items
|
2 |
+
#
|
3 |
+
# See documentation in:
|
4 |
+
# https://docs.scrapy.org/en/latest/topics/items.html
|
5 |
+
|
6 |
+
import scrapy
|
7 |
+
|
8 |
+
|
9 |
+
class DevganscrapItem(scrapy.Item):
|
10 |
+
# define the fields for your item here like:
|
11 |
+
# name = scrapy.Field()
|
12 |
+
pass
|
devganscrap/devganscrap/middlewares.py
ADDED
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Define here the models for your spider middleware
|
2 |
+
#
|
3 |
+
# See documentation in:
|
4 |
+
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
5 |
+
|
6 |
+
from scrapy import signals
|
7 |
+
|
8 |
+
# useful for handling different item types with a single interface
|
9 |
+
from itemadapter import is_item, ItemAdapter
|
10 |
+
|
11 |
+
|
12 |
+
class DevganscrapSpiderMiddleware:
|
13 |
+
# Not all methods need to be defined. If a method is not defined,
|
14 |
+
# scrapy acts as if the spider middleware does not modify the
|
15 |
+
# passed objects.
|
16 |
+
|
17 |
+
@classmethod
|
18 |
+
def from_crawler(cls, crawler):
|
19 |
+
# This method is used by Scrapy to create your spiders.
|
20 |
+
s = cls()
|
21 |
+
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
22 |
+
return s
|
23 |
+
|
24 |
+
def process_spider_input(self, response, spider):
|
25 |
+
# Called for each response that goes through the spider
|
26 |
+
# middleware and into the spider.
|
27 |
+
|
28 |
+
# Should return None or raise an exception.
|
29 |
+
return None
|
30 |
+
|
31 |
+
def process_spider_output(self, response, result, spider):
|
32 |
+
# Called with the results returned from the Spider, after
|
33 |
+
# it has processed the response.
|
34 |
+
|
35 |
+
# Must return an iterable of Request, or item objects.
|
36 |
+
for i in result:
|
37 |
+
yield i
|
38 |
+
|
39 |
+
def process_spider_exception(self, response, exception, spider):
|
40 |
+
# Called when a spider or process_spider_input() method
|
41 |
+
# (from other spider middleware) raises an exception.
|
42 |
+
|
43 |
+
# Should return either None or an iterable of Request or item objects.
|
44 |
+
pass
|
45 |
+
|
46 |
+
def process_start_requests(self, start_requests, spider):
|
47 |
+
# Called with the start requests of the spider, and works
|
48 |
+
# similarly to the process_spider_output() method, except
|
49 |
+
# that it doesn’t have a response associated.
|
50 |
+
|
51 |
+
# Must return only requests (not items).
|
52 |
+
for r in start_requests:
|
53 |
+
yield r
|
54 |
+
|
55 |
+
def spider_opened(self, spider):
|
56 |
+
spider.logger.info('Spider opened: %s' % spider.name)
|
57 |
+
|
58 |
+
|
59 |
+
class DevganscrapDownloaderMiddleware:
|
60 |
+
# Not all methods need to be defined. If a method is not defined,
|
61 |
+
# scrapy acts as if the downloader middleware does not modify the
|
62 |
+
# passed objects.
|
63 |
+
|
64 |
+
@classmethod
|
65 |
+
def from_crawler(cls, crawler):
|
66 |
+
# This method is used by Scrapy to create your spiders.
|
67 |
+
s = cls()
|
68 |
+
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
69 |
+
return s
|
70 |
+
|
71 |
+
def process_request(self, request, spider):
|
72 |
+
# Called for each request that goes through the downloader
|
73 |
+
# middleware.
|
74 |
+
|
75 |
+
# Must either:
|
76 |
+
# - return None: continue processing this request
|
77 |
+
# - or return a Response object
|
78 |
+
# - or return a Request object
|
79 |
+
# - or raise IgnoreRequest: process_exception() methods of
|
80 |
+
# installed downloader middleware will be called
|
81 |
+
return None
|
82 |
+
|
83 |
+
def process_response(self, request, response, spider):
|
84 |
+
# Called with the response returned from the downloader.
|
85 |
+
|
86 |
+
# Must either;
|
87 |
+
# - return a Response object
|
88 |
+
# - return a Request object
|
89 |
+
# - or raise IgnoreRequest
|
90 |
+
return response
|
91 |
+
|
92 |
+
def process_exception(self, request, exception, spider):
|
93 |
+
# Called when a download handler or a process_request()
|
94 |
+
# (from other downloader middleware) raises an exception.
|
95 |
+
|
96 |
+
# Must either:
|
97 |
+
# - return None: continue processing this exception
|
98 |
+
# - return a Response object: stops process_exception() chain
|
99 |
+
# - return a Request object: stops process_exception() chain
|
100 |
+
pass
|
101 |
+
|
102 |
+
def spider_opened(self, spider):
|
103 |
+
spider.logger.info('Spider opened: %s' % spider.name)
|
devganscrap/devganscrap/pipelines.py
ADDED
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Define your item pipelines here
|
2 |
+
#
|
3 |
+
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
|
4 |
+
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
5 |
+
|
6 |
+
|
7 |
+
# useful for handling different item types with a single interface
|
8 |
+
from itemadapter import ItemAdapter
|
9 |
+
|
10 |
+
|
11 |
+
class DevganscrapPipeline:
|
12 |
+
def process_item(self, item, spider):
|
13 |
+
return item
|
devganscrap/devganscrap/settings.py
ADDED
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Scrapy settings for devganscrap project
|
2 |
+
#
|
3 |
+
# For simplicity, this file contains only settings considered important or
|
4 |
+
# commonly used. You can find more settings consulting the documentation:
|
5 |
+
#
|
6 |
+
# https://docs.scrapy.org/en/latest/topics/settings.html
|
7 |
+
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
8 |
+
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
9 |
+
|
10 |
+
BOT_NAME = 'devganscrap'
|
11 |
+
|
12 |
+
SPIDER_MODULES = ['devganscrap.spiders']
|
13 |
+
NEWSPIDER_MODULE = 'devganscrap.spiders'
|
14 |
+
|
15 |
+
|
16 |
+
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
17 |
+
#USER_AGENT = 'devganscrap (+http://www.yourdomain.com)'
|
18 |
+
|
19 |
+
# Obey robots.txt rules
|
20 |
+
ROBOTSTXT_OBEY = True
|
21 |
+
|
22 |
+
# Configure maximum concurrent requests performed by Scrapy (default: 16)
|
23 |
+
#CONCURRENT_REQUESTS = 32
|
24 |
+
|
25 |
+
# Configure a delay for requests for the same website (default: 0)
|
26 |
+
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
|
27 |
+
# See also autothrottle settings and docs
|
28 |
+
#DOWNLOAD_DELAY = 3
|
29 |
+
# The download delay setting will honor only one of:
|
30 |
+
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
|
31 |
+
#CONCURRENT_REQUESTS_PER_IP = 16
|
32 |
+
|
33 |
+
# Disable cookies (enabled by default)
|
34 |
+
#COOKIES_ENABLED = False
|
35 |
+
|
36 |
+
# Disable Telnet Console (enabled by default)
|
37 |
+
#TELNETCONSOLE_ENABLED = False
|
38 |
+
|
39 |
+
# Override the default request headers:
|
40 |
+
#DEFAULT_REQUEST_HEADERS = {
|
41 |
+
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
42 |
+
# 'Accept-Language': 'en',
|
43 |
+
#}
|
44 |
+
|
45 |
+
# Enable or disable spider middlewares
|
46 |
+
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
47 |
+
#SPIDER_MIDDLEWARES = {
|
48 |
+
# 'devganscrap.middlewares.DevganscrapSpiderMiddleware': 543,
|
49 |
+
#}
|
50 |
+
|
51 |
+
# Enable or disable downloader middlewares
|
52 |
+
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
53 |
+
#DOWNLOADER_MIDDLEWARES = {
|
54 |
+
# 'devganscrap.middlewares.DevganscrapDownloaderMiddleware': 543,
|
55 |
+
#}
|
56 |
+
|
57 |
+
# Enable or disable extensions
|
58 |
+
# See https://docs.scrapy.org/en/latest/topics/extensions.html
|
59 |
+
#EXTENSIONS = {
|
60 |
+
# 'scrapy.extensions.telnet.TelnetConsole': None,
|
61 |
+
#}
|
62 |
+
|
63 |
+
# Configure item pipelines
|
64 |
+
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
65 |
+
#ITEM_PIPELINES = {
|
66 |
+
# 'devganscrap.pipelines.DevganscrapPipeline': 300,
|
67 |
+
#}
|
68 |
+
|
69 |
+
# Enable and configure the AutoThrottle extension (disabled by default)
|
70 |
+
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
|
71 |
+
#AUTOTHROTTLE_ENABLED = True
|
72 |
+
# The initial download delay
|
73 |
+
#AUTOTHROTTLE_START_DELAY = 5
|
74 |
+
# The maximum download delay to be set in case of high latencies
|
75 |
+
#AUTOTHROTTLE_MAX_DELAY = 60
|
76 |
+
# The average number of requests Scrapy should be sending in parallel to
|
77 |
+
# each remote server
|
78 |
+
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
79 |
+
# Enable showing throttling stats for every response received:
|
80 |
+
#AUTOTHROTTLE_DEBUG = False
|
81 |
+
|
82 |
+
# Enable and configure HTTP caching (disabled by default)
|
83 |
+
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
84 |
+
#HTTPCACHE_ENABLED = True
|
85 |
+
#HTTPCACHE_EXPIRATION_SECS = 0
|
86 |
+
#HTTPCACHE_DIR = 'httpcache'
|
87 |
+
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
88 |
+
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
|
devganscrap/devganscrap/spiders/__init__.py
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# This package will contain the spiders of your Scrapy project
|
2 |
+
#
|
3 |
+
# Please refer to the documentation for information on how to create and manage
|
4 |
+
# your spiders.
|
devganscrap/devganscrap/spiders/__pycache__/__init__.cpython-38.pyc
ADDED
Binary file (182 Bytes). View file
|
|
devganscrap/devganscrap/spiders/__pycache__/devgan.cpython-38.pyc
ADDED
Binary file (1.37 kB). View file
|
|
devganscrap/devganscrap/spiders/devgan.py
ADDED
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import scrapy
|
2 |
+
|
3 |
+
class QuotesSpider(scrapy.Spider):
|
4 |
+
name = "devgan"
|
5 |
+
allowed_domains = ["devgan.in"]
|
6 |
+
|
7 |
+
def start_requests(self):
|
8 |
+
urls = [
|
9 |
+
'http://devgan.in/all_sections_ipc.php',
|
10 |
+
]
|
11 |
+
for url in urls:
|
12 |
+
yield scrapy.Request(url=url, callback=self.parse_mainpage)
|
13 |
+
|
14 |
+
def parse_mainpage(self, response):
|
15 |
+
# identify the links to the individual section pages
|
16 |
+
sections = response.css('div#content').css('a')#.getall()
|
17 |
+
# for each section
|
18 |
+
for section in sections:
|
19 |
+
# loc var
|
20 |
+
loc = {
|
21 |
+
'title' : section.xpath('@title').extract(),
|
22 |
+
'link' : 'http://devgan.in' + section.xpath('@href').extract()[0],
|
23 |
+
'section': section.css('span.sectionlink::text').extract(),
|
24 |
+
}
|
25 |
+
# traverse again and extract the description
|
26 |
+
yield scrapy.Request(loc['link'], callback=self.parse_section,
|
27 |
+
cb_kwargs=dict(meta=loc))
|
28 |
+
|
29 |
+
def parse_section(self, response, meta):
|
30 |
+
# extract the description
|
31 |
+
meta['description'] = " ".join(response.css('tr.mys-desc').css('::text').extract())
|
32 |
+
# return
|
33 |
+
return meta
|
34 |
+
|
35 |
+
|
devganscrap/scrapy.cfg
ADDED
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Automatically created by: scrapy startproject
|
2 |
+
#
|
3 |
+
# For more information about the [deploy] section see:
|
4 |
+
# https://scrapyd.readthedocs.io/en/latest/deploy.html
|
5 |
+
|
6 |
+
[settings]
|
7 |
+
default = devganscrap.settings
|
8 |
+
|
9 |
+
[deploy]
|
10 |
+
#url = http://localhost:6800/
|
11 |
+
project = devganscrap
|
devganscrap/sections_desc.csv
ADDED
The diff for this file is too large to render.
See raw diff
|
|
requirements.txt
ADDED
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
1 |
+
scrapy==2.6.1
|
2 |
+
streamlit
|
3 |
+
txtai
|
4 |
+
pandas
|