Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+twitter_scraper_without_API/firefox-geckodriver/geckodriver.exe filter=lfs diff=lfs merge=lfs -text

Hands-on-WebScraping-master.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8634d80c9a416f3346c02ce5b6f25f96fda953cc24212a45f461a510d145f04c
+size 15838

Hands-on-WebScraping-master/Hands-on-WebScraping-master/.gitignore

@@ -0,0 +1,241 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+
+
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# Next.js build output
+.next
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+

Hands-on-WebScraping-master/Hands-on-WebScraping-master/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Amit Upreti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Hands-on-WebScraping-master/Hands-on-WebScraping-master/README.md

@@ -0,0 +1,2 @@
+# Hands-on-WebScraping (NO LONGER MAINTAINED)
+This repo is a part of blog series on several web scraping projects where we will explore scraping techniques to crawl data from simple websites to websites using advanced protection.

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/Readme.md

@@ -0,0 +1,118 @@
+#### Depricated. No longer maintained.
+
+# Twitter Hashtag crawler
+> A fast and unofficial twitter crawler to collect tweets using hashtag search.
+
+> Notice: The crawler is meant to be used for collecting data purely for academic and research purpose only. I am not responsible for any legal issue that might arise for any unintended use of this crawler
+
+[![Python 3](https://img.shields.io/badge/python-3.6-blue.svg)](https://www.python.org/downloads/release/python-360/)
+[![twitter crawler](https://img.shields.io/badge/twittercrawler-1.0-green)](https://github.com/amitupreti/Hands-on-WebScraping/tree/master/project1_twitter_hashtag_crawler)
+
+This is written using scrapy and python. The logic is straight forward. We are simply sending get requests to the mobile version of the twitter(mobile.twitter.com) to collect the list of tweets and sending get requests to the web version to parse tweet details.
+![](header.png)
+
+## Installation
+
+OS X & Linux:
+
+1. Download the project
+
+```sh
+git clone https://github.com/amitupreti/Hands-on-WebScraping
+
+cd Hands-on-WebScraping/project1_twitter_hashtag_crawler
+```
+2. Install the dependencies
+
+```sh
+pip install -r requirements.txt --user
+```
+
+3. Verify the crawler spider exists
+
+```sh
+scrapy list
+```
+if you see `twittercrawler` than you are all set.
+
+
+Windows:
+1. Install [python3](https://www.python.org/downloads/) if you haven't already
+2. Download the project. https://github.com/amitupreti/Hands-on-WebScraping/archive/master.zip
+3. Extract the project 
+4. Open cmd and navigate inside the project directory
+```sh
+cd Hands-on-WebScraping/project1_twitter_hashtag_crawler
+```
+5. Follow step 2 and 3 from Mac/Linux installation
+
+
+
+## Usage example
+
+1. Put the hashtags in a csv file seperated by new line. For example, I have included `myhashtags.csv` as a sample.
+
+![Hashtags file](https://i.paste.pics/225079df0d3dc27d66430b1553b2ac39.png)
+
+2. Run the crawler with your hashtag file and the desired [output formats] (https://docs.scrapy.org/en/latest/topics/feed-exports.html)(JSON,JSON lines,CSV,XML)
+
+* For csv
+   ```sh
+    scrapy crawl twittercrawler -a filename=myhashtags.csv -o mydata.csv
+
+   ```
+   
+* For JSON
+   ```sh
+    scrapy crawl twittercrawler -a filename=myhashtags.csv -o mydata.json
+
+   ```
+![sample images](https://i.paste.pics/4a5826a6a090522e5326bb11838258df.png)
+![sample images](https://i.paste.pics/68a64bab743150e00af4cd9eea9af8dc.png)
+
+
+### Speeding up the crawls
+If you feel like the crawler is a little slow then find the hashtag.py file in the project and edit the custom settings.
+```py
+custom_settings = {
+    'USER_AGENT': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36',
+    'CONCURRENT_REQUESTS': 2, 'DOWNLOAD_DELAY': 1, 'LOG_LEVEL': 'INFO'}
+```
+> Here CONCURRENT_REQUESTS is the number of URLs that will be processed parallelly and DOWNLOAD_DELAY is a wait between each  request. So, Increase CONCURRENT_REQUESTS and decrease DOWNLOAD_DELAY (minimum value for download delay is 0).
+
+
+## Data Columns
+* username
+* full_name
+* twitter_url
+* tweet_text
+* tweet_time
+* number_of_likes
+* no_of_retweets
+* no_of_replies
+* mentions
+* no_of_mentions
+* hashtags
+* no_of_hashtags
+* call_to_action
+* image_url
+
+## Release History
+
+* 1.0.0
+    * first release crawl by hashtags
+
+## Meta
+
+Amit Upreti – [@amitupreti](https://www.linkedin.com/in/amitupreti/) 
+
+Distributed under the MIT license. See ``LICENSE`` for more information.
+
+
+## Contributing
+
+1. Fork it (<https://github.com/amitupreti/Hands-on-WebScraping/fork>)
+2. Create your feature branch (`git checkout -b feature/fooBar`)
+3. Commit your changes (`git commit -am 'Add some fooBar'`)
+4. Push to the branch (`git push origin feature/fooBar`)
+5. Create a new Pull Request

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/items.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class TwitterhashtagcrawlerItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/middlewares.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+
+class TwitterhashtagcrawlerSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Response, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class TwitterhashtagcrawlerDownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/pipelines.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+class TwitterhashtagcrawlerPipeline(object):
+    def process_item(self, item, spider):
+        return item

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/settings.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+
+# Scrapy settings for TwitterHashTagCrawler project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://doc.scrapy.org/en/latest/topics/settings.html
+#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'TwitterHashTagCrawler'
+
+SPIDER_MODULES = ['TwitterHashTagCrawler.spiders']
+NEWSPIDER_MODULE = 'TwitterHashTagCrawler.spiders'
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'TwitterHashTagCrawler (+http://www.yourdomain.com)'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = True
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'TwitterHashTagCrawler.middlewares.TwitterhashtagcrawlerSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'TwitterHashTagCrawler.middlewares.TwitterhashtagcrawlerDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://doc.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    'TwitterHashTagCrawler.pipelines.TwitterhashtagcrawlerPipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/spiders/hashtag.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import ipdb
+import re
+from dateutil import parser
+import sys
+from scrapy.crawler import CrawlerProcess
+from utils import get_links, get_hashtags, get_mentions
+import logging
+
+class HashtagSpider(scrapy.Spider):
+    name = 'twittercrawler'
+    allowed_domains = ["twitter.com"]
+
+    # custom settings for user agent and proxy. Default will get chrome as user agent  and use a proxypool of 50 .
+    # Override here
+    custom_settings = {
+        'USER_AGENT': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36',
+        'CONCURRENT_REQUESTS': 5, 'DOWNLOAD_DELAY': 0, 'LOG_LEVEL': 'INFO'}
+
+    def __init__(self, filename=''):
+        if not filename:
+            sys.exit('Please provide the input filename also. Example \n\n$ python3 hashtags.py myinput.csv')
+        self.filename = filename
+
+    # the crawler will execute start_requests function at first.
+    def start_requests(self):
+        with open(self.filename, 'r') as f:
+            hashtags = f.read().splitlines()
+        if len(hashtags) == 0:
+            sys.exit('Emplty File detected.Please provide hashtags separated by newlines')
+        else:
+            logging.info(f'{len(hashtags)} hashtags found')
+        for hashtag in hashtags:
+            if hashtag:
+                search_url = "https://mobile.twitter.com/hashtag/" + hashtag.lower()
+
+                yield scrapy.Request(search_url, callback=self.find_tweets, dont_filter=True)
+
+    def find_tweets(self, response):
+        tweets = response.xpath('//table[@class="tweet  "]/@href').getall()
+        logging.info(f'{len(tweets)} tweets found')
+        for tweet_id in tweets:
+            tweet_id = re.findall("\d+", tweet_id)[-1]
+            tweet_url = 'https://twitter.com/anyuser/status/' + \
+                        str(tweet_id)
+            yield scrapy.Request(tweet_url, callback=self.parse_tweet)
+
+        # finding and visiting next page
+        next_page = response.xpath(
+            '//*[@class="w-button-more"]/a/@href').get(default='')
+        logging.info('Next page found:')
+        if next_page != '':
+            next_page = 'https://mobile.twitter.com' + next_page
+            yield scrapy.Request(next_page, callback=self.find_tweets)
+
+    def parse_tweet(self, response):
+        logging.info('Processing --> ' + response.url)
+        username = response.xpath(
+            '//*[@class="permalink-inner permalink-tweet-container"]//*[@class="username u-dir u-textTruncate"]/b/text()').get(
+            default='')
+        full_name = response.xpath(
+            '//*[@class="permalink-inner permalink-tweet-container"]//*[@class="FullNameGroup"]/strong/text()').get(
+            default='')
+        
+        try:
+            tweet_text = response.xpath('//title/text()').get(default='').split(':')[1].strip()
+            
+        except:
+            tweet_text = ' '.join(response.xpath(
+                '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-text-container"]/p//text()').getall()).strip()
+        image_list = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="AdaptiveMediaOuterContainer"]//img/@src').getall()
+        date_time = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-details-fixer tweet-details-fixer"]/div[@class="client-and-actions"]/span[@class="metadata"]/span/text()').get(
+            default='')
+
+        date_time = parser.parse(date_time.replace('-', '')).strftime('%Y-%m-%d %H:%M:%S')
+        retweets = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-details-fixer tweet-details-fixer"]/div[@class="js-tweet-stats-container tweet-stats-container"]//*[@class="js-stat-count js-stat-retweets stat-count"]/a/strong/text()').get(
+            default='')
+
+        likes = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-details-fixer tweet-details-fixer"]/div[@class="js-tweet-stats-container tweet-stats-container"]//*[@class="js-stat-count js-stat-favorites stat-count"]/a/strong/text()').get(
+            default='')
+        replies = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[contains(@id,"profile-tweet-action-reply-count")]/parent::span/@data-tweet-stat-count').get(
+            default='')
+        
+        mentions = get_mentions(tweet_text)
+        hashtags = get_hashtags(tweet_text)
+        cta = get_links(tweet_text)
+
+        result = {
+            'username': username.lower(),
+            'full_name': full_name,
+            'twitter_url': response.url,
+            'tweet_text': tweet_text,
+            'tweet_time': str(date_time),
+            'number_of_likes': str(likes),
+            'no_of_retweets': str(retweets),
+            'no_of_replies': str(replies),
+            'mentions': ' | '.join(mentions),
+            'no_of_mentions': str(len(mentions)),
+            'hashtags': ' | '.join(hashtags),
+            'no_of_hashtags': str(len(hashtags)),
+            'call_to_action': ' | '.join(cta),
+            'image_url': ' | '.join(image_list),
+
+        }
+        yield result
+        

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/myhashtags.csv

@@ -0,0 +1,2 @@
+cats
+dogs

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/requirements.txt

@@ -0,0 +1,2 @@
+scrapy
+dateutil

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/sampledata.csv

@@ -0,0 +1,12 @@
+username,full_name,twitter_url,tweet_text,tweet_time,number_of_likes,no_of_retweets,no_of_replies,mentions,no_of_mentions,hashtags,no_of_hashtags,call_to_action,image_url
+cctvasiapacific,CCTV Asia Pacific,https://twitter.com/CCTVAsiaPacific/status/1212269072328491008,"Turning off the stereotype of political faces, Hou Yanqi, the  # ChineseAmbssdor  to  # Nepal , amazes Nepalese and gains popularity on twitter by posting her ad-like photos and wishes: ""True beauty always touches the deep heart"", said Hou.
+२०२० नेपाल भ्रमाण वर्ष सफलताको शुभकामना pic.twitter.com/z0N8ru2vNd",2019-12-31 23:07:00,804,171,35,,0,,0,,https://pbs.twimg.com/media/ENLYSqlU4AAgiFh.jpg | https://pbs.twimg.com/media/ENLYSqoVAAASSS-.jpg | https://pbs.twimg.com/media/ENLYSqmU0AAZEyK.jpg
+,,https://twitter.com/BishowParajuli/status/1213037950549626882,"Zimbabwe is beautiful! Glad to hear your mountain climbing adventure ; If you wish to climb further higher, another beautiful place is  # Nepal ! You will you can also enjoy some terrific historical spots: pic.twitter.com/ofsCppyp0O",2020-01-03 02:02:00,27,3,1,,0,,0,,https://pbs.twimg.com/media/ENWTkzmUEAEKS1k.jpg | https://pbs.twimg.com/media/ENWTkznU4AAtVxK.jpg | https://pbs.twimg.com/media/ENWTkzoUwAEgMpX.jpg | https://pbs.twimg.com/media/ENWTkzlU4AEYxor.jpg
+kopinoora,kpila,https://twitter.com/kopinoora/status/1213481511967690752,# VisitNepal2020  official inauguration at London Nepal Embassy. # pic.twitter.com/e4N9XulBH7,2020-01-04 07:25:00,3,,0,,0,,0,,https://pbs.twimg.com/media/ENcnABiXsAE7_sw.jpg | https://pbs.twimg.com/media/ENcnABsXUAAnuBL.jpg
+mahbub_nazif,Nazif Mahbub,https://twitter.com/mahbub_nazif/status/1213328288271089664,"The joy of being Innocent. Durbar square, kathmandu, nepal pic.twitter.com/sbsfxTzeHN",2020-01-03 21:16:00,4,,0,,0,,0,,https://pbs.twimg.com/media/ENabn-uWwAcbUfb.jpg
+prabhumteverest,Prastuti_प्रश्तुती,https://twitter.com/PrabhuMteverest/status/1213178026457878528,"Visit nepal2020. where heaven meets and you won't feel regret choosing Nepal as your destination   
+
+We are eager to welcome you with our beautiful destinations and warm hospitality  pic.twitter.com/l7GQfk2ha6",2020-01-03 11:19:00,5,,0,,0,,0,,https://pbs.twimg.com/media/ENYS_CLUwAAVypp.jpg
+kashishds,Kashish Das Shrestha,https://twitter.com/kashishds/status/1213120581412876295,"Marpha bazaar, Mustang, Nepal. Today. 
+
+Requested my friend & Marpha resident Dipesh Hirachan for this clip. This is just outside his Apple orchard there. pic.twitter.com/oOFy88ylIt",2020-01-03 07:30:00,123,20,4,,0,,0,,

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = TwitterHashTagCrawler.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = TwitterHashTagCrawler

Hands-on-WebScraping-master/Hands-on-WebScraping-master/project1_twitter_hashtag_crawler/utils.py

@@ -0,0 +1,67 @@
+import re
+
+
+def find_emails(text):
+    """
+    It will parse the given string and return a list of emails if found
+
+    Example:
+    >>find_emails('hello\n find me here\nemail@gmail.com')
+    ['email@gmail.com']
+
+    :param text: string
+    :return: list
+    """
+    return re.findall(r"([a-zA-Z0-9+._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)", text)
+
+
+def get_mentions(text):
+    """
+    It will return mentions from the text i.e @someone
+
+    :param text: string
+    :return: list
+
+    example
+    >>> get_mentions('Hi @hero, How are you? I hope @hero2 is fine. BTW say hi to @heroine for me')
+    ['hero','hero2','heroine']
+    """
+    result = re.findall("(^|[^@\w])@(\w{1,15})", text)
+    if len(result) != 0:
+        result = [i[1] for i in result]
+    return result
+
+
+def get_hashtags(text):
+    """
+      It will return hashtags from the text i.e #something
+
+      :param text: string
+      :return: list
+
+      example
+      >>> get_hashtags('my first code #programmer #python #awesome #grepsr')
+      ['programmer','python','awesome','grepsr']
+      """
+
+    result = re.findall("(^|[^@\w])#(\w{1,15})", text)
+    if len(result) != 0:
+        result = [i[1] for i in result]
+    return result
+
+
+def get_links(text):
+    """
+         It will return website links from the text
+
+         :param text: string
+         :return: list
+
+         example
+         >>> message = 'http://twitter.com Project URL: https://app.grepsr.com/app/project/message/70454'
+         >>> get_links(message)
+        ['http://twitter.com', 'https://app.grepsr.com/app/project/message/70454']
+
+         """
+    result = re.findall(r"(?P<url>https?://[^\s]+)", text)
+    return result

Hands-on-WebScraping/.gitignore

@@ -0,0 +1,241 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+
+
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# Next.js build output
+.next
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and not Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+# Stores VSCode versions used for testing VSCode extensions
+.vscode-test
+

Hands-on-WebScraping/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Amit Upreti
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Hands-on-WebScraping/README.md

@@ -0,0 +1,2 @@
+# Hands-on-WebScraping (NO LONGER MAINTAINED)
+This repo is a part of blog series on several web scraping projects where we will explore scraping techniques to crawl data from simple websites to websites using advanced protection.

Hands-on-WebScraping/project1_twitter_hashtag_crawler/Readme.md

@@ -0,0 +1,118 @@
+#### Depricated. No longer maintained.
+
+# Twitter Hashtag crawler
+> A fast and unofficial twitter crawler to collect tweets using hashtag search.
+
+> Notice: The crawler is meant to be used for collecting data purely for academic and research purpose only. I am not responsible for any legal issue that might arise for any unintended use of this crawler
+
+[![Python 3](https://img.shields.io/badge/python-3.6-blue.svg)](https://www.python.org/downloads/release/python-360/)
+[![twitter crawler](https://img.shields.io/badge/twittercrawler-1.0-green)](https://github.com/amitupreti/Hands-on-WebScraping/tree/master/project1_twitter_hashtag_crawler)
+
+This is written using scrapy and python. The logic is straight forward. We are simply sending get requests to the mobile version of the twitter(mobile.twitter.com) to collect the list of tweets and sending get requests to the web version to parse tweet details.
+![](header.png)
+
+## Installation
+
+OS X & Linux:
+
+1. Download the project
+
+```sh
+git clone https://github.com/amitupreti/Hands-on-WebScraping
+
+cd Hands-on-WebScraping/project1_twitter_hashtag_crawler
+```
+2. Install the dependencies
+
+```sh
+pip install -r requirements.txt --user
+```
+
+3. Verify the crawler spider exists
+
+```sh
+scrapy list
+```
+if you see `twittercrawler` than you are all set.
+
+
+Windows:
+1. Install [python3](https://www.python.org/downloads/) if you haven't already
+2. Download the project. https://github.com/amitupreti/Hands-on-WebScraping/archive/master.zip
+3. Extract the project 
+4. Open cmd and navigate inside the project directory
+```sh
+cd Hands-on-WebScraping/project1_twitter_hashtag_crawler
+```
+5. Follow step 2 and 3 from Mac/Linux installation
+
+
+
+## Usage example
+
+1. Put the hashtags in a csv file seperated by new line. For example, I have included `myhashtags.csv` as a sample.
+
+![Hashtags file](https://i.paste.pics/225079df0d3dc27d66430b1553b2ac39.png)
+
+2. Run the crawler with your hashtag file and the desired [output formats] (https://docs.scrapy.org/en/latest/topics/feed-exports.html)(JSON,JSON lines,CSV,XML)
+
+* For csv
+   ```sh
+    scrapy crawl twittercrawler -a filename=myhashtags.csv -o mydata.csv
+
+   ```
+   
+* For JSON
+   ```sh
+    scrapy crawl twittercrawler -a filename=myhashtags.csv -o mydata.json
+
+   ```
+![sample images](https://i.paste.pics/4a5826a6a090522e5326bb11838258df.png)
+![sample images](https://i.paste.pics/68a64bab743150e00af4cd9eea9af8dc.png)
+
+
+### Speeding up the crawls
+If you feel like the crawler is a little slow then find the hashtag.py file in the project and edit the custom settings.
+```py
+custom_settings = {
+    'USER_AGENT': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36',
+    'CONCURRENT_REQUESTS': 2, 'DOWNLOAD_DELAY': 1, 'LOG_LEVEL': 'INFO'}
+```
+> Here CONCURRENT_REQUESTS is the number of URLs that will be processed parallelly and DOWNLOAD_DELAY is a wait between each  request. So, Increase CONCURRENT_REQUESTS and decrease DOWNLOAD_DELAY (minimum value for download delay is 0).
+
+
+## Data Columns
+* username
+* full_name
+* twitter_url
+* tweet_text
+* tweet_time
+* number_of_likes
+* no_of_retweets
+* no_of_replies
+* mentions
+* no_of_mentions
+* hashtags
+* no_of_hashtags
+* call_to_action
+* image_url
+
+## Release History
+
+* 1.0.0
+    * first release crawl by hashtags
+
+## Meta
+
+Amit Upreti – [@amitupreti](https://www.linkedin.com/in/amitupreti/) 
+
+Distributed under the MIT license. See ``LICENSE`` for more information.
+
+
+## Contributing
+
+1. Fork it (<https://github.com/amitupreti/Hands-on-WebScraping/fork>)
+2. Create your feature branch (`git checkout -b feature/fooBar`)
+3. Commit your changes (`git commit -am 'Add some fooBar'`)
+4. Push to the branch (`git push origin feature/fooBar`)
+5. Create a new Pull Request

Hands-on-WebScraping/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/items.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class TwitterhashtagcrawlerItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    pass

Hands-on-WebScraping/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/middlewares.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+
+class TwitterhashtagcrawlerSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Response, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class TwitterhashtagcrawlerDownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

Hands-on-WebScraping/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/pipelines.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+class TwitterhashtagcrawlerPipeline(object):
+    def process_item(self, item, spider):
+        return item

Hands-on-WebScraping/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/settings.py

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+
+# Scrapy settings for TwitterHashTagCrawler project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://doc.scrapy.org/en/latest/topics/settings.html
+#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'TwitterHashTagCrawler'
+
+SPIDER_MODULES = ['TwitterHashTagCrawler.spiders']
+NEWSPIDER_MODULE = 'TwitterHashTagCrawler.spiders'
+
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'TwitterHashTagCrawler (+http://www.yourdomain.com)'
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+#DOWNLOAD_DELAY = 3
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+#COOKIES_ENABLED = False
+
+# Disable Telnet Console (enabled by default)
+#TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+#DEFAULT_REQUEST_HEADERS = {
+#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+#   'Accept-Language': 'en',
+#}
+
+# Enable or disable spider middlewares
+# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'TwitterHashTagCrawler.middlewares.TwitterhashtagcrawlerSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'TwitterHashTagCrawler.middlewares.TwitterhashtagcrawlerDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://doc.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
+#ITEM_PIPELINES = {
+#    'TwitterHashTagCrawler.pipelines.TwitterhashtagcrawlerPipeline': 300,
+#}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

Hands-on-WebScraping/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

Hands-on-WebScraping/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/spiders/hashtag.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import ipdb
+import re
+from dateutil import parser
+import sys
+from scrapy.crawler import CrawlerProcess
+from utils import get_links, get_hashtags, get_mentions
+import logging
+
+class HashtagSpider(scrapy.Spider):
+    name = 'hashtag'
+    allowed_domains = ["twitter.com"]
+
+    # custom settings for user agent and proxy. Default will get chrome as user agent  and use a proxypool of 50 .
+    # Override here
+    custom_settings = {
+        'USER_AGENT': 'my-cool-project (http://example.com)',
+        'CONCURRENT_REQUESTS': 5, 'DOWNLOAD_DELAY': 0, 'LOG_LEVEL': 'INFO'}
+
+    def __init__(self, filename=''):
+        if not filename:
+            sys.exit('Please provide the input filename also. Example \n\n$ python3 hashtags.py myinput.csv')
+        self.filename = filename
+
+    # the crawler will execute start_requests function at first.
+    def start_requests(self):
+        #with open(self.filename, 'r') as f:
+            #hashtags = ['danaher']
+        hashtags= ['danaher']
+        if len(hashtags) == 0:
+            sys.exit('Emplty File detected.Please provide hashtags separated by newlines')
+        else:
+            logging.info(f'{len(hashtags)} hashtags found')
+            print('hashtag is..',hashtags)
+        for hashtag in hashtags:
+            if hashtag:
+                search_url = "https://twitter.com/hashtag/" + hashtag.lower()
+                print('search_url is...', search_url)
+
+                yield scrapy.Request(search_url, callback=self.find_tweets, dont_filter=True)
+
+    def find_tweets(self, response):
+        print("I am in find_tweets")
+        tweets = response.xpath('//*[@data-testid="tweetText"]/span[1]/text()').getall()
+        print("Tweets is...", tweets)
+        print(f'{len(tweets)} tweets found')
+        for tweet in tweets:
+            # tweet_id = re.findall("\d+", tweet_id)[-1]
+            # tweet_url = 'https://twitter.com/anyuser/status/' + \
+            #             str(tweet_id)
+            print(tweet)
+            yield scrapy.Request(tweet_url, callback=self.parse_tweet)
+
+        # finding and visiting next page
+        next_page = response.xpath(
+            '//*[@class="w-button-more"]/a/@href').get(default='')
+        logging.info('Next page found:')
+        if next_page != '':
+            next_page = 'https://mobile.twitter.com' + next_page
+            yield scrapy.Request(next_page, callback=self.find_tweets)
+
+    def parse_tweet(self, response):
+        logging.info('Processing --> ' + response.url)
+        username = response.xpath(
+            '//*[@class="permalink-inner permalink-tweet-container"]//*[@class="username u-dir u-textTruncate"]/b/text()').get(
+            default='')
+        full_name = response.xpath(
+            '//*[@class="permalink-inner permalink-tweet-container"]//*[@class="FullNameGroup"]/strong/text()').get(
+            default='')
+        
+        try:
+            tweet_text = response.xpath('//title/text()').get(default='').split(':')[1].strip()
+            
+        except:
+            tweet_text = ' '.join(response.xpath(
+                '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-text-container"]/p//text()').getall()).strip()
+        image_list = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="AdaptiveMediaOuterContainer"]//img/@src').getall()
+        date_time = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-details-fixer tweet-details-fixer"]/div[@class="client-and-actions"]/span[@class="metadata"]/span/text()').get(
+            default='')
+
+        date_time = parser.parse(date_time.replace('-', '')).strftime('%Y-%m-%d %H:%M:%S')
+        retweets = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-details-fixer tweet-details-fixer"]/div[@class="js-tweet-stats-container tweet-stats-container"]//*[@class="js-stat-count js-stat-retweets stat-count"]/a/strong/text()').get(
+            default='')
+
+        likes = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-details-fixer tweet-details-fixer"]/div[@class="js-tweet-stats-container tweet-stats-container"]//*[@class="js-stat-count js-stat-favorites stat-count"]/a/strong/text()').get(
+            default='')
+        replies = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[contains(@id,"profile-tweet-action-reply-count")]/parent::span/@data-tweet-stat-count').get(
+            default='')
+        
+        mentions = get_mentions(tweet_text)
+        hashtags = get_hashtags(tweet_text)
+        cta = get_links(tweet_text)
+
+        result = {
+            'username': username.lower(),
+            'full_name': full_name,
+            'twitter_url': response.url,
+            'tweet_text': tweet_text,
+            'tweet_time': str(date_time),
+            'number_of_likes': str(likes),
+            'no_of_retweets': str(retweets),
+            'no_of_replies': str(replies),
+            'mentions': ' | '.join(mentions),
+            'no_of_mentions': str(len(mentions)),
+            'hashtags': ' | '.join(hashtags),
+            'no_of_hashtags': str(len(hashtags)),
+            'call_to_action': ' | '.join(cta),
+            'image_url': ' | '.join(image_list),
+
+        }
+        yield result
+        

Hands-on-WebScraping/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/spiders/hashtag2.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+import scrapy
+import ipdb
+import re
+from dateutil import parser
+import sys
+from scrapy.crawler import CrawlerProcess
+from utils import get_links, get_hashtags, get_mentions
+from scrapy.http.request import Request
+import logging
+
+class HashtagSpider(scrapy.Spider):
+    name = 'hashtag2'
+    allowed_domains = ["twitter.com"]
+
+    # custom settings for user agent and proxy. Default will get chrome as user agent  and use a proxypool of 50 .
+    # Override here
+    custom_settings = {
+        'USER_AGENT': 'my-cool-project (http://example.com)',
+        'CONCURRENT_REQUESTS': 5, 'DOWNLOAD_DELAY': 0, 'LOG_LEVEL': 'INFO'}
+
+    def __init__(self, filename=''):
+        if not filename:
+            sys.exit('Please provide the input filename also. Example \n\n$ python3 hashtags.py myinput.csv')
+        self.filename = filename
+
+    # the crawler will execute start_requests function at first.
+    def start_requests(self):
+        #with open(self.filename, 'r') as f:
+            #hashtags = ['danaher']
+        hashtags= ['danaher']
+        if len(hashtags) == 0:
+            sys.exit('Emplty File detected.Please provide hashtags separated by newlines')
+        else:
+            logging.info(f'{len(hashtags)} hashtags found')
+            print('hashtag is..',hashtags)
+        for hashtag in hashtags:
+            if hashtag:
+                search_url = "https://twitter.com/hashtag/" + hashtag.lower()
+                print('search_url is...', search_url)
+        try:
+            yield Request(search_url, callback=self.find_tweets, dont_filter=True)
+        except Exception as e:
+            print(e)
+
+    def find_tweets(self, response):
+        print("I am in find_tweets")
+        tweets = response.xpath('//*[@data-testid="tweetText"]/span[1]/text()').getall()
+        print("Tweets is...", tweets)
+        print(f'{len(tweets)} tweets found')
+        for tweet in tweets:
+            # tweet_id = re.findall("\d+", tweet_id)[-1]
+            # tweet_url = 'https://twitter.com/anyuser/status/' + \
+            #             str(tweet_id)
+            print(tweet)
+            #yield scrapy.Request(tweet_url, callback=self.parse_tweet)
+
+        # finding and visiting next page
+        next_page = response.xpath(
+            '//*[@class="w-button-more"]/a/@href').get(default='')
+        logging.info('Next page found:')
+        if next_page != '':
+            next_page = 'https://mobile.twitter.com' + next_page
+            yield scrapy.Request(next_page, callback=self.find_tweets)
+
+    def parse_tweet(self, response):
+        logging.info('Processing --> ' + response.url)
+        username = response.xpath(
+            '//*[@class="permalink-inner permalink-tweet-container"]//*[@class="username u-dir u-textTruncate"]/b/text()').get(
+            default='')
+        full_name = response.xpath(
+            '//*[@class="permalink-inner permalink-tweet-container"]//*[@class="FullNameGroup"]/strong/text()').get(
+            default='')
+        
+        try:
+            tweet_text = response.xpath('//title/text()').get(default='').split(':')[1].strip()
+            
+        except:
+            tweet_text = ' '.join(response.xpath(
+                '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-text-container"]/p//text()').getall()).strip()
+        image_list = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="AdaptiveMediaOuterContainer"]//img/@src').getall()
+        date_time = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-details-fixer tweet-details-fixer"]/div[@class="client-and-actions"]/span[@class="metadata"]/span/text()').get(
+            default='')
+
+        date_time = parser.parse(date_time.replace('-', '')).strftime('%Y-%m-%d %H:%M:%S')
+        retweets = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-details-fixer tweet-details-fixer"]/div[@class="js-tweet-stats-container tweet-stats-container"]//*[@class="js-stat-count js-stat-retweets stat-count"]/a/strong/text()').get(
+            default='')
+
+        likes = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[@class="js-tweet-details-fixer tweet-details-fixer"]/div[@class="js-tweet-stats-container tweet-stats-container"]//*[@class="js-stat-count js-stat-favorites stat-count"]/a/strong/text()').get(
+            default='')
+        replies = response.xpath(
+            '//*[contains(@class,"permalink-inner permalink-tweet-container")]//*[contains(@id,"profile-tweet-action-reply-count")]/parent::span/@data-tweet-stat-count').get(
+            default='')
+        
+        mentions = get_mentions(tweet_text)
+        hashtags = get_hashtags(tweet_text)
+        cta = get_links(tweet_text)
+
+        result = {
+            'username': username.lower(),
+            'full_name': full_name,
+            'twitter_url': response.url,
+            'tweet_text': tweet_text,
+            'tweet_time': str(date_time),
+            'number_of_likes': str(likes),
+            'no_of_retweets': str(retweets),
+            'no_of_replies': str(replies),
+            'mentions': ' | '.join(mentions),
+            'no_of_mentions': str(len(mentions)),
+            'hashtags': ' | '.join(hashtags),
+            'no_of_hashtags': str(len(hashtags)),
+            'call_to_action': ' | '.join(cta),
+            'image_url': ' | '.join(image_list),
+
+        }
+        yield result
+        

Hands-on-WebScraping/project1_twitter_hashtag_crawler/TwitterHashTagCrawler/spiders/hashtag3.py

@@ -0,0 +1,142 @@
+
+from operator import concat
+import scrapy
+import time
+import pandas as pd
+from scrapy.http.request import Request
+from scrapy import signals
+#from sentence_transformers import SentenceTransformer, util
+#import numpy as np
+#import yake
+##import nltk.data
+#import nltk
+#nltk.download('punkt')
+#from nltk.tokenize import sent_tokenize
+#from keybert import KeyBERT
+#from statistics import mean   
+#from urllib.parse import urlparse  
+#import random
+
+
+from transformers import AutoTokenizer, AutoModel
+
+import torch
+
+from sklearn.metrics.pairwise import cosine_similarity
+
+
+#text =[]
+text2 =''
+res =[]
+len_res = 0
+len_res2 = 0
+
+list1 = []
+df = pd.DataFrame()
+df_sim = pd.DataFrame()
+allowed_domains = []
+list_start_urls = []
+list_companies = []
+index = 0
+len_df = 0
+mean_embedding = []
+list_df_sim = []
+
+class BioSpider(scrapy.Spider):
+    name = "hashtag3"
+    
+    custom_settings = {'CONCURRENT_REQUESTS': '1', 'CONCURRENT_REQUESTS_PER_DOMAIN':'1', 'ROBOTSTXT_OBEY' : False \
+                        , "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7", 'USER_AGENT':'my-cool-project (http://example.com)'}
+    
+    
+    global allowed_domains
+    allowed_domains = ["twitter.com"]
+    global list_start_urls
+    global list_companies
+    global res
+    global index
+    list_start_urls2 = []
+    global len_df
+    global df
+  
+    #data=pd.read_excel("C:\\Users\\RSPRASAD\OneDrive - Danaher\\Bec_LS\\2023\\D_and_B_Project\\Segmentation\\Customer focus list 2023 NGW.xlsx", sheet_name='Sheet1')
+    
+    #df= data[['Company', 'Website']]
+    #df.drop_duplicates(inplace = True)
+    #df['Content'] = ''
+
+    i = 0
+    len_df = 1
+    # for i in range(0, len(df)):
+    #     #df.loc[i, 'company_name']= df.loc[i, 'Company']
+    #     #df.loc[i, 'company_website']= df.loc[i, 'Website']
+    #     list_start_urls.append(df.loc[i, 'Website'])
+    #     list_companies.append(df.loc[i, 'Company'])
+    #     domain = urlparse(df.loc[i, 'Website']).netloc
+    #     allowed_domains.append(domain)
+    # print(allowed_domains)
+    # upper_len_websites = 5
+    start_index = 0
+    # if(len_df >upper_len_websites):
+    #     list_start_urls= list_start_urls[start_index:upper_len_websites]
+    #     df = df.iloc[start_index:upper_len_websites,:]
+
+
+    list_start_urls = ['https://twitter.com/hashtag/danaher/']
+    # df = df.iloc[start_index:upper_len_websites,:]
+    # df = df.reset_index()
+    # df = df.drop('index', axis = 1)
+    # len_df = len(df)
+
+    # print("Dataframe for crawling website is ..")
+    # print(df)
+
+    
+
+    print(list_start_urls)
+   
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = super(BioSpider, cls).from_crawler(crawler, *args, **kwargs)
+        #crawler.signals.connect(spider.spider_opened, signals.spider_opened)
+        crawler.signals.connect(spider.spider_closed, signals.spider_closed)
+        return spider
+
+    def start_requests(self):
+       
+        global list_start_urls
+        global index
+        global res
+        
+ 
+        index =0
+        index2 = len(list_start_urls)
+        print(" i am in start_requests")
+        
+            
+          
+        try:
+            
+            yield Request(list_start_urls[0].strip(), callback = self.parse)#, meta={'priority': index2})
+            
+        except Exception as e:
+            print("There is exception and exception is..",e)
+
+    
+    
+
+    def parse(self, response):
+        print("I am in parse..")
+        print("I am in find_tweets")
+        tweets = response.xpath('//*[@data-testid="tweetText"]/span[1]/text()').getall()
+        print("Tweets is...", tweets)
+        print(f'{len(tweets)} tweets found')
+        for tweet in tweets:
+            print(tweet)
+            count += 1
+            if (count >5):
+                break
+
+    def spider_closed(self, spider):
+        print("I am in spider closed...")

Hands-on-WebScraping/project1_twitter_hashtag_crawler/myhashtags.csv

@@ -0,0 +1 @@
+danaher

Hands-on-WebScraping/project1_twitter_hashtag_crawler/requirements.txt

@@ -0,0 +1,2 @@
+scrapy
+dateutil

Hands-on-WebScraping/project1_twitter_hashtag_crawler/sampledata.csv

@@ -0,0 +1,12 @@
+username,full_name,twitter_url,tweet_text,tweet_time,number_of_likes,no_of_retweets,no_of_replies,mentions,no_of_mentions,hashtags,no_of_hashtags,call_to_action,image_url
+cctvasiapacific,CCTV Asia Pacific,https://twitter.com/CCTVAsiaPacific/status/1212269072328491008,"Turning off the stereotype of political faces, Hou Yanqi, the  # ChineseAmbssdor  to  # Nepal , amazes Nepalese and gains popularity on twitter by posting her ad-like photos and wishes: ""True beauty always touches the deep heart"", said Hou.
+२०२० नेपाल भ्रमाण वर्ष सफलताको शुभकामना pic.twitter.com/z0N8ru2vNd",2019-12-31 23:07:00,804,171,35,,0,,0,,https://pbs.twimg.com/media/ENLYSqlU4AAgiFh.jpg | https://pbs.twimg.com/media/ENLYSqoVAAASSS-.jpg | https://pbs.twimg.com/media/ENLYSqmU0AAZEyK.jpg
+,,https://twitter.com/BishowParajuli/status/1213037950549626882,"Zimbabwe is beautiful! Glad to hear your mountain climbing adventure ; If you wish to climb further higher, another beautiful place is  # Nepal ! You will you can also enjoy some terrific historical spots: pic.twitter.com/ofsCppyp0O",2020-01-03 02:02:00,27,3,1,,0,,0,,https://pbs.twimg.com/media/ENWTkzmUEAEKS1k.jpg | https://pbs.twimg.com/media/ENWTkznU4AAtVxK.jpg | https://pbs.twimg.com/media/ENWTkzoUwAEgMpX.jpg | https://pbs.twimg.com/media/ENWTkzlU4AEYxor.jpg
+kopinoora,kpila,https://twitter.com/kopinoora/status/1213481511967690752,# VisitNepal2020  official inauguration at London Nepal Embassy. # pic.twitter.com/e4N9XulBH7,2020-01-04 07:25:00,3,,0,,0,,0,,https://pbs.twimg.com/media/ENcnABiXsAE7_sw.jpg | https://pbs.twimg.com/media/ENcnABsXUAAnuBL.jpg
+mahbub_nazif,Nazif Mahbub,https://twitter.com/mahbub_nazif/status/1213328288271089664,"The joy of being Innocent. Durbar square, kathmandu, nepal pic.twitter.com/sbsfxTzeHN",2020-01-03 21:16:00,4,,0,,0,,0,,https://pbs.twimg.com/media/ENabn-uWwAcbUfb.jpg
+prabhumteverest,Prastuti_प्रश्तुती,https://twitter.com/PrabhuMteverest/status/1213178026457878528,"Visit nepal2020. where heaven meets and you won't feel regret choosing Nepal as your destination   
+
+We are eager to welcome you with our beautiful destinations and warm hospitality  pic.twitter.com/l7GQfk2ha6",2020-01-03 11:19:00,5,,0,,0,,0,,https://pbs.twimg.com/media/ENYS_CLUwAAVypp.jpg
+kashishds,Kashish Das Shrestha,https://twitter.com/kashishds/status/1213120581412876295,"Marpha bazaar, Mustang, Nepal. Today. 
+
+Requested my friend & Marpha resident Dipesh Hirachan for this clip. This is just outside his Apple orchard there. pic.twitter.com/oOFy88ylIt",2020-01-03 07:30:00,123,20,4,,0,,0,,

Hands-on-WebScraping/project1_twitter_hashtag_crawler/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = TwitterHashTagCrawler.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = TwitterHashTagCrawler

Hands-on-WebScraping/project1_twitter_hashtag_crawler/utils.py

@@ -0,0 +1,67 @@
+import re
+
+
+def find_emails(text):
+    """
+    It will parse the given string and return a list of emails if found
+
+    Example:
+    >>find_emails('hello\n find me here\nemail@gmail.com')
+    ['email@gmail.com']
+
+    :param text: string
+    :return: list
+    """
+    return re.findall(r"([a-zA-Z0-9+._-]+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)", text)
+
+
+def get_mentions(text):
+    """
+    It will return mentions from the text i.e @someone
+
+    :param text: string
+    :return: list
+
+    example
+    >>> get_mentions('Hi @hero, How are you? I hope @hero2 is fine. BTW say hi to @heroine for me')
+    ['hero','hero2','heroine']
+    """
+    result = re.findall("(^|[^@\w])@(\w{1,15})", text)
+    if len(result) != 0:
+        result = [i[1] for i in result]
+    return result
+
+
+def get_hashtags(text):
+    """
+      It will return hashtags from the text i.e #something
+
+      :param text: string
+      :return: list
+
+      example
+      >>> get_hashtags('my first code #programmer #python #awesome #grepsr')
+      ['programmer','python','awesome','grepsr']
+      """
+
+    result = re.findall("(^|[^@\w])#(\w{1,15})", text)
+    if len(result) != 0:
+        result = [i[1] for i in result]
+    return result
+
+
+def get_links(text):
+    """
+         It will return website links from the text
+
+         :param text: string
+         :return: list
+
+         example
+         >>> message = 'http://twitter.com Project URL: https://app.grepsr.com/app/project/message/70454'
+         >>> get_links(message)
+        ['http://twitter.com', 'https://app.grepsr.com/app/project/message/70454']
+
+         """
+    result = re.findall(r"(?P<url>https?://[^\s]+)", text)
+    return result

README.md

@@ -1,12 +1,6 @@
 ---
-title: Twitter Sentiment
-emoji: 📉
-colorFrom: pink
-colorTo: red
+title: twitter_sentiment
+app_file: test.py
 sdk: gradio
 sdk_version: 4.31.1
-app_file: app.py
-pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

requirements.txt

@@ -0,0 +1,4 @@
+streamlit
+transformers==4.40.2
+tensorflow==2.16.1
+tweetnlp

scrapper.ipynb

@@ -0,0 +1,168 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import re\n",
+    "import csv\n",
+    "from getpass import getpass\n",
+    "from time import sleep\n",
+    "from selenium.webdriver.common.keys import Keys\n",
+    "from selenium.common.exceptions import NoSuchElementException\n",
+    "from msedge.selenium_tools import Edge, EdgeOptions "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_tweet_data(card):\n",
+    "    \"\"\"Extract data from tweet card\"\"\"\n",
+    "    username = card.find_element_by_xpath('.//span').text\n",
+    "    try:\n",
+    "        handle = card.find_element_by_xpath('.//span[contains(text(), \"@\")]').text\n",
+    "    except NoSuchElementException:\n",
+    "        return\n",
+    "    \n",
+    "    try:\n",
+    "        postdate = card.find_element_by_xpath('.//time').get_attribute('datetime')\n",
+    "    except NoSuchElementException:\n",
+    "        return\n",
+    "    \n",
+    "    comment = card.find_element_by_xpath('.//div[2]/div[2]/div[1]').text\n",
+    "    responding = card.find_element_by_xpath('.//div[2]/div[2]/div[2]').text\n",
+    "    text = comment + responding\n",
+    "    reply_cnt = card.find_element_by_xpath('.//div[@data-testid=\"reply\"]').text\n",
+    "    retweet_cnt = card.find_element_by_xpath('.//div[@data-testid=\"retweet\"]').text\n",
+    "    like_cnt = card.find_element_by_xpath('.//div[@data-testid=\"like\"]').text\n",
+    "\n",
+    "    \n",
+    "    tweet = (username, handle, postdate, text, reply_cnt, retweet_cnt, like_cnt)\n",
+    "    return tweet  "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "search_term = input('search term: ')\n",
+    "\n",
+    "# create instance of web driver\n",
+    "options = EdgeOptions()\n",
+    "options.use_chromium = True\n",
+    "driver = Edge(options=options)\n",
+    "\n",
+    "# navigate to login screen\n",
+    "driver.get('https://twitter.com/search')\n",
+    "driver.maximize_window()\n",
+    "sleep(5)\n",
+    "\n",
+    "# find search input and search for term\n",
+    "search_input = driver.find_element_by_xpath('//input[@aria-label=\"Search query\"]')\n",
+    "search_input.send_keys(search_term)\n",
+    "search_input.send_keys(Keys.RETURN)\n",
+    "sleep(1)\n",
+    "\n",
+    "# navigate to historical 'latest' tab\n",
+    "driver.find_element_by_link_text('Latest').click()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# get all tweets on the page\n",
+    "data = []\n",
+    "tweet_ids = set()\n",
+    "last_position = driver.execute_script(\"return window.pageYOffset;\")\n",
+    "scrolling = True\n",
+    "\n",
+    "while scrolling:\n",
+    "    page_cards = driver.find_elements_by_xpath('//article[@data-testid=\"tweet\"]')\n",
+    "    for card in page_cards[-15:]:\n",
+    "        tweet = get_tweet_data(card)\n",
+    "        if tweet:\n",
+    "            tweet_id = ''.join(tweet)\n",
+    "            if tweet_id not in tweet_ids:\n",
+    "                tweet_ids.add(tweet_id)\n",
+    "                data.append(tweet)\n",
+    "            \n",
+    "    scroll_attempt = 0\n",
+    "    while True:\n",
+    "        # check scroll position\n",
+    "        driver.execute_script('window.scrollTo(0, document.body.scrollHeight);')\n",
+    "        sleep(2)\n",
+    "        curr_position = driver.execute_script(\"return window.pageYOffset;\")\n",
+    "        if last_position == curr_position:\n",
+    "            scroll_attempt += 1\n",
+    "            \n",
+    "            # end of scroll region\n",
+    "            if scroll_attempt >= 3:\n",
+    "                scrolling = False\n",
+    "                break\n",
+    "            else:\n",
+    "                sleep(2) # attempt another scroll\n",
+    "        else:\n",
+    "            last_position = curr_position\n",
+    "            break\n",
+    "\n",
+    "# close the web driver\n",
+    "driver.close()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Saving the tweet data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('turkcell_tweets.csv', 'w', newline='', encoding='utf-8') as f:\n",
+    "    header = ['UserName', 'Handle', 'Timestamp', 'Text', 'Comments', 'Likes', 'Retweets']\n",
+    "    writer = csv.writer(f)\n",
+    "    writer.writerow(header)\n",
+    "    writer.writerows(data)"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "306b4709344c791e982a258cf5494139869959872aa39c2c4102a54cca0d2138"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.7.0 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.0"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}