obsei-demo / config.yaml
obsei's picture
Correcting imports
cc6c8d1
source:
Youtube Scrapper:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/Youtube.png"
_help_:
- '`video url` is Youtube video url.'
source:
_target_: obsei.source.youtube_scrapper.YoutubeScrapperSource
config:
_target_: obsei.source.youtube_scrapper.YoutubeScrapperConfig
video_url: "https://www.youtube.com/watch?v=uZfns0JIlFk"
lookup_period: "1Y"
max_comments: 10
Appstore Scrapper:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/appstore.png"
_help_:
- '`app_url` is application url on app store.
- For example for Xcode - https://apps.apple.com/us/app/xcode/id497799835'
source:
_target_: obsei.source.appstore_scrapper.AppStoreScrapperSource
config:
_target_: obsei.source.appstore_scrapper.AppStoreScrapperConfig
app_url: "https://apps.apple.com/us/app/gmail-email-by-google/id422689480"
lookup_period: "1h"
max_count: 5
Playstore Scrapper:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/playstore.png"
_help_:
- '`app_url` is application url on play store'
- 'For example for Gmail - https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en_IN&gl=US'
source:
_target_: obsei.source.playstore_scrapper.PlayStoreScrapperSource
config:
_target_: obsei.source.playstore_scrapper.PlayStoreScrapperConfig
app_url: "https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en_IN&gl=US"
lookup_period: "1h"
max_count: 5
Maps Reviews Scrapper:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/google_maps.png"
_help_:
- 'Collect `api_key` from https://outscraper.com/'
- ''
- 'For `queries` enter google maps urls or place ids, for example'
- "https://www.google.co.in/maps/place/Taj+Mahal/@27.1751496,78.0399535,17z/data=!4m5!3m4!1s0x39747121d702ff6d:0xdd2ae4803f767dde!8m2!3d27.1751448!4d78.0421422"
source:
_target_: obsei.source.google_maps_reviews.OSGoogleMapsReviewsSource
config:
_target_: obsei.source.google_maps_reviews.OSGoogleMapsReviewsConfig
api_key: ''
queries:
- "https://www.google.co.in/maps/place/Taj+Mahal/@27.1751496,78.0399535,17z/data=!4m5!3m4!1s0x39747121d702ff6d:0xdd2ae4803f767dde!8m2!3d27.1751448!4d78.0421422"
number_of_reviews: 5
Reddit Scrapper:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/reddit.png"
_help_:
- 'Reddit subreddit, search etc rss url. For proper url refer following link -'
- 'https://www.reddit.com/r/pathogendavid/comments/tv8m9/pathogendavids_guide_to_rss_and_reddit/'
source:
_target_: obsei.source.reddit_source.RedditScrapperSource
config:
_target_: obsei.source.reddit_source.RedditScrapperConfig
url: 'https://www.reddit.com/r/wallstreetbets/comments/.rss?sort=new'
lookup_period: "1h"
Twitter:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/twitter.png"
_help_:
- '`query` accept search string, @user or #hashtags also'
- ''
- 'Need twitter `consumer key` and `secret`, get it from https://developer.twitter.com/en/apply-for-access'
source:
_target_: obsei.source.twitter_source.TwitterSource
config:
_target_: obsei.source.twitter_source.TwitterSourceConfig
query: "@Twitter"
lookup_period: "1h"
max_tweets: 10
cred_info:
_target_: obsei.source.twitter_source.TwitterCredentials
consumer_key: ''
consumer_secret: ''
Facebook:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/facebook.png"
_help_:
- '`page_id` is id of your facebook page'
- ''
- 'Need facebook app_id, app_secret and long_term_token. Get it from https://developers.facebook.com/apps/'
source:
_target_: obsei.source.facebook_source.FacebookSource
config:
_target_: obsei.source.facebook_source.FacebookSourceConfig
page_id: "110844591144719"
lookup_period: "1h"
cred_info:
_target_: obsei.source.facebook_source.FacebookCredentials
app_id: ''
app_secret: ''
long_term_token: ''
Email:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/gmail.png"
_help_:
- 'List of IMAP servers for most commonly used email providers https://www.systoolsgroup.com/imap/'
- ''
- 'Also, if you are using a `Gmail` account then make sure you allow less secure apps on your account'
- 'https://myaccount.google.com/lesssecureapps?pli=1'
- 'Also enable IMAP access - https://mail.google.com/mail/u/0/#settings/fwdandpop'
source:
_target_: obsei.source.email_source.EmailSource
config:
_target_: obsei.source.email_source.EmailConfig
imap_server: 'imap.gmail.com'
cred_info:
_target_: obsei.source.email_source.EmailCredInfo
username: ''
password: ''
lookup_period: "1h"
Reddit:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/reddit.png"
_help_:
- 'Reddit account `username` and `password` require'
- 'Enter list of `subreddits`'
source:
_target_: obsei.source.reddit_source.RedditSource
config:
_target_: obsei.source.reddit_source.RedditConfig
subreddits:
- 'wallstreetbets'
cred_info:
_target_: obsei.source.reddit_source.RedditCredInfo
username: ''
password: ''
lookup_period: "1h"
Google News:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/googlenews.png"
_help_:
- '`fetch_article` use crawler to fetch full article'
source:
_target_: obsei.source.google_news_source.GoogleNewsSource
config:
_target_: obsei.source.google_news_source.GoogleNewsConfig
query: "bitcoin"
max_results: 3
lookup_period: "1d"
fetch_article: true
Website Crawler:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/webcrawler.png"
_help_:
- '`package name` can be found at the end of the url of app in play store.'
- ''
- 'For example - https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en&gl=US'
- '`com.google.android.gm` is the `package name` for xcode and `us` is `country`.'
source:
_target_: obsei.source.website_crawler_source.TrafilaturaCrawlerSource
config:
_target_: obsei.source.website_crawler_source.TrafilaturaCrawlerConfig
urls:
- 'https://obsei.github.io/obsei/'
sink:
Panda Dataframe:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/pandas.svg"
_help_: null
sink:
_target_: obsei.sink.pandas_sink.PandasSink
config:
_target_: obsei.sink.pandas_sink.PandasSinkConfig
Logger:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/logger.png"
_help_: null
sink:
_target_: obsei.sink.logger_sink.LoggerSink
config:
_target_: obsei.sink.logger_sink.LoggerSinkConfig
Jira:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/jira.png"
_help_:
- 'For testing purpose you can start jira server locally'
- 'Refer https://developer.atlassian.com/server/framework/atlassian-sdk/atlas-run-standalone/'
- ''
- 'Provide `server url`, `username` and `password` of the user'
- ''
- '`type` of issue to be created, for more information refer -'
- 'https://support.atlassian.com/jira-cloud-administration/docs/what-are-issue-types/'
- ''
- '`project` in which issue to be created, for more information refer -'
- 'https://support.atlassian.com/jira-software-cloud/docs/what-is-a-jira-software-project/'
sink:
_target_: obsei.sink.jira_sink.JiraSink
config:
_target_: obsei.sink.jira_sink.JiraSinkConfig
url: 'http://localhost:2990/jira'
username: ''
password: ''
issue_type:
name: "Task"
project:
key: ""
Zendesk:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/zendesk.png"
_help_:
- 'For custom domain refer http://docs.facetoe.com.au/zenpy.html#custom-domains'
- 'Provide zendesk `domain`'
- ''
- 'Provide `subdomain` if you have one'
- ''
- 'Provide zendesk account `email` and `password`'
sink:
_target_: obsei.sink.zendesk_sink.ZendeskSink
config:
_target_: obsei.sink.zendesk_sink.ZendeskSinkConfig
domain: "zendesk.com"
subdomain: null
cred_info:
_target_: obsei.sink.zendesk_sink.ZendeskCredInfo
email: ''
password: ''
Slack:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/slack.svg"
_help_:
- 'Provide slack bot/app `token`, for more detail refer -'
- 'https://slack.com/intl/en-de/help/articles/215770388-Create-and-regenerate-API-tokens'
- ''
- 'To get `channel id` refer -'
- 'https://stackoverflow.com/questions/40940327/what-is-the-simplest-way-to-find-a-slack-team-id-and-a-channel-id'
sink:
_target_: obsei.sink.slack_sink.SlackSink
config:
_target_: obsei.sink.slack_sink.SlackSinkConfig
slack_token: ''
channel_id: ''
jinja_template: |
:bell: Hi there!, a new notification by *Obsei*
>Content:
```
{%- for key, value in payload.items() recursive%}
{%- if value is mapping -%}
{{loop(value.items())}}
{%- else %}
{{key}}: {{value}}
{%- endif %}
{%- endfor%}
```
Elastic:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/elastic.png"
_help_:
- 'For testing purpose you can start Elasticsearch server locally via docker'
- '`docker run -d --name elasticsearch -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.9.2`'
- ''
- ' Provide server `hostname`, `port` along with `index` to be used'
sink:
_target_: obsei.sink.elasticsearch_sink.ElasticSearchSink
config:
_target_: obsei.sink.elasticsearch_sink.ElasticSearchSinkConfig
host: "localhost"
port: 9200
index_name: "test"
Http:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/http_api.png"
_help_:
- 'For testing purpose you can create mock http server via postman, refer -'
- 'https://learning.postman.com/docs/designing-and-developing-your-api/mocking-data/setting-up-mock/'
- ''
- 'Provide http server `url` and `headers`'
sink:
_target_: obsei.sink.http_sink.HttpSink
config:
_target_: obsei.sink.http_sink.HttpSinkConfig
url: 'https://localhost:8080/api/path'
headers:
Content-type: "application/json"
analyzer:
Sentiment:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/sentiment.png"
Transformer:
_help_:
- 'For supported models refer https://huggingface.co/models?filter=zero-shot-classification'
- ''
- 'Possible device values are `auto` (cuda:0 if available otherwise cpu), `cpu` and `cuda:{id}` (cuda device id)'
config:
_target_: obsei.analyzer.sentiment_analyzer.TransformersSentimentAnalyzerConfig
labels:
- "positive"
- "negative"
multi_class_classification: false
analyzer:
_target_: obsei.analyzer.sentiment_analyzer.TransformersSentimentAnalyzer
model_name_or_path: "typeform/mobilebert-uncased-mnli"
device: "auto"
Vader:
_help_:
- 'Vader is less resource hungry dictionary based Vader Sentiment detector'
analyzer:
_target_: obsei.analyzer.sentiment_analyzer.VaderSentimentAnalyzer
Classification:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/classification.png"
Transformer:
_help_:
- 'For supported models refer https://huggingface.co/models?filter=zero-shot-classification'
- ''
- 'Provide classification `labels`, two labels "positive" and "negative" are added by default'
- ''
- 'Possible device values are `auto` (cuda:0 if available otherwise cpu), `cpu` and `cuda:{id}` (cuda device id)'
config:
_target_: obsei.analyzer.classification_analyzer.ClassificationAnalyzerConfig
labels:
- "service"
- "content"
- "interface"
multi_class_classification: true
analyzer:
_target_: obsei.analyzer.classification_analyzer.ZeroShotClassificationAnalyzer
model_name_or_path: "typeform/mobilebert-uncased-mnli"
device: "auto"
Named Entity Recognition:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/ner.png"
Transformer:
_help_:
- 'For supported models refer https://huggingface.co/models?filter=token-classification'
- ''
- 'Possible device values are `auto` (cuda:0 if available otherwise cpu), `cpu` and `cuda:{id}` (cuda device id)'
analyzer:
_target_: obsei.analyzer.ner_analyzer.TransformersNERAnalyzer
model_name_or_path: "elastic/distilbert-base-cased-finetuned-conll03-english"
device: "auto"
Spacy:
_help_:
- 'For supported models refer https://spacy.io/models'
analyzer:
_target_: obsei.analyzer.ner_analyzer.SpacyNERAnalyzer
model_name_or_path: "en_core_web_sm"
Translation:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/translator.png"
Transformer:
_help_:
- 'For supported models refer https://huggingface.co/models?pipeline_tag=translation'
- ''
- 'Possible device values are `auto` (cuda:0 if available otherwise cpu), `cpu` and `cuda:{id}` (cuda device id)'
analyzer:
_target_: obsei.analyzer.translation_analyzer.TranslationAnalyzer
model_name_or_path: "Helsinki-NLP/opus-mt-en-hi"
device: "auto"
PII Anonymizer:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/pii.png"
Presidio:
_help_:
- '`analyze_only` decide whether to return only pii analysis or anonymize text'
- ''
- '`return_decision_process` decide whether to return detail information about anonymization decision'
- ''
- 'For `nlp_engine_name` spacy and stanza nlp engines are supported, For more info refer -'
- 'https://microsoft.github.io/presidio/analyzer/developing_recognizers/#utilize-spacy-or-stanza'
- ''
- 'Provide `model_name` and `lang_code` of the model'
config:
_target_: obsei.analyzer.pii_analyzer.PresidioPIIAnalyzerConfig
analyze_only: false
return_decision_process: false
analyzer:
_target_: obsei.analyzer.pii_analyzer.PresidioPIIAnalyzer
engine_config:
_target_: obsei.analyzer.pii_analyzer.PresidioEngineConfig
nlp_engine_name: "spacy"
models:
- _target_: obsei.analyzer.pii_analyzer.PresidioModelConfig
model_name: "en_core_web_md"
lang_code: "en"
Dummy:
_icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/dummy.png"
Dummy:
_help_:
- 'Dummy Analyzer, do nothing it simply used for transforming input to output'
config:
_target_: obsei.analyzer.dummy_analyzer.DummyAnalyzerConfig
analyzer:
_target_: obsei.analyzer.dummy_analyzer.DummyAnalyzer