config.yaml

source:
  Youtube Scrapper:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/Youtube.png"
    _help_:
      - '`video url` is Youtube video url.'
    source:
      _target_: obsei.source.youtube_scrapper.YoutubeScrapperSource
    config:
      _target_: obsei.source.youtube_scrapper.YoutubeScrapperConfig
      video_url: "https://www.youtube.com/watch?v=uZfns0JIlFk"
      lookup_period: "1Y"
      max_comments: 10
  Appstore Scrapper:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/appstore.png"
    _help_:
      - '`app_url` is application url on app store.
      - For example for Xcode - https://apps.apple.com/us/app/xcode/id497799835'
    source:
      _target_: obsei.source.AppStoreScrapperSource
    config:
      _target_: obsei.source.AppStoreScrapperConfig
      app_url: "https://apps.apple.com/us/app/gmail-email-by-google/id422689480"
      lookup_period: "1h"
      max_count: 5
  Playstore Scrapper:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/playstore.png"
    _help_:
      - '`app_url` is application url on play store'
      - 'For example for Gmail - https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en_IN&gl=US'
    source:
      _target_: obsei.source.PlayStoreScrapperSource
    config:
      _target_: obsei.source.PlayStoreScrapperConfig
      app_url: "https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en_IN&gl=US"
      lookup_period: "1h"
      max_count: 5
  Maps Reviews Scrapper:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/google_maps.png"
    _help_:
      - 'Collect `api_key` from https://outscraper.com/'
      - ''
      - 'For `queries` enter google maps urls or place ids, for example'
      - "https://www.google.co.in/maps/place/Taj+Mahal/@27.1751496,78.0399535,17z/data=!4m5!3m4!1s0x39747121d702ff6d:0xdd2ae4803f767dde!8m2!3d27.1751448!4d78.0421422"
    source:
      _target_: obsei.source.OSGoogleMapsReviewsSource
    config:
      _target_: obsei.source.OSGoogleMapsReviewsConfig
      api_key: ''
      queries:
        - "https://www.google.co.in/maps/place/Taj+Mahal/@27.1751496,78.0399535,17z/data=!4m5!3m4!1s0x39747121d702ff6d:0xdd2ae4803f767dde!8m2!3d27.1751448!4d78.0421422"
      number_of_reviews: 5
  Reddit Scrapper:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/reddit.png"
    _help_:
      - 'Reddit subreddit, search etc rss url. For proper url refer following link -'
      - 'https://www.reddit.com/r/pathogendavid/comments/tv8m9/pathogendavids_guide_to_rss_and_reddit/'
    source:
      _target_: obsei.source.RedditScrapperSource
    config:
      _target_: obsei.source.RedditScrapperConfig
      url: 'https://www.reddit.com/r/wallstreetbets/comments/.rss?sort=new'
      lookup_period: "1h"
  Twitter:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/twitter.png"
    _help_:
      - '`query` accept search string, @user or #hashtags also'
      - ''
      - 'Need twitter `consumer key` and `secret`, get it from https://developer.twitter.com/en/apply-for-access'
    source:
      _target_: obsei.source.TwitterSource
    config:
      _target_: obsei.source.TwitterSourceConfig
      query: "@Twitter"
      lookup_period: "1h"
      max_tweets: 10
      cred_info:
        _target_: obsei.source.TwitterCredentials
        consumer_key: ''
        consumer_secret: ''
  Facebook:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/facebook.png"
    _help_:
      - '`page_id` is id of your facebook page'
      - ''
      - 'Need facebook app_id, app_secret and long_term_token. Get it from https://developers.facebook.com/apps/'
    source:
      _target_: obsei.source.FacebookSource
    config:
      _target_: obsei.source.FacebookSourceConfig
      page_id: "110844591144719"
      lookup_period: "1h"
      cred_info:
        _target_: obsei.source.FacebookCredentials
        app_id: ''
        app_secret: ''
        long_term_token: ''
  Email:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/gmail.png"
    _help_:
      - 'List of IMAP servers for most commonly used email providers https://www.systoolsgroup.com/imap/'
      - ''
      - 'Also, if you are using a `Gmail` account then make sure you allow less secure apps on your account'
      - 'https://myaccount.google.com/lesssecureapps?pli=1'
      - 'Also enable IMAP access - https://mail.google.com/mail/u/0/#settings/fwdandpop'
    source:
      _target_: obsei.source.EmailSource
    config:
      _target_: obsei.source.EmailConfig
      imap_server: 'imap.gmail.com'
      cred_info:
        _target_: obsei.source.EmailCredInfo
        username: ''
        password: ''
      lookup_period: "1h"
  Reddit:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/reddit.png"
    _help_:
      - 'Reddit account `username` and `password` require'
      - 'Enter list of `subreddits`'
    source:
      _target_: obsei.source.RedditSource
    config:
      _target_: obsei.source.RedditConfig
      subreddits:
        - 'wallstreetbets'
      cred_info:
        _target_: obsei.source.RedditCredInfo
        username: ''
        password: ''
      lookup_period: "1h"
  Google News:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/googlenews.png"
    _help_:
      - '`fetch_article` use crawler to fetch full article'
    source:
      _target_: obsei.source.GoogleNewsSource
    config:
      _target_: obsei.source.GoogleNewsConfig
      query: "bitcoin"
      max_results: 3
      lookup_period: "1d"
      fetch_article: true
  Website Crawler:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/webcrawler.png"
    _help_:
      - '`package name` can be found at the end of the url of app in play store.'
      - ''
      - 'For example - https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en&gl=US'
      - '`com.google.android.gm` is the `package name` for xcode and `us` is `country`.'
    source:
      _target_: obsei.source.TrafilaturaCrawlerSource
    config:
      _target_: obsei.source.TrafilaturaCrawlerConfig
      urls:
        - 'https://obsei.github.io/obsei/'
sink:
  Panda Dataframe:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/pandas.svg"
    _help_: null
    sink:
      _target_: obsei.sink.pandas_sink.PandasSink
    config:
      _target_: obsei.sink.pandas_sink.PandasSinkConfig
  Logger:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/logger.png"
    _help_: null
    sink:
      _target_: obsei.sink.LoggerSink
    config:
      _target_: obsei.sink.LoggerSinkConfig
  Jira:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/jira.png"
    _help_:
      - 'For testing purpose you can start jira server locally'
      - 'Refer https://developer.atlassian.com/server/framework/atlassian-sdk/atlas-run-standalone/'
      - ''
      - 'Provide `server url`, `username`  and `password` of the user'
      - ''
      - '`type` of issue to be created, for more information refer -'
      - 'https://support.atlassian.com/jira-cloud-administration/docs/what-are-issue-types/'
      - ''
      - '`project` in which issue to be created, for more information refer -'
      - 'https://support.atlassian.com/jira-software-cloud/docs/what-is-a-jira-software-project/'
    sink:
      _target_: obsei.sink.JiraSink
    config:
      _target_: obsei.sink.JiraSinkConfig
      url: 'http://localhost:2990/jira'
      username: ''
      password: ''
      issue_type:
        name: "Task"
      project:
        key: ""
  Zendesk:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/zendesk.png"
    _help_:
      - 'For custom domain refer http://docs.facetoe.com.au/zenpy.html#custom-domains'
      - 'Provide zendesk `domain`'
      - ''
      - 'Provide `subdomain` if you have one'
      - ''
      - 'Provide zendesk account `email` and `password`'
    sink:
      _target_: obsei.sink.ZendeskSink
    config:
      _target_: obsei.sink.ZendeskSinkConfig
      domain: "zendesk.com"
      subdomain: null
      cred_info:
        _target_: obsei.sink.ZendeskCredInfo
        email: ''
        password: ''
  Slack:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/slack.svg"
    _help_:
      - 'Provide slack bot/app `token`, for more detail refer -'
      - 'https://slack.com/intl/en-de/help/articles/215770388-Create-and-regenerate-API-tokens'
      - ''
      - 'To get `channel id` refer -'
      - 'https://stackoverflow.com/questions/40940327/what-is-the-simplest-way-to-find-a-slack-team-id-and-a-channel-id'
    sink:
      _target_: obsei.sink.SlackSink
    config:
      _target_: obsei.sink.SlackSinkConfig
      slack_token: ''
      channel_id: ''
      jinja_template: |
        :bell: Hi there!, a new notification by *Obsei*
        >Content:
        ```
          {%- for key, value in payload.items() recursive%}
            {%- if value is mapping -%}
              {{loop(value.items())}}
            {%- else %}
              {{key}}: {{value}}
            {%- endif %}
          {%- endfor%}
        ```
  Elastic:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/elastic.png"
    _help_:
      - 'For testing purpose you can start Elasticsearch server locally via docker'
      - '`docker run -d --name elasticsearch -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.9.2`'
      - ''
      - ' Provide server `hostname`, `port` along with `index` to be used'
    sink:
      _target_: obsei.sink.ElasticSearchSink
    config:
      _target_: obsei.sink.ElasticSearchSinkConfig
      host: "localhost"
      port: 9200
      index_name: "test"
  Http:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/http_api.png"
    _help_:
      - 'For testing purpose you can create mock http server via postman, refer -'
      - 'https://learning.postman.com/docs/designing-and-developing-your-api/mocking-data/setting-up-mock/'
      - ''
      - 'Provide http server `url` and `headers`'
    sink:
      _target_: obsei.sink.HttpSink
    config:
      _target_: obsei.sink.HttpSinkConfig
      url: 'https://localhost:8080/api/path'
      headers:
        Content-type: "application/json"
analyzer:
  Sentiment:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/sentiment.png"
    Transformer:
      _help_:
        - 'For supported models refer https://huggingface.co/models?filter=zero-shot-classification'
        - ''
        - 'Possible device values are `auto` (cuda:0 if available otherwise cpu), `cpu` and `cuda:{id}` (cuda device id)'
      config:
        _target_: obsei.analyzer.TransformersSentimentAnalyzerConfig
        labels:
          - "positive"
          - "negative"
        multi_class_classification: false
      analyzer:
        _target_: obsei.analyzer.TransformersSentimentAnalyzer
        model_name_or_path: "typeform/mobilebert-uncased-mnli"
        device: "auto"
    Vader:
      _help_:
        - 'Vader is less resource hungry dictionary based Vader Sentiment detector'
      analyzer:
        _target_: obsei.analyzer.VaderSentimentAnalyzer
  Classification:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/classification.png"
    Transformer:
      _help_:
        - 'For supported models refer https://huggingface.co/models?filter=zero-shot-classification'
        - ''
        - 'Provide classification `labels`, two labels "positive" and "negative" are added by default'
        - ''
        - 'Possible device values are `auto` (cuda:0 if available otherwise cpu), `cpu` and `cuda:{id}` (cuda device id)'
      config:
        _target_: obsei.analyzer.ClassificationAnalyzerConfig
        labels:
          - "service"
          - "content"
          - "interface"
        multi_class_classification: true
      analyzer:
        _target_: obsei.analyzer.ZeroShotClassificationAnalyzer
        model_name_or_path: "typeform/mobilebert-uncased-mnli"
        device: "auto"
  Named Entity Recognition:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/ner.png"
    Transformer:
      _help_:
        - 'For supported models refer https://huggingface.co/models?filter=token-classification'
        - ''
        - 'Possible device values are `auto` (cuda:0 if available otherwise cpu), `cpu` and `cuda:{id}` (cuda device id)'
      analyzer:
        _target_: obsei.analyzer.TransformersNERAnalyzer
        model_name_or_path: "elastic/distilbert-base-cased-finetuned-conll03-english"
        device: "auto"
    Spacy:
      _help_:
        - 'For supported models refer https://spacy.io/models'
      analyzer:
        _target_: obsei.analyzer.SpacyNERAnalyzer
        model_name_or_path: "en_core_web_sm"
  Translation:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/translator.png"
    Transformer:
      _help_:
        - 'For supported models refer https://huggingface.co/models?pipeline_tag=translation'
        - ''
        - 'Possible device values are `auto` (cuda:0 if available otherwise cpu), `cpu` and `cuda:{id}` (cuda device id)'
      analyzer:
        _target_: obsei.analyzer.TranslationAnalyzer
        model_name_or_path: "Helsinki-NLP/opus-mt-en-hi"
        device: "auto"
  PII Anonymizer:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/pii.png"
    Presidio:
      _help_:
        - '`analyze_only` decide whether to return only pii analysis or anonymize text'
        - ''
        - '`return_decision_process` decide whether to return detail information about anonymization decision'
        - ''
        - 'For `nlp_engine_name` spacy and stanza nlp engines are supported, For more info refer -'
        - 'https://microsoft.github.io/presidio/analyzer/developing_recognizers/#utilize-spacy-or-stanza'
        - ''
        - 'Provide `model_name` and `lang_code` of the model'
      config:
        _target_: obsei.analyzer.PresidioPIIAnalyzerConfig
        analyze_only: false
        return_decision_process: false
      analyzer:
        _target_: obsei.analyzer.PresidioPIIAnalyzer
        engine_config:
          _target_: obsei.analyzer.PresidioEngineConfig
          nlp_engine_name: "spacy"
          models:
            - _target_: obsei.analyzer.PresidioModelConfig
              model_name: "en_core_web_md"
              lang_code: "en"
  Dummy:
    _icon_: "https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/dummy.png"
    Dummy:
      _help_:
        - 'Dummy Analyzer, do nothing it simply used for transforming input to output'
      config:
        _target_: obsei.analyzer.DummyAnalyzerConfig
      analyzer:
        _target_: obsei.analyzer.DummyAnalyzer