File size: 3,789 Bytes
579b090
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
from typing import List, Tuple
from typing_extensions import Literal
import logging
import pandas as pd
from pandas import DataFrame, Series
from utils.config import getconfig
from utils.preprocessing import processingpipeline
import streamlit as st
from transformers import pipeline

# Labels dictionary ###
_lab_dict = {
                      'GHG':'GHG',
                      'NOT_GHG':'NON GHG TRANSPORT TARGET',
                      'NEGATIVE':'OTHERS',
                      }
        

@st.cache_resource
def load_ghgClassifier(config_file:str = None, classifier_name:str = None):
    """
    loads the document classifier using haystack, where the name/path of model
    in HF-hub as string is used to fetch the model object.Either configfile or 
    model should be passed.
    1. https://docs.haystack.deepset.ai/reference/document-classifier-api
    2. https://docs.haystack.deepset.ai/docs/document_classifier
    Params
    --------
    config_file: config file path from which to read the model name
    classifier_name: if modelname is passed, it takes a priority if not \
    found then will look for configfile, else raise error.
    Return: document classifier model
    """
    if not classifier_name:
        if not config_file:
            logging.warning("Pass either model name or config file")
            return
        else:
            config = getconfig(config_file)
            classifier_name = config.get('ghg','MODEL')
    
    logging.info("Loading ghg classifier")    
    doc_classifier = pipeline("text-classification", 
                            model=classifier_name, 
                            top_k =1)

    return doc_classifier


@st.cache_data
def ghg_classification(haystack_doc:pd.DataFrame,
                        threshold:float = 0.5, 
                        classifier_model:pipeline= None
                        )->Tuple[DataFrame,Series]:
    """
    Text-Classification on the list of texts provided. Classifier provides the 
    most appropriate label for each text. these labels are in terms of if text 
    belongs to which particular Sustainable Devleopment Goal (SDG).
    Params
    ---------
    haystack_doc: List of haystack Documents. The output of Preprocessing Pipeline 
    contains the list of paragraphs in different format,here the list of 
    Haystack Documents is used.
    threshold: threshold value for the model to keep the results from classifier
    classifiermodel: you can pass the classifier model directly,which takes priority
    however if not then looks for model in streamlit session.
    In case of streamlit avoid passing the model directly.
    Returns
    ----------
    df: Dataframe with two columns['SDG:int', 'text']
    x: Series object with the unique SDG covered in the document uploaded and 
    the number of times it is covered/discussed/count_of_paragraphs. 
    """
    logging.info("Working on GHG Extraction")
    haystack_doc['GHG Label'] = 'NA'
    haystack_doc['GHG Score'] = 0.0
    # applying GHG Identifier to only 'Target' paragraphs.
    temp = haystack_doc[haystack_doc['Target Label'] == 'TARGET']
    temp = temp.reset_index(drop=True)
    df = haystack_doc[haystack_doc['Target Label'] == 'NEGATIVE']
    df = df.reset_index(drop=True)

    if not classifier_model:
        classifier_model = st.session_state['ghg_classifier']
    
    results = classifier_model(list(temp.text))
    labels_= [(l[0]['label'],l[0]['score']) for l in results]
    temp['GHG Label'],temp['GHG Score'] = zip(*labels_)
    temp['GHG Label'] = temp['GHG Label'].apply(lambda x: _lab_dict[x])
    # merge back Target and non-Target dataframe
    df = pd.concat([df,temp])
    df = df.reset_index(drop =True)
    df['GHG Score'] = df['GHG Score'].round(2)
    df.index += 1

    return df