File size: 3,471 Bytes
e42ecfa
a0f03b5
fd58ad7
 
b64b72d
cf64ea1
b871eea
 
 
 
32e8749
 
 
 
 
 
cf64ea1
 
32e8749
e42ecfa
fba8174
32e8749
412b9d9
32e8749
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
fba8174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
165f772
fba8174
 
6ca040b
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import zipfile
import sys
import os

sys.path.append('https://huggingface.co/spaces/PradeepJha/ISCO-code-predictor-api/resolve/main/models.zip')

# Check current directory and list files
print("Current Directory:", os.getcwd())
print("Files in Directory:", os.listdir())

import numpy as np
import tensorflow as tf
import tensorflow_hub as hub
import tf_keras as keras
import pandas as pd
from tensorflow.keras.models import load_model
from official.nlp.data import classifier_data_lib
from official.nlp.tools import tokenization
import joblib

import gradio as gr

model = load_model('ISCO-Coder-BERT.h5', custom_objects={'KerasLayer': hub.KerasLayer})

vocab_file = model.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = model.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file,do_lower_case)

# Parameters
max_seq_length = 128
label_list = 424
dummy_label = 100


# Define a function to preprocess the new data
def get_feature_new(text, max_seq_length, tokenizer, dummy_label):
    example = classifier_data_lib.InputExample(guid=None,
                                               text_a=text.numpy().decode('utf-8'),
                                               text_b=None,
                                               label=dummy_label)  # Use a valid dummy label
    feature = classifier_data_lib.convert_single_example(0, example, label_list, max_seq_length, tokenizer)
    return feature.input_ids, feature.input_mask, feature.segment_ids

def get_feature_map_new(text):
    input_ids, input_mask, segment_ids = tf.py_function(
        lambda text: get_feature_new(text, max_seq_length, tokenizer, dummy_label),
        inp=[text],
        Tout=[tf.int32, tf.int32, tf.int32]
    )
    input_ids.set_shape([max_seq_length])
    input_mask.set_shape([max_seq_length])
    segment_ids.set_shape([max_seq_length])
    
    x = {'input_word_ids': input_ids,
         'input_mask': input_mask,
         'input_type_ids': segment_ids}
    
    return x

def preprocess_new_data(texts):
    dataset = tf.data.Dataset.from_tensor_slices((texts,))
    dataset = dataset.map(get_feature_map_new,
                          num_parallel_calls=tf.data.experimental.AUTOTUNE)
    dataset = dataset.batch(32, drop_remainder=False)
    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
    
    return dataset
def launch(input):
    # Load the label encoder
    label_encoder = joblib.load('label_encoder.joblib')

    # Preprocess the new data
    sample_example = [input]
    new_data_dataset = preprocess_new_data(sample_example)

    # Assuming you have a model already loaded (add model loading code if needed)
    # Make predictions on the new data
    predictions = model.predict(new_data_dataset)

    # Decode the predictions
    predicted_classes = [label_list[np.argmax(pred)] for pred in predictions]

    # Print the predicted classes
    print(predicted_classes)

    # Calculate the highest probabilities
    highest_probabilities = [max(instance) for instance in predictions]

    # Decode labels using the label encoder
    decoded_labels = label_encoder.inverse_transform(predicted_classes)

    print("Most likely ISCO code is {} and probability is {}".format(decoded_labels,highest_probabilities))

# Gradio Interface
iface = gr.Interface(fn=launch,inputs=gr.inputs.Textbox(lines=2, placeholder="Enter job title and description here..."),outputs="text")

# Launch the Gradio app
iface.launch()