setting language pipes

app.py

@@ -0,0 +1,56 @@
+import gradio as gr
+from transformers import pipeline
+
+get_completion = pipeline("summarization",model="sshleifer/distilbart-cnn-12-6")
+get_ner = pipeline("ner", model="dslim/bert-base-NER")
+get_caption = pipeline("image-to-text")
+def summarize_text(input):
+    output = get_completion(input)
+    return output[0]['summary_text']
+
+def merge_tokens(tokens):
+    merged_tokens = []
+    for token in tokens:
+        if merged_tokens and token['entity'].startswith('I-') and merged_tokens[-1]['entity'].endswith(token['entity'][2:]):
+            # If current token continues the entity of the last one, merge them
+            last_token = merged_tokens[-1]
+            last_token['word'] += token['word'].replace('##', '')
+            last_token['end'] = token['end']
+            last_token['score'] = (last_token['score'] + token['score']) / 2
+        else:
+            # Otherwise, add the token to the list
+            merged_tokens.append(token)
+        return merged_tokens
+
+def named_entity_recognition(input):
+    output = get_ner(input)
+    merged_output = merge_tokens(output)
+    return {"text": input, "entities": output}
+
+
+interface_summarise = gr.Interface(fn=summarize_text,
+ inputs=[gr.Textbox(label="Text to summarise", lines=5)],
+ outputs=[gr.Textbox(label="Summary")],
+ title="Text Summarizer",
+ description="Summary of text via `distillBART-CNN` model!")
+
+interface_ner = gr.Interface(fn=named_entity_recognition,
+                    inputs=[gr.Textbox(label="Text to find entities", lines=2)],
+                    outputs=[gr.HighlightedText(label="Text with entities")],
+                    title="NER with dslim/bert-base-NER",
+                    description="Find entities using the `dslim/bert-base-NER` model under the hood!",
+                    allow_flagging="never",
+                    examples=[
+                        "Tim Cook is the CEO of Apple, stays in California and makes iPhones ",
+                        "My name is Bose and I am a physicist living in Delhi"
+                    ])
+
+demo = gr.TabbedInterface([
+    interface_summarise, 
+    interface_ner],
+    ["Text Summary ",
+     "Named Entity Recognition"
+    ])
+
+if __name__ == "__main__":
+    demo.launch(enable_queue=True)

requirements.txt

@@ -0,0 +1,67 @@
+aiofiles==23.2.1
+altair==5.1.2
+annotated-types==0.6.0
+anyio==3.7.1
+attrs==23.1.0
+certifi==2023.7.22
+charset-normalizer==3.3.1
+click==8.1.7
+contourpy==1.1.1
+cycler==0.12.1
+exceptiongroup==1.1.3
+fastapi==0.104.0
+ffmpy==0.3.1
+filelock==3.13.0
+fonttools==4.43.1
+fsspec==2023.10.0
+gradio==3.50.2
+gradio_client==0.6.1
+h11==0.14.0
+httpcore==0.18.0
+httpx==0.25.0
+huggingface-hub==0.17.3
+idna==3.4
+importlib-resources==6.1.0
+Jinja2==3.1.2
+jsonschema==4.19.1
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.5
+MarkupSafe==2.1.3
+matplotlib==3.8.0
+mpmath==1.3.0
+networkx==3.2.1
+numpy==1.26.1
+orjson==3.9.10
+packaging==23.2
+pandas==2.1.2
+Pillow==10.1.0
+pydantic==2.4.2
+pydantic_core==2.10.1
+pydub==0.25.1
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
+rpds-py==0.10.6
+safetensors==0.4.0
+semantic-version==2.10.0
+six==1.16.0
+sniffio==1.3.0
+starlette==0.27.0
+sympy==1.12
+tokenizers==0.14.1
+toolz==0.12.0
+torch==2.1.0
+torchaudio==2.1.0
+torchvision==0.16.0
+tqdm==4.66.1
+transformers==4.34.1
+typing_extensions==4.8.0
+tzdata==2023.3
+urllib3==2.0.7
+uvicorn==0.23.2
+websockets==11.0.3