import pinecone # init connection to pinecone pinecone.init( api_key="0898750a-ee05-44f1-ac8a-98c5fef92f4a", # app.pinecone.io environment="asia-southeast1-gcp-free" # find next to api key ) index_name = "hybrid-image-search" index = pinecone.GRPCIndex(index_name) from datasets import load_dataset # load the dataset from huggingface datasets hub fashion = load_dataset( "ashraq/fashion-product-images-small", split='train[:1000]' ) images = fashion["image"] metadata = fashion.remove_columns("image") import requests with open('pinecone_text.py' ,'w') as fb: fb.write(requests.get('https://storage.googleapis.com/gareth-pinecone-datasets/pinecone_text.py').text) from transformers import BertTokenizerFast import pinecone_text # load bert tokenizer from huggingface tokenizer = BertTokenizerFast.from_pretrained( 'bert-base-uncased' ) def tokenize_func(text): token_ids = tokenizer( text, add_special_tokens=False )['input_ids'] return tokenizer.convert_ids_to_tokens(token_ids) bm25 = pinecone_text.BM25(tokenize_func) bm25.fit(metadata['productDisplayName']) from sentence_transformers import SentenceTransformer import transformers.models.clip.image_processing_clip import torch device = 'cuda' if torch.cuda.is_available() else 'cpu' # load a CLIP model from huggingface model = SentenceTransformer( 'sentence-transformers/clip-ViT-B-32', device=device ) def hybrid_scale(dense, sparse, alpha: float): if alpha < 0 or alpha > 1: raise ValueError("Alpha must be between 0 and 1") # scale sparse and dense vectors to create hybrid search vecs hsparse = { 'indices': sparse['indices'], 'values': [v * (1 - alpha) for v in sparse['values']] } hdense = [v * alpha for v in dense] return hdense, hsparse def text_to_image(query, alpha, k_results): sparse = bm25.transform_query(query) dense = model.encode(query).tolist() # scale sparse and dense vectors hdense, hsparse = hybrid_scale(dense, sparse, alpha=alpha) # search result = index.query( top_k=k_results, vector=hdense, sparse_vector=hsparse, include_metadata=True ) # used returned product ids to get images imgs = [images[int(r["id"])] for r in result["matches"]] description = [] for x in result["matches"]: description.append( x["metadata"]['productDisplayName'] ) return imgs, description counter = {"dir_num": 1} img_files = {'x':[]} def img_to_file_list(imgs): path = "searches" sub_path = './' + path + '/' + 'search' + '_' + str(counter["dir_num"]) # Check whether the specified path exists or not isExist = os.path.exists('.'+'/'+path) if not isExist: print("Directory does not exists") # Create a new directory because it does not exist os.makedirs('.'+'/'+path, exist_ok = True) print("The new directory is created!") # Check whether the specified path exists or not isExist = os.path.exists(sub_path) if isExist: shutil.rmtree(sub_path) os.makedirs(sub_path, exist_ok = True) img_files = {'search'+str(counter["dir_num"]):[]} i = 0 for img in imgs: img.save(sub_path+"/img_" + str(i) + ".png","PNG") img_files['search'+str(counter["dir_num"])].append(sub_path + '/' + 'img_'+ str(i) + ".png") i+=1 counter["dir_num"]+=1 return img_files['search'+str(counter["dir_num"]-1)] import gradio as gr from deep_translator import GoogleTranslator counter = {"dir_num": 1} img_files = {'x':[]} def fake_gan(text, alpha): text_eng=GoogleTranslator(source='iw', target='en').translate(text) imgs, descr = text_to_image(text_eng, alpha, 3) img_files = img_to_file_list(imgs) return img_files def fake_text(text, alpha): en_text = GoogleTranslator(source='iw', target='en').translate(text) img , descr = text_to_image(en_text, alpha, 3) return descr with gr.Blocks() as demo: with gr.Row(): text = gr.Textbox( value = "ג'ינס כחול לגברים", label="Enter the product characteristics:", ) alpha = gr.Slider(0, 1, step=0.01, label='Choose alpha:', value = 0.05) with gr.Row(): btn = gr.Button("Generate image") with gr.Row(): gallery = gr.Gallery( label="Generated images", show_label=False, elem_id="gallery" ).style(columns=[8], rows=[2], object_fit='scale-down', height='auto') with gr.Row(): selected = gr.Textbox(label="Product description: ", interactive=False, value = "-----> Description <-------",placeholder="Selected") btn.click(fake_gan, inputs=[text, alpha], outputs=gallery) def get_select_index(evt: gr.SelectData,text,alpha): print(evt.index) eng_text = fake_text(text, alpha)[evt.index] #heb_text = GoogleTranslator(source='en', target='iw').translate(eng_text) return eng_text gallery.select( fn=get_select_index, inputs=[text,alpha], outputs=selected ) demo.launch()