Package: duplicate-recognition
Duplicate-RecognitionThis Project uses the calculation of similarities scores of a set of entities in an edge list. To allow for versatile usage, it uses dependency injection to implement it into any application.UsageYou need to implement all the read/write methods, to keep the project versatile.For an example on how to use it, see theexample, or the following code block:"""This is an example implementation of the DuplicateRecognition class.It won't work. It's just to show, how it could be used."""importloggingimportosfromcollectionsimportdefaultdictfromtypingimportDict,SetfromtypingimportGenerator,Tuple,Anyfromitertoolsimportchain,islicefrommysql.connectorimportconnectfromduplicate_recognitionimportDuplicateRecognition,Algorithm,Comparisonlogging.basicConfig(level=logging.DEBUG)defchunks(iterable,size=1000):# https://stackoverflow.com/a/24527424iterator=iter(iterable)forfirstiniterator:yieldchain([first],islice(iterator,size-1))classEntity(DuplicateRecognition):ID_COLUMN:str="id"F_SCORES:Dict[str,float]=defaultdict(lambda:0,{"id":DuplicateRecognition.F_SCORE_FOR_EXACT_MATCH,"company":1,"postal_code":1,"country":0.5,})MATCHING_ALGORITHM:Dict[str,Algorithm]=defaultdict(lambda:Algorithm.EQUALITY,{"id":Algorithm.EQUALITY,"company":Algorithm.PHONETIC_DISTANCE,"postal_code":Algorithm.EQUALITY,"country":Algorithm.COUNTRY,})THRESHOLDS:Dict[str,float]=defaultdict(lambda:0,{"country":1,})NEGATIVE_FIELDS:Set[str]={"country"}def__init__(self):self.connection=connect(host=os.getenv("MYSQL_HOST"),port=os.getenv("MYSQL_PORT"),user=os.getenv("MYSQL_USER"),password=os.getenv("MYSQL_PASSWORD"),database="foo",)super().__init__()defget_relevant_entities(self)->Generator[Dict[str,Any],None,None]:cursor=self.connection.cursor(dictionary=True)cursor.execute("""SELECT DISTINCT * FROM entityORDER BY entity.id ASC""")returncursordefget_refresh_pairs(self)->Generator[Tuple[int,int],None,None]:cursor=self.connection.cursor(buffered=True)cursor.execute("""SELECT entity_edge_list.a, entity_edge_list.bFROM entity_edge_listINNER JOIN entityON entity.id = entity_edge_list.a OR entity.id = entity_edge_list.bWHERE entity.change_date > entity_edge_list.change_dateORDER BY entity_edge_list.a, entity_edge_list.b ASC""")returncursordefget_compared(self)->Generator[int,None,None]:cursor=self.connection.cursor(buffered=True)cursor.execute("SELECT DISTINCT a FROM entity_edge_list")forrowincursor:yieldrow[0]defget_uncompared(self)->Generator[int,None,None]:cursor=self.connection.cursor(buffered=True)cursor.execute("""SELECT DISTINCT entity.idFROM entityLEFT JOIN entity_edge_listON entity.id = entity_edge_list.aWHERE entity_edge_list.a IS NULLORDER BY entity.id ASC""")forrowincursor:yieldrow[0]defwrite_comparisons(self,comparisons:Generator[Comparison,None,None]):cursor=self.connection.cursor()query=f"""INSERT INTO entity_edge_list (a, b, score, count, f_score_sum, change_date) VALUE (%s, %s, %s, %s, %s, NOW())ON DUPLICATE KEY UPDATE score=VALUES(score), count=VALUES(count), f_score_sum=VALUES(f_score_sum), change_date=NOW();"""# execute in batches of 1000forchunkinchunks(comparisons,size=1000):cursor.executemany(query,[(c.entity[self.ID_COLUMN],c.other_entity[self.ID_COLUMN],c.score,c.count,c.f_score_sum)forcinchunk])self.connection.commit()if__name__=="__main__":Entity().execute(limit=None)