File size: 976 Bytes
5434c4b aef1dbe 6503e4e a8a595d aab3281 5434c4b 6503e4e a8a595d 86f1b98 5434c4b aab3281 216d66f aef1dbe 6676c5a 216d66f aef1dbe 5f3a4af |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 |
import diff_match_patch as dmp_module
from tqdm import tqdm
from api_wrappers import hf_data_loader
def get_annotated_diff(start_text, end_text):
dmp = dmp_module.diff_match_patch()
dmp_mapping = {
-1: '-',
0: None,
1: '+'
}
diff = dmp.diff_main(start_text, end_text)
dmp.diff_cleanupSemantic(diff)
result = [[w, dmp_mapping[t]] for t, w in diff]
return result
def annotated_diff_for_row(row):
if "commit_msg_start" in row:
start = row['commit_msg_start']
else:
start = row["G_text"]
if "commit_msg_end" in row:
end = row['commit_msg_end']
else:
end = row["E_text"]
return get_annotated_diff(start, end)
def data_with_annotated_diffs():
tqdm.pandas()
df = hf_data_loader.load_synthetic_as_pandas()
df = df.loc[df.is_related].copy()
annotated = df.progress_apply(annotated_diff_for_row, axis=1)
df['annotated_diff'] = annotated
return df
|