File size: 1,029 Bytes
5434c4b aef1dbe 6503e4e a8a595d aab3281 5434c4b 6503e4e a8a595d 6503e4e 5434c4b aab3281 0c136d8 aef1dbe 6676c5a aef1dbe 5434c4b 5f3a4af aef1dbe 6676c5a aef1dbe 5f3a4af |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
import diff_match_patch as dmp_module
from tqdm import tqdm
from api_wrappers import hf_data_loader
def get_annotated_diff(start_text, end_text):
dmp = dmp_module.diff_match_patch()
dmp_mapping = {
-1: '-',
0: None,
1: '+'
}
diff = dmp.diff_main(start_text, end_text)
dmp.diff_cleanupSemantic(diff)
result = [[w, dmp_mapping[t]] for t, w in diff]
return result
def annotated_diff_for_row(row):
start = row['commit_msg_start']
end = row['commit_msg_end']
return get_annotated_diff(start, end)
def manual_data_with_annotated_diffs():
tqdm.pandas()
df = hf_data_loader.load_raw_rewriting_as_pandas()
annotated = df.progress_apply(annotated_diff_for_row, axis=1)
df['annotated_diff'] = annotated
return df
def synthetic_data_with_annotated_diffs():
tqdm.pandas()
df = hf_data_loader.load_synthetic_as_pandas()
annotated = df.progress_apply(annotated_diff_for_row, axis=1)
df['annotated_diff'] = annotated
return df
|