Seonglae ChoGPT
baseline topk-1 gpt
psgs_w100.dpr_nq.1_gpt-3.5-turbo{'exact_match': 26.925207756232687, 'f1': 36.13242325784241, 'psgs_tokens': 135.95096952908588, 'read_fp': 21.689750692520775, 'read_tn': 1.10803324099723, 'ret_em': 47.50692520775623}
top-k2
{'exact_match': 29.61218836565097, 'f1': 39.97846382276714, 'psgs_tokens': 272.75650969529084, 'read_fp': 27.950138504155124, 'read_tn': 0.5817174515235457, 'ret_em': 56.98060941828255}
baseline topk-4 gpt
{'exact_match': 30.81967213114754, 'f1': 40.40037545072675, 'psgs_tokens': 544.1409836065574, 'read_fp': 33.9344262295082, 'read_tn': 0.6557377049180327, 'ret_em': 64.09836065573771}
baseline topk-8 gpt
{'exact_match': 31.606648199445985, 'f1': 42.93274362444592, 'psgs_tokens': 1094.1650969529087, 'read_fp': 41.52354570637119, 'read_tn': 0.221606648199446, 'ret_em': 72.90858725761773}
baseline topk-16 gpt
{'exact_match': 31.994459833795013, 'f1': 42.9381781336727, 'psgs_tokens': 2189.9739612188364, 'reader_fp': 59.541446208112866, 'reader_fn': 1.032258064516129}
32부터는 토큰 부족
baseline topk-10 gpt
{'exact_match': 33.22968605724839, 'f1': 43.10231143125888, 'psgs_tokens': 1368.1897506925209, 'read_fp': 41.13573407202216, 'read_tn': 0.08310249307479224, 'ret_em': 74.73684210526315}
33.22968605724839
{'exact_match': 37.43721144967682, 'f1': 47.18106390377866, 'psgs_tokens': 1368.240166204986, 'summary_tokens': 114.34709141274239, 'sum_tn': 1.440443213296399, 'sum_rc': 78.78563495001852, 'sum_fn': 15.87257617728532, 'ret_em': 74.81994459833795, 'sum_em': 60.387811634349035, 'read_fp': 22.60387811634349, 'read_tn': 0.221606648199446}
GPT Summarized
1→1
{'exact_match': 28.25484764542936, 'f1': 37.79880322816614, 'psgs_tokens': 135.9382271468144, 'summary_tokens': 78.43905817174515, 'sum_tn': 6.256627783669141, 'sum_fn': 12.296983758700696, 'ret_em': 47.75623268698061, 'sum_em': 45.15235457063712, 'read_fp': 38.282208588957054, 'read_tn': 0.7070707070707071}
2→1
{'exact_match': 31.218836565096954, 'f1': 41.1978051188578, 'psgs_tokens': 272.69168975069255, 'summary_tokens': 147.0595567867036, 'sum_tn': 5.6172436316133245, 'sum_fn': 11.255411255411255, 'ret_em': 57.59002770083102, 'sum_em': 53.49030470914128, 'read_fp': 41.843604350077676, 'read_tn': 0.23823704586063135}
4→1
{'exact_match': 33.37950138504155, 'f1': 43.50731017213017, 'psgs_tokens': 546.4903047091412, 'summary_tokens': 161.60775623268697, 'sum_tn': 3.9867109634551494, 'sum_fn': 16.084788029925186, 'ret_em': 66.64819944598338, 'sum_em': 57.25761772853185, 'read_fp': 42.23512336719884, 'read_tn': 0.712896953985742}
8→1
{'exact_match': 35.40166204986149, 'f1': 46.4832925615475, 'psgs_tokens': 1094.1232686980609, 'summary_tokens': 243.28254847645428, 'sum_tn': 5.82726326742976, 'sum_fn': 16.874292185730464, 'ret_em': 73.37950138504155, 'sum_em': 62.548476454293635, 'read_fp': 43.622674933569535, 'read_tn': 0.3698224852071006}
16 → 1
{'exact_match': 36.094182825484765, 'f1': 46.92736784040187, 'psgs_tokens': 2189.9326869806096, 'summary_tokens': 233.7049861495845, 'sum_tn': 3.8809831824062093, 'sum_fn': 21.92456820585125, 'ret_em': 78.58725761772854, 'sum_em': 62.18836565096952, 'read_fp': 42.1826280623608, 'read_tn': 0.3663003663003663}