Update results.py

results.py

@@ -150,6 +150,49 @@ fig.update_layout(
 
 different_document_dup_counts_global = fig
 
+##graph 4
+
+# Data
+data = {
+    "2014": [[1, 2, 3, 4, 5, 6, 7], [16.82572561037536, 15.568688184472002, 15.81492345088457, 15.206355227978303, 14.957766124315622, 15.0626618458479, 14.868694970954875]],
+    "2015": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], [16.89591942700286, 16.065453332424525, 16.27127382267357, 15.98729389349345, 15.820859416672139, 15.607600911192797, 15.110138119917048, 14.775887397593593, 14.660655022362318, 14.588547400309, 14.55020529201701, 14.60127048285861, 14.629467286160864, 14.724440862889]],
+    "2016": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], [16.937063110995666, 16.159401045667217, 16.24178377816029, 16.073494730595225, 15.774737597940442, 15.621947075605162, 15.652955728810767, 15.490807185265608, 15.107547930881259, 14.976802818244652, 14.84517846815987, 14.839768640933237, 14.794471716989547, 14.63826024444852, 14.57725826735028, 14.446983381957654, 14.357863327746747, 14.370146480850329, 14.207796791759758, 14.067617540013867, 13.968037234662484, 14.015386030957798, 14.240341030736081]],
+    "2017": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34], [16.731010662045737, 16.534529481773635, 16.624818021435388, 16.6027968727952, 16.309053610063657, 15.897351021398148, 15.28125612085656, 15.247650153402024, 15.254523671401472, 14.914419299947483, 14.734687782223004, 14.78413990847839, 14.632065773173599, 14.553246204352645, 14.566566717229817, 14.598997883895558, 14.630837564582592, 14.577208610859856, 14.519413074334954, 14.378498447103155, 14.230943065441672, 14.21486380216727, 14.267422050116286, 14.281606945252474, 14.285535565035358, 14.267266452303197, 14.183136371592417, 14.148268300539986, 14.21036055571916, 14.403658436211263, 14.07740471177247, 13.978242496152737, 14.002633023111342, 13.921442192327655]],
+    "2018": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46], [16.36546565307699, 16.40198820766892, 16.223802437472045, 15.876190858008776, 15.67866795074444, 15.06202559354935, 14.90395971116697, 14.678203166785968, 14.584307457124408, 14.566603376547835, 14.716527825876883, 15.325114798824192, 15.153635738494962, 15.02749650458142, 15.001501640976104, 14.928437622685198, 14.82066263298361, 14.821925418558614, 14.766538823073754, 14.742836908980854, 14.56365471694568, 14.676889493065413, 14.554327268897188, 14.532997117657752, 14.424665661602356, 14.498427111442304, 14.503622356394782, 14.465294436509943, 14.238209588803269, 14.138531026657803, 14.11633073136879, 14.139167312709546, 14.018054330979824, 13.99555243574117, 13.944770385859002, 13.893353445566834, 14.062344563726617, 13.93602183204734, 14.021518977400955, 14.205273479299835, 14.476114932821364, 14.707612238290547, 14.403724822460271, 14.134464626849528, 14.444143421676507, 14.372513155481114]], [0.13507625272331159, 0.5185185185185183, 0.7843137254901962]],
+    "2019": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58], [16.430122616908218, 16.959894024725404, 17.530494343918114, 16.868158683774006, 15.881925908038214, 15.534071764082064, 15.131599277920825, 14.912091189771328, 14.670101527122705, 14.591504400695127, 14.405999306200146, 14.445437111619501, 14.460150974610382, 14.484377024024335, 14.454923933668082, 14.419367200283267, 14.413528998792588, 14.362242243379736, 14.326113076490996, 14.296366861932851, 14.241137636937816, 14.199018730937293, 14.229939247452181, 14.136900232594229, 14.118747592622253, 14.084639441555339, 14.215359965064522, 14.321335291582407, 14.324483466296755, 14.22250351319463, 14.086153725088337, 13.835546707594524, 13.861133395338289, 13.943395906095482, 13.924621051048792, 13.978711084778347, 13.905837106100599, 13.895058375110484, 13.965442215811176, 13.960110799963521, 13.99641930380663, 14.036529641608142, 14.138294825947872, 14.3360579115842, 14.381214824082278, 14.389155929858507, 14.645175007732961, 14.477402464547152, 14.51444505845359, 14.452153618691407, 14.369430039316658, 14.777857326947053, 14.919588044831785, 14.44053007434645, 15.18596756335163, 15.362545102466955, 14.19095014510728, 14.974116009038374]], [0.10806100217864933, 0.4148148148148145, 0.6274509803921567]],
+    "2020": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67], [16.370879996210345, 15.8912555858172, 15.30157381561316, 15.461362783916728, 15.499602726814674, 15.329200065422526, 15.15238010628145, 14.763158730707703, 14.554444724050978, 14.391542531017906, 14.36062417910266, 14.38408773153761, 14.326398992080678, 14.30559398460406, 14.296620966953196, 14.257641523080219, 14.239785079623704, 14.257963344315925, 14.242187947481078, 14.227247960684823, 14.246528806504662, 14.218691858857001, 14.119027315863725, 14.16049126937658, 14.1393355330565, 14.045446781499932, 14.157717833319223, 14.118844952908322, 14.196031529646438, 14.213151244097208, 13.961708881578405, 13.933434907040512, 13.851967361230258, 13.837032186918211, 13.83609308408103, 13.74084041793037, 13.725470779946555, 13.828238575588477, 13.821753201986834, 13.81152543072784, 13.848104780631868, 13.867032605563987, 13.91567734841802, 13.951714207206178, 14.049401288158215, 14.000020142532282, 14.097054408814992, 13.973915015915077, 14.001305792897133, 14.144447551433583, 14.259794771973013, 14.451580305620148, 14.51523695971965, 14.491781843544896, 14.282130970472108, 15.107471010795585, 14.507809743534436, 14.521069179252333, 14.434674191986367, 14.608533166251355, 14.638444470251414, 14.533496524040576, 14.798278071146985, 14.426231926251468, 14.697215649677831, 16.000970311873413, 15.962112261396376]], [0.08104575163398697, 0.31111111111111095, 0.47058823529411764]],
+    "2021": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76], [16.40998076014569, 16.214064988829055, 16.102423371338, 15.61960664445389, 15.365806380257993, 15.258505702187641, 14.987691202287733, 14.702934989974425, 14.47618699872759, 14.27294780183996, 14.149902488372312, 14.205153888507418, 14.180406523176302, 14.179162306409303, 14.148172641684653, 14.146789562089962, 14.13207546325995, 14.095045629570848, 14.11059784727951, 14.099774964675055, 14.123404475264994, 14.04205645064162, 14.071957340879374, 14.076002230293538, 13.997579723886533, 14.055982816834012, 13.969566421026498, 14.044197137615983, 13.970800321234117, 13.991254850488748, 13.882773884993162, 13.878676595695538, 13.738534890032913, 13.731264526843741, 13.743931319054433, 13.744385448034985, 13.654269807446708, 13.649793785589242, 13.659669286701138, 13.70981039304929, 13.706017327034093, 13.793638036031938, 13.820253019972634, 13.895844222751752, 13.805532005211788, 13.879969410141696, 13.918667434527839, 13.905080860878972, 13.928649229766597, 13.92971106121914, 14.027211058825408, 13.869387194129342, 14.002439547556614, 14.197189244986761, 14.171599593405029, 14.189026952927275, 14.317088413682809, 14.266884102594418, 14.419819049388389, 14.394102801112203, 14.28486572885178, 14.463889605992794, 14.33813105749612, 14.49991185507524, 14.143521529230846, 14.020378845184027, 13.929518285869944, 14.785566241242854, 14.273112202537183, 14.214773231874327, 15.148250280334482, 14.563912283938823, 14.552655401047055, 15.103655560583926, 15.10169368308145, 15.51226670578053]], [0.05403050108932467, 0.20740740740740726, 0.31372549019607837]],
+    "2022": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82], [15.816438075969113, 16.09580666529207, 15.996333190110686, 15.943848815283268, 15.656093894572393, 15.117394092950262, 14.78287160043243, 14.543141634926663, 14.305058018637094, 14.125611753367533, 14.020067247129745, 14.009478965058692, 14.001669618595649, 13.985248005578073, 13.973728782674742, 13.977117237954532, 13.937202607945661, 13.935625722728988, 13.910945770197134, 13.893835251097759, 13.89679037463924, 13.881064193374637, 13.904299030382344, 13.904090626213797, 13.888526913768459, 13.838568567514612, 13.829178463025315, 13.835043064902585, 13.81757570730985, 13.776370281729609, 13.762638676519028, 13.755074572847866, 13.70045870380896, 13.636805800875507, 13.621671132216942, 13.633589926779942, 13.602080944483301, 13.555848610415586, 13.558904660155596, 13.515784046498322, 13.609062507319349, 13.583137427458137, 13.483909540769114, 13.552547586090368, 13.598431457952788, 13.553603210491028, 13.57678073518515, 13.638974685844074, 13.536340332636364, 13.665029112653803, 13.792584632059397, 13.694618901117751, 13.76826442085512, 13.712141215124436, 13.606927085734855, 13.715251358141709, 13.832372298562165, 13.833057817510603, 14.158282794215733, 13.720855464980609, 13.803417413649743, 13.969854384335488, 13.892171370159991, 13.778269401162506, 14.338216122354686, 13.954993084366778, 14.124621205791534, 14.23737489353975, 13.735373769168353, 14.680642990647039, 14.955805632680471, 14.657580957949671, 14.515390831490835, 14.664661248141265, 14.497750142907295, 14.662641057721332, 14.889760996859867, 14.044358215057608, 14.201835355776206, 15.739648390380614, 16.209835613545337, 16.152241961523004]], [0.027015250544662334, 0.10370370370370363, 0.15686274509803919]],
+    "2023": [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87], [14.789164834058647, 14.816940361326127, 14.859128791010018, 14.874353138456074, 14.799558338831533, 14.521955875294703, 14.344439116861626, 14.188768591243738, 14.068214649484007, 13.975247139727212, 13.892361806680233, 13.873812682757855, 13.855623423924795, 13.834034581531931, 13.820704041290954, 13.809345726828388, 13.799809305012891, 13.783733425983497, 13.777650159371621, 13.769295261894937, 13.759922637457876, 13.763157283741176, 13.746892077566258, 13.76043892787152, 13.780799936451544, 13.769219249583701, 13.776107653632701, 13.791395904208231, 13.796017873163873, 13.793511195440795, 13.752832872061946, 13.744140909741626, 13.708291014183953, 13.71791539682566, 13.687888739966342, 13.639779981608498, 13.60820604942688, 13.647619221630219, 13.586732110007599, 13.578824230397977, 13.56286775189147, 13.54887132723798, 13.528005768240323, 13.562278263274335, 13.50423334091739, 13.534814940452101, 13.500524731734556, 13.50415611423685, 13.539792624454682, 13.524280636462276, 13.508635254011224, 13.559898859515048, 13.52340866000508, 13.580987242413414, 13.55406078682323, 13.51949982056399, 13.554946096777044, 13.582384634669271, 13.596405485832927, 13.56397157046417, 13.50076890156882, 13.5996552161972, 13.640177272640555, 13.488306430506505, 13.561144538501607, 13.538229936728902, 13.523274401746107, 13.549803559768709, 13.473822808409837, 13.604196078834873, 13.495970010370977, 13.511897788189435, 13.480753572940852, 13.790758773339501, 13.507154643917646, 13.451282820788672, 13.690887252346345, 13.752272760476266, 13.844969190718858, 13.547043033265561, 14.011397551896307, 13.686917203544905, 13.87527368747695, 13.86828531076653, 14.200910595590095, 14.228639023300655, 14.830844952562549]], [0.0, 0.0, 0.0]]
+}
+
+# Create figure
+fig = go.Figure()
+
+# Add a line for each year
+for year, values in data.items():
+    x_values = values[0]  # Dumps duplication count
+    y_values = values[1]  # Average perplexity
+    
+    fig.add_trace(go.Scatter(
+        x=x_values, 
+        y=y_values, 
+        mode='lines+markers',
+        name=year
+    ))
+
+# Update layout
+fig.update_layout(
+    title="Perplexity Across Different Dump Duplication Counts (global)",
+    xaxis_title="Number of Dumps Duplication",
+    yaxis_title="Average Perplexity",
+    legend_title="Year"
+)
+
+# Show figure
+graph4 = fig
+
+
 
 intro_div = Div(
         H2("Perplexity Evaluation on Duplicate Data"),
@@ -177,12 +220,13 @@ perp1_div = Div(
             H3("Perplexity vs Document Duplication"),
             P("We can also break each bucket into distinct document counts. The graph becomes a bit noisy at the end because of insufficient samples with larger duplication counts."),
             Img(src="images/prep-across-diff-docs-dup-count-global.png", height = "300", width = "600" ),
-        plotly2fasthtml(different_document_dup_counts_global),
+            plotly2fasthtml(different_document_dup_counts_global),
         ),
     Section(
             H3("Perplexity vs Dump Duplication"),
             P("We are also interested in how the number of dumps a document is in affect data quality. From the graph below we can see that documents that are duplicated across around 40 - 60 dumps usually have lower perplexity."),
             Img(src="images/prep-across-diff-dump-dup-counts-global.png", height = "300", width = "600" ),
+            plotly2fasthtml(graph4),
         ),
     Section(
             H3("Perplexity vs Local Buckets"),