Hugging Face's logo

Spaces:
danieldux
/
isco_hierarchical_accuracy
Build error

App Files Community
danieldux commited on
Commit
bd9a05a
·
1 Parent(s): a5b84dd

cleaning

Browse files
Files changed (1) hide show
  1. tests.ipynb +8 -436
tests.ipynb CHANGED
@@ -334,88 +334,9 @@
334
  },
335
  {
336
  "cell_type": "code",
337
- "execution_count": 79,
338
- "metadata": {},
339
- "outputs": [
340
- {
341
- "data": {
342
- "text/plain": [
343
- "<pandas.core.groupby.generic.DataFrameGroupBy object at 0x7f2c318dd350>"
344
- ]
345
- },
346
- "execution_count": 79,
347
- "metadata": {},
348
- "output_type": "execute_result"
349
- }
350
- ],
351
- "source": [
352
- "grouped_df"
353
- ]
354
- },
355
- {
356
- "cell_type": "code",
357
- "execution_count": 77,
358
  "metadata": {},
359
- "outputs": [
360
- {
361
- "name": "stdout",
362
- "output_type": "stream",
363
- "text": [
364
- "Accuracy: 0.7149425287356321, Hierarchical Precision: 0.9314641744548287, Hierarchical Recall: 0.8898809523809523, Hierarchical F-measure: 0.9101978691019786\n",
365
- "Language: da\n",
366
- "Result: {'accuracy': 0.7149425287356321, 'hierarchical_precision': 0.9314641744548287, 'hierarchical_recall': 0.8898809523809523, 'hierarchical_fmeasure': 0.9101978691019786}\n",
367
- "\n",
368
- "Accuracy: 0.9075297225891678, Hierarchical Precision: 0.9578651685393258, Hierarchical Recall: 0.9742857142857143, Hierarchical F-measure: 0.9660056657223796\n",
369
- "Language: en\n",
370
- "Result: {'accuracy': 0.9075297225891678, 'hierarchical_precision': 0.9578651685393258, 'hierarchical_recall': 0.9742857142857143, 'hierarchical_fmeasure': 0.9660056657223796}\n",
371
- "\n",
372
- "Accuracy: 0.8794080604534005, Hierarchical Precision: 0.9774590163934426, Hierarchical Recall: 0.9655870445344129, Hierarchical F-measure: 0.9714867617107942\n",
373
- "Language: es\n",
374
- "Result: {'accuracy': 0.8794080604534005, 'hierarchical_precision': 0.9774590163934426, 'hierarchical_recall': 0.9655870445344129, 'hierarchical_fmeasure': 0.9714867617107942}\n",
375
- "\n",
376
- "Accuracy: 0.9286376274328082, Hierarchical Precision: 0.9591836734693877, Hierarchical Recall: 0.9733727810650887, Hierarchical F-measure: 0.9662261380323054\n",
377
- "Language: fi\n",
378
- "Result: {'accuracy': 0.9286376274328082, 'hierarchical_precision': 0.9591836734693877, 'hierarchical_recall': 0.9733727810650887, 'hierarchical_fmeasure': 0.9662261380323054}\n",
379
- "\n",
380
- "Accuracy: 0.5772994129158513, Hierarchical Precision: 0.8571428571428571, Hierarchical Recall: 0.8808864265927978, Hierarchical F-measure: 0.8688524590163934\n",
381
- "Language: fr\n",
382
- "Result: {'accuracy': 0.5772994129158513, 'hierarchical_precision': 0.8571428571428571, 'hierarchical_recall': 0.8808864265927978, 'hierarchical_fmeasure': 0.8688524590163934}\n",
383
- "\n",
384
- "Accuracy: 0.9332579185520362, Hierarchical Precision: 0.9616613418530351, Hierarchical Recall: 0.9525316455696202, Hierarchical F-measure: 0.9570747217806042\n",
385
- "Language: it\n",
386
- "Result: {'accuracy': 0.9332579185520362, 'hierarchical_precision': 0.9616613418530351, 'hierarchical_recall': 0.9525316455696202, 'hierarchical_fmeasure': 0.9570747217806042}\n",
387
- "\n",
388
- "Accuracy: 0.9313346228239845, Hierarchical Precision: 0.9816849816849816, Hierarchical Recall: 0.9710144927536232, Hierarchical F-measure: 0.97632058287796\n",
389
- "Language: kk\n",
390
- "Result: {'accuracy': 0.9313346228239845, 'hierarchical_precision': 0.9816849816849816, 'hierarchical_recall': 0.9710144927536232, 'hierarchical_fmeasure': 0.97632058287796}\n",
391
- "\n",
392
- "Accuracy: 0.9369047619047619, Hierarchical Precision: 0.9726962457337884, Hierarchical Recall: 0.9827586206896551, Hierarchical F-measure: 0.9777015437392795\n",
393
- "Language: ko\n",
394
- "Result: {'accuracy': 0.9369047619047619, 'hierarchical_precision': 0.9726962457337884, 'hierarchical_recall': 0.9827586206896551, 'hierarchical_fmeasure': 0.9777015437392795}\n",
395
- "\n",
396
- "Accuracy: 0.8936170212765957, Hierarchical Precision: 0.9591836734693877, Hierarchical Recall: 0.9563953488372093, Hierarchical F-measure: 0.957787481804949\n",
397
- "Language: pt\n",
398
- "Result: {'accuracy': 0.8936170212765957, 'hierarchical_precision': 0.9591836734693877, 'hierarchical_recall': 0.9563953488372093, 'hierarchical_fmeasure': 0.957787481804949}\n",
399
- "\n",
400
- "Accuracy: 0.9259259259259259, Hierarchical Precision: 0.971875, Hierarchical Recall: 0.9658385093167702, Hierarchical F-measure: 0.9688473520249222\n",
401
- "Language: ru\n",
402
- "Result: {'accuracy': 0.9259259259259259, 'hierarchical_precision': 0.971875, 'hierarchical_recall': 0.9658385093167702, 'hierarchical_fmeasure': 0.9688473520249222}\n",
403
- "\n",
404
- "Accuracy: 0.9726027397260274, Hierarchical Precision: 0.9927007299270073, Hierarchical Recall: 1.0, Hierarchical F-measure: 0.9963369963369962\n",
405
- "Language: sv\n",
406
- "Result: {'accuracy': 0.9726027397260274, 'hierarchical_precision': 0.9927007299270073, 'hierarchical_recall': 1.0, 'hierarchical_fmeasure': 0.9963369963369962}\n",
407
- "\n"
408
- ]
409
- },
410
- {
411
- "name": "stderr",
412
- "output_type": "stream",
413
- "text": [
414
- "/tmp/ipykernel_29614/1496722815.py:17: FutureWarning: The behavior of DataFrame concatenation with empty or all-NA entries is deprecated. In a future version, this will no longer exclude empty or all-NA columns when determining the result dtypes. To retain the old behavior, exclude the relevant entries before the concat operation.\n",
415
- " results_df = pd.concat([results_df, group_result_df], ignore_index=True)\n"
416
- ]
417
- }
418
- ],
419
  "source": [
420
  "\n",
421
  "results_df = pd.DataFrame(columns=['Language', 'Accuracy', 'Hierarchical Precision', 'Hierarchical Recall', 'Hierarchical F1'])\n",
@@ -455,179 +376,9 @@
455
  },
456
  {
457
  "cell_type": "code",
458
- "execution_count": 62,
459
  "metadata": {},
460
- "outputs": [
461
- {
462
- "data": {
463
- "text/html": [
464
- "<div>\n",
465
- "<style scoped>\n",
466
- " .dataframe tbody tr th:only-of-type {\n",
467
- " vertical-align: middle;\n",
468
- " }\n",
469
- "\n",
470
- " .dataframe tbody tr th {\n",
471
- " vertical-align: top;\n",
472
- " }\n",
473
- "\n",
474
- " .dataframe thead th {\n",
475
- " text-align: right;\n",
476
- " }\n",
477
- "</style>\n",
478
- "<table border=\"1\" class=\"dataframe\">\n",
479
- " <thead>\n",
480
- " <tr style=\"text-align: right;\">\n",
481
- " <th></th>\n",
482
- " <th>JOB</th>\n",
483
- " <th>DUTIES</th>\n",
484
- " <th>ISCO</th>\n",
485
- " <th>ISCO_REL</th>\n",
486
- " <th>LANGUAGE</th>\n",
487
- " </tr>\n",
488
- " </thead>\n",
489
- " <tbody>\n",
490
- " <tr>\n",
491
- " <th>0</th>\n",
492
- " <td>acopio</td>\n",
493
- " <td>recibe tarros con leche y despues hecha la lec...</td>\n",
494
- " <td>9333</td>\n",
495
- " <td>9333</td>\n",
496
- " <td>es</td>\n",
497
- " </tr>\n",
498
- " <tr>\n",
499
- " <th>5</th>\n",
500
- " <td>yo vivo con mi abuela y abuelo mi abuela o tr...</td>\n",
501
- " <td>mi mama trabaja en limpiar las casas</td>\n",
502
- " <td>9111</td>\n",
503
- " <td>9111</td>\n",
504
- " <td>es</td>\n",
505
- " </tr>\n",
506
- " <tr>\n",
507
- " <th>9</th>\n",
508
- " <td>dueña de casa</td>\n",
509
- " <td>mantiene el orden de la casa</td>\n",
510
- " <td>9701</td>\n",
511
- " <td>9701</td>\n",
512
- " <td>es</td>\n",
513
- " </tr>\n",
514
- " <tr>\n",
515
- " <th>10</th>\n",
516
- " <td>señora de casa</td>\n",
517
- " <td>trabaja en la lecheria con las bacas y terneros</td>\n",
518
- " <td>9701</td>\n",
519
- " <td>9701</td>\n",
520
- " <td>es</td>\n",
521
- " </tr>\n",
522
- " <tr>\n",
523
- " <th>11</th>\n",
524
- " <td>trabajadora agricolar</td>\n",
525
- " <td>aplicar liquidos ala plantas</td>\n",
526
- " <td>9211</td>\n",
527
- " <td>9211</td>\n",
528
- " <td>es</td>\n",
529
- " </tr>\n",
530
- " <tr>\n",
531
- " <th>...</th>\n",
532
- " <td>...</td>\n",
533
- " <td>...</td>\n",
534
- " <td>...</td>\n",
535
- " <td>...</td>\n",
536
- " <td>...</td>\n",
537
- " </tr>\n",
538
- " <tr>\n",
539
- " <th>113962</th>\n",
540
- " <td>Фотограф</td>\n",
541
- " <td>Рассылал снимки в журналы, получал за это гоно...</td>\n",
542
- " <td>3431</td>\n",
543
- " <td>3431</td>\n",
544
- " <td>ru</td>\n",
545
- " </tr>\n",
546
- " <tr>\n",
547
- " <th>114114</th>\n",
548
- " <td>Магазин</td>\n",
549
- " <td>У него есть всой магазин где он работает.</td>\n",
550
- " <td>5221</td>\n",
551
- " <td>5221</td>\n",
552
- " <td>ru</td>\n",
553
- " </tr>\n",
554
- " <tr>\n",
555
- " <th>114295</th>\n",
556
- " <td>цирк</td>\n",
557
- " <td>держал перши</td>\n",
558
- " <td>2659</td>\n",
559
- " <td>2659</td>\n",
560
- " <td>ru</td>\n",
561
- " </tr>\n",
562
- " <tr>\n",
563
- " <th>114317</th>\n",
564
- " <td>Человек-молкула</td>\n",
565
- " <td>Супер-герой</td>\n",
566
- " <td>9705</td>\n",
567
- " <td>9705</td>\n",
568
- " <td>ru</td>\n",
569
- " </tr>\n",
570
- " <tr>\n",
571
- " <th>114371</th>\n",
572
- " <td>Строительство заборов</td>\n",
573
- " <td>Ставит заборы дачникам и не только</td>\n",
574
- " <td>7111</td>\n",
575
- " <td>7111</td>\n",
576
- " <td>ru</td>\n",
577
- " </tr>\n",
578
- " </tbody>\n",
579
- "</table>\n",
580
- "<p>13055 rows × 5 columns</p>\n",
581
- "</div>"
582
- ],
583
- "text/plain": [
584
- " JOB \\\n",
585
- "0 acopio \n",
586
- "5 yo vivo con mi abuela y abuelo mi abuela o tr... \n",
587
- "9 dueña de casa \n",
588
- "10 señora de casa \n",
589
- "11 trabajadora agricolar \n",
590
- "... ... \n",
591
- "113962 Фотограф \n",
592
- "114114 Магазин \n",
593
- "114295 цирк \n",
594
- "114317 Человек-молкула \n",
595
- "114371 Строительство заборов \n",
596
- "\n",
597
- " DUTIES ISCO ISCO_REL \\\n",
598
- "0 recibe tarros con leche y despues hecha la lec... 9333 9333 \n",
599
- "5 mi mama trabaja en limpiar las casas 9111 9111 \n",
600
- "9 mantiene el orden de la casa 9701 9701 \n",
601
- "10 trabaja en la lecheria con las bacas y terneros 9701 9701 \n",
602
- "11 aplicar liquidos ala plantas 9211 9211 \n",
603
- "... ... ... ... \n",
604
- "113962 Рассылал снимки в журналы, получал за это гоно... 3431 3431 \n",
605
- "114114 У него есть всой магазин где он работает. 5221 5221 \n",
606
- "114295 держал перши 2659 2659 \n",
607
- "114317 Супер-герой 9705 9705 \n",
608
- "114371 Ставит заборы дачникам и не только 7111 7111 \n",
609
- "\n",
610
- " LANGUAGE \n",
611
- "0 es \n",
612
- "5 es \n",
613
- "9 es \n",
614
- "10 es \n",
615
- "11 es \n",
616
- "... ... \n",
617
- "113962 ru \n",
618
- "114114 ru \n",
619
- "114295 ru \n",
620
- "114317 ru \n",
621
- "114371 ru \n",
622
- "\n",
623
- "[13055 rows x 5 columns]"
624
- ]
625
- },
626
- "execution_count": 62,
627
- "metadata": {},
628
- "output_type": "execute_result"
629
- }
630
- ],
631
  "source": [
632
  "# create a dataframe with samples where ISCO and ISCO_REL the same\n",
633
  "isco_rel_df_same = isco_rel_df[isco_rel_df['ISCO'] == isco_rel_df['ISCO_REL']]\n",
@@ -637,179 +388,9 @@
637
  },
638
  {
639
  "cell_type": "code",
640
- "execution_count": 63,
641
  "metadata": {},
642
- "outputs": [
643
- {
644
- "data": {
645
- "text/html": [
646
- "<div>\n",
647
- "<style scoped>\n",
648
- " .dataframe tbody tr th:only-of-type {\n",
649
- " vertical-align: middle;\n",
650
- " }\n",
651
- "\n",
652
- " .dataframe tbody tr th {\n",
653
- " vertical-align: top;\n",
654
- " }\n",
655
- "\n",
656
- " .dataframe thead th {\n",
657
- " text-align: right;\n",
658
- " }\n",
659
- "</style>\n",
660
- "<table border=\"1\" class=\"dataframe\">\n",
661
- " <thead>\n",
662
- " <tr style=\"text-align: right;\">\n",
663
- " <th></th>\n",
664
- " <th>JOB</th>\n",
665
- " <th>DUTIES</th>\n",
666
- " <th>ISCO</th>\n",
667
- " <th>ISCO_REL</th>\n",
668
- " <th>LANGUAGE</th>\n",
669
- " </tr>\n",
670
- " </thead>\n",
671
- " <tbody>\n",
672
- " <tr>\n",
673
- " <th>4</th>\n",
674
- " <td>Asistente judirica</td>\n",
675
- " <td>gestionar casos de fiscalia</td>\n",
676
- " <td>3342</td>\n",
677
- " <td>3411</td>\n",
678
- " <td>es</td>\n",
679
- " </tr>\n",
680
- " <tr>\n",
681
- " <th>8</th>\n",
682
- " <td>lechera</td>\n",
683
- " <td>saca leche</td>\n",
684
- " <td>9212</td>\n",
685
- " <td>9211</td>\n",
686
- " <td>es</td>\n",
687
- " </tr>\n",
688
- " <tr>\n",
689
- " <th>14</th>\n",
690
- " <td>Mi madre es dueña de casa</td>\n",
691
- " <td>Realiza todos los quehaceres del hogar, y trab...</td>\n",
692
- " <td>9111</td>\n",
693
- " <td>9701</td>\n",
694
- " <td>es</td>\n",
695
- " </tr>\n",
696
- " <tr>\n",
697
- " <th>34</th>\n",
698
- " <td>algricultura</td>\n",
699
- " <td>algricultura</td>\n",
700
- " <td>9705</td>\n",
701
- " <td>9211</td>\n",
702
- " <td>es</td>\n",
703
- " </tr>\n",
704
- " <tr>\n",
705
- " <th>38</th>\n",
706
- " <td>en la agricultura</td>\n",
707
- " <td>produce alimentos de vegetacion</td>\n",
708
- " <td>633</td>\n",
709
- " <td>9211</td>\n",
710
- " <td>es</td>\n",
711
- " </tr>\n",
712
- " <tr>\n",
713
- " <th>...</th>\n",
714
- " <td>...</td>\n",
715
- " <td>...</td>\n",
716
- " <td>...</td>\n",
717
- " <td>...</td>\n",
718
- " <td>...</td>\n",
719
- " </tr>\n",
720
- " <tr>\n",
721
- " <th>111656</th>\n",
722
- " <td>gerente de ventas</td>\n",
723
- " <td>ropa</td>\n",
724
- " <td>5222</td>\n",
725
- " <td>1221</td>\n",
726
- " <td>es</td>\n",
727
- " </tr>\n",
728
- " <tr>\n",
729
- " <th>111700</th>\n",
730
- " <td>policia jubilado</td>\n",
731
- " <td>capitan</td>\n",
732
- " <td>5412</td>\n",
733
- " <td>9703</td>\n",
734
- " <td>es</td>\n",
735
- " </tr>\n",
736
- " <tr>\n",
737
- " <th>111792</th>\n",
738
- " <td>Vendiendo comida</td>\n",
739
- " <td>Mi padrastro vende comida</td>\n",
740
- " <td>5223</td>\n",
741
- " <td>5212</td>\n",
742
- " <td>es</td>\n",
743
- " </tr>\n",
744
- " <tr>\n",
745
- " <th>112817</th>\n",
746
- " <td>Собственник ювелирного магазина</td>\n",
747
- " <td>Продавал ювелирные изделия</td>\n",
748
- " <td>7313</td>\n",
749
- " <td>5221</td>\n",
750
- " <td>ru</td>\n",
751
- " </tr>\n",
752
- " <tr>\n",
753
- " <th>113081</th>\n",
754
- " <td>Предприниматель</td>\n",
755
- " <td>Вещи продовал (продукты)</td>\n",
756
- " <td>5221</td>\n",
757
- " <td>112</td>\n",
758
- " <td>ru</td>\n",
759
- " </tr>\n",
760
- " </tbody>\n",
761
- "</table>\n",
762
- "<p>1958 rows × 5 columns</p>\n",
763
- "</div>"
764
- ],
765
- "text/plain": [
766
- " JOB \\\n",
767
- "4 Asistente judirica \n",
768
- "8 lechera \n",
769
- "14 Mi madre es dueña de casa \n",
770
- "34 algricultura \n",
771
- "38 en la agricultura \n",
772
- "... ... \n",
773
- "111656 gerente de ventas \n",
774
- "111700 policia jubilado \n",
775
- "111792 Vendiendo comida \n",
776
- "112817 Собственник ювелирного магазина \n",
777
- "113081 Предприниматель \n",
778
- "\n",
779
- " DUTIES ISCO ISCO_REL \\\n",
780
- "4 gestionar casos de fiscalia 3342 3411 \n",
781
- "8 saca leche 9212 9211 \n",
782
- "14 Realiza todos los quehaceres del hogar, y trab... 9111 9701 \n",
783
- "34 algricultura 9705 9211 \n",
784
- "38 produce alimentos de vegetacion 633 9211 \n",
785
- "... ... ... ... \n",
786
- "111656 ropa 5222 1221 \n",
787
- "111700 capitan 5412 9703 \n",
788
- "111792 Mi padrastro vende comida 5223 5212 \n",
789
- "112817 Продавал ювелирные изделия 7313 5221 \n",
790
- "113081 Вещи продовал (продукты) 5221 112 \n",
791
- "\n",
792
- " LANGUAGE \n",
793
- "4 es \n",
794
- "8 es \n",
795
- "14 es \n",
796
- "34 es \n",
797
- "38 es \n",
798
- "... ... \n",
799
- "111656 es \n",
800
- "111700 es \n",
801
- "111792 es \n",
802
- "112817 ru \n",
803
- "113081 ru \n",
804
- "\n",
805
- "[1958 rows x 5 columns]"
806
- ]
807
- },
808
- "execution_count": 63,
809
- "metadata": {},
810
- "output_type": "execute_result"
811
- }
812
- ],
813
  "source": [
814
  "# create a dataframe with samples where ISCO and ISCO_REL are different\n",
815
  "isco_rel_df_diff = isco_rel_df[isco_rel_df['ISCO'] != isco_rel_df['ISCO_REL']]\n",
@@ -830,18 +411,9 @@
830
  },
831
  {
832
  "cell_type": "code",
833
- "execution_count": 66,
834
  "metadata": {},
835
- "outputs": [
836
- {
837
- "name": "stdout",
838
- "output_type": "stream",
839
- "text": [
840
- "Accuracy: 0.8695796975954173, Hierarchical Precision: 0.9876106194690265, Hierarchical Recall: 0.9911190053285968, Hierarchical F-measure: 0.9893617021276595\n",
841
- "Evaluation results saved to isco_rel_results.json\n"
842
- ]
843
- }
844
- ],
845
  "source": [
846
  "# Compute the hierarchical accuracy\n",
847
  "reliability_results = hierarchical_accuracy.compute(predictions=coder2, references=coder1)\n",
 
334
  },
335
  {
336
  "cell_type": "code",
337
+ "execution_count": null,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
338
  "metadata": {},
339
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
340
  "source": [
341
  "\n",
342
  "results_df = pd.DataFrame(columns=['Language', 'Accuracy', 'Hierarchical Precision', 'Hierarchical Recall', 'Hierarchical F1'])\n",
 
376
  },
377
  {
378
  "cell_type": "code",
379
+ "execution_count": null,
380
  "metadata": {},
381
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
382
  "source": [
383
  "# create a dataframe with samples where ISCO and ISCO_REL the same\n",
384
  "isco_rel_df_same = isco_rel_df[isco_rel_df['ISCO'] == isco_rel_df['ISCO_REL']]\n",
 
388
  },
389
  {
390
  "cell_type": "code",
391
+ "execution_count": null,
392
  "metadata": {},
393
+ "outputs": [],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
394
  "source": [
395
  "# create a dataframe with samples where ISCO and ISCO_REL are different\n",
396
  "isco_rel_df_diff = isco_rel_df[isco_rel_df['ISCO'] != isco_rel_df['ISCO_REL']]\n",
 
411
  },
412
  {
413
  "cell_type": "code",
414
+ "execution_count": null,
415
  "metadata": {},
416
+ "outputs": [],
 
 
 
 
 
 
 
 
 
417
  "source": [
418
  "# Compute the hierarchical accuracy\n",
419
  "reliability_results = hierarchical_accuracy.compute(predictions=coder2, references=coder1)\n",