Add files using upload-large-folder tool

.gitattributes

@@ -46,3 +46,16 @@ omni_speech/deploy/question_audios/1645526475106087602_1646614490536_1.wav filte
 omni_speech/deploy/question_audios/1645531431473521186_1646710283001_4.wav filter=lfs diff=lfs merge=lfs -text
 omni_speech/deploy/question_audios/1645707155649472210_1646714061719_4.wav filter=lfs diff=lfs merge=lfs -text
 omni_speech/deploy/question_audios/1646106634502747851_1646895238093_4.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/output.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/helpful_base_1.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/helpful_base_2.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/helpful_base_3.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/helpful_base_4.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/helpful_base_5.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/vicuna_1.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/vicuna_2.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/vicuna_3.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/vicuna_4.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/examples/question_wav/vicuna_5.wav filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/fairseq/alignment_train_cpu_binding.cpython-312-x86_64-linux-gnu.so filter=lfs diff=lfs merge=lfs -text
+omni_speech/infer/fairseq/docs/fairseq.gif filter=lfs diff=lfs merge=lfs -text

omni_speech/infer/examples/answer.json

@@ -0,0 +1,62 @@
+[
+  {
+    "question_id": "helpful_base_1",
+    "prediction": "The United States of America got its name from the Latin phrase \"United States,\" which means \"United States of America.\" The term \"states\" refers to the individual states that make up the country, while \"America\" is a shortened version of the Latin word \"America,\" which was used by Christopher Columbus to describe the New World he had discovered.",
+    "prediction_units": "202 991 202 946 393 946 258 436 139 575 179 961 62 837 81 934 599 333 523 555 85 589 600 702 874 822 89 194 664 545 85 510 700 362 932 148 498 338 877 338 466 70 319 501 107 137 494 173 382 319 263 914 416 836 384 879 901 944 825 685 333 437 85 589 884 116 281 428 822 89 194 627 915 143 390 422 515 330 647 501 246 896 627 202 393 946 734 781 645 761 430 70 185 501 107 137 333 747 352 390 313 62 663 194 896 599 161 523 555 233 589 330 576 592 103 969 660 67 940 85 297 675 237 286 499 666 544 772 497 63 665 991 535 271 523 437 552 326 243 172 536 877 485 948 620 758 545 711 124 362 935 812 328 915 143 268 934 599 333 161 555 85 519 589 600 702 874 576 822 89 194 664 506 545 85 510 362 932 148 565 734 498 172 338 877 384 466 70 501 107 137 572 494 87 164 447 726 44 902 819 286 499 415 497 102 497 63 644 991 393 946 734 105 244 583 576 663 969 896 627 168 85 589 600 702 576 822 89 194 664 506 545 85 510 297 243 156 824 366 390 422 330 776 663 969 873 828 711 510 884 799 220 846 202 393 946 734 327 812 328 915 721 250 998 692 526 559 333 523 93 325 852 74 366 716 205 521 98 519 589 702 874 576 822 89 194 664 506 686 545 85 510 884 459 945 29 73 172 871 877 822 89 664 990 107 137 413 238 268 404 876 393 734 263 914 445 469 167 104 650 816 274 477 728 663 53 321 458 942 115 286 6 111 544 63 665 991 101 741 246 693 521 877 338 877 384 466 70 501 107 445 137 865 641 124 362 565 734 432 742 26 531 323 592 103 969 660 166 599 333 523 555 233 526 559 663 969 817 146 283 352 931 932 148 202 393 946 734 575 116 281 250 70 137 955 944 272 281 428 969 961 794 680 243 871 877 879 70 319 501 107 445 137 87 164 447 726 942 115 286 499 63 497 63 665 991 535 271 523 437 552 326 243 935 101 741 173 641 124 258 139 340 347 940 233 243 850 547 833 368 837 81 319 45 914 119 647 333 32 683 940 600 702 788 59 998 390 515 663 969 870 263 914 978 607 104 650 816 409 765 978 559 6 791 380 683 67 884 79 868 220 340 382 483 721 338 998 523 166 647 503 81 664 29 884 202 393 946 734 575 116 372 950 139 948 340 483 470 955 663 969 664 233 821 534 948 86 470 152 555 537 721 250 432 337 243 137 167 663 474 108 577 154 559 663 377 555 208 944 755 286 277 415 772 497 39",
+    "answer": null
+  },
+  {
+    "question_id": "helpful_base_2",
+    "prediction": "Sure, I'd be happy to help. Kickball is a fun and easy game that can be played with two teams of 5 players each. The game is played on a rectangular field with a goal at each end. The object of the game is to kick a ball into the goal using your foot. The game is usually played in four innings, with each team getting a turn to kick the ball into the goal. If a player successfully kicks the ball into the goal, their team gets a point. If the ball hits the goal and goes out of bounds, the other team gets a point. The game continues until one team reaches 10 points.",
+    "prediction_units": "969 447 726 44 752 497 63 644 254 504 530 733 555 290 943 485 321 948 86 470 821 761 108 404 757 323 56 903 86 539 79 868 220 846 470 821 167 384 693 521 611 268 876 417 755 237 544 286 111 415 497 63 662 914 445 333 487 319 219 107 29 290 978 833 246 764 521 453 503 865 641 124 362 734 319 390 479 422 330 776 167 655 104 650 816 325 915 931 428 222 915 734 39 488 948 319 263 350 836 822 89 194 896 627 459 173 945 233 29 914 445 137 576 761 173 620 112 915 943 485 948 86 251 412 260 712 593 822 89 194 787 935 271 333 523 196 884 850 412 244 808 220 139 340 557 244 680 534 485 321 948 627 168 758 545 711 510 362 148 479 330 776 655 837 81 664 219 506 148 337 850 260 712 593 822 89 284 663 969 198 711 510 337 362 469 384 907 693 828 711 67 297 265 755 237 307 286 415 772 497 63 665 991 393 946 734 319 350 416 836 822 89 194 627 168 428 865 641 124 337 850 213 260 712 593 822 89 194 664 521 555 944 493 361 931 428 565 734 870 156 824 56 822 487 219 416 477 161 969 713 716 205 521 208 390 479 330 776 803 327 693 521 233 935 271 523 196 918 393 946 734 319 45 416 836 607 908 592 521 453 29 258 950 321 948 86 539 67 940 326 531 327 384 879 488 443 325 758 555 208 417 755 44 544 497 63 644 991 202 393 946 734 327 905 27 108 404 595 705 11 576 384 879 219 952 315 944 878 932 148 202 393 946 734 319 350 416 836 822 89 194 627 168 865 641 683 337 884 79 808 220 340 846 390 45 330 593 647 366 565 734 870 290 978 833 246 693 453 483 25 825 771 46 812 222 274 799 220 202 393 946 734 319 416 836 908 592 103 521 858 258 436 139 340 347 376 398 212 455 258 436 592 103 969 390 479 330 776 435 485 803 791 611 660 506 686 208 417 755 237 752 415 497 63 665 991 393 946 734 319 350 416 836 822 89 194 627 168 428 865 641 124 258 436 139 340 660 11 614 716 205 518 56 948 143 251 676 260 712 593 167 822 89 693 194 664 555 944 812 222 915 143 390 479 330 435 592 103 969 466 812 53 398 53 832 758 711 510 297 265 675 237 307 626 111 544 63 665 991 271 333 523 196 290 56 948 813 86 539 105 552 326 850 260 485 948 627 143 416 836 398 212 455 428 734 523 105 244 583 821 663 969 896 627 143 38 79 799 220 340 846 119 387 246 333 70 506 107 29 202 393 946 734 870 290 978 833 246 693 521 317 453 25 825 771 46 812 222 915 274 799 220 202 393 946 734 319 416 836 908 592 103 521 208 726 755 115 415 497 63 644 389 864 771 685 333 958 66 362 734 870 676 260 712 593 821 89 485 352 143 38 326 531 362 853 487 742 522 295 586 576 6 70 835 431 243 330 716 518 948 86 45 914 445 137 333 664 107 545 85 510 884 202 393 734 870 290 978 833 246 693 521 453 503 771 46 812 222 915 274 799 220 202 393 946 734 319 416 668 908 103 521 942 115 286 111 544 497 63 665 991 881 331 333 663 969 793 105 244 583 534 485 321 948 627 290 978 836 366 173 523 555 545 85 510 124 362 6 734 870 251 0 260 41 740 592 908 797 81 194 664 93 506 686 613 417 755 752 415 497 63 644 389 864 685 333 958 66 884 202 393 946 734 870 290 978 833 246 693 470 333 523 506 952 85 884 202 393 946 734 870 978 836 331 908 764 103 521 453 483 222 136 233 850 416 836 803 380 711 975 362 761 907 597 944 932 148 290 978 833 592 592 655 764 81 650 194 325 758 711 85 297 417 675 237 286 111 415 497 63 665 991 202 393 946 734 327 905 954 108 938 663 969 523 867 251 676 244 680 534 485 948 627 290 836 384 879 945 555 545 85 124 362 734 870 676 412 260 41 740 592 246 89 81 194 664 385 506 686 613 417 755 752 415 497 63 665 991 393 946 734 319 263 416 836 822 89 194 896 627 469 274 794 680 314 333 488 179 207 950 139 340 198 711 124 362 6 812 352 915 143 244 583 205 521 935 101 741 246 104 650 816 38 244 821 534 485 321 948 896 627 156 824 442 321 948 664 539 552 326 955 326 711 124 259 781 645 435 384 879 443 357 251 412 260 41 740 592 908 741 81 194 664 274 686 545 85 297 265 675 755 415 497",
+    "answer": null
+  },
+  {
+    "question_id": "helpful_base_3",
+    "prediction": "Sure, I can help you with that. To start, I suggest creating a survey to gather feedback from your target customers. This will give you an idea of what they like and don't like about your product. Additionally, you can use customer interviews to gain more in-depth insights into their preferences. Finally, you can analyze customer reviews to identify areas for improvement.",
+    "prediction_units": "969 726 755 752 63 499 63 644 254 504 530 733 263 914 445 137 576 284 905 907 488 620 325 915 470 821 268 404 876 323 258 436 139 340 787 935 271 333 523 918 743 881 331 430 945 506 686 613 417 755 237 752 415 497 102 63 662 79 868 220 340 742 98 519 589 702 874 576 655 764 969 506 686 613 417 755 237 752 411 497 499 63 644 254 530 733 742 768 204 280 314 734 523 196 705 11 576 384 879 430 70 835 67 940 118 233 850 914 119 113 284 89 664 398 212 455 565 734 432 98 519 26 204 280 614 663 692 428 559 6 822 89 194 105 79 799 808 220 139 340 382 45 914 836 167 761 907 430 901 921 503 663 969 390 479 330 776 647 485 321 948 86 290 978 833 6 462 907 430 70 107 390 479 515 647 954 896 627 258 436 103 105 244 583 821 655 764 969 350 836 333 793 29 914 445 469 167 830 70 835 731 940 600 702 788 59 748 872 336 663 969 198 711 510 297 265 675 237 307 415 497 63 665 991 881 241 331 333 32 683 337 243 935 101 741 37 205 521 867 537 836 333 885 692 258 483 384 933 179 620 931 530 837 81 194 664 166 250 113 284 905 494 87 932 148 935 101 741 246 945 29 881 331 228 781 645 837 81 319 990 107 137 812 222 721 549 908 246 795 38 29 233 645 655 837 81 990 107 137 734 849 907 597 660 233 258 436 635 592 103 969 251 412 973 288 750 246 934 921 556 238 87 70 219 952 686 613 755 544 752 63 644 196 166 250 333 817 146 353 947 716 205 518 53 321 458 942 115 286 499 752 497 63 665 991 162 258 436 139 340 263 914 445 137 576 384 907 430 620 112 915 207 258 436 139 340 347 32 124 337 850 914 469 167 104 70 731 600 702 788 59 748 627 336 663 969 466 526 366 488 328 274 992 788 59 663 870 692 526 258 436 139 340 198 711 510 297 884 79 868 220 382 45 416 836 822 89 194 620 112 915 889 324 826 592 103 969 812 328 915 660 721 549 384 879 108 404 595 256 743 955 333 488 222 915 912 519 26 204 280 668 167 837 81 194 664 506 686 85 297 204 280 362 6 812 222 274 799 220 202 881 331 663 969 676 481 973 288 796 167 761 70 390 776 663 488 832 758 912 866 586 955 6 377 53 198 711 510 297 265 675 755 307 286 544 497 63 665 780 330 776 167 655 837 81 179 961 62 518 321 458 942 115 286 499 63 497 63 665 991 258 436 139 340 319 263 45 445 137 576 384 907 488 620 112 878 538 423 384 761 650 931 62 645 655 837 81 664 641 124 337 850 445 469 167 104 70 683 731 600 702 788 59 336 663 969 156 824 442 851 998 692 526 559 436 139 340 198 711 510 337 884 79 868 220 846 483 733 196 166 549 879 443 274 794 680 366 870 958 66 776 655 837 81 327 466 964 113 327 865 198 711 510 337 243 479 592 103 969 503 328 409 481 973 288 796 677 340 692 747 671 377 443 93 208 613 417 755 237 193 415 772 497",
+    "answer": null
+  },
+  {
+    "question_id": "helpful_base_4",
+    "prediction": "Sure, I can help you with that. To create a successful online marketing campaign, there are several steps you should take. First, you need to identify your target audience and understand their needs. Then, you should create a strategy that will help you reach them. After that, you should develop content that resonates with your target audience. Finally, you should measure the success of your campaign and make adjustments as needed.",
+    "prediction_units": "497 592 969 164 726 44 752 497 63 644 254 530 733 319 45 914 445 137 576 284 384 907 488 620 112 915 470 821 693 268 876 323 258 436 139 340 787 935 362 333 918 743 881 331 238 907 430 945 686 613 417 755 237 752 497 63 662 79 868 220 846 867 45 914 119 678 113 284 327 89 664 137 565 734 742 768 204 280 6 998 319 219 522 295 586 668 576 879 70 835 67 295 243 330 435 716 205 521 453 493 361 915 781 645 655 837 81 620 915 324 338 359 655 764 969 934 185 501 137 523 398 212 455 143 445 137 167 761 430 896 409 757 323 821 822 89 194 664 385 343 942 936 308 286 499 544 63 665 881 331 663 466 579 969 742 98 519 26 204 280 576 6 384 879 577 692 154 663 716 205 521 828 98 519 589 702 874 576 384 879 70 404 876 85 510 700 955 258 436 139 32 742 431 531 220 523 555 29 105 244 583 576 822 89 194 664 990 107 417 755 752 415 497 63 665 780 330 776 803 969 828 835 67 940 118 613 755 237 286 497 499 63 665 991 258 436 139 575 179 372 948 86 29 79 868 220 340 530 81 733 196 166 549 384 443 274 794 788 59 958 66 776 655 837 81 664 258 436 592 103 969 793 105 244 583 821 655 764 969 934 350 836 161 793 403 794 583 821 655 764 901 944 964 113 920 327 488 832 758 545 85 510 700 362 6 684 136 428 325 801 549 663 432 742 519 589 600 702 781 576 384 907 430 684 136 915 881 331 663 969 575 372 321 948 53 86 555 545 85 297 265 675 237 307 286 111 415 772 497 63 922 991 162 881 331 384 55 488 620 112 385 654 726 942 115 752 286 499 497 63 665 991 258 436 139 340 431 531 643 220 523 555 914 119 678 113 284 89 194 664 944 734 742 683 702 728 227 647 761 430 185 921 503 523 705 11 534 321 633 86 459 173 945 29 935 101 741 205 521 470 821 167 693 382 268 876 258 436 139 340 156 824 485 321 948 86 539 552 326 531 884 202 881 331 384 907 597 896 627 168 385 942 544 193 286 415 772 497 63 644 254 823 27 761 430 70 429 595 315 794 788 663 523 202 881 331 238 907 430 945 506 686 208 613 417 755 237 752 286 411 497 63 665 991 258 436 930 431 531 643 6 523 555 721 250 734 998 692 526 558 384 246 317 453 734 870 404 876 233 850 914 313 367 246 650 816 274 794 75 583 874 576 384 879 488 443 93 274 208 613 417 243 884 459 173 945 233 156 824 384 879 347 975 362 59 179 961 428 89 194 664 506 545 85 297 243 935 271 333 523 196 233 258 436 635 592 103 969 867 105 244 583 821 655 764 969 934 350 836 333 523 403 794 583 167 655 764 901 944 964 113 920 488 832 758 545 711 510 297 265 675 237 307 111 544 772 497 63 662 780 330 776 167 655 837 81 961 62 402 518 53 321 458 524 942 44 115 286 499 300 63 497 63 665 991 258 436 139 930 431 531 643 220 523 555 29 889 172 338 359 384 879 70 953 663 541 202 393 946 734 742 98 204 280 6 998 219 522 295 586 668 576 384 879 70 835 67 22 700 878 538 932 148 258 436 592 103 969 867 45 914 445 137 576 761 650 816 409 757 323 821 822 89 194 620 112 428 46 812 222 915 498 172 871 6 822 89 219 990 107 137 161 523 105 705 11 668 167 104 70 835 67 940 337 243 747 671 877 832 758 85 510 878 538 823 27 173 641 124 243 116 372 321 948 86 166 599 549 377 993 198 555 208 755 237 193 415 772",
+    "answer": null
+  },
+  {
+    "question_id": "helpful_base_5",
+    "prediction": "You can dice without slicing your finger by using a sharp knife and a cutting board. First, make sure the knife is sharp and clean. Then, place the food you want to dice on the cutting board. Hold the knife at a 45-degree angle and cut the food into small pieces. Make sure to keep your fingers away from the blade and move the knife in a sawing motion.",
+    "prediction_units": "340 319 263 45 445 137 576 761 384 761 907 620 112 721 549 837 81 787 271 333 523 918 921 556 238 761 907 597 506 85 297 243 645 935 303 81 664 835 67 552 326 398 212 455 258 436 592 103 969 870 390 479 330 776 333 212 455 350 836 663 969 86 870 290 833 368 837 81 664 258 436 139 340 347 376 398 212 455 428 565 734 742 431 531 668 655 969 404 876 233 747 281 137 62 822 89 81 664 958 526 362 352 915 931 944 734 319 45 445 469 821 655 104 837 185 921 398 212 455 143 290 978 822 89 969 377 705 377 969 164 447 942 115 286 544 415 772 497 63 662 780 479 330 776 803 969 873 828 835 67 940 118 613 417 755 237 752 286 415 497 499 63 665 991 162 172 871 822 89 219 107 431 531 614 663 969 523 196 202 393 946 734 870 105 961 281 62 655 837 81 194 664 86 990 107 655 865 873 32 683 589 683 589 337 126 323 576 655 764 969 283 488 352 915 143 850 914 119 887 593 485 321 53 321 385 942 115 499 286 499 497 63 662 991 537 978 881 331 384 907 488 620 112 385 208 613 726 942 44 752 286 499 497 63 662 213 260 593 89 194 664 835 67 940 884 202 393 946 734 390 479 330 776 253 340 555 944 233 258 436 139 787 935 101 741 246 29 244 79 868 220 340 846 196 498 721 250 321 333 32 835 67 22 124 700 362 493 361 393 946 734 45 914 445 469 104 523 185 398 212 455 143 290 978 833 384 907 933 103 969 611 916 942 115 286 499 415 772 497 63 991 470 821 908 205 521 660 202 393 946 734 156 281 26 204 280 879 81 194 325 107 173 945 944 565 734 793 390 479 330 435 592 969 660 166 680 56 903 86 390 422 776 655 837 81 885 721 250 56 11 878 668 6 879 249 969 93 325 409 526 586 955 327 488 684 136 915 912 787 914 445 137 167 70 185 506 29 202 393 946 215 113 870 390 479 330 776 6 340 878 25 825 46 812 222 915 274 79 799 220 340 742 98 519 589 126 324 789 246 382 412 260 323 534 948 882 924 866 586 362 955 366 377 198 711 510 297 265 675 237 307 286 544 415 497 63 991 162 172 871 822 664 219 107 431 531 614 663 969 523 105 79 868 808 220 139 340 867 45 445 476 534 485 86 539 876 323 258 436 635 592 103 969 390 781 303 333 948 455 901 545 711 362 734 382 787 101 741 822 89 194 664 390 479 515 647 246 896 627 202 393 946 734 870 290 978 833 655 764 837 81 664 46 812 222 915 143 156 826 789 677 340 355 692 148 243 202 393 946 734 523 196 705 11 822 89 380 664 692 483 812 222 915 931 734 742 98 519 26 204 76 668 167 655 398 212 455 498 324 789 402 384 430 380 692 817 146 283 377 385 942 755 237 286 499 544 497",
+    "answer": null
+  },
+  {
+    "question_id": "vicuna_1",
+    "prediction": "Time management is an important skill that can help you achieve more in less time. One way to improve your time management skills is to set realistic goals and break them down into smaller tasks. You should also create a schedule and stick to it, as well as prioritize tasks based on their importance. Additionally, you can use tools such as calendars, reminders, and task lists to help you stay organized and focused.",
+    "prediction_units": "583 655 837 896 627 384 761 179 961 428 523 555 243 671 877 443 93 455 208 428 25 865 641 124 362 384 761 430 620 112 931 428 878 25 771 328 409 0 41 740 592 103 969 199 774 832 915 143 912 519 589 126 702 137 120 635 693 205 521 925 881 459 173 945 233 45 914 445 137 576 907 430 488 620 112 915 470 821 167 693 382 268 876 258 436 139 483 734 523 793 105 326 531 534 321 948 86 885 148 243 498 324 592 103 969 466 503 812 222 915 781 645 384 474 879 70 835 67 589 702 583 874 167 655 837 81 377 627 385 942 544 603 286 111 544 497 63 991 162 535 935 101 741 246 650 816 325 112 915 787 101 741 822 89 194 664 539 105 79 868 220 846 483 825 333 328 409 481 973 288 796 677 340 885 692 148 258 436 635 592 103 244 583 821 655 837 81 896 627 498 172 338 359 384 761 430 328 961 428 523 705 243 671 877 398 832 93 915 208 85 297 589 126 137 791 693 205 521 711 510 169 25 389 825 771 865 641 683 337 884 79 808 220 340 742 204 280 576 384 879 901 945 233 156 824 56 327 635 259 303 503 333 432 835 67 940 118 702 15 6 333 487 990 29 914 416 836 908 103 521 711 510 362 812 222 915 143 38 290 978 647 822 89 664 990 107 233 202 881 331 384 879 896 627 915 721 549 238 761 907 597 112 659 25 710 825 771 46 812 222 274 79 799 220 742 519 337 243 126 324 789 246 259 317 453 366 663 969 523 691 105 244 583 167 761 907 430 70 835 67 940 107 545 85 297 265 675 237 755 415 415 772 497 499 63 665 991 162 258 436 930 431 531 220 555 944 556 246 310 540 295 76 614 37 382 45 914 119 678 56 284 89 664 944 734 432 742 519 337 126 137 384 879 705 11 716 205 521 483 503 366 832 222 915 912 519 600 702 15 6 333 487 219 952 79 799 220 846 503 377 437 506 686 613 417 755 237 752 415 772 63 644 254 823 27 175 173 641 347 124 243 787 935 101 741 384 693 317 453 556 907 430 641 124 481 973 288 796 167 104 81 327 635 894 466 969 523 403 244 583 167 837 81 664 198 711 940 884 244 583 647 761 907 430 70 835 67 940 118 107 545 85 297 337 243 850 978 822 89 194 664 835 67 940 118 944 493 361 881 331 663 969 466 503 771 328 409 38 41 740 592 103 969 199 774 832 758 545 85 297 265 675 755 237 752 415 772 497 63 644 389 823 523 196 166 250 333 817 146 283 353 205 518 53 321 458 942 115 308 286 626 499 544 300 497 63 665 991 258 436 139 930 263 45 914 445 137 576 384 907 430 488 620 112 915 428 258 436 139 340 347 683 337 884 244 240 808 220 253 205 521 828 98 519 26 204 280 6 366 185 552 362 668 6 173 641 124 337 850 914 445 137 821 761 430 317 453 366 443 801 549 663 969 198 711 510 297 675 755 237 307 286 277 544 63 991 824 734 754 748 872 338 359 655 837 81 325 801 549 366 663 969 198 711 510 297 265 675 755 286 626 752 63 644 823 175 684 136 143 244 583 576 167 761 430 70 835 67 940 126 107 850 781 645 333 873 835 67 85 589 884 79 868 220 340 846 470 821 246 693 382 268 876 323 258 436 139 32 742 519 589 702 576 822 89 406 187 594 592 103 969 934 350 836 179 961 62 655 81 194 198 711 940 118 944 6 812 222 915 143 390 422 330 776 435 6 908 791 382 934 501 860 137 333 377 32 835 67 940 118 613 755 415 772 497",
+    "answer": null
+  },
+  {
+    "question_id": "vicuna_2",
+    "prediction": "There are many effective ways to deal with stress, including exercising regularly, getting enough sleep, eating a healthy diet, practicing mindfulness and meditation, engaging in activities that bring joy and relaxation, and seeking professional help if needed.",
+    "prediction_units": "466 969 498 172 871 877 488 179 961 428 950 633 565 958 390 422 776 88 761 70 219 952 471 737 366 885 148 787 935 101 741 89 194 664 198 711 510 884 79 868 220 340 846 196 721 250 113 327 635 693 205 521 787 935 271 333 523 918 479 330 702 728 314 333 430 70 835 67 297 265 237 307 666 544 63 644 864 771 328 409 45 468 340 660 166 398 212 455 555 522 586 362 663 432 742 924 866 668 167 837 81 664 347 376 398 212 455 143 156 824 879 487 350 836 161 484 228 259 453 663 969 259 518 321 458 942 410 115 286 499 544 63 497 63 662 416 836 384 879 945 398 455 428 46 812 222 931 62 238 104 70 390 66 519 589 884 781 303 321 948 86 876 755 237 307 286 626 666 544 63 644 389 771 181 948 86 398 212 455 428 565 734 470 821 167 693 521 918 743 910 485 321 948 86 537 721 549 238 104 837 81 327 905 377 506 686 208 613 755 237 752 415 698 497 63 662 213 973 288 796 761 430 70 219 952 315 680 15 866 398 455 889 338 359 655 837 81 194 325 915 233 479 330 435 205 521 961 931 62 366 377 198 835 22 700 362 684 172 871 877 166 503 161 523 403 794 583 822 89 194 664 817 146 283 377 352 385 343 942 115 286 499 544 497 63 644 812 222 915 143 350 836 822 89 194 664 705 398 212 455 46 812 361 931 428 879 219 952 471 737 15 161 234 161 50 321 948 198 711 510 884 459 173 945 29 260 978 426 663 466 663 212 455 705 11 74 614 592 103 81 377 53 327 488 352 915 912 156 824 614 777 905 362 219 589 837 194 6 748 671 146 877 283 352 385 942 286 6 666 544 300 63 497 63 644 254 823 175 684 136 143 519 26 204 280 534 485 86 501 398 212 455 143 38 481 973 288 750 663 998 958 390 66 776 167 879 70 817 146 353 716 205 521 470 821 167 693 521 268 876 323 333 998 958 148 243 116 372 321 948 86 166 599 245 377 555 208 726 755 752 286 415 63",
+    "answer": null
+  },
+  {
+    "question_id": "vicuna_3",
+    "prediction": "Hensel's and Protagorean languages are two different types of programming languages. Hensel's language is a low-level language, meaning it is closer to the hardware and provides more control over the computer's resources. It is often used for system programming, such as writing operating systems or device drivers. Protagorean language is a high-level language, meaning it is closer to human language and abstracts away many details of the computer's architecture. It is often used for general-purpose programming, such as web development, game development, and data science.",
+    "prediction_units": "991 758 912 545 85 589 26 204 716 205 521 711 124 362 222 915 143 38 481 973 288 796 750 803 791 380 660 105 244 583 227 954 487 219 350 836 791 969 283 361 352 112 915 781 645 761 430 488 816 325 350 852 741 333 523 705 11 6 333 198 711 124 878 362 579 969 541 105 244 808 220 139 340 196 721 250 333 998 958 66 776 663 488 443 93 274 208 105 244 583 167 655 837 81 664 876 85 297 169 878 932 148 412 213 288 750 803 791 24 382 517 556 384 907 896 627 398 212 455 781 645 761 430 488 816 325 765 852 741 333 523 705 11 955 333 198 711 510 297 265 675 237 307 286 544 772 63 497 63 991 470 821 384 879 488 443 325 758 912 545 711 866 204 76 362 716 205 521 198 711 510 337 243 781 645 761 879 430 488 816 325 968 852 387 333 198 555 705 169 25 865 641 124 362 734 781 645 803 791 380 382 969 664 912 835 67 6 118 233 781 645 761 430 81 816 325 350 968 852 387 741 333 705 326 531 417 675 237 307 286 42 111 544 63 665 991 162 172 536 485 948 179 961 398 212 455 428 459 333 437 25 825 685 865 641 124 337 243 850 914 119 887 593 803 791 380 828 346 540 866 586 76 828 67 969 660 884 79 799 220 202 393 946 734 470 821 655 764 969 555 233 470 935 101 741 822 249 969 812 684 136 143 38 481 288 750 870 692 154 558 655 837 81 664 683 337 124 243 324 826 592 103 969 867 263 45 445 328 915 143 477 728 647 908 103 521 453 663 969 692 154 663 196 202 393 946 734 263 914 469 328 409 0 260 323 534 436 139 340 166 788 663 969 198 711 510 700 243 824 56 321 948 707 882 924 295 76 614 592 969 828 540 866 586 245 377 198 711 510 297 265 675 499 286 544 415 772 497 63 644 864 771 685 437 865 641 124 362 167 104 246 70 66 702 794 75 15 59 620 352 915 258 436 139 340 835 683 67 940 118 233 479 515 435 592 103 969 742 768 519 204 280 314 6 432 882 731 600 702 788 15 366 627 143 251 676 973 288 750 908 791 382 517 323 556 384 907 896 627 168 398 212 455 385 942 115 286 499 544 63 665 780 204 280 88 523 185 552 668 576 384 879 173 641 124 243 156 824 741 246 837 555 166 398 212 455 27 579 246 268 404 757 41 663 466 837 81 664 398 212 455 912 98 26 204 280 314 366 798 432 882 731 600 702 874 366 627 758 711 510 362 614 592 103 969 662 721 250 734 870 692 526 558 655 104 837 81 664 835 866 204 85 912 143 213 973 824 556 166 523 466 503 484 933 901 664 921 549 663 969 198 711 510 297 265 675 499 415 497 63 662 213 973 288 796 750 402 660 244 583 459 366 173 523 263 836 263 156 824 663 380 284 488 620 112 659 781 645 761 907 81 488 816 325 350 968 852 387 741 377 53 198 555 705 955 169 865 641 124 362 734 470 821 655 837 81 664 259 781 645 104 577 154 302 521 29 781 645 384 488 816 325 968 101 387 377 555 705 531 417 675 237 307 286 111 544 63 497 63 665 991 162 172 536 485 948 179 961 398 212 455 333 437 865 641 124 337 850 272 119 887 593 576 803 791 660 828 700 828 663 969 523 337 105 79 799 220 340 867 821 258 436 139 340 748 872 336 877 488 352 915 781 645 761 384 488 325 350 968 852 741 333 523 555 705 955 352 136 245 487 108 404 595 589 884 728 227 647 167 430 70 219 952 208 340 545 85 204 362 734 101 821 761 597 390 73 935 172 871 384 879 179 961 207 950 948 86 721 250 86 403 244 583 821 693 521 711 510 878 148 202 393 946 734 263 914 272 445 469 328 409 0 260 323 534 436 139 340 166 788 663 969 198 711 510 362 469 812 487 219 501 526 137 647 161 523 961 62 761 430 70 219 952 727 326 531 663 377 969 164 447 942 115 286 646 499 544 772 497 63 644 864 771 685 437 865 641 124 362 167 246 620 390 66 59 620 915 258 436 139 340 347 940 118 233 243 479 515 435 592 103 969 196 537 705 11 576 384 879 933 961 62 466 716 205 521 382 260 258 241 167 6 835 683 337 243 850 213 973 288 750 402 24 382 517 238 384 907 896 627 168 398 212 455 343 942 936 308 646 499 544 63 665 780 204 280 88 523 552 641 124 243 935 101 741 384 879 173 901 404 290 29 721 250 998 692 526 558 6 246 317 366 268 876 29 671 877 488 443 93 274 208 613 417 237 286 499 497 63 662 416 836 822 89 194 627 915 721 250 998 692 526 558 6 104 246 317 402 24 337 29 747 671 377 443 93 274 208 613 417 755 237 286 497 63 644 254 823 175 684 136 915 721 250 822 89 194 664 944 366 828 683 519 204 280 668 576 167 655 333 81 113 920 377 832 758 545 85 297 265 675 755 307 666 415 497",
+    "answer": null
+  },
+  {
+    "question_id": "vicuna_4",
+    "prediction": "There are several ways to increase your productivity while working from home. Firstly, it is important to create a designated workspace that is free from distractions and clutter. Secondly, it is important to set realistic goals and break down tasks into smaller chunks. Thirdly, it is important to take regular breaks and to stay connected with colleagues and friends. Finally, it is important to use technology to your advantage, such as using online collaboration tools and setting up an automated system for reminders.",
+    "prediction_units": "300 991 331 663 466 579 969 742 519 26 204 280 668 576 6 879 577 154 663 205 521 787 101 741 822 89 194 198 711 510 884 79 868 220 846 483 25 771 328 143 914 119 678 485 948 32 835 683 67 337 243 436 635 592 103 969 867 251 412 973 288 750 246 211 660 921 6 173 246 952 315 737 314 333 885 234 523 50 321 86 787 935 101 741 655 246 693 521 382 787 935 101 741 969 867 351 45 501 398 212 455 143 390 479 515 330 647 246 896 627 168 470 821 908 611 896 627 168 385 726 584 544 224 286 499 497 63 662 330 776 969 873 835 67 866 295 940 518 53 321 458 524 942 115 752 286 499 752 497 63 644 864 771 685 437 865 641 124 878 25 771 409 740 576 592 969 199 774 832 352 915 79 868 220 914 119 113 284 327 89 664 734 196 721 250 384 879 901 958 347 975 362 59 390 179 817 166 89 194 599 758 233 721 589 798 664 576 243 850 822 89 194 664 884 459 173 430 945 944 771 865 641 124 337 243 479 330 647 56 321 948 633 86 390 479 422 330 647 246 896 627 250 432 742 589 600 702 728 647 761 430 219 727 817 146 283 832 758 711 510 362 684 143 914 119 593 645 246 70 390 921 663 969 726 755 237 752 193 415 497 499 497 63 665 780 204 280 384 879 185 501 137 484 488 352 915 274 518 53 321 630 458 942 902 308 286 499 544 497 63 644 864 825 771 685 437 825 865 641 124 878 25 771 328 409 878 62 592 969 199 774 832 352 915 79 868 220 340 32 98 519 26 280 88 6 879 945 233 156 824 56 327 635 303 333 835 67 940 118 702 15 333 487 990 29 416 836 908 103 521 711 510 362 812 222 915 143 38 290 978 426 647 822 89 664 990 107 29 244 549 761 907 597 816 112 915 143 244 583 576 167 761 907 70 835 67 940 545 85 297 265 169 675 46 812 222 143 79 799 220 358 742 98 519 243 324 789 246 259 317 453 366 663 969 105 326 531 668 167 246 650 816 325 143 545 85 297 265 675 755 415 497 63 662 780 256 668 576 803 969 660 555 518 53 321 458 942 902 752 497 63 644 864 771 685 437 865 641 124 878 25 771 328 409 0 740 592 969 199 774 832 352 915 143 79 868 220 340 846 105 244 583 822 89 194 664 990 107 228 824 303 384 879 487 350 836 74 161 228 259 453 663 969 870 251 290 978 647 466 822 89 664 990 107 85 510 700 362 6 812 684 915 143 38 79 808 220 340 846 32 742 519 589 702 576 822 89 664 319 501 914 445 469 179 961 428 384 879 219 952 315 599 6 333 523 555 915 787 935 271 333 366 523 196 921 243 914 272 204 469 380 852 167 907 219 70 921 56 663 321 948 198 711 510 700 362 6 684 136 390 479 330 647 384 907 488 443 325 758 545 711 510 297 675 237 307 286 544 497 63 662 780 330 776 167 655 837 81 179 961 62 37 518 53 321 458 524 410 115 752 286 499 752 497 63 644 864 771 685 437 865 641 124 878 25 771 328 409 41 740 592 969 199 774 832 915 143 38 79 868 220 340 846 258 436 139 340 347 376 337 884 244 583 576 384 879 219 501 747 62 246 854 59 196 705 11 321 86 105 79 799 220 340 846 258 436 592 103 969 466 663 523 196 692 526 559 384 907 430 443 93 274 794 75 15 955 198 523 705 531 417 675 237 307 666 544 300 497 63 780 204 280 6 523 793 185 552 668 576 384 430 173 641 124 258 436 139 340 347 376 398 212 455 143 538 27 493 361 915 781 645 655 837 81 620 915 143 272 313 607 246 645 453 108 577 290 302 663 466 822 89 194 817 146 283 352 915 143 79 244 583 808 220 253 205 521 711 510 362 39 812 222 98 519 26 204 280 576 384 879 901 185 921 398 212 455 905 238 108 404 876 821 384 430 620 931 538 27 579 246 901 944 366 754 748 872 336 877 89 664 599 523 555 912 85 519 26 204 280 314 6 333 432 170 731 600 702 788 874 366 896 627 143 390 479 515 592 103 969 501 156 824 442 663 754 498 872 336 359 655 837 325 801 549 377 663 969 377 198 711 510 297 265 675 237 111 415 772 497",
+    "answer": null
+  },
+  {
+    "question_id": "vicuna_5",
+    "prediction": "Quantum computing is a type of computing that uses the principles of quantum mechanics to process and store data. It is based on the idea that a quantum bit (qubit) can exist in multiple states at the same time, allowing for more efficient processing of data than traditional computers. This means that qubits can represent more information than a single bit in a traditional computer, which can lead to faster and more powerful computations.",
+    "prediction_units": "274 944 366 896 627 263 914 469 328 409 323 534 436 139 340 86 398 212 455 428 865 641 124 362 6 734 793 105 244 583 821 655 837 81 664 86 876 932 148 914 272 469 328 409 0 323 534 436 139 340 86 398 212 455 915 881 459 173 945 29 477 258 139 340 347 376 975 955 641 124 884 202 393 946 734 870 251 481 288 796 333 488 816 912 540 295 586 59 870 757 716 205 521 711 510 878 538 932 148 850 272 119 387 246 816 274 794 788 59 366 627 889 172 877 879 319 914 445 137 576 761 961 931 428 503 487 990 107 85 297 884 79 799 220 340 846 251 676 481 288 750 246 346 540 866 295 586 668 576 384 879 70 835 22 866 700 352 915 912 98 519 884 702 614 592 103 969 196 721 250 822 89 664 921 944 291 87 164 447 942 115 286 646 499 544 415 497 63 644 389 771 685 437 865 641 124 850 978 833 822 89 664 835 940 118 944 493 361 393 946 113 327 905 837 81 664 166 250 113 284 327 618 513 196 459 173 945 565 734 45 272 119 227 246 614 650 325 794 614 627 143 233 544 290 978 333 506 208 613 417 755 544 63 689 662 272 119 387 677 253 355 290 978 337 914 445 137 576 430 620 931 428 487 219 522 975 230 333 432 882 683 67 866 118 702 944 955 6 812 222 915 889 324 789 246 521 660 351 794 788 59 870 0 757 740 716 205 521 828 519 431 531 702 874 6 822 89 194 664 506 545 85 700 362 6 945 29 202 393 946 734 742 768 26 204 280 668 6 822 89 194 852 627 915 143 244 583 167 655 837 81 53 627 168 385 726 942 936 308 286 646 499 544 300 497 63 254 734 317 645 238 907 894 828 398 455 143 479 330 435 592 103 969 498 324 826 592 103 969 565 998 958 390 422 330 776 245 432 882 531 283 832 93 208 233 850 213 288 750 655 246 70 540 295 586 668 88 798 70 835 866 586 398 212 455 659 878 538 932 148 721 250 822 89 194 664 921 944 503 494 541 881 331 488 684 143 777 227 663 59 196 166 250 333 817 146 283 353 716 205 521 38 914 272 469 328 409 0 323 534 436 139 340 660 166 788 663 969 198 711 510 297 265 675 499 307 286 544 415 497 63 662 991 881 331 333 873 32 683 337 243 172 536 950 948 758 545 711 510 884 459 173 945 233 45 914 119 607 677 253 290 978 333 523 545 85 337 243 544 445 137 576 907 430 620 915 156 824 384 879 404 757 288 33 432 347 975 576 384 879 443 274 208 613 233 243 850 324 826 592 103 969 406 25 771 328 143 390 422 330 435 592 103 969 748 806 336 877 89 194 817 146 283 352 301 881 331 384 173 684 931 428 734 742 98 519 26 204 280 314 6 212 455 325 350 968 716 205 521 382 290 978 6 333 793 506 686 944 747 377 812 684 222 931 565 734 793 105 777 227 59 523 166 250 333 817 146 283 353 716 205 521 382 45 272 313 469 328 409 38 260 323 534 436 139 340 660 166 549 663 969 164 447 726 942 115 544 286 499 341 63 665 991 535 271 437 552 326 850 914 445 137 576 761 430 650 620 112 915 781 645 384 879 523 555 29 79 799 220 139 340 390 479 330 776 167 761 430 70 67 940 600 702 788 663 969 488 684 136 915 324 826 592 103 969 251 676 444 167 761 907 894 387 969 390 422 330 716 205 521 263 272 313 367 246 816 409 757 323 534 284 179 259 139 340 194 817 146 283 832 758 711 297 675 237 307 621 277 415 772 497",
+    "answer": null
+  }
+]

omni_speech/infer/examples/answer_multiturn.json

@@ -0,0 +1,32 @@
+[
+  {
+    "question_id": "helpful_base_1",
+    "prediction": "The President of the United States is Donald Trump. He was born in 1946 and has been the leader of the country since January 20, 2017. He is the 45th president of the United States.",
+    "prediction_units": "33 879 879 932 340 523 641 325 641 641 969 969 932 523 161 523",
+    "answer": null
+  },
+  {
+    "question_id": "helpful_base_1",
+    "prediction": "Yes, you asked me who the President of the United States is. The answer is Donald Trump.",
+    "prediction_units": "932 523 641",
+    "answer": null
+  },
+  {
+    "question_id": "helpful_base_1",
+    "prediction": "Understood, stopping now.",
+    "prediction_units": "663 663 555 523 377 793 398 415",
+    "answer": null
+  },
+  {
+    "question_id": "helpful_base_2",
+    "prediction": "Washington, D.C.",
+    "prediction_units": "892 755",
+    "answer": null
+  },
+  {
+    "question_id": "helpful_base_2",
+    "prediction": "Paris",
+    "prediction_units": "225 736",
+    "answer": null
+  }
+]

omni_speech/infer/examples/asr/answer.json

@@ -0,0 +1,27 @@
+[
+  {
+    "question_id": "9701887_12850291_68-1139163",
+    "prediction": "whether as a package on a unit by unit basis or for a full turnkey tank battery",
+    "answer": "whether as a package on a unit by unit basis or for a full turnkey tank battery"
+  },
+  {
+    "question_id": "22130068_8582956_1400282_002121023-124198",
+    "prediction": "hi l e eco",
+    "answer": "hi l e eco"
+  },
+  {
+    "question_id": "4136660_5226863_001950290-327047",
+    "prediction": "but many of those who did not get on his backside have paid a heavy price",
+    "answer": "but many of those who did not get on his bad side have paid a heavy price"
+  },
+  {
+    "question_id": "9183489_12773116_68-1129988",
+    "prediction": "how frequently you plan on evaluating increases and distributions moving forward will this be a decision that's evaluated quarterly semiannually or annually going forward",
+    "answer": "how frequently you plan on evaluating increases and distributions moving forward will this be a decision that's evaluated quarterly semi annually or annually going forward"
+  },
+  {
+    "question_id": "12129038_21717467_2469779_102530044-277436",
+    "prediction": "they are great at taking advantage of unexpected situations",
+    "answer": "they are great at take taking advantage of unexpected situations"
+  }
+]

omni_speech/infer/examples/asr/question.json

@@ -0,0 +1,72 @@
+[
+    {
+      "conversations": [
+        {
+          "from": "human",
+          "value": "<speech>\nPlease transcribe this audio into text."
+        },
+        {
+          "from": "gpt",
+          "value": "whether as a package on a unit by unit basis or for a full turnkey tank battery"
+        }
+      ],
+      "id": "9701887_12850291_68-1139163",
+      "speech": "/data1/speech/speechData/data_En/audio_En/English-copora/extractAll/spgispeech/train/8beac2a5cb0bd40b198e403650ed8041/68.wav"
+    },
+    {
+      "conversations": [
+        {
+          "from": "human",
+          "value": "<speech>\nPlease transcribe this audio into text."
+        },
+        {
+          "from": "gpt",
+          "value": "hi l e eco"
+        }
+      ],
+      "id": "22130068_8582956_1400282_002121023-124198",
+      "speech": "/data1/speech/speechData/data_En/audio_En/OceanSpeech_2/English/King-ASR-390/DATA/CHANNEL0/WAVE/SPEAKER0212/SESSION1/002121023.WAV"
+    },
+    {
+      "conversations": [
+        {
+          "from": "human",
+          "value": "<speech>\nPlease transcribe this audio into text."
+        },
+        {
+          "from": "gpt",
+          "value": "but many of those who did not get on his bad side have paid a heavy price"
+        }
+      ],
+      "id": "4136660_5226863_001950290-327047",
+      "speech": "/data1/speech/speechData/rawData/Public_Data/IMDA-National_Speech_Corpus/IMDA-National_Speech_Corpus/PART1/DATA/CHANNEL0/WAVE/SPEAKER0195/SESSION0/001950290.WAV"
+    },
+    {
+      "conversations": [
+        {
+          "from": "human",
+          "value": "<speech>\nPlease transcribe this audio into text."
+        },
+        {
+          "from": "gpt",
+          "value": "how frequently you plan on evaluating increases and distributions moving forward will this be a decision that's evaluated quarterly semi annually or annually going forward"
+        }
+      ],
+      "id": "9183489_12773116_68-1129988",
+      "speech": "/data1/speech/speechData/data_En/audio_En/English-copora/extractAll/spgispeech/train/d32cdf6cd528b99c8e8585b8ede2989d/68.wav"
+    },
+    {
+      "conversations": [
+        {
+          "from": "human",
+          "value": "<speech>\nPlease transcribe this audio into text."
+        },
+        {
+          "from": "gpt",
+          "value": "they are great at take taking advantage of unexpected situations"
+        }
+      ],
+      "id": "12129038_21717467_2469779_102530044-277436",
+      "speech": "/data1/speech/speechData/data_En/audio_En/OceanSpeech_2/English/King-ASR-076/DATA/CHANNEL1/WAVE/SPEAKER0253/SESSION0/102530044.WAV"
+    }
+]

omni_speech/infer/examples/fc/answer.json

@@ -0,0 +1,27 @@
+[
+  {
+    "question_id": 18819,
+    "prediction": "<tool_call>\n{\"name\": \"calculate_area\", \"arguments\": '{\"shape\": \"rectangle\", \"measurements\": {\"length\": 5, \"width\": 3}}'}\n</tool_call>",
+    "answer": null
+  },
+  {
+    "question_id": 18819,
+    "prediction": "<tool_call>\n{\"name\": \"calculate_area\", \"arguments\": '{\"shape\": \"circle\", \"measurements\": {\"radius\": 4}}'}\n</tool_call>",
+    "answer": null
+  },
+  {
+    "question_id": 18821,
+    "prediction": "<tool_call>\n{\"name\": \"check_email_availability\", \"arguments\": '{\"email\": \"john.doe@gmail.com\"}'}\n</tool_call>",
+    "answer": null
+  },
+  {
+    "question_id": 18821,
+    "prediction": "<tool_call>\n{\"name\": \"check_email_availability\", \"arguments\": '{\"email\": \"john.doe123@gmail.com\"}'}\n</tool_call>",
+    "answer": null
+  },
+  {
+    "question_id": 18823,
+    "prediction": "<tool_call>\n{\"name\": \"calculate_age\", \"arguments\": '{\"date_of_birth\": \"1990-05-15\"}'}\n</tool_call>",
+    "answer": null
+  }
+]

omni_speech/infer/examples/fc/question.json

@@ -0,0 +1,71 @@
+[
+   {  
+    "conversations": [
+      {
+        "from": "human",
+        "value": "<speech>\nPlease directly answer the questions in the user's speech."
+      },
+      {
+        "from": "gpt",
+        "value": "<tool_call>\n{\"name\": \"calculate_area\", \"arguments\": '{\"shape\": \"rectangle\", \"measurements\": {\"length\": 5, \"width\": 3}}'}\n</tool_call>"
+      },
+      {
+        "from": "human",
+        "value": "<speech>\nPlease directly answer the questions in the user's speech."
+      },
+      {
+        "from": "gpt",
+        "value": "<tool_call>\n{\"name\": \"calculate_area\", \"arguments\": '{\"shape\": \"circle\", \"measurements\": {\"radius\": 4}}'}\n</tool_call>"
+      }
+    ],
+    "id": 18819,
+    "speech": [
+      "/data1/speech/anhnmt2/dataset/s2s/english/fc/audios/tts_fc_18819_1.wav",
+      "/data1/speech/anhnmt2/dataset/s2s/english/fc/audios/tts_fc_18819_2.wav"
+    ],
+    "tools": "{\n    \"name\": \"calculate_area\",\n    \"description\": \"Calculate the area of a given shape\",\n    \"parameters\": {\n        \"type\": \"object\",\n        \"properties\": {\n            \"shape\": {\n                \"type\": \"string\",\n                \"description\": \"The shape for which the area is to be calculated\"\n            },\n            \"measurements\": {\n                \"type\": \"object\",\n                \"properties\": {\n                    \"length\": {\n                        \"type\": \"number\",\n                        \"description\": \"The length of the shape\"\n                    },\n                    \"width\": {\n                        \"type\": \"number\",\n                        \"description\": \"The width of the shape\"\n                    },\n                    \"radius\": {\n                        \"type\": \"number\",\n                        \"description\": \"The radius of the shape\"\n                    }\n                },\n                \"required\": [\n                    \"length\",\n                    \"width\",\n                    \"radius\"\n                ]\n            }\n        },\n        \"required\": [\n            \"shape\",\n            \"measurements\"\n        ]\n    }\n}\n\n{\n    \"name\": \"schedule_meeting\",\n    \"description\": \"Schedule a meeting with specified participants and time\",\n    \"parameters\": {\n        \"type\": \"object\",\n        \"properties\": {\n            \"participants\": {\n                \"type\": \"array\",\n                \"items\": {\n                    \"type\": \"string\"\n                },\n                \"description\": \"The participants of the meeting\"\n            },\n            \"time\": {\n                \"type\": \"string\",\n                \"format\": \"date-time\",\n                \"description\": \"The date and time of the meeting\"\n            }\n        },\n        \"required\": [\n            \"participants\",\n            \"time\"\n        ]\n    }\n}\n\n"
+  },
+  {
+    "conversations": [
+      {
+        "from": "human",
+        "value": "<speech>\nPlease directly answer the questions in the user's speech."
+      },
+      {
+        "from": "gpt",
+        "value": "<tool_call>\n{\"name\": \"check_email_availability\", \"arguments\": '{\"email\": \"john.doe@gmail.com\"}'}\n</tool_call>"
+      },
+      {
+        "from": "human",
+        "value": "<speech>\nPlease directly answer the questions in the user's speech."
+      },
+      {
+        "from": "gpt",
+        "value": "<tool_call>\n{\"name\": \"check_email_availability\", \"arguments\": '{\"email\": \"john.doe123@gmail.com\"}'}\n</tool_call>"
+      }
+    ],
+    "id": 18821,
+    "speech": [
+      "/data1/speech/anhnmt2/dataset/s2s/english/fc/audios/tts_fc_18821_1.wav",
+      "/data1/speech/anhnmt2/dataset/s2s/english/fc/audios/tts_fc_18821_2.wav"
+    ],
+    "tools": "{\n    \"name\": \"check_email_availability\",\n    \"description\": \"Check if an email address is available or already taken\",\n    \"parameters\": {\n        \"type\": \"object\",\n        \"properties\": {\n            \"email\": {\n                \"type\": \"string\",\n                \"description\": \"The email address to be checked\"\n            }\n        },\n        \"required\": [\n            \"email\"\n        ]\n    }\n}\n"
+  },
+  {
+    "conversations": [
+      {
+        "from": "human",
+        "value": "<speech>\nPlease directly answer the questions in the user's speech."
+      },
+      {
+        "from": "gpt",
+        "value": "<tool_call>\n{\"name\": \"calculate_age\", \"arguments\": '{\"date_of_birth\": \"1990-05-15\"}'}\n</tool_call>"
+      }
+    ],
+    "id": 18823,
+    "speech": [
+      "/data1/speech/anhnmt2/dataset/s2s/english/fc/audios/tts_fc_18823_1.wav"
+    ],
+    "tools": "{\n    \"name\": \"calculate_age\",\n    \"description\": \"Calculate the age of a person\",\n    \"parameters\": {\n        \"type\": \"object\",\n        \"properties\": {\n            \"date_of_birth\": {\n                \"type\": \"string\",\n                \"format\": \"date\",\n                \"description\": \"The date of birth of the person\"\n            }\n        },\n        \"required\": [\n            \"date_of_birth\"\n        ]\n    }\n}\n"
+  }
+]

omni_speech/infer/examples/multiturn/answer.json

@@ -0,0 +1,27 @@
+[
+  {
+    "question_id": 112244,
+    "prediction": "There are several potential problems that could arise from using a chatbot. Firstly, chatbots can be biased and may not always provide accurate information. Secondly, chatbots can be easily manipulated by malicious actors who may try to exploit them for their own purposes. Thirdly, chatbots can be vulnerable to cyber-attacks and may expose user data to malicious actors. Finally, chatbots may not be able to understand complex queries or provide detailed explanations.",
+    "answer": null
+  },
+  {
+    "question_id": 112244,
+    "prediction": "To address these problems, chatbot developers can implement measures to ensure that their chatbots are unbiased and accurate. They can also create algorithms to detect malicious activities and protect user data from cyber-attacks. Additionally, they can use natural language processing (NLP) techniques to better understand complex queries and provide detailed explanations.",
+    "answer": null
+  },
+  {
+    "question_id": 13203,
+    "prediction": "Sure! For example, a chatbot can be used to answer frequently asked questions (FAQs) about a product or service. It can also be used to provide personalized recommendations and advice to customers. Additionally, a chatbot can help with customer service by providing quick answers to common queries, taking orders, and helping customers find the best products for their needs.",
+    "answer": null
+  },
+  {
+    "question_id": 13203,
+    "prediction": "Chatbots can provide a number of benefits in customer service, such as 24/7 availability, cost savings, improved customer experience, and better customer engagement. Chatbots are also able to quickly respond to customer queries and provide personalized recommendations, which can help build trust and loyalty with customers.",
+    "answer": null
+  },
+  {
+    "question_id": 13203,
+    "prediction": "Chatbots have some limitations when it comes to customer service, such as difficulty in understanding complex queries or providing detailed explanations. Additionally, chatbots may not be able to provide the same level of personalization and empathy as a human customer service representative. Furthermore, chatbots may not be able to handle unexpected situations or emergencies.",
+    "answer": null
+  }
+]

omni_speech/infer/examples/multiturn/question.json

@@ -0,0 +1,61 @@
+[
+    {
+      "conversations": [
+        {
+          "from": "human",
+          "value": "<speech>\nPlease directly answer the questions in the user's speech."
+        },
+        {
+          "from": "gpt",
+          "value": "There are a few potential problems that could arise when using a chatbot. First, chatbots may not always be able to adequately understand and interpret natural language. This can lead to misinterpretations or misunderstandings that can be difficult to resolve. Second, chatbots are limited by the amount of data they can access and the computational power available to them. This can lead to inaccurate results or incomplete solutions. Third, chatbots may not be able to accurately interpret certain types of input, such as audio or visual data. Fourth, chatbots are vulnerable to malicious attacks and may be susceptible to malicious code. Fifth, chatbots may be unable to recognize and respond to more complex tasks, such as making decisions."
+        },
+        {
+          "from": "human",
+          "value": "<speech>\nPlease directly answer the questions in the user's speech."
+        },
+        {
+          "from": "gpt",
+          "value": "These problems can be addressed in a few ways. First, chatbot designers can implement methods to improve the chatbot's understanding of natural language. This can be done by using more sophisticated algorithms, training the chatbot on larger datasets, and using human input to help the chatbot improve its understanding. Second, developers can increase the amount of data available to the chatbot and give it access to more powerful computing resources to increase its accuracy and performance. Third, developers can implement methods to improve the chatbot's ability to interpret audio and visual input. Finally, developers can implement methods to protect the chatbot from malicious attacks and ensure that it is able to recognize and respond to more complex tasks."
+        }
+      ],
+      "id": 112244,
+      "speech": [
+        "/data1/speech/anhnmt2/dataset/s2s/english/qna/moss/audios/id_112244_user_turn_0.wav",
+        "/data1/speech/anhnmt2/dataset/s2s/english/qna/moss/audios/id_112244_user_turn_1.wav"
+      ]
+    },
+    {
+      "conversations": [
+        {
+          "from": "human",
+          "value": "<speech>\nPlease directly answer the questions in the user's speech."
+        },
+        {
+          "from": "gpt",
+          "value": "Yes, absolutely. Chatbots have been used to great success in a variety of customer service settings. For example, some companies have implemented chatbots to manage and answer customer inquiries, allowing them to quickly provide customers with the answers they need. Chatbots can also be used to automate customer service tasks, such as order updates, shipping confirmations, and product recommendations. Additionally, chatbots can be used to handle customer complaints and provide solutions in a timely manner."
+        },
+        {
+          "from": "human",
+          "value": "<speech>\nPlease directly answer the questions in the user's speech."
+        },
+        {
+          "from": "gpt",
+          "value": "The benefits of using chatbots in customer service are numerous. Chatbots can provide customers with an immediate response to inquiries, allowing them to quickly get the answers they need. Additionally, chatbots can provide more personalized service by analyzing customer data and providing tailored responses. Furthermore, chatbots can help automate customer service tasks, saving time and resources. Finally, chatbots can help improve customer satisfaction by providing quick and reliable customer service."
+        },
+        {
+          "from": "human",
+          "value": "<speech>\nPlease directly answer the questions in the user's speech."
+        },
+        {
+          "from": "gpt",
+          "value": "One of the main limitations of using chatbots in customer service is the difficulty of providing human-level intelligence. While chatbots can provide an immediate response to simple queries, they may not be able to handle more complex inquiries. Additionally, chatbots may not be able to understand customer sentiment or emotions. Finally, chatbots may not be able to provide the same level of personalized service as a human customer service agent."
+        }
+      ],
+      "id": 13203,
+      "speech": [
+        "/data1/speech/anhnmt2/dataset/s2s/english/qna/moss/audios/id_13203_user_turn_0.wav",
+        "/data1/speech/anhnmt2/dataset/s2s/english/qna/moss/audios/id_13203_user_turn_1.wav",
+        "/data1/speech/anhnmt2/dataset/s2s/english/qna/moss/audios/id_13203_user_turn_2.wav"
+      ]
+    }
+]

omni_speech/infer/examples/question.json

@@ -0,0 +1,102 @@
+[
+    {
+        "id": "helpful_base_1",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/helpful_base_1.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    },
+    {
+        "id": "helpful_base_2",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/helpful_base_2.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    },
+    {
+        "id": "helpful_base_3",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/helpful_base_3.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    },
+    {
+        "id": "helpful_base_4",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/helpful_base_4.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    },
+    {
+        "id": "helpful_base_5",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/helpful_base_5.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    },
+    {
+        "id": "vicuna_1",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/vicuna_1.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    },
+    {
+        "id": "vicuna_2",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/vicuna_2.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    },
+    {
+        "id": "vicuna_3",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/vicuna_3.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    },
+    {
+        "id": "vicuna_4",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/vicuna_4.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    },
+    {
+        "id": "vicuna_5",
+        "speech": "/data1/speech/anhnmt2/Speech2Speech/half-streaming-speech-nlp/omni_speech/infer/examples/question_wav/vicuna_5.wav",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            }
+        ]
+    }
+]

omni_speech/infer/examples/question_multiturn.json

@@ -0,0 +1,54 @@
+[
+    {
+        "id": "helpful_base_1",
+        "speech": ["/data1/speech/anhnmt2/datapreparation/egs/whisperspeech/test_whisperspeech_turn_1_2607-seven_poor_travellers_0808_librivox_64kb_mp3-sevenpoortravellers_02_dickens_64kb_.wav", "/data1/speech/anhnmt2/datapreparation/egs/whisperspeech/test_whisperspeech_turn_2_1212-goldenbowl_0905_librivox_64kb_mp3-goldenbowl_3-16_james_64kb_.wav", "/data1/speech/anhnmt2/datapreparation/egs/whisperspeech/test_whisperspeech_interrupt_3318-lincoln_story_book2_0902_librivox_64kb_mp3-lincolnstorybook_307_williams_64kb_.wav"],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            },
+            {
+                "from": "gpt",
+                "value": "The president of the United States is Joe Biden"
+            },
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            },
+            {
+                "from": "gpt",
+                "value": "The president of the United States is Joe Biden"
+            },
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            },
+            {
+                "from": "gpt",
+                "value": "The president of the United States is Joe Biden"
+            }
+        ]
+    },
+    {
+        "id": "helpful_base_2",
+        "speech": ["/data1/speech/anhnmt2/datapreparation/egs/whisperspeech/test_whisperspeech_turn_1_681-tale_of_major_monkey_lb_librivox_64kb_mp3-taleofmajormonkey_08_bailey_64kb_.wav", "/data1/speech/anhnmt2/datapreparation/egs/whisperspeech/test_whisperspeech_turn_2_94-revelations_of_divine_love_drb_librivox_64kb_mp3-revelations_57_julian_64kb_.wav"],
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            },
+            {
+                "from": "gpt",
+                "value": "The capital of the United States is Washington D.C"
+            },
+            {
+                "from": "human",
+                "value": "<speech>\nPlease directly answer the questions in the user's speech."
+            },
+            {
+                "from": "gpt",
+                "value": "The president of the United States is Joe Biden"
+            }
+        ]
+    }
+]

omni_speech/infer/examples/question_wav/helpful_base_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adffb5da5f2f60328f7228a83d7b7baf0d0dc2d4100ef8697fcf26b1d3f8032c
+size 190544

omni_speech/infer/examples/question_wav/helpful_base_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a68c2ca9ae64900a9078320936a4242f712a4b9fddd7988e15ae2d56a931c2c
+size 742480

omni_speech/infer/examples/question_wav/helpful_base_3.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e471fe0d6cc9cdc6b290a0d4f79296baed41ee85214303e657ce8f70c5a991b
+size 329808

omni_speech/infer/examples/question_wav/helpful_base_4.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cbcc24b5fc7b07d603eb182d032078ffeb4143268dad80802945a19194b0d1d
+size 169040

omni_speech/infer/examples/question_wav/helpful_base_5.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87181fc0c2d727a5b62f525ff98b1ce5f1ab40ce808fcc116b4a32fac81df061
+size 218192

omni_speech/infer/examples/question_wav/vicuna_1.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89c090e7e58430f36eec6de88b1a73d8124cec7b5c81a6def6cf8cd1d2d047f7
+size 241744

omni_speech/infer/examples/question_wav/vicuna_2.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:059d747a0423e4cc3d35e9f3a482cac468000f162ec049e720833af82d8778d5
+size 254032

omni_speech/infer/examples/question_wav/vicuna_3.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20b6d20153a1ea91feb07f52d0f691bfe13acf624f9f03295e003d439c691d89
+size 354384

omni_speech/infer/examples/question_wav/vicuna_4.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4aff89f7a03904d551cf584bc49a125eb3da1f1c5db1f6fc8de2914438b16d3
+size 269392

omni_speech/infer/examples/question_wav/vicuna_5.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12960e00213e261762d2ceb4beb2c9885149a43099ad63c987b6ddb96ab89ceb
+size 232528

omni_speech/infer/examples/tsv2jsonl.py

@@ -0,0 +1,45 @@
+import csv
+import json
+
+# Define the path to your TSV file and output JSON file
+tsv_file = '/data1/speech/anhnmt2/dataset/s2s/asr/viet_bud500/test_data.tsv'  # Replace with your TSV file path
+json_file = 'asr/viet-bud/question.json'  # Replace with your desired output JSON file path
+
+# Initialize an empty list to store the JSON data
+output_data = []
+
+# Open and read the TSV file
+with open(tsv_file, 'r', encoding='utf-8') as tsv:
+    reader = csv.DictReader(tsv, delimiter='\t')
+    
+    # Process each row in the TSV file
+    for idx, row in enumerate(reader):
+        # Extract necessary data from the TSV row
+        path = row['PATH']
+        transcript = row['TRANSCRIPT']
+        file_idx = path.split("/")[-1].replace(".wav","")
+        
+        # Create the required format for each row
+        conversation = {
+            "conversations": [
+                {
+                    "from": "human",
+                    "value": "<speech>\nPlease transcribe this audio into text."
+                },
+                {
+                    "from": "gpt",
+                    "value": transcript
+                }
+            ],
+            "id": file_idx,
+            "speech": path
+        }
+        
+        # Append the formatted conversation to the output data list
+        output_data.append(conversation)
+
+# Write the output data to the JSON file
+with open(json_file, 'w', encoding='utf-8') as json_out:
+    json.dump(output_data, json_out, indent=4, ensure_ascii=False)
+
+print(f"Conversion complete! Output saved to {json_file}.")

omni_speech/infer/fairseq/.circleci/config.yml

@@ -0,0 +1,128 @@
+# Use 2.1 for orbs
+version: 2.1
+
+# -------------------------------------------------------------------------------------
+# Environments to run the jobs in
+# -------------------------------------------------------------------------------------
+gpu: &gpu
+  environment:
+    CUDA_VERSION: "11.2"
+  machine:
+    image: ubuntu-2004-cuda-11.2:202103-01
+  resource_class: gpu.nvidia.medium.multi
+
+
+# -------------------------------------------------------------------------------------
+# Re-usable commands
+# -------------------------------------------------------------------------------------
+cache_key: &cache_key cache-key-{{ .Environment.CIRCLE_JOB }}-{{ checksum ".circleci/config.yml" }}-{{ checksum "setup.py"}}
+
+install_dep_pt1_10: &install_dep_pt1_10
+  - run:
+      name: Install Pytorch Dependencies
+      command: |
+        source activate fairseq
+        pip install --upgrade setuptools
+        pip install torch==1.10.1+cu111 torchaudio==0.10.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html
+        python -c 'import torch; print("Torch version:", torch.__version__)'
+
+install_dep_pt1_12: &install_dep_pt1_12
+  - run:
+      name: Install Pytorch Dependencies
+      command: |
+        source activate fairseq
+        pip install --upgrade setuptools
+        pip install torch==1.12.1+cu116 torchaudio==0.12.1+cu116 -f https://download.pytorch.org/whl/torch_stable.html
+        python -c 'import torch; print("Torch version:", torch.__version__)'
+
+install_repo: &install_repo
+  - run:
+      name: Install Repository
+      command: |
+        source activate fairseq
+        python -m pip install fairscale
+        python -m pip install -e '.[dev,docs]'
+        python -c 'import torch; print("Torch version:", torch.__version__)'
+
+run_unittests: &run_unittests
+  - run:
+      name: Run Unit Tests
+      command: |
+        source activate fairseq
+        pytest tests/gpu/test_binaries_gpu.py
+
+check_nvidia_driver: &check_nvidia_driver
+  - run:
+      name: Check NVIDIA Driver
+      working_directory: ~/
+      command: |
+        pyenv versions
+        nvidia-smi
+
+create_conda_env: &create_conda_env
+  - run:
+      name: Install and Create Conda Environment
+      command: |
+        curl -o ~/miniconda.sh -O  https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+        chmod +x ~/miniconda.sh
+        bash ~/miniconda.sh -b -p $HOME/miniconda
+        rm ~/miniconda.sh
+        echo 'export PATH=$HOME/miniconda/bin:$PATH' >> $BASH_ENV
+        source $BASH_ENV
+        if [ ! -d ~/miniconda/envs/fairseq ]
+        then
+          conda create -y -n fairseq python=3.8
+        fi
+        source activate fairseq
+        python --version
+        pip install --upgrade pip
+# -------------------------------------------------------------------------------------
+# Jobs to run
+# -------------------------------------------------------------------------------------
+
+jobs:
+
+  gpu_tests_pt1_10:
+    <<: *gpu
+
+    working_directory: ~/fairseq-py
+
+    steps:
+      - checkout
+      - <<: *check_nvidia_driver
+      - <<: *create_conda_env
+      - restore_cache:
+          key: *cache_key
+      - <<: *install_dep_pt1_10
+      - save_cache:
+          paths:
+            - ~/miniconda/
+          key: *cache_key
+      - <<: *install_repo
+      - <<: *run_unittests
+
+  gpu_tests_pt1_12:
+    <<: *gpu
+
+    working_directory: ~/fairseq-py
+
+    steps:
+      - checkout
+      - <<: *check_nvidia_driver
+      - <<: *create_conda_env
+      - restore_cache:
+          key: *cache_key
+      - <<: *install_dep_pt1_12
+      - save_cache:
+          paths:
+            - ~/miniconda/
+          key: *cache_key
+      - <<: *install_repo
+      - <<: *run_unittests
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - gpu_tests_pt1_12
+      - gpu_tests_pt1_10

omni_speech/infer/fairseq/.github/CODEOWNERS

@@ -0,0 +1,21 @@
+# Setting up CODEOWNERS for UST related codebase
+# Documentation for open sourced models relevant to UST
+examples/speech_to_text     @kahne @sravyapopuri388 @jmp84
+examples/speech_to_speech   @an918tw @sravyapopuri388 @jmp84
+examples/speech_synthesis   @kahne @jmp84
+examples/simultaneous_translation   @kahne @jmp84
+examples/speech_text_joint_to_text  @yuntang @jmp84
+
+# Speech related models relevant to UST
+fairseq/models/speech_to_speech @sravyapopuri388 @jmp84
+fairseq/models/speech_to_text   @kahne @sravyapopuri388 @jmp84
+fairseq/models/text_to_speech   @kahne @jmp84
+
+# CONFORMER IMPLEMENTATION
+fairseq/modules/conformer_layer.py @sravyapopuri388 @jmp84
+fairseq/modules/espnet_multihead_attention.py @sravyapopuri388 @jmp84
+fairseq/modules/rotary_positional_embedding.py @sravyapopuri388 @jmp84
+fairseq/modules/positional_encoding.py @sravyapopuri388 @jmp84
+
+# Machine Translation/NLLB
+fairseq/tasks/translation.py @gwenzek

omni_speech/infer/fairseq/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,3 @@
+## 👉 [Please follow one of these issue templates](https://github.com/pytorch/fairseq/issues/new/choose) 👈
+
+Note: to keep the backlog clean and actionable, issues may be immediately closed if they do not follow one of the above issue templates.

omni_speech/infer/fairseq/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,43 @@
+---
+name: 🐛 Bug Report
+about: Submit a bug report to help us improve
+labels: 'bug, needs triage'
+---
+
+## 🐛 Bug
+
+<!-- A clear and concise description of what the bug is. -->
+
+### To Reproduce
+
+Steps to reproduce the behavior (**always include the command you ran**):
+
+1. Run cmd '....'
+2. See error
+
+<!-- If you have a code sample, error messages, stack traces, please provide it here as well -->
+
+
+#### Code sample
+<!-- Ideally attach a minimal code sample to reproduce the decried issue.
+Minimal means having the shortest code but still preserving the bug. -->
+
+### Expected behavior
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+### Environment
+
+ - fairseq Version (e.g., 1.0 or main):
+ - PyTorch Version (e.g., 1.0)
+ - OS (e.g., Linux):
+ - How you installed fairseq (`pip`, source):
+ - Build command you used (if compiling from source):
+ - Python version:
+ - CUDA/cuDNN version:
+ - GPU models and configuration:
+ - Any other relevant information:
+
+### Additional context
+
+<!-- Add any other context about the problem here. -->

omni_speech/infer/fairseq/.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,15 @@
+---
+name: 📚 Documentation/Typos
+about: Report an issue related to documentation or a typo
+labels: 'documentation, needs triage'
+---
+
+## 📚 Documentation
+
+For typos and doc fixes, please go ahead and:
+
+1. Create an issue.
+2. Fix the typo.
+3. Submit a PR.
+
+Thanks!

omni_speech/infer/fairseq/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,24 @@
+---
+name: 🚀 Feature Request
+about: Submit a proposal/request for a new feature
+labels: 'enhancement, help wanted, needs triage'
+---
+
+## 🚀 Feature Request
+<!-- A clear and concise description of the feature proposal -->
+
+### Motivation
+
+<!-- Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too -->
+
+### Pitch
+
+<!-- A clear and concise description of what you want to happen. -->
+
+### Alternatives
+
+<!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->
+
+### Additional context
+
+<!-- Add any other context or screenshots about the feature request here. -->

omni_speech/infer/fairseq/.github/ISSUE_TEMPLATE/how-to-question.md

@@ -0,0 +1,33 @@
+---
+name: ❓ Questions/Help
+about: If you have questions, please first search existing issues and docs
+labels: 'question, needs triage'
+---
+
+## ❓ Questions and Help
+
+### Before asking:
+1. search the issues.
+2. search the docs.
+
+<!-- If you still can't find what you need: -->
+
+#### What is your question?
+
+#### Code
+
+<!-- Please paste a code snippet if your question requires it! -->
+
+#### What have you tried?
+
+#### What's your environment?
+
+ - fairseq Version (e.g., 1.0 or main):
+ - PyTorch Version (e.g., 1.0)
+ - OS (e.g., Linux):
+ - How you installed fairseq (`pip`, source):
+ - Build command you used (if compiling from source):
+ - Python version:
+ - CUDA/cuDNN version:
+ - GPU models and configuration:
+ - Any other relevant information:

omni_speech/infer/fairseq/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+# Before submitting
+
+- [ ] Was this discussed/approved via a Github issue? (no need for typos, doc improvements)
+- [ ] Did you read the [contributor guideline](https://github.com/pytorch/fairseq/blob/main/CONTRIBUTING.md)?
+- [ ] Did you make sure to update the docs?
+- [ ] Did you write any new necessary tests?
+
+## What does this PR do?
+Fixes # (issue).
+
+## PR review
+Anyone in the community is free to review the PR once the tests have passed.
+If we didn't discuss your PR in Github issues there's a high chance it will not be merged.
+
+## Did you have fun?
+Make sure you had fun coding 🙃

omni_speech/infer/fairseq/.github/stale.yml

@@ -0,0 +1,30 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+# Mostly copied from github.com/facebook/react/blob/master/.github/stale.yml
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 90
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - bug
+# Label to use when marking an issue as stale
+staleLabel: stale
+issues:
+  # Comment to post when marking an issue as stale.
+  markComment: >
+    This issue has been automatically marked as stale.
+    **If this issue is still affecting you, please leave any comment** (for example, "bump"), and we'll keep it open.
+    We are sorry that we haven't been able to prioritize it yet. If you have any new additional information, please include it with your comment!
+  # Comment to post when closing a stale issue.
+  closeComment: >
+    Closing this issue after a prolonged period of inactivity. If this issue is still present in the latest release, please create a new issue with up-to-date information. Thank you!
+pulls:
+  # Comment to post when marking a pull request as stale.
+  markComment: >
+    This pull request has been automatically marked as stale.
+    **If this pull request is still relevant, please leave any comment** (for example, "bump"), and we'll keep it open.
+    We are sorry that we haven't been able to prioritize reviewing it yet. Your contribution is very much appreciated.
+  # Comment to post when closing a stale pull request.
+  closeComment: >
+    Closing this pull request after a prolonged period of inactivity. If this issue is still present in the latest release, please ask for this pull request to be reopened. Thank you!
+

omni_speech/infer/fairseq/.github/workflows/build.yml

@@ -0,0 +1,81 @@
+name: build
+
+on:
+  # Trigger the workflow on push to main or any pull request
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build:
+
+    strategy:
+      max-parallel: 4
+      matrix:
+        platform: [ubuntu-latest, macos-latest]
+        python-version: [3.8, 3.9]
+
+    runs-on: ${{ matrix.platform }}
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Conditionally install pytorch
+      if: matrix.platform == 'windows-latest'
+      run: pip3 install torch -f https://download.pytorch.org/whl/torch_stable.html
+
+    - name: Install locally
+      run: |
+        python -m pip install --upgrade pip
+        git submodule update --init --recursive
+        python -m pip install .
+
+    - name: Check installation
+      working-directory: /tmp
+      run: python $GITHUB_WORKSPACE/scripts/check_installation.py
+
+    - name: Install optional test requirements
+      run: |
+        python -m pip install '.[dev,docs]'
+        python -m pip install iopath transformers pyarrow
+        python -m pip install git+https://github.com/facebookresearch/fairscale.git@main
+        python -m pip install pygit2 pgzip
+        
+    - name: Install xformers for Macos
+      if: matrix.platform == 'macos-latest'
+      run: |
+        brew install llvm libomp
+        CC=/usr/local/opt/llvm/bin/clang CXX=clang++ pip install git+https://github.com/facebookresearch/xformers.git@main
+
+    - name: Install xformers for non-MacOS
+      if: matrix.platform != 'macos-latest'
+      run: |
+        python -m pip install --progress-bar off git+https://github.com/facebookresearch/xformers.git@main
+
+    - name: Lint with black
+      run: black --check --diff .
+
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+    - name: Build doc
+      run: make singlehtml
+      working-directory: docs/
+
+    - name: Run tests
+      # When installing in non-editable mode, the .so files will be generated in 'site-packages/fairseq'.
+      # But by default, pytest import machinery will load local fairseq, and won't see the .so.
+      # Use --import-mode=append to favorize the 'site-packages/fairseq'.
+      # https://docs.pytest.org/en/7.1.x/explanation/pythonpath.html
+      run: pytest --import-mode=append -vvv tests/
+

omni_speech/infer/fairseq/.github/workflows/release.yml

@@ -0,0 +1,161 @@
+name: Fairseq Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      name:
+        description: 'Release Type'
+        default: 'patch'
+        required: true
+
+jobs:
+
+  get_next_version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout-repo-content
+        uses: actions/checkout@v2
+
+      - name: setup-python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: get next version and tag
+        id: get-next-version-and-tag
+        run: |
+          output=$(python3 release_utils.py --release-type ${{ github.event.inputs.name }}) 
+          echo $output
+          new_version=$(echo $output | awk '{print $1}')
+          new_tag=$(echo $output | awk '{print $2}')
+          echo "new version is $new_version"
+          echo "new tag is $new_tag"
+          echo ::set-output name=version::$new_version
+          echo ::set-output name=tag::$new_tag
+          echo ::set-output name=branch_name::$new_version-release
+          echo "NEW_TAG=$new_tag" >> $GITHUB_ENV
+          echo "NEW_BRANCH=$new_version-release" >> $GITHUB_ENV
+
+
+      # update the version number in version.txt
+      - name: update version
+        id: update-version
+        run : |
+          echo "current folder = $PWD"
+          echo "current branch = $(git branch --show-current)"
+          output=$(python3 release_utils.py --release-type ${{ github.event.inputs.name }} --update-version)
+
+      - name: add and commit
+        uses: EndBug/add-and-commit@v9
+        with:
+          author_name: ${{ secrets.AUTHOR_NAME }}
+          author_email: ${{ secrets.AUTHOR_EMAIL }}
+
+          # TODO: change this to main once shipit is disabled.
+          new_branch: '${{ env.NEW_BRANCH }}'
+          default_author: github_actor
+          message: '${{ env.NEW_TAG }} release'
+          pathspec_error_handling: exitAtEnd
+
+          # Arguments for the git pull command. Use NO-PULL to avoid the action pulling at all.
+          # pull: 'NO-PULL'
+          tag: '${{ env.NEW_TAG }}'
+
+    outputs:
+      new_version: ${{ steps.get-next-version-and-tag.outputs.version }}
+      new_tag: ${{ steps.get-next-version-and-tag.outputs.tag }}
+      branch_name: ${{ steps.get-next-version-and-tag.outputs.branch_name }}
+
+  create_sdist:
+    runs-on: ubuntu-latest
+    name: Create Source Distribution
+    needs: get_next_version
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.get_next_version.outputs.branch_name }}
+
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+
+      - name: Upgrade pip
+        run: |
+          python3 -m pip install --upgrade pip
+
+      - name: Create Source Distribution
+        run: |
+          python3 -m pip install setuptools wheel twine torch
+          python3 setup.py sdist
+ 
+      - uses: actions/upload-artifact@v2
+        with:
+          path: dist/*.tar.gz
+
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs: get_next_version
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.get_next_version.outputs.branch_name }}
+
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+
+      - name: Upgrade pip
+        run: |
+          python3 -m pip install --upgrade pip
+
+      - name: Install cibuildwheel
+        run: |
+          python3 -m pip install cibuildwheel
+
+      - name: Build wheels for CPython
+        run: |
+          python3 -m cibuildwheel --output-dir dist
+        env:
+          CIBW_BUILD: "cp38-*64"
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
+          CIBW_BEFORE_BUILD: git submodule update --init --recursive && pip install .
+          # Install system library
+          CIBW_BEFORE_BUILD_LINUX: (yum install -y libffi-devel || apt-get install -y libffi-devel || apk add --update --no-cache libffi-devel || true) && (yum install -y libc6 || apt-get install -y libc6 || apk add --update --no-cache libc6 || true)
+          CIBW_ENVIRONMENT: "PIP_ONLY_BINARY=numpy"
+          CIBW_SKIP: "*musllinux*"
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: dist
+
+  upload:
+    name: Upload to PyPi and create release
+    runs-on: ubuntu-latest
+    needs: [build_wheels, create_sdist, get_next_version]
+    steps:
+      - uses: actions/download-artifact@v2
+        with:
+          name: artifact
+          path: dist
+
+      # build the PyPI package and upload it
+      - name: upload
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          pip install setuptools wheel twine
+          python3 -m twine upload --repository pypi dist/*
+
+      # create the release on github
+      - name: create release on github
+        uses: ncipollo/release-action@v1
+        with:
+          tag: '${{ needs.get_next_version.outputs.new_tag }}'

omni_speech/infer/fairseq/.gitignore

@@ -0,0 +1,141 @@
+# JetBrains PyCharm IDE
+.idea/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# macOS dir files
+.DS_Store
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Checkpoints
+checkpoints
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# Generated files
+/fairseq/temporal_convolution_tbc
+/fairseq/modules/*_layer/*_forward.cu
+/fairseq/modules/*_layer/*_backward.cu
+/fairseq/version.py
+
+# data
+data-bin/
+
+# reranking
+/examples/reranking/rerank_data
+
+# Cython-generated C++ source files
+/fairseq/data/data_utils_fast.cpp
+/fairseq/data/token_block_utils_fast.cpp
+
+# VSCODE
+.vscode/ftp-sync.json
+.vscode/settings.json
+
+# Experimental Folder
+experimental/*
+
+# Weights and Biases logs
+wandb/
+
+# Hydra artifacts
+nohup.out
+multirun
+outputs

omni_speech/infer/fairseq/.gitmodules

@@ -0,0 +1,4 @@
+[submodule "fairseq/model_parallel/megatron"]
+    path = fairseq/model_parallel/megatron
+    url = https://github.com/ngoyal2707/Megatron-LM
+    branch = fairseq

omni_speech/infer/fairseq/.pre-commit-config.yaml

@@ -0,0 +1,40 @@
+exclude: 'build|stubs'
+
+default_language_version:
+    python: python3
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: check-ast
+    -   id: check-merge-conflict
+    -   id: no-commit-to-branch
+        args: ['--branch=master']
+    -   id: check-added-large-files
+        args: ['--maxkb=500']
+    -   id: end-of-file-fixer
+
+-   repo: https://github.com/ambv/black
+    rev: 22.3.0
+    hooks:
+    - id: black
+      language_version: python3.8
+
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+    -   id: flake8
+        args: [
+            # only error for syntax errors and undefined names
+            "--select=E9,F63,F7,F82",
+        ]
+
+-   repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+    -   id: isort
+        exclude: README.md
+        additional_dependencies: [toml]
+        args: ["--profile", "black"]

omni_speech/infer/fairseq/CODE_OF_CONDUCT.md

@@ -0,0 +1,77 @@
+# Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at <conduct@pytorch.org>. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq
+

omni_speech/infer/fairseq/CONTRIBUTING.md

@@ -0,0 +1,82 @@
+# Contributing to Facebook AI Research Sequence-to-Sequence Toolkit (fairseq)
+We want to make contributing to this project as easy and transparent as
+possible.
+
+## Pull Requests
+We actively welcome your pull requests.
+
+1. Fork the repo and create your branch from `main`.
+2. If you've added code that should be tested, add tests.
+3. If you've changed APIs, update the documentation.
+4. Ensure the test suite passes.
+5. Make sure your code lints.
+6. If you haven't already, complete the Contributor License Agreement ("CLA").
+
+## Contributor License Agreement ("CLA")
+In order to accept your pull request, we need you to submit a CLA. You only need
+to do this once to work on any of Facebook's open source projects.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
+## Issues
+We use GitHub issues to track public bugs. Please ensure your description is
+clear and has sufficient instructions to be able to reproduce the issue.
+
+## License
+By contributing to Facebook AI Research Sequence-to-Sequence Toolkit (fairseq),
+you agree that your contributions will be licensed under the LICENSE file in
+the root directory of this source tree.
+
+## Pre-commit hooks
+In order to ensure your code lints, there are pre-commit hooks configured in the repository which you can install.
+After installation, they will automatically run each time you commit.
+An abbreviated guide is given below; for more information, refer to [the offical pre-commit documentation](https://pre-commit.com/).
+
+### Installation
+```
+pip install pre-commit
+pre-commit install
+```
+
+### Usage
+Just commit your changes:
+```
+git commit -m "My informative commit message"
+```
+
+If there was a failure, you will get feedback
+```
+[INFO] Initializing environment for https://github.com/PyCQA/flake8.
+[INFO] Installing environment for https://github.com/pre-commit/pre-commit-hooks.
+[INFO] Once installed this environment will be reused.
+[INFO] This may take a few minutes...
+[INFO] Installing environment for https://github.com/PyCQA/flake8.
+[INFO] Once installed this environment will be reused.
+[INFO] This may take a few minutes...
+Trim Trailing Whitespace.................................................Failed
+- hook id: trailing-whitespace
+- exit code: 1
+- files were modified by this hook
+Fixing examples/nllb/modeling/wmt15_benchmark/eval_langs2.sh
+Fix End of Files.........................................................Failed
+- hook id: end-of-file-fixer
+- exit code: 1
+- files were modified by this hook
+Fixing examples/few_shot/scripts/schedule_jobs_few_shot.py
+flake8...................................................................Passed
+```
+
+Certain hooks modify your files to comply.
+To include these modifications, you will need to add them (i.e. `git add ...`) and commit again.
+
+If all is well, you should see something like:
+```
+Trim Trailing Whitespace.................................................Passed
+Fix End of Files.........................................................Passed
+flake8...................................................................Passed
+[gshard-fix-ci 8698644e1] Fix lint, add pre-commit hooks
+ 10 files changed, 148 insertions(+), 110 deletions(-)
+ create mode 100644 .flake8
+ create mode 100644 .pre-commit-config.yaml
+ rename examples/nllb/modeling/wmt15_benchmark/{eval_langs2.py => eval_langs2.sh} (99%)
+ ```

omni_speech/infer/fairseq/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Facebook, Inc. and its affiliates.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

omni_speech/infer/fairseq/MANIFEST.in

@@ -0,0 +1 @@
+include fairseq/version.txt

omni_speech/infer/fairseq/README.md

@@ -0,0 +1,242 @@
+<p align="center">
+  <img src="docs/fairseq_logo.png" width="150">
+  <br />
+  <br />
+  <a href="https://opensource.fb.com/support-ukraine"><img alt="Support Ukraine" src="https://img.shields.io/badge/Support-Ukraine-FFD500?style=flat&labelColor=005BBB" /></a>
+  <a href="https://github.com/pytorch/fairseq/blob/main/LICENSE"><img alt="MIT License" src="https://img.shields.io/badge/license-MIT-blue.svg" /></a>
+  <a href="https://github.com/pytorch/fairseq/releases"><img alt="Latest Release" src="https://img.shields.io/github/release/pytorch/fairseq.svg" /></a>
+  <a href="https://github.com/pytorch/fairseq/actions?query=workflow:build"><img alt="Build Status" src="https://github.com/pytorch/fairseq/workflows/build/badge.svg" /></a>
+  <a href="https://fairseq.readthedocs.io/en/latest/?badge=latest"><img alt="Documentation Status" src="https://readthedocs.org/projects/fairseq/badge/?version=latest" /></a>
+  <a href="https://app.circleci.com/pipelines/github/facebookresearch/fairseq/"><img alt="CicleCI Status" src="https://circleci.com/gh/facebookresearch/fairseq.svg?style=shield" /></a>
+</p>
+
+--------------------------------------------------------------------------------
+
+Fairseq(-py) is a sequence modeling toolkit that allows researchers and
+developers to train custom models for translation, summarization, language
+modeling and other text generation tasks.
+
+We provide reference implementations of various sequence modeling papers:
+
+<details><summary>List of implemented papers</summary><p>
+
+* **Convolutional Neural Networks (CNN)**
+  + [Language Modeling with Gated Convolutional Networks (Dauphin et al., 2017)](examples/language_model/conv_lm/README.md)
+  + [Convolutional Sequence to Sequence Learning (Gehring et al., 2017)](examples/conv_seq2seq/README.md)
+  + [Classical Structured Prediction Losses for Sequence to Sequence Learning (Edunov et al., 2018)](https://github.com/pytorch/fairseq/tree/classic_seqlevel)
+  + [Hierarchical Neural Story Generation (Fan et al., 2018)](examples/stories/README.md)
+  + [wav2vec: Unsupervised Pre-training for Speech Recognition (Schneider et al., 2019)](examples/wav2vec/README.md)
+* **LightConv and DynamicConv models**
+  + [Pay Less Attention with Lightweight and Dynamic Convolutions (Wu et al., 2019)](examples/pay_less_attention_paper/README.md)
+* **Long Short-Term Memory (LSTM) networks**
+  + Effective Approaches to Attention-based Neural Machine Translation (Luong et al., 2015)
+* **Transformer (self-attention) networks**
+  + Attention Is All You Need (Vaswani et al., 2017)
+  + [Scaling Neural Machine Translation (Ott et al., 2018)](examples/scaling_nmt/README.md)
+  + [Understanding Back-Translation at Scale (Edunov et al., 2018)](examples/backtranslation/README.md)
+  + [Adaptive Input Representations for Neural Language Modeling (Baevski and Auli, 2018)](examples/language_model/README.adaptive_inputs.md)
+  + [Lexically constrained decoding with dynamic beam allocation (Post & Vilar, 2018)](examples/constrained_decoding/README.md)
+  + [Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context (Dai et al., 2019)](examples/truncated_bptt/README.md)
+  + [Adaptive Attention Span in Transformers (Sukhbaatar et al., 2019)](examples/adaptive_span/README.md)
+  + [Mixture Models for Diverse Machine Translation: Tricks of the Trade (Shen et al., 2019)](examples/translation_moe/README.md)
+  + [RoBERTa: A Robustly Optimized BERT Pretraining Approach (Liu et al., 2019)](examples/roberta/README.md)
+  + [Facebook FAIR's WMT19 News Translation Task Submission (Ng et al., 2019)](examples/wmt19/README.md)
+  + [Jointly Learning to Align and Translate with Transformer Models (Garg et al., 2019)](examples/joint_alignment_translation/README.md )
+  + [Multilingual Denoising Pre-training for Neural Machine Translation (Liu et at., 2020)](examples/mbart/README.md)
+  + [Neural Machine Translation with Byte-Level Subwords (Wang et al., 2020)](examples/byte_level_bpe/README.md)
+  + [Unsupervised Quality Estimation for Neural Machine Translation (Fomicheva et al., 2020)](examples/unsupervised_quality_estimation/README.md)
+  + [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations (Baevski et al., 2020)](examples/wav2vec/README.md)
+  + [Generating Medical Reports from Patient-Doctor Conversations Using Sequence-to-Sequence Models (Enarvi et al., 2020)](examples/pointer_generator/README.md)
+  + [Linformer: Self-Attention with Linear Complexity (Wang et al., 2020)](examples/linformer/README.md)
+  + [Cross-lingual Retrieval for Iterative Self-Supervised Training (Tran et al., 2020)](examples/criss/README.md)
+  + [Deep Transformers with Latent Depth (Li et al., 2020)](examples/latent_depth/README.md)
+  + [Unsupervised Cross-lingual Representation Learning for Speech Recognition (Conneau et al., 2020)](https://arxiv.org/abs/2006.13979)
+  + [Self-training and Pre-training are Complementary for Speech Recognition (Xu et al., 2020)](https://arxiv.org/abs/2010.11430)
+  + [Robust wav2vec 2.0: Analyzing Domain Shift in Self-Supervised Pre-Training (Hsu, et al., 2021)](https://arxiv.org/abs/2104.01027)
+  + [Unsupervised Speech Recognition (Baevski, et al., 2021)](https://arxiv.org/abs/2105.11084)
+  + [Simple and Effective Zero-shot Cross-lingual Phoneme Recognition (Xu et al., 2021)](https://arxiv.org/abs/2109.11680)
+  + [VideoCLIP: Contrastive Pre-training for Zero-shot Video-Text Understanding (Xu et. al., 2021)](https://arxiv.org/pdf/2109.14084.pdf)
+  + [VLM: Task-agnostic Video-Language Model Pre-training for Video Understanding (Xu et. al., 2021)](https://aclanthology.org/2021.findings-acl.370.pdf)
+  + [NormFormer: Improved Transformer Pretraining with Extra Normalization (Shleifer et. al, 2021)](examples/normformer/README.md)
+* **Non-autoregressive Transformers**
+  + Non-Autoregressive Neural Machine Translation (Gu et al., 2017)
+  + Deterministic Non-Autoregressive Neural Sequence Modeling by Iterative Refinement (Lee et al. 2018)
+  + Insertion Transformer: Flexible Sequence Generation via Insertion Operations (Stern et al. 2019)
+  + Mask-Predict: Parallel Decoding of Conditional Masked Language Models (Ghazvininejad et al., 2019)
+  + [Levenshtein Transformer (Gu et al., 2019)](examples/nonautoregressive_translation/README.md)
+* **Finetuning**
+  + [Better Fine-Tuning by Reducing Representational Collapse (Aghajanyan et al. 2020)](examples/rxf/README.md)
+
+</p></details>
+
+### What's New:
+* May 2023 [Released models for Scaling Speech Technology to 1,000+ Languages  (Pratap, et al., 2023)](examples/mms/README.md)
+* June 2022 [Released code for wav2vec-U 2.0 from Towards End-to-end Unsupervised Speech Recognition (Liu, et al., 2022)](examples/wav2vec/unsupervised/README.md)
+* May 2022 [Integration with xFormers](https://github.com/facebookresearch/xformers)
+* December 2021 [Released Direct speech-to-speech translation code](examples/speech_to_speech/README.md)
+* October 2021 [Released VideoCLIP and VLM models](examples/MMPT/README.md)
+* October 2021 [Released multilingual finetuned XLSR-53 model](examples/wav2vec/README.md)
+* September 2021 [`master` branch renamed to `main`](https://github.com/github/renaming).
+* July 2021 [Released DrNMT code](examples/discriminative_reranking_nmt/README.md)
+* July 2021 [Released Robust wav2vec 2.0 model](examples/wav2vec/README.md)
+* June 2021 [Released XLMR-XL and XLMR-XXL models](examples/xlmr/README.md)
+* May 2021 [Released Unsupervised Speech Recognition code](examples/wav2vec/unsupervised/README.md)
+* March 2021 [Added full parameter and optimizer state sharding + CPU offloading](examples/fully_sharded_data_parallel/README.md)
+* February 2021 [Added LASER training code](examples/laser/README.md)
+* December 2020: [Added Adaptive Attention Span code](examples/adaptive_span/README.md)
+* December 2020: [GottBERT model and code released](examples/gottbert/README.md)
+* November 2020: Adopted the [Hydra](https://github.com/facebookresearch/hydra) configuration framework
+  * [see documentation explaining how to use it for new and existing projects](docs/hydra_integration.md)
+* November 2020: [fairseq 0.10.0 released](https://github.com/pytorch/fairseq/releases/tag/v0.10.0)
+* October 2020: [Added R3F/R4F (Better Fine-Tuning) code](examples/rxf/README.md)
+* October 2020: [Deep Transformer with Latent Depth code released](examples/latent_depth/README.md)
+* October 2020: [Added CRISS models and code](examples/criss/README.md)
+
+<details><summary>Previous updates</summary><p>
+
+* September 2020: [Added Linformer code](examples/linformer/README.md)
+* September 2020: [Added pointer-generator networks](examples/pointer_generator/README.md)
+* August 2020: [Added lexically constrained decoding](examples/constrained_decoding/README.md)
+* August 2020: [wav2vec2 models and code released](examples/wav2vec/README.md)
+* July 2020: [Unsupervised Quality Estimation code released](examples/unsupervised_quality_estimation/README.md)
+* May 2020: [Follow fairseq on Twitter](https://twitter.com/fairseq)
+* April 2020: [Monotonic Multihead Attention code released](examples/simultaneous_translation/README.md)
+* April 2020: [Quant-Noise code released](examples/quant_noise/README.md)
+* April 2020: [Initial model parallel support and 11B parameters unidirectional LM released](examples/megatron_11b/README.md)
+* March 2020: [Byte-level BPE code released](examples/byte_level_bpe/README.md)
+* February 2020: [mBART model and code released](examples/mbart/README.md)
+* February 2020: [Added tutorial for back-translation](https://github.com/pytorch/fairseq/tree/main/examples/backtranslation#training-your-own-model-wmt18-english-german)
+* December 2019: [fairseq 0.9.0 released](https://github.com/pytorch/fairseq/releases/tag/v0.9.0)
+* November 2019: [VizSeq released (a visual analysis toolkit for evaluating fairseq models)](https://facebookresearch.github.io/vizseq/docs/getting_started/fairseq_example)
+* November 2019: [CamemBERT model and code released](examples/camembert/README.md)
+* November 2019: [BART model and code released](examples/bart/README.md)
+* November 2019: [XLM-R models and code released](examples/xlmr/README.md)
+* September 2019: [Nonautoregressive translation code released](examples/nonautoregressive_translation/README.md)
+* August 2019: [WMT'19 models released](examples/wmt19/README.md)
+* July 2019: fairseq relicensed under MIT license
+* July 2019: [RoBERTa models and code released](examples/roberta/README.md)
+* June 2019: [wav2vec models and code released](examples/wav2vec/README.md)
+
+</p></details>
+
+### Features:
+
+* multi-GPU training on one machine or across multiple machines (data and model parallel)
+* fast generation on both CPU and GPU with multiple search algorithms implemented:
+  + beam search
+  + Diverse Beam Search ([Vijayakumar et al., 2016](https://arxiv.org/abs/1610.02424))
+  + sampling (unconstrained, top-k and top-p/nucleus)
+  + [lexically constrained decoding](examples/constrained_decoding/README.md) (Post & Vilar, 2018)
+* [gradient accumulation](https://fairseq.readthedocs.io/en/latest/getting_started.html#large-mini-batch-training-with-delayed-updates) enables training with large mini-batches even on a single GPU
+* [mixed precision training](https://fairseq.readthedocs.io/en/latest/getting_started.html#training-with-half-precision-floating-point-fp16) (trains faster with less GPU memory on [NVIDIA tensor cores](https://developer.nvidia.com/tensor-cores))
+* [extensible](https://fairseq.readthedocs.io/en/latest/overview.html): easily register new models, criterions, tasks, optimizers and learning rate schedulers
+* [flexible configuration](docs/hydra_integration.md) based on [Hydra](https://github.com/facebookresearch/hydra) allowing a combination of code, command-line and file based configuration
+* [full parameter and optimizer state sharding](examples/fully_sharded_data_parallel/README.md)
+* [offloading parameters to CPU](examples/fully_sharded_data_parallel/README.md)
+
+We also provide [pre-trained models for translation and language modeling](#pre-trained-models-and-examples)
+with a convenient `torch.hub` interface:
+
+``` python
+en2de = torch.hub.load('pytorch/fairseq', 'transformer.wmt19.en-de.single_model')
+en2de.translate('Hello world', beam=5)
+# 'Hallo Welt'
+```
+
+See the PyTorch Hub tutorials for [translation](https://pytorch.org/hub/pytorch_fairseq_translation/)
+and [RoBERTa](https://pytorch.org/hub/pytorch_fairseq_roberta/) for more examples.
+
+# Requirements and Installation
+
+* [PyTorch](http://pytorch.org/) version >= 1.10.0
+* Python version >= 3.8
+* For training new models, you'll also need an NVIDIA GPU and [NCCL](https://github.com/NVIDIA/nccl)
+* **To install fairseq** and develop locally:
+
+``` bash
+git clone https://github.com/pytorch/fairseq
+cd fairseq
+pip install --editable ./
+
+# on MacOS:
+# CFLAGS="-stdlib=libc++" pip install --editable ./
+
+# to install the latest stable release (0.10.x)
+# pip install fairseq
+```
+
+* **For faster training** install NVIDIA's [apex](https://github.com/NVIDIA/apex) library:
+
+``` bash
+git clone https://github.com/NVIDIA/apex
+cd apex
+pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" \
+  --global-option="--deprecated_fused_adam" --global-option="--xentropy" \
+  --global-option="--fast_multihead_attn" ./
+```
+
+* **For large datasets** install [PyArrow](https://arrow.apache.org/docs/python/install.html#using-pip): `pip install pyarrow`
+* If you use Docker make sure to increase the shared memory size either with `--ipc=host` or `--shm-size`
+ as command line options to `nvidia-docker run` .
+
+# Getting Started
+
+The [full documentation](https://fairseq.readthedocs.io/) contains instructions
+for getting started, training new models and extending fairseq with new model
+types and tasks.
+
+# Pre-trained models and examples
+
+We provide pre-trained models and pre-processed, binarized test sets for several tasks listed below,
+as well as example training and evaluation commands.
+
+* [Translation](examples/translation/README.md): convolutional and transformer models are available
+* [Language Modeling](examples/language_model/README.md): convolutional and transformer models are available
+
+We also have more detailed READMEs to reproduce results from specific papers:
+
+* [XLS-R: Self-supervised Cross-lingual Speech Representation Learning at Scale (Babu et al., 2021)](examples/wav2vec/xlsr/README.md)
+* [Cross-lingual Retrieval for Iterative Self-Supervised Training (Tran et al., 2020)](examples/criss/README.md)
+* [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations (Baevski et al., 2020)](examples/wav2vec/README.md)
+* [Unsupervised Quality Estimation for Neural Machine Translation (Fomicheva et al., 2020)](examples/unsupervised_quality_estimation/README.md)
+* [Training with Quantization Noise for Extreme Model Compression ({Fan*, Stock*} et al., 2020)](examples/quant_noise/README.md)
+* [Neural Machine Translation with Byte-Level Subwords (Wang et al., 2020)](examples/byte_level_bpe/README.md)
+* [Multilingual Denoising Pre-training for Neural Machine Translation (Liu et at., 2020)](examples/mbart/README.md)
+* [Reducing Transformer Depth on Demand with Structured Dropout (Fan et al., 2019)](examples/layerdrop/README.md)
+* [Jointly Learning to Align and Translate with Transformer Models (Garg et al., 2019)](examples/joint_alignment_translation/README.md)
+* [Levenshtein Transformer (Gu et al., 2019)](examples/nonautoregressive_translation/README.md)
+* [Facebook FAIR's WMT19 News Translation Task Submission (Ng et al., 2019)](examples/wmt19/README.md)
+* [RoBERTa: A Robustly Optimized BERT Pretraining Approach (Liu et al., 2019)](examples/roberta/README.md)
+* [wav2vec: Unsupervised Pre-training for Speech Recognition (Schneider et al., 2019)](examples/wav2vec/README.md)
+* [Mixture Models for Diverse Machine Translation: Tricks of the Trade (Shen et al., 2019)](examples/translation_moe/README.md)
+* [Pay Less Attention with Lightweight and Dynamic Convolutions (Wu et al., 2019)](examples/pay_less_attention_paper/README.md)
+* [Understanding Back-Translation at Scale (Edunov et al., 2018)](examples/backtranslation/README.md)
+* [Classical Structured Prediction Losses for Sequence to Sequence Learning (Edunov et al., 2018)](https://github.com/pytorch/fairseq/tree/classic_seqlevel)
+* [Hierarchical Neural Story Generation (Fan et al., 2018)](examples/stories/README.md)
+* [Scaling Neural Machine Translation (Ott et al., 2018)](examples/scaling_nmt/README.md)
+* [Convolutional Sequence to Sequence Learning (Gehring et al., 2017)](examples/conv_seq2seq/README.md)
+* [Language Modeling with Gated Convolutional Networks (Dauphin et al., 2017)](examples/language_model/README.conv.md)
+
+# Join the fairseq community
+
+* Twitter: https://twitter.com/fairseq
+* Facebook page: https://www.facebook.com/groups/fairseq.users
+* Google group: https://groups.google.com/forum/#!forum/fairseq-users
+
+# License
+
+fairseq(-py) is MIT-licensed.
+The license applies to the pre-trained models as well.
+
+# Citation
+
+Please cite as:
+
+``` bibtex
+@inproceedings{ott2019fairseq,
+  title = {fairseq: A Fast, Extensible Toolkit for Sequence Modeling},
+  author = {Myle Ott and Sergey Edunov and Alexei Baevski and Angela Fan and Sam Gross and Nathan Ng and David Grangier and Michael Auli},
+  booktitle = {Proceedings of NAACL-HLT 2019: Demonstrations},
+  year = {2019},
+}
+```

omni_speech/infer/fairseq/RELEASE.md

@@ -0,0 +1,13 @@
+# Creating a New Release
+
+In order to create a new release:
+
+1. Navigate to the [Fairseq Workflows](https://github.com/facebookresearch/fairseq/actions) and find the one named _Fairseq Release_. 
+
+2. Under _Run Workflow_ choose the branch `main` and for _Release Type_ enter either `major`, `minor`, or `patch`.  
+
+3. A branch named `$new_version-release` will be created where the `version.txt` file is updated. Merge those changes into `main`.
+
+4. Make sure that a [new PYPI package](https://pypi.org/project/fairseq/) has been uploaded.
+
+5. Make sure that a [new github release](https://github.com/facebookresearch/fairseq/releases) has been created.

omni_speech/infer/fairseq/alignment_train_cpu_binding.cpython-312-x86_64-linux-gnu.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:765fbdd89dffac55bffce40b75c4cc44d07fb2463dcf00e76525611e8aa6a086
+size 152600

omni_speech/infer/fairseq/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = python -msphinx
+SPHINXPROJ    = fairseq
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)