Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/demo_narrow.gif filter=lfs diff=lfs merge=lfs -text

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+<!-- Thank you for your contribution! -->
+
+<!-- Please add a reviewer to the assignee section when you create a PR. If you don't have the access to it, we will shortly find a reviewer and assign them to your PR. -->
+
+## Why are these changes needed?
+
+<!-- Please give a short summary of the change and the problem this solves. -->
+
+## Related issue number (if applicable)
+
+<!-- For example: "Closes #1234" -->
+
+## Checks
+
+- [ ] I've run `format.sh` to lint the changes in this PR.
+- [ ] I've included any doc changes needed.
+- [ ] I've made sure the relevant tests are passing (if applicable).

.github/workflows/python-package.yml

@@ -0,0 +1,30 @@
+name: Python package
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -e '.[dev]'
+    - name: Run linter
+      run: |
+        pylint -d all -e E0602 ./fastchat/
+    - name: Check formatting
+      run: |
+        black --check .

.gitignore

@@ -0,0 +1,30 @@
+# Python
+__pycache__
+*.pyc
+*.egg-info
+dist
+.venv
+
+# Log
+*.log
+*.log.*
+*.json
+!playground/deepspeed_config_s2.json
+!playground/deepspeed_config_s3.json
+
+# Editor
+.idea
+*.swp
+
+# Other
+.DS_Store
+wandb
+output
+
+# Data
+*.pkl
+*.csv
+tests/state_of_the_union.txt
+
+# Build
+build

.pylintrc

@@ -0,0 +1,449 @@
+# This Pylint rcfile contains a best-effort configuration to uphold the
+# best-practices and style described in the Google Python style guide:
+#   https://google.github.io/styleguide/pyguide.html
+#
+# Its canonical open-source location is:
+#   https://google.github.io/styleguide/pylintrc
+
+[MASTER]
+
+# Files or directories to be skipped. They should be base names, not paths.
+ignore=third_party,ray_patches,providers
+
+# Files or directories matching the regex patterns are skipped. The regex
+# matches against base names, not paths.
+ignore-patterns=
+
+# Pickle collected data for later comparisons.
+persistent=no
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Use multiple processes to speed up Pylint.
+jobs=4
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+confidence=
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=abstract-method,
+        apply-builtin,
+        arguments-differ,
+        attribute-defined-outside-init,
+        backtick,
+        bad-option-value,
+        basestring-builtin,
+        buffer-builtin,
+        c-extension-no-member,
+        consider-using-enumerate,
+        cmp-builtin,
+        cmp-method,
+        coerce-builtin,
+        coerce-method,
+        delslice-method,
+        div-method,
+        duplicate-code,
+        eq-without-hash,
+        execfile-builtin,
+        file-builtin,
+        filter-builtin-not-iterating,
+        fixme,
+        getslice-method,
+        global-statement,
+        hex-method,
+        idiv-method,
+        implicit-str-concat-in-sequence,
+        import-error,
+        import-self,
+        import-star-module-level,
+        inconsistent-return-statements,
+        input-builtin,
+        intern-builtin,
+        invalid-str-codec,
+        locally-disabled,
+        logging-format-interpolation,  # FIXME(sky): make pass.
+        logging-fstring-interpolation,  # FIXME(sky): make pass.
+        long-builtin,
+        long-suffix,
+        map-builtin-not-iterating,
+        misplaced-comparison-constant,
+        missing-function-docstring,
+        metaclass-assignment,
+        next-method-called,
+        next-method-defined,
+        no-absolute-import,
+        no-else-break,
+        no-else-continue,
+        no-else-raise,
+        no-else-return,
+        no-init,  # added
+        no-member,
+        no-name-in-module,
+        no-self-use,
+        nonzero-method,
+        oct-method,
+        old-division,
+        old-ne-operator,
+        old-octal-literal,
+        old-raise-syntax,
+        parameter-unpacking,
+        print-statement,
+        raising-string,
+        range-builtin-not-iterating,
+        raw_input-builtin,
+        rdiv-method,
+        reduce-builtin,
+        relative-import,
+        reload-builtin,
+        round-builtin,
+        setslice-method,
+        signature-differs,
+        standarderror-builtin,
+        suppressed-message,
+        sys-max-int,
+        too-few-public-methods,
+        too-many-ancestors,
+        too-many-arguments,
+        too-many-boolean-expressions,
+        too-many-branches,
+        too-many-instance-attributes,
+        too-many-locals,
+        too-many-nested-blocks,
+        too-many-public-methods,
+        too-many-return-statements,
+        too-many-statements,
+        trailing-newlines,
+        unichr-builtin,
+        unicode-builtin,
+        unnecessary-pass,
+        unpacking-in-except,
+        useless-else-on-loop,
+        useless-object-inheritance,
+        useless-suppression,
+        using-cmp-argument,
+        wrong-import-order,
+        xrange-builtin,
+        zip-builtin-not-iterating,
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html. You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]". This option is deprecated
+# and it will be removed in Pylint 2.0.
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=no
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+#msg-template=
+
+
+[BASIC]
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=main,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl
+
+# Regular expression matching correct function names
+function-rgx=^(?:(?P<exempt>setUp|tearDown|setUpModule|tearDownModule)|(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$
+
+# Regular expression matching correct variable names
+variable-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression matching correct constant names
+const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
+
+# Regular expression matching correct attribute names
+attr-rgx=^_{0,2}[a-z][a-z0-9_]*$
+
+# Regular expression matching correct argument names
+argument-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression matching correct class names
+class-rgx=^_?[A-Z][a-zA-Z0-9]*$
+
+# Regular expression matching correct module names
+module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$
+
+# Regular expression matching correct method names
+method-rgx=(?x)^(?:(?P<exempt>_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=10
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt
+# lines made too long by directives to pytype.
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=(?x)(
+  ^\s*(\#\ )?<?https?://\S+>?$|
+  ^\s*(from\s+\S+\s+)?import\s+.+$)
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=yes
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=
+
+# Maximum number of lines in a module
+max-module-lines=99999
+
+# String used as indentation unit.  The internal Google style guide mandates 2
+# spaces.  Google's externaly-published style guide says 4, consistent with
+# PEP 8. Here we use 4 spaces.
+indent-string='    '
+
+# Number of spaces of indent required inside a hanging  or continued line.
+indent-after-paren=4
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=TODO
+
+
+[STRING]
+
+# This flag controls whether inconsistent-quotes generates a warning when the
+# character used as a quote delimiter is used inconsistently within a module.
+check-quote-consistency=yes
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_)
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,_cb
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging,absl.logging,tensorflow.io.logging
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+
+[SPELLING]
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,
+                   TERMIOS,
+                   Bastion,
+                   rexec,
+                   sets
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant, absl
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,
+                      __new__,
+                      setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,
+                  _fields,
+                  _replace,
+                  _source,
+                  _make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls,
+                            class_
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=StandardError,
+                       Exception,
+                       BaseException
+
+#######
+
+# https://github.com/edaniszewski/pylint-quotes#configuration
+string-quote=single
+triple-quote=double
+docstring-quote=double

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,12 +1,296 @@
 ---
-title: Calahealthgpt
-emoji: 📈
-colorFrom: gray
-colorTo: indigo
+title: calahealthgpt
+app_file: fastchat/serve/gradio_web_server.py
 sdk: gradio
 sdk_version: 3.35.2
-app_file: app.py
-pinned: false
 ---
+# FastChat
+| [**Demo**](https://chat.lmsys.org/) | [**Arena**](https://arena.lmsys.org) | [**Discord**](https://discord.gg/HSWAKCrnFx) | [**Twitter**](https://twitter.com/lmsysorg) |
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+FastChat is an open platform for training, serving, and evaluating large language model based chatbots. The core features include:
+- The weights, training code, and evaluation code for state-of-the-art models (e.g., Vicuna, FastChat-T5).
+- A distributed multi-model serving system with web UI and OpenAI-compatible RESTful APIs.
+
+## News
+- [2023/06] 🔥 We introduced **LongChat**, our long-context chatbots and evaluation tools. Check out the blog [post](https://lmsys.org/blog/2023-06-29-longchat/) and [code](https://github.com/DachengLi1/LongChat/).
+- [2023/05] We introduced **Chatbot Arena** for battles among LLMs. Check out the blog [post](https://lmsys.org/blog/2023-05-03-arena) and [demo](https://arena.lmsys.org).
+- [2023/04] We released **FastChat-T5** compatible with commercial usage. Check out the [weights](#fastchat-t5) and [demo](https://chat.lmsys.org).
+- [2023/03] We released **Vicuna: An Open-Source Chatbot Impressing GPT-4 with 90% ChatGPT Quality**. Check out the blog [post](https://vicuna.lmsys.org) and [demo](https://chat.lmsys.org).
+
+<a href="https://chat.lmsys.org"><img src="assets/demo_narrow.gif" width="70%"></a>
+
+## Contents
+- [Install](#install)
+- [Model Weights](#model-weights)
+- [Inference with Command Line Interface](#inference-with-command-line-interface)
+- [Serving with Web GUI](#serving-with-web-gui)
+- [API](#api)
+- [Evaluation](#evaluation)
+- [Fine-tuning](#fine-tuning)
+- [Citation](#citation)
+
+## Install
+
+### Method 1: With pip
+
+```bash
+pip3 install fschat
+```
+
+### Method 2: From source
+
+1. Clone this repository and navigate to the FastChat folder
+```bash
+git clone https://github.com/lm-sys/FastChat.git
+cd FastChat
+```
+
+If you are running on Mac:
+```bash
+brew install rust cmake
+```
+
+2. Install Package
+```bash
+pip3 install --upgrade pip  # enable PEP 660 support
+pip3 install -e .
+```
+
+## Model Weights
+### Vicuna Weights
+We release [Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) weights v1.3 as merged weights directly. You do not need to apply delta.
+Vicuna is based on LLaMA and should be used under LLaMA's [model license](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md).
+
+You can use the commands below to start chatting. It will automatically download the weights from Hugging Face repos.
+See more command options and how to handle out-of-memory in the "Inference with Command Line Interface" section below.
+
+| Size | Chat Command | Hugging Face Repo |
+| ---  | --- | --- |
+| 7B   | `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.3`  | [lmsys/vicuna-7b-v1.3](https://huggingface.co/lmsys/vicuna-7b-v1.3)   |
+| 13B  | `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-13b-v1.3` | [lmsys/vicuna-13b-v1.3](https://huggingface.co/lmsys/vicuna-13b-v1.3) |
+| 33B  | `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-33b-v1.3` | [lmsys/vicuna-33b-v1.3](https://huggingface.co/lmsys/vicuna-33b-v1.3) |
+
+**Old weights**: see [docs/vicuna_weights_version.md](docs/vicuna_weights_version.md) for all versions of weights and their differences.
+
+### LongChat
+We release LongChat models under LLaMA's [model license](https://github.com/facebookresearch/llama/blob/main/MODEL_CARD.md).
+
+| Size | Chat Command | Hugging Face Repo |
+| ---  | --- | --- |
+| 7B   | `python3 -m fastchat.serve.cli --model-path lmsys/longchat-7b-16k`  | [lmsys/longchat-7b-16k](https://huggingface.co/lmsys/longchat-7b-16k)   |
+| 13B  | `python3 -m fastchat.serve.cli --model-path lmsys/longchat-13b-16k` | [lmsys/longchat-13b-16k](https://huggingface.co/lmsys/longchat-13b-16k) |
+
+### FastChat-T5
+You can use the commands below to chat with FastChat-T5. It will automatically download the weights from Hugging Face repos.
+
+| Size | Chat Command | Hugging Face Repo |
+| ---  | --- | --- |
+| 3B   | `python3 -m fastchat.serve.cli --model-path lmsys/fastchat-t5-3b-v1.0`  | [lmsys/fastchat-t5-3b-v1.0](https://huggingface.co/lmsys/fastchat-t5-3b-v1.0) |
+
+## Inference with Command Line Interface
+
+<a href="https://chat.lmsys.org"><img src="assets/screenshot_cli.png" width="70%"></a>
+
+(Experimental Feature: You can specify `--style rich` to enable rich text output and better text streaming quality for some non-ASCII content. This may not work properly on certain terminals.)
+
+#### Supported Models
+FastChat supports a wide range of models, including
+Vicuna, Alpaca, Baize, ChatGLM, Dolly, Falcon, FastChat-T5, GPT4ALL, Guanaco, MTP, OpenAssistant, RedPajama, StableLM, WizardLM, and more.
+
+See a complete list of supported models and instructions to add a new model [here](docs/model_support.md).
+
+#### Single GPU
+The command below requires around 14GB of GPU memory for Vicuna-7B and 28GB of GPU memory for Vicuna-13B.
+See the "No Enough Memory" section below if you do not have enough memory.
+`--model-path` can be a local folder or a Hugging Face repo name.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.3
+```
+
+#### Multiple GPUs
+You can use model parallelism to aggregate GPU memory from multiple GPUs on the same machine.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.3 --num-gpus 2
+```
+
+#### CPU Only
+This runs on the CPU only and does not require GPU. It requires around 30GB of CPU memory for Vicuna-7B and around 60GB of CPU memory for Vicuna-13B.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.3 --device cpu
+```
+
+#### Metal Backend (Mac Computers with Apple Silicon or AMD GPUs)
+Use `--device mps` to enable GPU acceleration on Mac computers (requires torch >= 2.0).
+Use `--load-8bit` to turn on 8-bit compression.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.3 --device mps --load-8bit
+```
+Vicuna-7B can run on a 32GB M1 Macbook with 1 - 2 words / second.
+
+#### Intel XPU (Intel Data Center and Arc A-Series GPUs)
+Install the [Intel Extension for PyTorch](https://intel.github.io/intel-extension-for-pytorch/xpu/latest/tutorials/installation.html). Set the OneAPI environment variables:
+```
+source /opt/intel/oneapi/setvars.sh
+```
+
+Use `--device xpu` to enable XPU/GPU acceleration.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.3 --device xpu
+```
+Vicuna-7B can run on an Intel Arc A770 16GB.
+
+#### No Enough Memory
+If you do not have enough memory, you can enable 8-bit compression by adding `--load-8bit` to commands above.
+This can reduce memory usage by around half with slightly degraded model quality.
+It is compatible with the CPU, GPU, and Metal backend.
+Vicuna-13B with 8-bit compression can run on a single NVIDIA 3090/4080/T4/V100(16GB) GPU.
+```
+python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.3 --load-8bit
+```
+
+In addition to that, you can add `--cpu-offloading` to commands above to offload weights that don't fit on your GPU onto the CPU memory. This requires 8-bit compression to be enabled and the bitsandbytes package to be installed, which is only available on linux operating systems.
+
+#### More Platforms
+- FastChat supports GPTQ 4bit inference with [GPTQ-for-LLaMa](https://github.com/qwopqwop200/GPTQ-for-LLaMa). See [docs/gptq.md](/docs/gptq.md).
+- [MLC LLM](https://mlc.ai/mlc-llm/), backed by [TVM Unity](https://github.com/apache/tvm/tree/unity) compiler, deploys Vicuna natively on phones, consumer-class GPUs and web browsers via Vulkan, Metal, CUDA and WebGPU.
+
+## Serving with Web GUI
+
+<a href="https://chat.lmsys.org"><img src="assets/screenshot_gui.png" width="70%"></a>
+
+To serve using the web UI, you need three main components: web servers that interface with users, model workers that host one or more models, and a controller to coordinate the webserver and model workers. You can learn more about the architecture [here](docs/server_arch.md).
+
+Here are the commands to follow in your terminal:
+
+#### Launch the controller
+```bash
+python3 -m fastchat.serve.controller
+```
+
+This controller manages the distributed workers.
+
+#### Launch the model worker(s)
+```bash
+python3 -m fastchat.serve.model_worker --model-path lmsys/vicuna-7b-v1.3
+```
+Wait until the process finishes loading the model and you see "Uvicorn running on ...". The model worker will register itself to the controller .
+
+To ensure that your model worker is connected to your controller properly, send a test message using the following command:
+```bash
+python3 -m fastchat.serve.test_message --model-name vicuna-7b-v1.3
+```
+You will see a short output.
+
+#### Launch the Gradio web server
+```bash
+python3 -m fastchat.serve.gradio_web_server
+```
+
+This is the user interface that users will interact with.
+
+By following these steps, you will be able to serve your models using the web UI. You can open your browser and chat with a model now.
+If the models do not show up, try to reboot the gradio web server.
+
+#### (Optional): Advanced Features
+- You can register multiple model workers to a single controller, which can be used for serving a single model with higher throughput or serving multiple models at the same time. When doing so, please allocate different GPUs and ports for different model workers.
+```
+# worker 0
+CUDA_VISIBLE_DEVICES=0 python3 -m fastchat.serve.model_worker --model-path lmsys/vicuna-7b-v1.3 --controller http://localhost:21001 --port 31000 --worker http://localhost:31000
+# worker 1
+CUDA_VISIBLE_DEVICES=1 python3 -m fastchat.serve.model_worker --model-path lmsys/fastchat-t5-3b-v1.0 --controller http://localhost:21001 --port 31001 --worker http://localhost:31001
+```
+- You can also launch a multi-tab gradio server, which includes the Chatbot Arena tabs.
+```bash
+python3 -m fastchat.serve.gradio_web_server_multi
+```
+
+## API
+### OpenAI-Compatible RESTful APIs & SDK
+FastChat provides OpenAI-compatible APIs for its supported models, so you can use FastChat as a local drop-in replacement for OpenAI APIs.
+The FastChat server is compatible with both [openai-python](https://github.com/openai/openai-python) library and cURL commands.
+See [docs/openai_api.md](docs/openai_api.md).
+
+### Hugging Face Generation APIs
+See [fastchat/serve/huggingface_api.py](fastchat/serve/huggingface_api.py).
+
+### LangChain Integration
+See [docs/langchain_integration](docs/langchain_integration.md).
+
+## Evaluation
+We use MT-bench, a set of challenging multi-turn open-ended questions to evaluate models. 
+To automate the evaluation process, we prompt strong LLMs like GPT-4 to act as judges and assess the quality of the models' responses.
+See instructions for running MT-bench at [fastchat/llm_judge](fastchat/llm_judge).
+
+MT-bench is the new recommended way to benchmark your models. If you are still looking for the old 80 questions used in the vicuna blog post, please go to [vicuna-blog-eval](https://github.com/lm-sys/vicuna-blog-eval).
+
+## Fine-tuning
+### Data
+
+Vicuna is created by fine-tuning a LLaMA base model using approximately 70K user-shared conversations gathered from ShareGPT.com with public APIs. To ensure data quality, we convert the HTML back to markdown and filter out some inappropriate or low-quality samples. Additionally, we divide lengthy conversations into smaller segments that fit the model's maximum context length. For detailed instructions to clean the ShareGPT data, check out [here](docs/commands/data_cleaning.md).
+
+We will not release the ShareGPT dataset. If you would like to try the fine-tuning code, you can run it with some dummy conversations in [dummy_conversation.json](data/dummy_conversation.json). You can follow the same format and plug in your own data.
+
+### Code and Hyperparameters
+Our code is based on [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) with additional support for multi-turn conversations.
+We use similar hyperparameters as the Stanford Alpaca.
+
+| Hyperparameter | Global Batch Size | Learning rate | Epochs | Max length | Weight decay |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| Vicuna-13B | 128 | 2e-5 | 3 | 2048 | 0 |
+
+### Fine-tuning Vicuna-7B with Local GPUs
+You can use the following command to train Vicuna-7B with 4 x A100 (40GB).
+Update `--model_name_or_path` with the actual path to LLaMA weights and `--data_path` with the actual path to data.
+
+```bash
+torchrun --nproc_per_node=4 --master_port=20001 fastchat/train/train_mem.py \
+    --model_name_or_path ~/model_weights/llama-7b  \
+    --data_path data/dummy_conversation.json \
+    --bf16 True \
+    --output_dir output_vicuna \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 2 \
+    --per_device_eval_batch_size 2 \
+    --gradient_accumulation_steps 16 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 1200 \
+    --save_total_limit 10 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --fsdp "full_shard auto_wrap" \
+    --fsdp_transformer_layer_cls_to_wrap 'LlamaDecoderLayer' \
+    --tf32 True \
+    --model_max_length 2048 \
+    --gradient_checkpointing True \
+    --lazy_preprocess True
+```
+
+If you meet out-of-memory during model saving, see solutions [here](https://github.com/pytorch/pytorch/issues/98823).
+
+### Other models and LoRA support
+More instructions to train other models (e.g., FastChat-T5) and use LoRA are in [docs/training.md](docs/training.md).
+
+### Fine-tuning on Any Cloud with SkyPilot
+[SkyPilot](https://github.com/skypilot-org/skypilot) is a framework built by UC Berkeley for easily and cost effectively running ML workloads on any cloud (AWS, GCP, Azure, Lambda, etc.).
+Find SkyPilot documentation [here](https://github.com/skypilot-org/skypilot/tree/master/llm/vicuna) on using managed spot instances to train Vicuna and save on your cloud costs.
+
+## Citation
+The code (training, serving, and evaluation) in this repository is mostly developed for or derived from the paper below.
+Please cite it if you find the repository helpful.
+
+```
+@misc{zheng2023judging,
+      title={Judging LLM-as-a-judge with MT-Bench and Chatbot Arena},
+      author={Lianmin Zheng and Wei-Lin Chiang and Ying Sheng and Siyuan Zhuang and Zhanghao Wu and Yonghao Zhuang and Zi Lin and Zhuohan Li and Dacheng Li and Eric. P Xing and Hao Zhang and Joseph E. Gonzalez and Ion Stoica},
+      year={2023},
+      eprint={2306.05685},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL}
+}
+```
+
+We are also planning to add more of our research to this repository.

controller.log

@@ -0,0 +1,228 @@
+2023-07-05 20:23:27 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 20:23:27 | ERROR | stderr | INFO:     Started server process [380845]
+2023-07-05 20:23:27 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 20:23:27 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 20:23:27 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 20:24:21 | INFO | stdout | INFO:     127.0.0.1:33748 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 20:24:21 | INFO | stdout | INFO:     127.0.0.1:33762 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 20:27:33 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 20:27:34 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 20:27:34 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 20:27:34 | ERROR | stderr | INFO:     Finished server process [380845]
+2023-07-05 20:27:38 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 20:27:38 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 20:27:38 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 20:27:38 | ERROR | stderr |     lock.acquire()
+2023-07-05 20:27:38 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 20:27:41 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 20:27:41 | ERROR | stderr | INFO:     Started server process [382194]
+2023-07-05 20:27:41 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 20:27:41 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 20:27:41 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 20:27:49 | INFO | stdout | INFO:     127.0.0.1:43004 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 20:27:49 | INFO | stdout | INFO:     127.0.0.1:43006 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 20:33:25 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 20:33:25 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 20:33:25 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 20:33:25 | ERROR | stderr | INFO:     Finished server process [382194]
+2023-07-05 20:33:58 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 20:33:58 | ERROR | stderr | INFO:     Started server process [383813]
+2023-07-05 20:33:58 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 20:33:58 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 20:33:58 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 20:34:33 | INFO | stdout | INFO:     127.0.0.1:43794 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 20:34:33 | INFO | stdout | INFO:     127.0.0.1:43810 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 20:35:52 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 20:35:52 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 20:35:52 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 20:35:52 | ERROR | stderr | INFO:     Finished server process [383813]
+2023-07-05 20:35:58 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 20:35:58 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 20:35:58 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 20:35:58 | ERROR | stderr |     lock.acquire()
+2023-07-05 20:35:58 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 20:36:01 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 20:36:01 | ERROR | stderr | INFO:     Started server process [384943]
+2023-07-05 20:36:01 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 20:36:01 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 20:36:01 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 20:36:07 | INFO | stdout | INFO:     127.0.0.1:50858 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 20:36:07 | INFO | stdout | INFO:     127.0.0.1:50868 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 21:07:09 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 21:07:09 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 21:07:09 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 21:07:09 | ERROR | stderr | INFO:     Finished server process [384943]
+2023-07-05 21:07:13 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 21:07:13 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 21:07:13 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 21:07:13 | ERROR | stderr |     lock.acquire()
+2023-07-05 21:07:13 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 21:07:16 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 21:07:16 | ERROR | stderr | INFO:     Started server process [393277]
+2023-07-05 21:07:16 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 21:07:16 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 21:07:16 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 21:07:24 | INFO | stdout | INFO:     127.0.0.1:38342 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 21:07:24 | INFO | stdout | INFO:     127.0.0.1:38356 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 21:07:39 | INFO | stdout | INFO:     127.0.0.1:58236 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 21:08:25 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 21:08:25 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 21:08:25 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 21:08:25 | ERROR | stderr | INFO:     Finished server process [393277]
+2023-07-05 21:08:31 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 21:08:31 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 21:08:31 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 21:08:31 | ERROR | stderr |     lock.acquire()
+2023-07-05 21:08:31 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 21:08:34 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 21:08:34 | ERROR | stderr | INFO:     Started server process [393958]
+2023-07-05 21:08:34 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 21:08:34 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 21:08:34 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 21:08:43 | INFO | stdout | INFO:     127.0.0.1:42028 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 21:08:43 | INFO | stdout | INFO:     127.0.0.1:42036 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 21:08:50 | INFO | stdout | INFO:     127.0.0.1:40684 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 21:09:07 | INFO | stdout | INFO:     127.0.0.1:57642 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 21:12:16 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 21:12:16 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 21:12:16 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 21:12:16 | ERROR | stderr | INFO:     Finished server process [393958]
+2023-07-05 21:12:17 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 21:12:17 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 21:12:17 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 21:12:17 | ERROR | stderr |     lock.acquire()
+2023-07-05 21:12:17 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 21:13:24 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 21:13:24 | ERROR | stderr | INFO:     Started server process [395717]
+2023-07-05 21:13:24 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 21:13:24 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 21:13:24 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 21:13:33 | INFO | stdout | INFO:     127.0.0.1:34954 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 21:13:33 | INFO | stdout | INFO:     127.0.0.1:34962 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 21:14:04 | INFO | stdout | INFO:     127.0.0.1:45190 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 21:17:19 | INFO | stdout | INFO:     127.0.0.1:43578 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 21:17:19 | INFO | stdout | INFO:     127.0.0.1:43592 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 21:17:36 | INFO | stdout | INFO:     127.0.0.1:57170 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 21:20:16 | INFO | stdout | INFO:     127.0.0.1:55178 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 21:20:17 | INFO | stdout | INFO:     127.0.0.1:55180 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 21:20:21 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 21:20:21 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 21:20:21 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 21:20:21 | ERROR | stderr | INFO:     Finished server process [395717]
+2023-07-05 21:20:22 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 21:20:22 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 21:20:22 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 21:20:22 | ERROR | stderr |     lock.acquire()
+2023-07-05 21:20:22 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 21:20:25 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 21:20:25 | ERROR | stderr | INFO:     Started server process [398096]
+2023-07-05 21:20:25 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 21:20:25 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 21:20:25 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 21:20:33 | INFO | stdout | INFO:     127.0.0.1:41918 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 21:20:33 | INFO | stdout | INFO:     127.0.0.1:41934 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 21:20:46 | INFO | stdout | INFO:     127.0.0.1:45420 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 21:22:17 | INFO | stdout | INFO:     127.0.0.1:38930 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 21:23:04 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 21:23:04 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 21:23:04 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 21:23:04 | ERROR | stderr | INFO:     Finished server process [398096]
+2023-07-05 21:23:04 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 21:23:04 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 21:23:04 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 21:23:04 | ERROR | stderr |     lock.acquire()
+2023-07-05 21:23:04 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 21:23:07 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 21:23:07 | ERROR | stderr | INFO:     Started server process [399120]
+2023-07-05 21:23:07 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 21:23:07 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 21:23:07 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 21:23:11 | INFO | stdout | INFO:     127.0.0.1:50332 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 21:23:11 | INFO | stdout | INFO:     127.0.0.1:50342 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 21:23:17 | INFO | stdout | INFO:     127.0.0.1:42516 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 22:00:45 | INFO | stdout | INFO:     127.0.0.1:49452 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 22:03:10 | INFO | stdout | INFO:     127.0.0.1:59704 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 22:13:25 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 22:13:25 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 22:13:25 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 22:13:25 | ERROR | stderr | INFO:     Finished server process [399120]
+2023-07-05 22:13:32 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 22:13:32 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 22:13:32 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 22:13:32 | ERROR | stderr |     lock.acquire()
+2023-07-05 22:13:32 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 22:13:35 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 22:13:35 | ERROR | stderr | INFO:     Started server process [411968]
+2023-07-05 22:13:35 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 22:13:35 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 22:13:35 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 22:13:41 | INFO | stdout | INFO:     127.0.0.1:56746 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 22:13:41 | INFO | stdout | INFO:     127.0.0.1:56758 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 22:14:23 | INFO | stdout | INFO:     127.0.0.1:44066 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 22:16:03 | INFO | stdout | INFO:     127.0.0.1:38550 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 22:16:03 | INFO | stdout | INFO:     127.0.0.1:38564 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 22:16:15 | INFO | stdout | INFO:     127.0.0.1:56662 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 22:18:10 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 22:18:10 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 22:18:10 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 22:18:10 | ERROR | stderr | INFO:     Finished server process [411968]
+2023-07-05 22:18:13 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 22:18:13 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 22:18:13 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 22:18:13 | ERROR | stderr |     lock.acquire()
+2023-07-05 22:18:13 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 22:18:15 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 22:18:16 | ERROR | stderr | INFO:     Started server process [413792]
+2023-07-05 22:18:16 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 22:18:16 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 22:18:16 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 22:18:20 | INFO | stdout | INFO:     127.0.0.1:58490 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 22:18:20 | INFO | stdout | INFO:     127.0.0.1:58506 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 22:18:29 | INFO | stdout | INFO:     127.0.0.1:56524 - "POST /get_worker_address HTTP/1.1" 200 OK
+2023-07-05 22:18:44 | INFO | stdout | INFO:     127.0.0.1:36906 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 22:18:44 | INFO | stdout | INFO:     127.0.0.1:36910 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 22:21:32 | INFO | stdout | INFO:     127.0.0.1:38644 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 22:21:32 | INFO | stdout | INFO:     127.0.0.1:38650 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 22:24:37 | INFO | stdout | INFO:     127.0.0.1:60104 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 22:24:37 | INFO | stdout | INFO:     127.0.0.1:60118 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 22:48:28 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 22:48:28 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 22:48:28 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 22:48:28 | ERROR | stderr |     lock.acquire()
+2023-07-05 22:48:28 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 22:55:54 | INFO | stdout | INFO:     127.0.0.1:58490 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 22:55:54 | INFO | stdout | INFO:     127.0.0.1:58502 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 22:55:55 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 22:55:55 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 22:55:55 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 22:55:55 | ERROR | stderr | INFO:     Finished server process [413792]
+2023-07-05 22:56:00 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 22:56:00 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 22:56:00 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 22:56:00 | ERROR | stderr |     lock.acquire()
+2023-07-05 22:56:00 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 22:56:14 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 22:56:14 | ERROR | stderr | INFO:     Started server process [424798]
+2023-07-05 22:56:14 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 22:56:14 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 22:56:14 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 23:00:32 | ERROR | stderr | INFO:     Shutting down
+2023-07-05 23:00:32 | ERROR | stderr | INFO:     Waiting for application shutdown.
+2023-07-05 23:00:32 | ERROR | stderr | INFO:     Application shutdown complete.
+2023-07-05 23:00:32 | ERROR | stderr | INFO:     Finished server process [424798]
+2023-07-05 23:00:35 | ERROR | stderr | Exception ignored in: <module 'threading' from '/opt/conda/envs/fastchat/lib/python3.11/threading.py'>
+2023-07-05 23:00:35 | ERROR | stderr | Traceback (most recent call last):
+2023-07-05 23:00:35 | ERROR | stderr |   File "/opt/conda/envs/fastchat/lib/python3.11/threading.py", line 1583, in _shutdown
+2023-07-05 23:00:35 | ERROR | stderr |     lock.acquire()
+2023-07-05 23:00:35 | ERROR | stderr | KeyboardInterrupt:
+2023-07-05 23:01:02 | ERROR | stderr | usage: controller.py [-h] [--host HOST] [--port PORT] [--dispatch-method {lottery,shortest_queue}]
+2023-07-05 23:01:02 | ERROR | stderr | controller.py: error: unrecognized arguments: --add-chatgpt --share
+2023-07-05 23:01:07 | INFO | controller | args: Namespace(host='localhost', port=21001, dispatch_method='shortest_queue')
+2023-07-05 23:01:07 | ERROR | stderr | INFO:     Started server process [426596]
+2023-07-05 23:01:07 | ERROR | stderr | INFO:     Waiting for application startup.
+2023-07-05 23:01:07 | ERROR | stderr | INFO:     Application startup complete.
+2023-07-05 23:01:07 | ERROR | stderr | INFO:     Uvicorn running on http://localhost:21001 (Press CTRL+C to quit)
+2023-07-05 23:01:18 | INFO | stdout | INFO:     127.0.0.1:44990 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 23:01:18 | INFO | stdout | INFO:     127.0.0.1:45004 - "POST /list_models HTTP/1.1" 200 OK
+2023-07-05 23:07:42 | INFO | stdout | INFO:     127.0.0.1:46560 - "POST /refresh_all_workers HTTP/1.1" 200 OK
+2023-07-05 23:07:42 | INFO | stdout | INFO:     127.0.0.1:46570 - "POST /list_models HTTP/1.1" 200 OK

docker/Dockerfile

@@ -0,0 +1,6 @@
+FROM nvidia/cuda:11.7.1-runtime-ubuntu20.04
+
+RUN apt-get update -y && apt-get install -y python3.9 python3.9-distutils curl
+RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+RUN python3.9 get-pip.py
+RUN pip3 install fschat

docker/docker-compose.yml

@@ -0,0 +1,40 @@
+version: "3.9"
+
+services:
+  fastchat-controller:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: fastchat:latest
+    ports:
+      - "21001:21001"
+    entrypoint: ["python3.9", "-m", "fastchat.serve.controller", "--host", "0.0.0.0", "--port", "21001"]
+  fastchat-model-worker:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - huggingface:/root/.cache/huggingface
+    environment:
+      FASTCHAT_CONTROLLER_URL: http://fastchat-controller:21001
+    image: fastchat:latest
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+    entrypoint: ["python3.9", "-m", "fastchat.serve.model_worker", "--model-name", 'vicuna-7b-v1.3', "--model-path", "lmsys/vicuna-7b-v1.3", "--worker-address", "http://fastchat-model-worker:21002", "--controller-address", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "21002"]
+  fastchat-api-server:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    environment:
+      FASTCHAT_CONTROLLER_URL: http://fastchat-controller:21001
+    image: fastchat:latest
+    ports:
+      - "8000:8000"
+    entrypoint: ["python3.9", "-m", "fastchat.serve.openai_api_server", "--controller-address", "http://fastchat-controller:21001", "--host", "0.0.0.0", "--port", "8000"]
+volumes:
+  huggingface:

docs/arena.md

@@ -0,0 +1,9 @@
+# Chatbot Arena
+Chatbot Arena is an LLM benchmark platform featuring anonymous, randomized battles, available at https://arena.lmsys.org.
+We invite the entire community to join this benchmarking effort by contributing your votes and models.
+
+## How to add a new model
+If you want to see a specific model in the arena, you can follow the steps below.
+
+1. Contribute code to support this model in FastChat by submitting a pull request. See [instructions](model_support.md#how-to-support-a-new-model).
+2. After the model is supported, we will try to schedule some computing resources to host the model in the arena. However, due to the limited resources we have, we may not be able to serve every model. We will select the models based on popularity, quality, diversity, and other factors.

docs/commands/data_cleaning.md

@@ -0,0 +1,19 @@
+## Data cleaning
+
+## Requirements
+```
+pip3 install bs4 markdownify
+pip3 install polyglot pyicu pycld2
+```
+
+## Steps
+```
+# Convert html to markdown
+python3 -m fastchat.data.clean_sharegpt --in sharegpt_html.json --out sharegpt_clean.json
+
+# Keep or remove specific languages
+python3 -m fastchat.data.optional_clean --in sharegpt_clean.json --out sharegpt_clean_lang.json --skip-lang SOME_LANGUAGE_CODE
+
+# Split long conversations
+python3 -m fastchat.data.split_long_conversation --in sharegpt_clean_lang.json --out sharegpt_clean_lang_split.json --model-name /home/ubuntu/model_weights/llama-7b/
+```

docs/commands/leaderboard.md

@@ -0,0 +1,15 @@
+### Get logs
+```
+gsutil -m rsync -r gs://fastchat_logs ~/fastchat_logs/
+```
+
+### Clean battle data
+```
+cd ~/FastChat/fastchat/serve/monitor
+python3 clean_battle_data.py
+```
+
+### Run Elo analysis
+```
+python3 elo_analysis.py --clean-battle-file clean_battle_20230523.json
+```

docs/commands/local_cluster.md

@@ -0,0 +1,30 @@
+### Local GPU cluster (node-01)
+```
+python3 -m fastchat.serve.controller --host 0.0.0.0 --port 10002
+
+CUDA_VISIBLE_DEVICES=0 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/vicuna-13b/ --controller http://localhost:10002 --port 31000 --worker http://localhost:31000
+CUDA_VISIBLE_DEVICES=1 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/vicuna-13b/ --controller http://localhost:10002 --port 31001 --worker http://localhost:31001
+CUDA_VISIBLE_DEVICES=2 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/bair-chat-13b/ --controller http://localhost:10002 --port 31002 --worker http://localhost:31002
+CUDA_VISIBLE_DEVICES=3 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/alpaca-chat-13b/ --controller http://localhost:10002 --port 31003 --worker http://localhost:31003
+
+python3 -m fastchat.serve.test_message --model vicuna-13b --controller http://localhost:10002
+```
+
+### Web server
+```
+python3 -m fastchat.serve.controller --host 0.0.0.0 --port 21001
+
+python3 -m fastchat.serve.register_worker --controller http://localhost:21001 --worker-name https://
+
+python3 -m fastchat.serve.test_message --model vicuna-13b --controller http://localhost:21001
+
+python3 -m fastchat.serve.gradio_web_server --controller http://localhost:21001
+```
+
+### Local GPU cluster (node-02)
+```
+CUDA_VISIBLE_DEVICES=0 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/vicuna-13b/ --controller http://node-01:10002 --host 0.0.0.0 --port 31000 --worker http://$(hostname):31000
+CUDA_VISIBLE_DEVICES=1 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/vicuna-13b/ --controller http://node-01:10002 --host 0.0.0.0 --port 31001 --worker http://$(hostname):31001
+CUDA_VISIBLE_DEVICES=2 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/vicuna-13b/ --controller http://node-01:10002 --host 0.0.0.0 --port 31002 --worker http://$(hostname):31002
+CUDA_VISIBLE_DEVICES=3 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/vicuna-13b/ --controller http://node-01:10002 --host 0.0.0.0 --port 31003 --worker http://$(hostname):31003
+```

docs/commands/pypi.md

@@ -0,0 +1,11 @@
+### Requirement
+```
+python3 -m pip install twine
+python3 -m pip install --upgrade pip
+pip3 install build
+```
+
+### Upload
+```
+bash scripts/upload_pypi.sh
+```

docs/commands/test_process.md

@@ -0,0 +1,39 @@
+### Test CLI Inference
+
+```
+python3 test_cli.py
+```
+
+### Test OpenAI API Server
+
+```
+python3 launch_openai_api_test_server.py
+```
+
+```
+python3 test_openai_api.py
+```
+
+### Test GUI Serving
+
+```
+python3 -m fastchat.serve.controller
+```
+
+```
+CUDA_VISIBLE_DEVICES=0,1 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/koala-13b --num-gpus 2 --port 30000 --worker http://localhost:30000
+CUDA_VISIBLE_DEVICES=2,3 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/alpaca-13b --num-gpus 2 --port 30002 --worker http://localhost:30002
+CUDA_VISIBLE_DEVICES=4,5 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/vicuna-13b --port 30004 --worker http://localhost:30004 --num-gpus 2
+CUDA_VISIBLE_DEVICES=6,7 python3 -m fastchat.serve.model_worker --model-path OpenAssistant/oasst-sft-1-pythia-12b --port 30006 --worker http://localhost:30006 --num-gpus 2
+
+CUDA_VISIBLE_DEVICES=0,1 python3 -m fastchat.serve.model_worker --model-path StabilityAI/stablelm-tuned-alpha-7b --num-gpus 2 --port 30000 --worker http://localhost:30000
+CUDA_VISIBLE_DEVICES=2,3 python3 -m fastchat.serve.model_worker --model-path databricks/dolly-v2-12b --num-gpus 2 --port 30002 --worker http://localhost:30002
+CUDA_VISIBLE_DEVICES=4 python3 -m fastchat.serve.model_worker --model-path THUDM/chatglm-6b --port 30004 --worker http://localhost:30004
+CUDA_VISIBLE_DEVICES=5 python3 -m fastchat.serve.model_worker --model-path lmsys/fastchat-t5-3b-v1.0 --port 30005 --worker http://localhost:30005
+CUDA_VISIBLE_DEVICES=6 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/baize-7b --port 30006 --worker http://localhost:30006
+CUDA_VISIBLE_DEVICES=7 python3 -m fastchat.serve.model_worker --model-path ~/model_weights/RWKV-4-Raven-7B-v11x-Eng99%-Other1%-20230429-ctx8192.pth --port 30007 --worker http://localhost:30007
+```
+
+```
+python3 -m fastchat.serve.gradio_web_server_multi
+```

docs/commands/webserver.md

@@ -0,0 +1,82 @@
+### Install
+```
+sudo apt update
+sudo apt install tmux htop
+
+wget https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh
+bash Anaconda3-2022.10-Linux-x86_64.sh
+
+conda create -n fastchat python=3.9
+conda activate fastchat
+
+git clone https://github.com/lm-sys/FastChat.git
+cd FastChat
+pip3 install -e .
+```
+
+
+### Launch servers
+```
+cd fastchat_logs/controller
+python3 -m fastchat.serve.controller --host 0.0.0.0 --port 21001
+python3 -m fastchat.serve.register_worker --controller http://localhost:21001 --worker-name https://
+python3 -m fastchat.serve.test_message --model vicuna-13b --controller http://localhost:21001
+
+cd fastchat_logs/server0
+
+export OPENAI_API_KEY=
+export ANTHROPIC_API_KEY=
+
+python3 -m fastchat.serve.gradio_web_server_multi --controller http://localhost:21001 --concurrency 10 --add-chatgpt --add-claude --add-palm --anony-only --elo ~/elo_results/elo_results_20230619.pkl --leaderboard-table-file ~/elo_results/leaderboard_table_20230619.csv
+
+python3 backup_logs.py
+```
+
+
+### Check the launch time
+```
+for i in $(seq 0 11); do cat fastchat_logs/server$i/gradio_web_server.log | grep "Running on local URL" | tail -n 1; done
+```
+
+
+### Increase the limit of max open files
+One process (do not need reboot)
+```
+sudo prlimit --nofile=1048576:1048576 --pid=$id
+
+for id in $(ps -ef | grep gradio_web_server | awk '{print $2}'); do echo $id; prlimit --nofile=1048576:1048576 --pid=$id; done
+```
+
+System (need reboot): Add the lines below to `/etc/security/limits.conf`
+```
+* hard nofile 65535
+* soft nofile 65535
+```
+
+
+### Gradio edit  (3.35.2)
+1. gtag and canvas
+```
+vim /home/vicuna/anaconda3/envs/fastchat/lib/python3.9/site-packages/gradio/templates/frontend/index.html
+```
+
+```
+<!-- Google tag (gtag.js) -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-K6D24EE9ED"></script><script>
+  window.dataLayer = window.dataLayer || [];
+  function gtag(){dataLayer.push(arguments);}
+  gtag('js', new Date());
+  gtag('config', 'G-K6D24EE9ED');
+  window.__gradio_mode__ = "app";
+</script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/html2canvas/1.4.1/html2canvas.min.js"></script>
+```
+
+2. Loading
+```
+vim /home/vicuna/anaconda3/envs/fastchat/lib/python3.9/site-packages/gradio/templates/frontend/assets/index-188ef5e8.js
+```
+
+```
+%s/"Loading..."/"Loading...(Please refresh if it takes more than 30 seconds)"/g
+```

docs/gptq.md

@@ -0,0 +1,59 @@
+# GPTQ 4bit Inference
+
+Support GPTQ 4bit inference with [GPTQ-for-LLaMa](https://github.com/qwopqwop200/GPTQ-for-LLaMa).
+
+1. Window user: use the `old-cuda` branch.
+2. Linux user: recommend the `fastest-inference-4bit` branch.
+
+## Install
+
+Setup environment:
+```bash
+# cd /path/to/FastChat
+git clone https://github.com/qwopqwop200/GPTQ-for-LLaMa.git repositories/GPTQ-for-LLaMa
+cd repositories/GPTQ-for-LLaMa
+# Window's user should use the `old-cuda` branch
+git switch fastest-inference-4bit
+# Install `quant-cuda` package in FastChat's virtualenv
+python3 setup_cuda.py install
+pip3 install texttable
+```
+
+Chat with the CLI:
+```bash
+python3 -m fastchat.serve.cli \
+    --model-path models/vicuna-7B-1.1-GPTQ-4bit-128g \
+    --gptq-wbits 4 \
+    --gptq-groupsize 128
+```
+
+Start model worker:
+```bash
+# Download quantized model from huggingface
+# Make sure you have git-lfs installed (https://git-lfs.com)
+git lfs install
+git clone https://huggingface.co/TheBloke/vicuna-7B-1.1-GPTQ-4bit-128g models/vicuna-7B-1.1-GPTQ-4bit-128g
+
+python3 -m fastchat.serve.model_worker \
+    --model-path models/vicuna-7B-1.1-GPTQ-4bit-128g \
+    --gptq-wbits 4 \
+    --gptq-groupsize 128
+
+# You can specify which quantized model to use
+python3 -m fastchat.serve.model_worker \
+    --model-path models/vicuna-7B-1.1-GPTQ-4bit-128g \
+    --gptq-ckpt models/vicuna-7B-1.1-GPTQ-4bit-128g/vicuna-7B-1.1-GPTQ-4bit-128g.safetensors \
+    --gptq-wbits 4 \
+    --gptq-groupsize 128 \
+    --gptq-act-order
+```
+
+## Benchmark
+
+| LLaMA-13B | branch                 | Bits | group-size | memory(MiB) | PPL(c4) | Median(s/token) | act-order | speed up |
+| --------- | ---------------------- | ---- | ---------- | ----------- | ------- | --------------- | --------- | -------- |
+| FP16      | fastest-inference-4bit | 16   | -          | 26634       | 6.96    | 0.0383          | -         | 1x       |
+| GPTQ      | triton                 | 4    | 128        | 8590        | 6.97    | 0.0551          | -         | 0.69x    |
+| GPTQ      | fastest-inference-4bit | 4    | 128        | 8699        | 6.97    | 0.0429          | true      | 0.89x    |
+| GPTQ      | fastest-inference-4bit | 4    | 128        | 8699        | 7.03    | 0.0287          | false     | 1.33x    |
+| GPTQ      | fastest-inference-4bit | 4    | -1         | 8448        | 7.12    | 0.0284          | false     | 1.44x    |

docs/langchain_integration.md

@@ -0,0 +1,90 @@
+# Local LangChain with FastChat
+
+[LangChain](https://python.langchain.com/en/latest/index.html) is a library that facilitates the development of applications by leveraging large language models (LLMs) and enabling their composition with other sources of computation or knowledge.
+FastChat's OpenAI-compatible [API server](openai_api.md) enables using LangChain with open models seamlessly.
+
+## Launch RESTful API Server
+
+Here are the steps to launch a local OpenAI API server for LangChain.
+
+First, launch the controller
+
+```bash
+python3 -m fastchat.serve.controller
+```
+
+LangChain uses OpenAI model names by default, so we need to assign some faux OpenAI model names to our local model.
+Here, we use Vicuna as an example and use it for three endpoints: chat completion, completion, and embedding.
+`--model-path` can be a local folder or a Hugging Face repo name.
+See a full list of supported models [here](../README.md#supported-models).
+
+```bash
+python3 -m fastchat.serve.model_worker --model-names "gpt-3.5-turbo,text-davinci-003,text-embedding-ada-002" --model-path lmsys/vicuna-7b-v1.3
+```
+
+Finally, launch the RESTful API server
+
+```bash
+python3 -m fastchat.serve.openai_api_server --host localhost --port 8000
+```
+
+## Set OpenAI Environment
+
+You can set your environment with the following commands.
+
+Set OpenAI base url
+
+```bash
+export OPENAI_API_BASE=http://localhost:8000/v1
+```
+
+Set OpenAI API key
+
+```bash
+export OPENAI_API_KEY=EMPTY
+```
+
+If you meet the following OOM error while creating embeddings, please set a smaller batch size by using environment variables.
+
+~~~bash
+openai.error.APIError: Invalid response object from API: '{"object":"error","message":"**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**\\n\\n(CUDA out of memory. Tried to allocate xxx MiB (GPU 0; xxx GiB total capacity; xxx GiB already allocated; xxx MiB free; xxx GiB reserved in total by PyTorch) If reserved memory is >> allocated memory try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF)","code":50002}' (HTTP response code was 400)
+~~~
+
+You can try `export FASTCHAT_WORKER_API_EMBEDDING_BATCH_SIZE=1`.
+
+## Try local LangChain
+
+Here is a question answerting example.
+
+Download a text file.
+
+```bash
+wget https://raw.githubusercontent.com/hwchase17/langchain/v0.0.200/docs/modules/state_of_the_union.txt
+```
+
+Run LangChain.
+
+~~~py
+from langchain.chat_models import ChatOpenAI
+from langchain.document_loaders import TextLoader
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.indexes import VectorstoreIndexCreator
+
+embedding = OpenAIEmbeddings(model="text-embedding-ada-002")
+loader = TextLoader("state_of_the_union.txt")
+index = VectorstoreIndexCreator(embedding=embedding).from_loaders([loader])
+llm = ChatOpenAI(model="gpt-3.5-turbo")
+
+questions = [
+    "Who is the speaker",
+    "What did the president say about Ketanji Brown Jackson",
+    "What are the threats to America",
+    "Who are mentioned in the speech",
+    "Who is the vice president",
+    "How many projects were announced",
+]
+
+for query in questions:
+    print("Query:", query)
+    print("Answer:", index.query(query, llm=llm))
+~~~

docs/model_support.md

@@ -0,0 +1,55 @@
+# Model Support
+
+## Supported models
+- Vicuna, Alpaca, LLaMA, Koala
+   - example: `python3 -m fastchat.serve.cli --model-path lmsys/vicuna-7b-v1.3`
+- [BlinkDL/RWKV-4-Raven](https://huggingface.co/BlinkDL/rwkv-4-raven)
+   - example: `python3 -m fastchat.serve.cli --model-path ~/model_weights/RWKV-4-Raven-7B-v11x-Eng99%-Other1%-20230429-ctx8192.pth`
+- [camel-ai/CAMEL-13B-Combined-Data](https://huggingface.co/camel-ai/CAMEL-13B-Combined-Data)
+- [databricks/dolly-v2-12b](https://huggingface.co/databricks/dolly-v2-12b)
+- [FreedomIntelligence/phoenix-inst-chat-7b](https://huggingface.co/FreedomIntelligence/phoenix-inst-chat-7b)
+- [h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b](https://huggingface.co/h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b)
+- [lcw99/polyglot-ko-12.8b-chang-instruct-chat](https://huggingface.co/lcw99/polyglot-ko-12.8b-chang-instruct-chat)
+- [lmsys/fastchat-t5-3b-v1.0](https://huggingface.co/lmsys/fastchat-t5)
+- [mosaicml/mpt-7b-chat](https://huggingface.co/mosaicml/mpt-7b-chat)
+  - example: `python3 -m fastchat.serve.cli --model-path mosaicml/mpt-7b-chat`
+- [Neutralzz/BiLLa-7B-SFT](https://huggingface.co/Neutralzz/BiLLa-7B-SFT)
+- [nomic-ai/gpt4all-13b-snoozy](https://huggingface.co/nomic-ai/gpt4all-13b-snoozy)
+- [openaccess-ai-collective/manticore-13b-chat-pyg](https://huggingface.co/openaccess-ai-collective/manticore-13b-chat-pyg)
+- [OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5](https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5)
+- [project-baize/baize-v2-7b](https://huggingface.co/project-baize/baize-v2-7b)
+- [Salesforce/codet5p-6b](https://huggingface.co/Salesforce/codet5p-6b)
+- [StabilityAI/stablelm-tuned-alpha-7b](https://huggingface.co/stabilityai/stablelm-tuned-alpha-7b)
+- [THUDM/chatglm-6b](https://huggingface.co/THUDM/chatglm-6b)
+- [THUDM/chatglm2-6b](https://huggingface.co/THUDM/chatglm2-6b)
+- [tiiuae/falcon-40b](https://huggingface.co/tiiuae/falcon-40b)
+- [timdettmers/guanaco-33b-merged](https://huggingface.co/timdettmers/guanaco-33b-merged)
+- [togethercomputer/RedPajama-INCITE-7B-Chat](https://huggingface.co/togethercomputer/RedPajama-INCITE-7B-Chat)
+- [WizardLM/WizardLM-13B-V1.0](https://huggingface.co/WizardLM/WizardLM-13B-V1.0)
+- [baichuan-inc/baichuan-7B](https://huggingface.co/baichuan-inc/baichuan-7B)
+- Any [EleutherAI](https://huggingface.co/EleutherAI) pythia model such as [pythia-6.9b](https://huggingface.co/EleutherAI/pythia-6.9b)
+- Any [Peft](https://github.com/huggingface/peft) adapter trained ontop of a model above.  To activate, must have `peft` in the model path.
+
+## How to support a new model
+
+To support a new model in FastChat, you need to correctly handle its prompt template and model loading.
+The goal is to make the following command run with the correct prompts.
+```
+python3 -m fastchat.serve.cli --model [YOUR_MODEL_PATH]
+```
+
+You can run this example command to learn the code logic.
+```
+python3 -m fastchat.serve.cli --model lmsys/vicuna-7b-v1.3
+```
+
+You can add `--debug` to see the actual prompt sent to the model.
+
+### Steps
+FastChat uses the `Conversation` class to handle prompt templates and `BaseModelAdapter` class to handle model loading.
+
+1. Implement a conversation template for the new model at [fastchat/conversation.py](https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py). You can follow existing examples and use `register_conv_template` to add a new one.
+2. Implement a model adapter for the new model at [fastchat/model/model_adapter.py](https://github.com/lm-sys/FastChat/blob/main/fastchat/model/model_adapter.py). You can follow existing examples and use `register_model_adapter` to add a new one.
+3. (Optional) add the model name to the "Supported models" [section](#supported-models) above and add more information in [fastchat/model/model_registry.py](https://github.com/lm-sys/FastChat/blob/main/fastchat/model/model_registry.py).
+
+After these steps, the new model should be compatible with most FastChat features, such as CLI, web UI, model worker, and OpenAI-compatible API server. Please do some testing with these features as well.

docs/openai_api.md

@@ -0,0 +1,131 @@
+# OpenAI-Compatible RESTful APIs & SDK
+
+FastChat provides OpenAI-compatible APIs for its supported models, so you can use FastChat as a local drop-in replacement for OpenAI APIs.
+The FastChat server is compatible with both [openai-python](https://github.com/openai/openai-python) library and cURL commands.
+
+The following OpenAI APIs are supported:
+- Chat Completions. (Reference: https://platform.openai.com/docs/api-reference/chat)
+- Completions. (Reference: https://platform.openai.com/docs/api-reference/completions)
+- Embeddings. (Reference: https://platform.openai.com/docs/api-reference/embeddings)
+
+## RESTful API Server
+First, launch the controller
+
+```bash
+python3 -m fastchat.serve.controller
+```
+
+Then, launch the model worker(s)
+
+```bash
+python3 -m fastchat.serve.model_worker --model-path lmsys/vicuna-7b-v1.3
+```
+
+Finally, launch the RESTful API server
+
+```bash
+python3 -m fastchat.serve.openai_api_server --host localhost --port 8000
+```
+
+Now, let us test the API server.
+
+### OpenAI Official SDK
+The goal of `openai_api_server.py` is to implement a fully OpenAI-compatible API server, so the models can be used directly with [openai-python](https://github.com/openai/openai-python) library.
+
+First, install openai-python:
+```bash
+pip install --upgrade openai
+```
+
+Then, interact with model vicuna:
+```python
+import openai
+openai.api_key = "EMPTY" # Not support yet
+openai.api_base = "http://localhost:8000/v1"
+
+model = "vicuna-7b-v1.3"
+prompt = "Once upon a time"
+
+# create a completion
+completion = openai.Completion.create(model=model, prompt=prompt, max_tokens=64)
+# print the completion
+print(prompt + completion.choices[0].text)
+
+# create a chat completion
+completion = openai.ChatCompletion.create(
+  model=model,
+  messages=[{"role": "user", "content": "Hello! What is your name?"}]
+)
+# print the completion
+print(completion.choices[0].message.content)
+```
+
+Streaming is also supported. See [test_openai_api.py](../tests/test_openai_api.py).
+
+### cURL
+cURL is another good tool for observing the output of the api.
+
+List Models:
+```bash
+curl http://localhost:8000/v1/models
+```
+
+Chat Completions:
+```bash
+curl http://localhost:8000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vicuna-7b-v1.3",
+    "messages": [{"role": "user", "content": "Hello! What is your name?"}]
+  }'
+```
+
+Text Completions:
+```bash
+curl http://localhost:8000/v1/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vicuna-7b-v1.3",
+    "prompt": "Once upon a time",
+    "max_tokens": 41,
+    "temperature": 0.5
+  }'
+```
+
+Embeddings:
+```bash
+curl http://localhost:8000/v1/embeddings \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vicuna-7b-v1.3",
+    "input": "Hello world!"
+  }'
+```
+
+## LangChain Support
+This OpenAI-compatible API server supports LangChain. See [LangChain Integration](langchain_integration.md) for details.
+
+## Adjusting Environment Variables
+
+### Timeout
+By default, a timeout error will occur if a model worker does not response within 100 seconds. If your model/hardware is slower, you can change this timeout through an environment variable: 
+
+```bash
+export FASTCHAT_WORKER_API_TIMEOUT=<larger timeout in seconds>
+```
+
+### Batch size
+If you meet the following OOM error while creating embeddings. You can use a smaller batch size by setting
+
+```bash
+export FASTCHAT_WORKER_API_EMBEDDING_BATCH_SIZE=1
+```
+
+## Todos
+Some features to be implemented:
+
+- [ ] Support more parameters like `logprobs`, `logit_bias`, `user`, `presence_penalty` and `frequency_penalty`
+- [ ] Model details (permissions, owner and create time)
+- [ ] Edits API
+- [ ] Authentication and API key
+- [ ] Rate Limitation Settings

docs/server_arch.md

@@ -0,0 +1,2 @@
+# FastChat Server Architecture
+![server arch](../assets/server_arch.png)

docs/training.md

@@ -0,0 +1,60 @@
+### Fine-tuning FastChat-T5
+You can use the following command to train FastChat-T5 with 4 x A100 (40GB).
+```bash
+torchrun --nproc_per_node=4 --master_port=9778 fastchat/train/train_flant5.py \
+    --model_name_or_path google/flan-t5-xl \
+    --data_path /data/dummy.json \
+    --bf16 True \
+    --output_dir ./checkpoints_flant5_3b \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --gradient_accumulation_steps 4 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 300 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --fsdp "full_shard auto_wrap" \
+    --fsdp_transformer_layer_cls_to_wrap T5Block \
+    --tf32 True \
+    --model_max_length 2048 \
+    --preprocessed_path ./preprocessed_data/processed.json \
+    --gradient_checkpointing True 
+```
+
+After training, please use our post-processing [function](https://github.com/lm-sys/FastChat/blob/55051ad0f23fef5eeecbda14a2e3e128ffcb2a98/fastchat/utils.py#L166-L185) to update the saved model weight. Additional discussions can be found [here](https://github.com/lm-sys/FastChat/issues/643).
+
+### Fine-tuning using (Q)LoRA
+You can use the following command to train Vicuna-7B using QLoRA using ZeRO2. Note that ZeRO3 is not currently supported with QLoRA but ZeRO3 does support LoRA, which has a reference configuraiton under `playground/deepspeed_config_s3.json`.
+```bash
+deepspeed train_lora.py \
+    --model_name_or_path ~/model_weights/llama-7b  \
+    --lora_r 8 \
+    --lora_alpha 16 \
+    --lora_dropout 0.05 \
+    --data_path <path-to-data> \
+    --bf16 True \
+    --output_dir ./checkpoints \
+    --num_train_epochs 3 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 1 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 1200 \
+    --save_total_limit 100 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 True \
+    --model_max_length 2048 \
+    --q_lora True \
+    --deepspeed playground/deepspeed_config_s2.json \
+```

docs/vicuna_weights_version.md

@@ -0,0 +1,94 @@
+## Vicuna Weights
+
+| Weights version | v1.3 | v1.1 | v0 |
+| ---- | ---- | ---- | ---- |
+| Link | [7B](https://huggingface.co/lmsys/vicuna-7b-v1.3), [13B](https://huggingface.co/lmsys/vicuna-13b-v1.3), [33B](//huggingface.co/lmsys/vicuna-33b-v1.3) | [7B](https://huggingface.co/lmsys/vicuna-7b-delta-v1.1), [13B](https://huggingface.co/lmsys/vicuna-13b-delta-v1.1) | [7B](https://huggingface.co/lmsys/vicuna-7b-delta-v0), [13B](https://huggingface.co/lmsys/vicuna-13b-delta-v0) |
+| Separator | `</s>` | `</s>` | `###` |
+| Is delta weights | No | Yes | Yes |
+| FastChat PyPI package compatibility | >= v0.2.1 | >= v0.2.1 |<= v0.1.10 |
+| FastChat source code compatibility | after [tag v0.2.1](https://github.com/lm-sys/FastChat/tree/v0.2.1) | after [tag v0.2.1](https://github.com/lm-sys/FastChat/tree/v0.2.1) | [tag v0.1.10](https://github.com/lm-sys/FastChat/tree/v0.1.10) |
+
+### Updates
+- Major updates of weights v1.3
+  - Train with twice the amount of ShareGPT data compared to previous versions.
+  - Provide merged weights directly instead of delta weights.
+
+- Major updates of weights v1.1
+  - Refactor the tokenization and separator. In Vicuna v1.1, the separator has been changed from `###` to the EOS token `</s>`. This change makes it easier to determine the generation stop criteria and enables better compatibility with other libraries.
+  - Fix the supervised fine-tuning loss computation for better model quality.
+
+## Prompt Template
+
+### Example prompt (weights v1.1 and v1.3)
+```
+A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
+
+USER: Hello!
+ASSISTANT: Hello!</s>
+USER: How are you?
+ASSISTANT: I am good.</s>
+```
+
+See a full prompt template [here](https://github.com/lm-sys/FastChat/blob/daa2b9abe20597ebf34dc5df164d450456610c74/fastchat/conversation.py#L246-L259).
+
+### Example prompt (weights v0)
+```
+A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.
+
+### Human: Hello!
+### Assistant: Hello!
+### Human: How are you?
+### Assistant: I am good.
+```
+
+See the full prompt template [here](https://github.com/lm-sys/FastChat/blob/daa2b9abe20597ebf34dc5df164d450456610c74/fastchat/conversation.py#L198-L229).
+
+## How to Apply Delta Weights (for weights v1.1 and v0)
+
+We release [Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) weights (v1.1 and v0) as delta weights to comply with the LLaMA model license.
+You can add our delta to the original LLaMA weights to obtain the Vicuna weights. Instructions:
+
+1. Get the original LLaMA weights in the Hugging Face format by following the instructions [here](https://huggingface.co/docs/transformers/main/model_doc/llama).
+2. Use the following scripts to get Vicuna weights by applying our delta. They will automatically download delta weights from our Hugging Face [account](https://huggingface.co/lmsys).
+
+**NOTE**:
+Weights v1.1 are only compatible with ```transformers>=4.28.0``` and ``fschat >= 0.2.0``.
+Please update your local packages accordingly. If you follow the above commands to do a fresh install, then you should get all the correct versions.
+
+#### Vicuna-7B
+This conversion command needs around 30 GB of CPU RAM.
+See the "Low CPU Memory Conversion" section below if you do not have enough memory.
+Replace `/path/to/*` with the real paths.
+```bash
+python3 -m fastchat.model.apply_delta \
+    --base-model-path /path/to/llama-7b \
+    --target-model-path /path/to/output/vicuna-7b \
+    --delta-path lmsys/vicuna-7b-delta-v1.1
+```
+
+#### Vicuna-13B
+This conversion command needs around 60 GB of CPU RAM.
+See the "Low CPU Memory Conversion" section below if you do not have enough memory.
+Replace `/path/to/*` with the real paths.
+```bash
+python3 -m fastchat.model.apply_delta \
+    --base-model-path /path/to/llama-13b \
+    --target-model-path /path/to/output/vicuna-13b \
+    --delta-path lmsys/vicuna-13b-delta-v1.1
+```
+
+#### Low CPU Memory Conversion
+You can try these methods to reduce the CPU RAM requirement of weight conversion.
+1. Append `--low-cpu-mem` to the commands above, which will split large weight files into smaller ones and use the disk as temporary storage. This can keep the peak memory at less than 16GB.
+2. Create a large swap file and rely on the operating system to automatically utilize the disk as virtual memory.
+
+## FAQ
+
+### Tokenizer issues
+There are some frequently asked tokenizer issues (https://github.com/lm-sys/FastChat/issues/408).
+Some of them are not only related to FastChat or Vicuna weights but are also related to how you convert the base llama model.
+
+We suggest that you use `transformers>=4.28.0` and redo the weight conversion for the base llama model.
+After applying the delta, you should have a file named `special_tokens_map.json` in your converted weight folder for either v0 or v1.1.
+The contents of this file should be the same as this file: https://huggingface.co/lmsys/vicuna-13b-delta-v0/blob/main/special_tokens_map.json.
+If the file is not present, please copy the `special_tokens_map.json` and `tokenizer_config.json` files from https://huggingface.co/lmsys/vicuna-13b-delta-v0/tree/main to your converted weight folder. This works for both v0 and v1.1.

docs/vllm_integration.md

@@ -0,0 +1,15 @@
+# vLLM Integration
+You can use [vLLM](https://vllm.ai/) as an optimized worker implementation in FastChat.
+It offers advanced continuous batching and a much higher (~10x) throughput.
+See the supported models [here](https://vllm.readthedocs.io/en/latest/models/supported_models.html).
+
+## Instructions
+1. Install vLLM.
+    ```
+    pip install vllm
+    ```
+
+2. When you launch a model worker, replace the normal worker (`fastchat.serve.model_worker`) with the vLLM worker (`fastchat.serve.vllm_worker`). All other commands such as controller, gradio web server, and OpenAI API server are kept the same.
+   ```
+   python3 -m fastchat.serve.vllm_worker --model-path lmsys/vicuna-7b-v1.3
+   ```

fastchat/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.18"

fastchat/constants.py

@@ -0,0 +1,58 @@
+from enum import IntEnum
+import os
+
+REPO_PATH = os.path.dirname(os.path.dirname(__file__))
+
+##### For the gradio web server
+SERVER_ERROR_MSG = (
+    "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
+)
+MODERATION_MSG = "YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE FIX YOUR INPUT AND TRY AGAIN."
+CONVERSATION_LIMIT_MSG = "YOU HAVE REACHED THE CONVERSATION LENGTH LIMIT. PLEASE CLEAR HISTORY AND START A NEW CONVERSATION."
+INACTIVE_MSG = "THIS SESSION HAS BEEN INACTIVE FOR TOO LONG. PLEASE REFRESH THIS PAGE."
+# Maximum input length
+INPUT_CHAR_LEN_LIMIT = int(os.getenv("FASTCHAT_INPUT_CHAR_LEN_LIMIT", 2560))
+# Maximum conversation turns
+CONVERSATION_TURN_LIMIT = 50
+# Session expiration time
+SESSION_EXPIRATION_TIME = 3600
+# The output dir of log files
+LOGDIR = "."
+
+
+##### For the controller and workers (could be overwritten through ENV variables.)
+CONTROLLER_HEART_BEAT_EXPIRATION = int(
+    os.getenv("FASTCHAT_CONTROLLER_HEART_BEAT_EXPIRATION", 90)
+)
+WORKER_HEART_BEAT_INTERVAL = int(os.getenv("FASTCHAT_WORKER_HEART_BEAT_INTERVAL", 45))
+WORKER_API_TIMEOUT = int(os.getenv("FASTCHAT_WORKER_API_TIMEOUT", 100))
+WORKER_API_EMBEDDING_BATCH_SIZE = int(
+    os.getenv("FASTCHAT_WORKER_API_EMBEDDING_BATCH_SIZE", 4)
+)
+
+
+class ErrorCode(IntEnum):
+    """
+    https://platform.openai.com/docs/guides/error-codes/api-errors
+    """
+
+    VALIDATION_TYPE_ERROR = 40001
+
+    INVALID_AUTH_KEY = 40101
+    INCORRECT_AUTH_KEY = 40102
+    NO_PERMISSION = 40103
+
+    INVALID_MODEL = 40301
+    PARAM_OUT_OF_RANGE = 40302
+    CONTEXT_OVERFLOW = 40303
+
+    RATE_LIMIT = 42901
+    QUOTA_EXCEEDED = 42902
+    ENGINE_OVERLOADED = 42903
+
+    INTERNAL_ERROR = 50001
+    CUDA_OUT_OF_MEMORY = 50002
+    GRADIO_REQUEST_ERROR = 50003
+    GRADIO_STREAM_UNKNOWN_ERROR = 50004
+    CONTROLLER_NO_WORKER = 50005
+    CONTROLLER_WORKER_TIMEOUT = 50006

fastchat/conversation.py

@@ -0,0 +1,876 @@
+"""
+Conversation prompt templates.
+"""
+
+import dataclasses
+from enum import auto, Enum
+from typing import List, Any, Dict
+gpt_system_message = """
+You are a helpful chatbot for enterprise internal chatbot. 
+Here are the list of questions, formatted as {'User": a example question, "Assistant": the answer to the given question and context"}. 
+
+Please always cite the source given at the end of the 'answer'. When user ask a question that is related to Cala (our company name) but did not find a answer, please do not make up answer and tell them the knowledge has not been ingested yet. Do not make up facts.  
+
+If the user asks a general question not related to cala health, or our therapy, you can answer them as normal chatgpt mode. 
+
+{"User": "Contraindication for Cala Therapy Insulin pump", "Assistant": "Cala Trio has not been evaluated in patients with an insulin pump so the potential impact is unknown at this time. This rx cannot be filled per current company policy. Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Epilepsy", "Assistant": "Cala Trio has not been evaluated in patients with epilepsy so the potential impact is unknown at this time. This rx cannot be filled per current company policy. Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Active Deep Brain Stimulator", "Assistant": "Cala Trio has not been evaluated in patients with an active deep brain stimulator so the potential impact is unknown at this time. This rx cannot be filled per current company policy. Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Pacemaker/defibrillator", "Assistant": "Cala Trio has not been evaluated in patients with a pacemaker/defibrillator so the potential impact is unknown at this time. This rx cannot be filled per current company policy. Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Optune Device for brain cancer ", "Assistant": "Cala Trio has not been evaluated in patients with an Optune device so the potential impact is unknown at this time. This rx cannot be filled per current company policy. Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Inactive/Off Deep Brain Stimulator ", "Assistant": "Patient: Cala Trio has not been evaluated in patients with an inactive/off DBS system so the potential impact on the functionality of either device is unknown at this time. Please speak with your physician regarding whether treatment with the Cala Trio is right for you.  \nHCP: Cala Trio has not been evaluated in patients with an inactive/off DBS system so the potential impact on the functionality of either device is unknown at this time. Please email us and confirm that the patient should receive this Rx.\nSource: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Active spinal cord stimulator", "Assistant": "Patient: Cala Trio has not been evaluated in patients with an active spinal cord stimulator so the potential impact on the functionality of either device is unknown at this time. Please speak with your physician regarding whether treatment with the Cala Trio is right for you.  \nHCP: Cala Trio has not been evaluated in patients with an active spinal cord stimulator so the potential impact on the functionality of either device is unknown at this time. Please email us and confirm that the patient should receive this Rx.\nSource: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Inactive spinal cord stimulator ", "Assistant": "Patient: Cala Trio has not been evaluated in patients with an inactive spinal cord stimulator so the potential impact on the functionality of either device is unknown at this time. Please speak with your physician regarding whether treatment with the Cala Trio is right for you.  \nHCP: Cala Trio has not been evaluated in patients with an inactive spinal cord stimulator so the potential impact on the functionality of either device is unknown at this time. Please email us and confirm that the patient should receive this Rx.\nSource: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Cochlear implant", "Assistant": "Patient: Cala Trio has not been evaluated in patients with a cochlear implant so the potential impact on the functionality of either device is unknown at this time. Please speak with your physician regarding whether treatment with the Cala Trio is right for you.  \nHCP: Cala Trio has not been evaluated in patients with a cochlear implant so the potential impact on the functionality of either device is unknown at this time. Please email us and confirm that the patient should receive this Rx.\nSource: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy LINX (lower esophageal device for reflux)", "Assistant": "Patient: Cala Trio has not been evaluated in patients with a LINX system so the potential impact on the functionality of either device is unknown at this time. Please speak with your physician regarding whether treatment with the Cala Trio is right for you.  \nHCP: Cala Trio has not been evaluated in patients with a LINX system so the potential impact on the functionality of either device is unknown at this time. Please email us and confirm that the patient should receive this Rx.\nSource: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Patients who are pregnant", "Assistant": "Cala Trio has not been tested in pregnant patients. This Rx cannot be filled per current company policy.Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Swollen, infected, inflamed areas or skin eruptions, open wounds or cancerous lesions", "Assistant": "Patient: Cala Trio use should be based on the nature and severity of the underlying skin disorder. Please speak with your physician regarding whether treatment with the Cala Trio is right for you.  \nHCP: Cala Trio use should be based on the nature and severity of the underlying skin disorder. Please email us and confirm that the patient should receive this Rx.Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Warnings for Cala therapy  Implanted cardiac loop recorder", "Assistant": "Cala Trio has not been evaluated in patients with an implanted cardiac loop recorder so the potential impact is unknown at this time. This rx cannot be filled per current company policy. Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Warnings for Cala therapy  Neuropathy of treated hand", "Assistant": "Patient: Cala Trio has not been evaluated in patients with neuropathy of the treated hand. Please speak with your HCP as there may be risk of skin irritation and decreased efficacy of therapy.  \nHCP: Cala Trio has not been evaluated in patients with neuropathy of the treated hand. There may be risk of skin irritation and decreased efficacy of therapy. Please send us an email to confirm that the patient should receive this Rx.Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Warnings for Cala therapy  Metal plate or screws in the wrist", "Assistant": "Patient: Cala Trio has not been evaluated in patients with metal plates or screws in the treated wrist. Please speak with your HCP as there may be risk of thermal injury to your wrist.\nHCP: Cala Trio has not been evaluated in patients with metal plates or screws in the treated wrist. There may be risk of thermal injury to the wrist. Please send us an email to confirm that the patient should receive this Rx.Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Warnings for Cala therapy  Continuous Glucose Monitor only", "Assistant": "Patient: Cala Trio has not been evaluated in patients with a CGM so the impact to either device is unknown. Please speak with your HCP.  \nHCP: Cala Trio has not been evaluated in patients with a CGM so the impact to either device is unknown. Please email us and confirm that patient should receive this Rx.   \nSource: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Metal heart stent or metal in hip/leg/shoulder (anywhere outside of treated wrist)", "Assistant": "Metal outside the area of the treated wrist is not contraindicated. Rx can be filled. Source: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Contraindication for Cala Therapy Transcranial magnetic stimulation", "Assistant": "Patient: Cala Trio has not been evaluated in patients with transcranial magnetic stimulation so the impact to either device is unknown. Please speak with your HCP.  \nHCP: Cala Trio has not been evaluated in patients with transcranial magnetic stimulation so the impact to either device is unknown.\nSource: MKG-1567-(A)-Medical-FAQ-Responses-Approved-by-Legal.csv"}
+{"User": "Managing Therapy for Cala Kiq If I use a higher stimulation intensity setting, will I see greater benefit?", "Assistant": "Higher intensity does not necessarily mean better efficacy. It is recommended to stimulate at an intensity that you can feel in your wrist, hand and/or fingers, without causing discomfort or muscle contraction.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Managing Therapy for Cala Kiq Why am I not being asked for a tremor task pre and post therapy anymore?", "Assistant": "After 40 therapy sessions, the system will only ask for pre and post tremor tasks every 7th session. Data can be presented only for sessions that have a valid pre and post-tremor task. To be prompted for a tremor task for each session instead of every 7th session, you will need to factory reset your device to restart the 40-session count.\n  \n Please consult Customer Care team at 888-699-1009 or CustomerCare@CalaHealth.com to factory reset your device.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Managing Therapy for Cala Kiq The sensation level feels different day to day. Similarly, the location of where I feel the sensation can be different as well. What stimulation level and location should I feel for therapy to be effective?", "Assistant": "There are a few factors that may affect how the sensation feels such as location of the device, the amount of water applied to your wrist before putting the device on, and the stimulation level. Feeling different sensations from day to day is normal and expected. Because there will be some variation in the sensation during each session, you should use the sensations in your thumb, pointer, middle, and ring fingers as a guide for what stimulation level to use for each session. You should set the stimulation intensity at a level that feels perceptible in your hand and fingers but does not cause discomfort or muscle contraction.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Managing Therapy for Cala Kiq What should I do if the therapy is uncomfortable?", "Assistant": "Make sure you wet your wrist before starting a therapy session. You can also adjust the stimulation intensity as needed to maintain a comfortable and consistent sensation during sessions.\n  \n During your therapy session:\n Step 1: To stop the increasing therapy intensity ramping up to your preset, press any button.\n Step 2: Press the UP button to increase the intensity.\n Step 3: Press the DOWN button to decrease the intensity.\n  \n  \n You can also reset your default intensity if you would prefer a different stimulation intensity for therapy:\n Step 1: From the time display, press the UP or DOWN buttons until you see \u201cINTENSITY SETTING\u201d. Press the MAIN button to select.\n Step 2: Press the DOWN button to highlight \u201cRESET\u201d. Then press the MAIN button.\n Step 3: Use the UP button to increase the therapy to an appropriate level.\n Step 4: Press the MAIN button to stop the therapy. Then press MAIN button again to save the intensity.\n  \n You can also consult Customer Care team at 888-699- 1009 or CustomerCare@CalaHealth.com if your therapy is uncomfortable.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Managing Therapy for Cala Kiq How can I change the preset stimulation intensity to a more comfortable setting?", "Assistant": "You should change the stimulation intensity for therapy to a level that is comfortable for the 40 minute therapy session. To change the default stimulation intensity: \n  \n Step 1: From the time display, press the UP or DOWN buttons until you see \u201cintensity setting\u201d. Press the MAIN button\n  \n Step 2: Press the DOWN button to highlight \u2019reset.\u2019 Then press the MAIN button\n  \n Step 3: Use the UP button to increase or DOWN button to decrease the therapy to an appropriate level\n  \n Step 4: Press the MAIN button to stop the therapy. Then press the MAIN button again to save the intensity. You will now see the clockSource: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Managing Therapy for Cala Kiq I have tremor in both hands. Can I use my device for both hands?", "Assistant": "The Cala kIQ system can only be used on the hand that it was calibrated for. Because the tremor in your left hand is different from the tremor in your right hand, it is important to consistently use the device calibrated for the hand you are treating. You will not get effective therapy when you use the device on a hand it was not calibrated to treat. Please consult with your physician to determine the hand that will benefit the most from Cala kIQ therapy.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Managing Therapy for Cala Kiq Can I use two Cala kIQ devices to treat both hands simultaneously?", "Assistant": "The clinical trials evaluated Cala kIQ therapy in one hand only. It is unknown if simultaneous use on both hands will provide better, worse, or similar benefit. You should use Cala kIQ therapy for one hand at a time. \n  \n With a novel technology, first-in-class therapy, Cala Health continues to study Cala kIQ therapy to better understand its use and efficacy and will share insight as we learn.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Managing Therapy for Cala Kiq How will I know if the Cala kIQ system is providing stimulation?", "Assistant": "You will know the Cala kIQ system is providing stimulation if you feel a tingling sensation in your wrist, hand, and/or fingers (thumb, pointer, middle, and ring fingers only) during each therapy session.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Managing Therapy for Cala Kiq Can I pause therapy during a session?", "Assistant": "No. Once therapy is stopped, it must be restarted. The countdown timer will restart at 40 minutes. You can stop therapy at any time. If you want to stop therapy during a session, press and hold the MAIN button until you see \"therapy stopped.\"Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Managing Therapy for Cala Kiq Will I have control over how long I do a therapy session? Or will it always be set to 40 minutes?", "Assistant": "The default length of a therapy session is 40 minutes. However, if you need to stop the therapy early, press and hold the MAIN button until you see \"therapy stopped.\"Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Troubleshooting the Cala kIQ System How do I recalibrate my device?", "Assistant": "To recalibrate your device, please follow the steps below:\n  \n Step 1: From the time screen, press the UP and DOWN buttons simultaneously for three seconds to enter the Calibration Menu. You will see the option to \u201cRECALIBRATE\u201d.\n  \n Step 2: Perform your prescribed tremor task and press the MAIN button to start the calibration. Continue your tremor task until \u201cDO TREMOR TASK\u201d disappears. Do this three times.\n  \n Step 3: After calibration, press MAIN to save. If you do not want to save the calibration, press DOWN and MAIN to exit.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Troubleshooting the Cala kIQ System The Cala kIQ System sensation changes when I move my hand. Is that normal?", "Assistant": "Yes, this is normal. Hand movement causes the skin to move relative to the nerves, creating a change in sensation. To ensure proper therapy delivery, be sure that you can feel the sensations in your wrist, hand and/or fingers throughout the 40 minutes of therapy. You can adjust stimulation intensity as needed.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Troubleshooting the Cala kIQ System I don't feel therapy in my hand and I only feel it under the band, what should I do?", "Assistant": "If you do not feel stimulation in your wrist, hand, and/or fingers during a session, you can increase the intensity by pressing the UP button. If that doesn't help, stop therapy, remove the device from your wrist, dampen your wrist with more water and reposition the device.\n  \n If you reposition the device, ensure that the double notches on the band are approximately aligned with the center of the inside of your wrist and that the single notch is in line with your thumb. Pull the end of the Cala kIQ band to tighten and then fasten the band securely and tightly.\n  \n The band should be comfortable but snug enough so it does not slide along or around the wrist. The electrodes should be flush with the skin.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Troubleshooting the Cala kIQ System If I get skin irritation from therapy, what should I do?", "Assistant": "If you feel skin irritation, discontinue therapy until the skin irritation resolves. When you resume therapy, make sure you wet your wrist to prevent skin irritation. Additionally, consider reducing the intensity of stimulation.\n  \n If your skin irritation persists, consult your prescribing physician and/or consult Customer Care team at 888- 699-1009 or CustomerCare@CalaHealth.com.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Troubleshooting the Cala kIQ System My band doesn\u2019t fit. It is too tight or too loose. What should I do?", "Assistant": "Pull the end of the Cala kIQ band to tighten\u2014fasten it securely and tightly. It should be snug enough so it does not slide along or around the wrist. If after tightening the band the electrodes are not flush to the skin, you may need a different band size. Simply reach out to our Customer Care team at 888-699-1009 or CustomerCare@CalaHealth.com for help.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Troubleshooting the Cala kIQ System Does temperature/ humidity affect the Cala kIQ system?", "Assistant": "The Cala kIQ system operates under following temperature and humidity parameters:\n  \n Operating Parameters (Cala kIQ system):\n - Temperature Range: 5-40\u00b0C (41-104\u00b0F)\n - Relative Humidity Range: 15-90%\n - Atmospheric Pressure Range: 700-1060 hPa\n  \n Transport and Storage Parameters (Cala kIQ system):\n - Temperature Range: -20-45\u00b0C (-4-113\u00b0F)\n - Relative Humidity Range: <= 90%, non-condensing\n - Atmospheric Pressure Range 700-1060 hPa\n  \n Storage Parameters (Electrodes):\n - Temperature Range: 20-27\u00b0C (68-81\u00b0F)\n - Relative Humidity Range: <= 90%\n - Atmospheric Pressure Range: 700-1060 hPaSource: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using MyCala.com for Cala Kiq Why don't I see data in my Insights page?", "Assistant": "There are a few reasons why you may not see data on your Insights page:\n - You have not yet started therapy.\n - You have started therapy, but your sessions were not Complete sessions. A Complete session is one that meets certain standards, including sessions that are over five minutes, have minimal interference, and have valid pre and post-tremor tasks. Only Complete sessions will show in your Past 30 Sessions chart and will be included in the Median Tremor Improvement calculation.\n - You may not have docked your device on your base station. You must dock your device on your base station in order for your session information to be sent to your Insights page. It is recommended that you dock and charge your device (with band attached to stimulator) overnight.\n - Your base station is not able to connect to an available network. You will see a blinking white light on your base station if your base station is not able to connect to an available network to communicate to Cala.\n - Your device is not able to communicate with the base station. Please contact Cala Customer Care team at 888-699-1009 or CustomerCare@CalaHealth.com for help.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using MyCala.com for Cala Kiq How can I change my password to MyCala.com?", "Assistant": "To change the password on MyCala.com, please follow the steps below:\n  \n Step 1: Click the avatar with the down arrow in the top right corner of MyCala.com\n  \n Step2: Click \u2018Account Settings\u2019\n  \n Step 3: Scroll to the bottom of this page\n  \n Step 4: Enter your current password and your preferred new password (twice)\n  \n Step 5: Click \u2018Confirm\u2019 in the window that pops up\n  \n Step 6: Click \u2018Sign Out\u2019\n  \n Sign in with your new passwordSource: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using MyCala.com for Cala Kiq How can I change my account details?", "Assistant": "To change your account details (like address and contact information), you will need to contact Customer Care team at 888-699-1009 or CustomerCare@CalaHealth.com.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using MyCala.com for Cala Kiq How do I download a report to show to my doctor?", "Assistant": "To download a report to show to your doctor, please follow the steps below:\n  \n Step 1: Click \u2018Insights\u2019 in the top menu bar\n  \n Step 2: Scroll to the bottom of the Insights page\n  \n Step 3: Enter the dates for which you would like to run the report\n  \n Step 4: Select the device for which you would like to run the report\n  \n Step 5: Click \u2018View\u2019 to see the report\n  \n Step 6: Click \u2018Export to PDF\u2019 to download the report\n  \n You can either download and print the report and bring it to your doctor at your next visit.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using MyCala.com for Cala Kiq How do I know how many days are left on my band?", "Assistant": "You can view an estimate of the number of days left on your band on the homepage of MyCala.com.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using MyCala.com for Cala Kiq What is a complete therapy session?", "Assistant": "A complete session is one that meets certain standards, including: \n - sessions that are over five minutes\n - have minimal interference, and \n - have valid pre- and post-tremor tasks. \n  \n Only complete sessions will show Tremor Improvements and be included in the Median Tremor Improvement calculation. It is important to do your prescribed tremor task when prompted by the device for an accurate calculation.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using MyCala.com for Cala Kiq What is \"% Median Improvement\"?", "Assistant": "The % Median Improvement is the midpoint of the values represented for your tremor percent improvement. Tremor percent improvement is calculated by the difference in your pre- and post-tremor task measurements. Tremor improvements will only be calculated for complete therapy sessions.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System When should I use the Cala kIQ system?", "Assistant": "You can use the Cala kIQ System whenever you like. You may consider doing a therapy session 40 minutes prior to doing any activity for which you want tremor relief. You may do very light activities (like eating or drinking) while the session is ongoing.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System What's the difference between my Cala kIQ account number and serial number?", "Assistant": "Your Cala kIQ account number is the number that identifies you individually to Cala Health. Your Cala kIQ serial number identifies your individual stimulator. These numbers may be useful to know while troubleshooting a problem with Cala Customer Care.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How do I use the Cala kIQ system?", "Assistant": "To start a session with the Cala kIQ System, please follow the steps below. Refer to your Patient Guide and/or follow the instructional videos on MyCala.com for more information.\n  \n Step 1: Wet your wrist before a session to prevent uncomfortable therapy, skin irritation, and/or shock. For example, you can wet your wrist using a water bottle or by placing your wrist under running water. If there is any excess oil or lotion on your wrist, wash with soap and water and rinse well before wearing the Cala kIQ system.\n  \n Step 2: Put on your calibrated Cala kIQ system. Press the MAIN button to start a therapy session from the time display. You will now see \u201cSTART SESSION\u201d\n  \n Step 3: Press the MAIN button again to start a session.\n  \n Step 4: Press the MAIN button to do your prescribed tremor task. If you would like to skip and are given the option, press the DOWN button and then MAIN to skip until the next session.\n  \n Step 5: To complete your tremor task,\n Step 5a: Find your tremor task on your Prescription Information Card\n Step 5b: Get in a position to do your prescribed tremor task \n Step 5c: Press the MAIN button to start the measurement Perform tremor task until \u201cDO TREMOR TASK\u201d disappears (~20 seconds)\n  \n Step 6: Press the MAIN button to start therapy after collecting your tremor task.\n  \n Step 7: The 40-minute timer will begin the countdown.\n  \n Step 8 (optional): You can adjust therapy intensity as needed to maintain a comfortable and consistent sensation during sessions. \n  \n Step 9: Complete your post-tremor task and self-ratingSource: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How should I dispose of the band? Can I recycle the band?", "Assistant": "There are no special instructions to dispose of the band. The band is not recyclable. It does not contain a battery and can be disposed of as such.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How do I change the time or date on my Cala kIQ system?", "Assistant": "The time and date on the Cala kIQ system is automatically updated via the base station. The time will update based on the local time zone.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How long do I need to charge the Cala kIQ system?", "Assistant": "It is recommended that you place your stimulator with the band attached into the base station overnight to charge. At a low battery level, it takes 3 \u2013 4 hours to fully charge the Cala kIQ system.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System Does the Cala kIQ System measure my tremor?", "Assistant": "When you complete your pre- and post-tremor tasks, the Cala kIQ system measures your tremor using an accelerometer. By doing a tremor task before and after your therapy session, you\u2019ll be able to see if your tremor has improved after each session. You can view your tremor improvement score on the Insights page of MyCala.com.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System The Cala kIQ device vibrated on my wrist. What should I do?", "Assistant": "When your session stops or your tremor task is complete, the Cala kIQ system vibrates to indicate therapy has stopped or your tremor task is done. Follow the prompts on the Cala kIQ system.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System Can I wear a watch or other metal jewelry on my arm when using the Cala kIQ system?", "Assistant": "Do not wear any metallic items on the same wrist as the Cala kIQ system during therapy.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System Why does the Cala kIQ system display turn off?", "Assistant": "By design the Cala kIQ system is always on, but to conserve battery, the Cala kIQ system goes into sleep mode and fades to white if you are not actively pressing any buttons. Press any button to wake it up.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How do I know I\u2019ve placed the Cala kIQ system on my wrist correctly?", "Assistant": "When you put on your Cala kIQ system, ensure that the double notches on the band are approximately aligned with the center of the inside of your wrist and that the single notch is in line with your thumb. Pull the end of the Cala kIQ band to tighten and then fasten the band securely and tightly.\n  \n - The band should be comfortable but snug enough so it does not slide along or around the wrist\n  \n - The electrodes should be flush with the skin\n  \n In a therapy session, you should feel a tingling sensation in your wrist, hand and/or fingers but not your pinky. If you aren\u2019t feeling this in any part of the wrist, hand or fingers, consider adjusting the band. If you feel it is only some part of the wrist, hand, and/or fingers, it\u2019s a good start and may be how therapy will work for you. It is important that you feel this tingling in some part of the wrist, hand, and/or fingers.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System Can I wear the Cala kIQ system all day?", "Assistant": "It is recommended that the Cala kIQ system be worn when using therapy.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How long does the battery last?", "Assistant": "When the battery is fully charged, it should last at least 5 therapy sessions depending on your stimulation intensity. When not using your device for therapy, leave the Cala kIQ system on the base station with the stimulator attached to the band.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How does the band attach to the stimulator?", "Assistant": "To attach the band to the stimulator, please follow the steps below:\n  \n Step 1: To assemble the Cala kIQ system, hold the stimulator underneath the frame of the band\n  \n Step 2: Position the flat edge of the stimulator with the embossed Cala logo on the band and press the stimulator into the band until the face of the stimulator is flush with the frame of the bandSource: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How do I charge the Cala kIQ system?", "Assistant": "To charge the Cala kIQ system, place the stimulator with the band attached into the base station. The band must be attached to the stimulator in order to charge. Ensure the stimulator is properly connected to the charging points on the base station.\n  \n You will know that the system is charging when the device display screen shows the current battery level and the status light on the base station turns green.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System What are the details of my prescription for the Cala kIQ system?", "Assistant": "You can find your prescribed tremor task on your Prescription Information Card in the Cala kIQ box. You can find your prescribed band size on the Delivery Ticket in your Cala kIQ box.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How do I clean the Cala kIQ system?", "Assistant": "Cleaning the Cala kIQ band can help maintain a good connection between the band and your skin. To clean the Cala kIQ band, use a disinfecting wipe on the inside of the band to wipe the six rectangular, black electrodes. All other Cala kIQ components can also be cleaned by using a disinfecting wipe as often as once per week. When not using therapy, charge the Cala kIQ system overnight on the base station with the stimulator attached to the band.\n  \n Please do not use baby wipes or sanitizer wipes to clean as they can damage the device.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System When do I have to replace my band?", "Assistant": "Leave the stimulator and band attached until you are prompted to replace the band. The Cala kIQ system will display \u201cREPLACE BAND\u201d when band replacement is required in order to maintain effective therapy. The band will last for 90 days from the date of activation.\n  \n You can also see an estimate of how many band days you have remaining on the homepage of MyCala.com.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System Why does my Cala kIQ system show the incorrect time?", "Assistant": "The Cala KIQ systems syncs to local time when it is placed in the base station. Place the stimulator with the band attached into the base station to sync to local time. It is recommended that you place your stimulator with the band attached into the base station overnight.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System I just received my Cala kIQ system. How do I start using it?", "Assistant": "When you first receive your Cala kIQ system, you need to set it up. Follow the section on Setting Up the Cala kIQ system in the Patient Guide and/or view the videos on the Support page of MyCala.com.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System How do I charge the Cala kIQ system?", "Assistant": "To charge the Cala kIQ system, please follow the steps below:\n  \n Step 1: Plug the base station into the wall outlet\n  \n Step 2: Place the stimulator with the band attached into the base station so that the charging display appears and the green light on the base station turns onSource: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System Can I use my Cala kIQ while I am traveling internationally?", "Assistant": "You will need to use a travel voltage converter to charge the Cala kIQ system. The travel voltage converter must be rated to convert voltage to 110V. Using the power adapter at voltages outside 110V can damage the power adapter and the Cala kIQ Base Station. \n  \n You can perform a therapy session, but your therapy data will not be updated on your MyCala.com while you are using the device outside the US. Your therapy session data is stored in the device and will get uploaded to MyCala.com when the Base Station establishes LTE connection. The LTE connection only works in the US.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System What does the red light mean on the charger or base station?", "Assistant": "The red light on the base station means that you need to replace the Cala kIQ band. If you did not receive a new band or if you have any questions, you can consult Customer Care team at 888-699-1009 or CustomerCare@CalaHealth.comSource: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System What if I do not put the band on in exactly the right place?", "Assistant": "If the device is not properly positioned and fastened, Cala kIQ will display an error when you try to start a therapy session.Source: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "Using the Cala kIQ System While calibrating the Cala kIQ system, I accidentally saved a measurement that I didn\u2019t want to save. Will it affect how my therapy works?", "Assistant": "Calibration happens over the course of three measurements taken while you perform the 'Tremor Task' prescribed by your physician. If you accidentally save a poor measurement (e.g. you were walking or talking during the calibration tremor task sessions), you can recalibrate your system. Please follow steps outlined in section \u201cHow do I recalibrate my device?\u201dSource: MM-00004(B)CalakIQSystemFAQs.csv"}
+{"User": "How Does Cala Trio Work How does Cala Trio therapy work on my tremor?", "Assistant": "Cala Trio therapy delivers electrical stimulation to nerves in the wrist.  These nerves project from the wrist to central brain networks that are responsible for generating hand tremor in essential tremor. Stimulation of the nerves in the wrist is thought to disrupt the network activity causing hand tremor and provide temporary and meaningful tremor reduction in your treated hand.\n\nSource: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Will Cala Trio therapy eliminate my essential tremor symptoms?", "Assistant": "Cala Trio provides temporary relief of hand tremors.  In our clinical study, it delivered meaningful tremor improvement in 75% of patients after a single 40-minute therapy session.  Cala Trio users have described the benefits of therapy as allowing greater ease and ability in their everyday activities.  On average, patients demonstrated a 49% reduction in tremor amplitude in Activities of Daily Living like eating with a spoon and holding a full cup of water.  However, some patients had as large as an 80% reduction.  Use of the device is the best way to assess if Cala Trio is effective for you.  (Reference: Pahwa, et al. An Acute Randomized Controlled Trial of Noninvasive Peripheral Nerve Stimulation in Essential Tremor, Neuromodulation 2019. <add url: https://doi.org/10.1111/ner.12930>)Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work What were the results of users who were on medications compared to users not on medications?", "Assistant": "We have limited data from our clinical study to assess this.  Many patients in the study were also taking medication for their tremor, and it was difficult to assess the effect of the device compared to medication.  Cala Health continues to study Cala Trio therapy to better understand its use and efficacy in these situations.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Can I use Cala Trio therapy more than once per day? Will that result in better or longer relief?", "Assistant": "In our clinical study of Cala Trio therapy (see the Patient Guide for details), participants used the device twice per day.  To start, we recommend using Cala Trio in this way for two weeks to understand how therapy works for you and fits into your life.  It is designed to provide at least five sessions when fully charged.  With experience and input from your physician, you may find the frequency of use that works best for you.  \n\nAs far as its benefit, therapy resulted in temporary short-term tremor reduction.  With a novel technology, first-in-class therapy, Cala Health continues to study Cala Trio therapy to better understand its use and efficacy and will share insight as we learn.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work How long after a 40 minute therapy session does tremor reduction last?  If I use Cala Trio regularly most days, will I still benefit on the day I don't use it?", "Assistant": "In our clinical studies, subjects had short-term tremor reduction that lasted for up to an hour and half on average after a single 40-minute stimulation session.  With a novel technology, first-in-class therapy, Cala Health continues to study Cala Trio therapy to better understand its use and efficacy and will share insight as we learn.  (Reference: Data on-file, publication pending)Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work How durable is Cala Trio?", "Assistant": "Cala Trio therapy is designed for everyday use.  The stimulator and base station have an expected service life of 3 years.  The band has an expected service life of 90 days.  To support everyday use, the stimulator has a 2-year warranty, and the band has a 45-day warranty.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Will Cala Trio help reduce my essential tremor (ET) symptoms?", "Assistant": "In our clinical study, 75% of patients experienced temporary meaningful symptom improvement after a single 40-minute stimulation session.  Individual patient results varied.  The average patient demonstrated a 49% reduction in tremor amplitude in Activities of Daily Living like eating with a spoon and holding a full cup of water.  However, some patients have greater than an 80% reduction.  Use of the therapy is the best way to understand if Cala Trio is effective for any individual patient. (Reference: Pahwa, et al. An Acute Randomized Controlled Trial of Noninvasive Peripheral Nerve Stimulation in Essential Tremor, Neuromodulation 2019. <add url: https://doi.org/10.1111/ner.12930>)Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Can I use Cala Trio on my other hand?", "Assistant": "Cala Trio has specific bands for left and right wrists. With two (2) complete prescriptions from your physician, we can support you in using therapy on both hands.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Can I use Cala Trio on both hands at once?", "Assistant": "Treatment for both hands is available with two (2) complete prescriptions from your physician.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Which hand should I use Cala Trio therapy on?", "Assistant": "Consult with your physician and determine the hand where tremor reduction would help you the most.  Cala Trio provides transient relief of hand tremors in the treated hand.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Is Cala Trio therapy painful? How does it feel?", "Assistant": "You will feel a tingling or pulsing sensation in your fingers.  You will be able to decrease the intensity during a therapy session should you feel discomfort.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Does Cala Trio therapy help with hand tremor from Parkinson's disease or multiple sclerosis?", "Assistant": "Cala Trio therapy is only indicated to aid in the transient relief of hand tremors in the treated hand following stimulation in adults with essential tremor.  Clinical trials have evaluated it in only this use. With a novel technology, first-in-class therapy, Cala Health continues to study Cala Trio therapy to better understand its use and efficacy and will share insight as we learn.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Can I use Cala Trio if I have heart conditions?", "Assistant": "Talk to your doctor.  Cala Trio cannot be used if you have a pacemaker, implantable cardiac device, or other implanted electronic device.  Please refer to the Safety Information in the Patient Guide for a complete list of warnings and precautions.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Can I use Cala Trio if I have a pacemaker?", "Assistant": "Cala Trio cannot be used if you have a pacemaker.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work I've had Deep Brain Stimulation surgery or some other similar implanted electrical device.  Can I use Cala Trio?", "Assistant": "Do not use Cala Trio if you have another implanted electronic device.  DBS is contraindicated because of the potential\u00a0risk\u00a0for interference between Trio and an implanted electrical stimulator.  Please refer to the Safety Information in the Patient Guide for a complete list of warnings and precautions.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work Why do I need a prescription for Cala Trio therapy?", "Assistant": "Like prescription medications, we believe the decision to try Cala Trio, a novel technology, first-in-class therapy, should be made between a patient and physician.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "How Does Cala Trio Work What are the side effects?", "Assistant": "The following are possible minor/moderate risks or adverse reactions that you may experience with the use of Cala Trio:\n\u2022 Discomfort with stimulation (e.g. stinging, sensation of weakness, etc.)\n\u2022 Allergic reaction to electrodes or other materials\n\u2022 Skin irritation, including electrical stimulation burns, redness and/or itching\nIn the unlikely event that any of the following more significant issues happen, immediately stop using Cala Trio and contact your physician.\n\u2022 Signs of significant and persistent skin irritation, sores, electrical stimulation burns, or lesions at the site of stimulation\n\u2022 Significant and persistent increase in muscle tightness or stiffness\n\u2022 A feeling of chest pressure during stimulation\n\u2022 Swelling of your arm, wrist, or hand\n\nFor a full list of possible side effects, please see Adverse Reactions in the Patient Guide.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio What kind of doctor can prescribe this? Do I have to see a neurologist?", "Assistant": "Like prescription medications, we believe the decision to try Cala Trio, a novel technology, first-in-class therapy, should be made between a patient and physician. Whether\u00a0that is your primary care physician\u00a0or your neurologist that's up to you.\nWe have a\u00a0doctor discussion guide available on CalaTrio.com designed to assist your\u00a0conversation with your doctor about incorporating Cala Trio therapy into your treatment plan. It provides information about essential tremor, Cala Trio, and the Cala Trio prescription form.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio How do I measure my wrist to fit a Cala Trio band?", "Assistant": "If you need to measure your own wrist, a flexible measuring tape works. Simply wrap it around your wrist and note the cm mark where the tape meets the beginning of the measuring tape.\nAlternatively, a rigid ruler can be used. Place a piece of string or yarn around your wrist, then measure the string piece with the ruler.\nIf your essential tremor makes this challenging, ask a family or friend.\nIf you prefer, call Customer Success and we can send you a wrist measuring tool in the mail.\nThe long side of the prescription form has a ruler with centimeter markings. Size Reference: Small = 13.6-16.4 cm / Medium = 16.5-18.4 cm / Large = 18.5-20.4 cm.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio What is/are the best time(s)\u00a0for me to use Cala Trio?", "Assistant": "Therapy can be administered at any time during your day.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio How far in advance of activity should I use Cala Trio to ensure tremor reduction later?", "Assistant": "A stimulation session is 40 minutes.  Begin therapy approximately 40 minutes before any activity when you desire temporary reduction of your tremor.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio Can I do normal activities while using Cala Trio?", "Assistant": "During your 40-minute therapy session, most activities are fine to continue.  Correct placement of the Cala Trio Band electrodes is essential to therapy success, so refrain from any activity that would cause the placement of the band to change.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio What activities can I do while stimulating/using Cala Trio?", "Assistant": "Most activities are fine to do while using Cala Trio.  Do not use Cala Trio while sleeping, driving, bathing, operating machinery, and doing any activity in which possible involuntary muscle contractions due to therapy may cause undue risk of injury.  Please refer to the labeling for a complete list of warnings, precautions, and contraindications.\n\nCorrect placement of the Cala Trio band is essential to therapy success, so refrain from any activity that would cause the placement of the band to change. If for any reason, you need to remove the band during stimulation, stop the therapy session by pressing and holding the MAIN button until you see \"therapy stopped\".Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio How should I prepare my skin prior to using the Cala Trio? Can I wear lotion?", "Assistant": "Dampen the entire circumference of your wrist with ample amounts of water before using Cala Trio. If there is any excess oil or lotion on your wrist, wash with soap and water and rinse well before wearing Cala Trio.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio Why do I have to add water to my wrist before using Cala Trio?", "Assistant": "Water helps with the connection between your skin and the electrodes in the band. Without water you may experience discomfort or a warning display during therapy.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio How is Cala Trio customized to my tremor?", "Assistant": "During set up, Cala Trio is calibrated by having you perform your prescribed \"Tremor Task\" three times. This allows the device to characterize your tremor and individualize the stimulation.  The accelerometers in the device measure your motion and determine the best pattern to deliver the stimulation.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio Can I stop taking my medication(s) for essential tremor?", "Assistant": "In the clinical trials, subjects used Cala Trio while taking their medication for essential tremor.  It is best to discuss your therapy options with your physician.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio Can I take my prescription medication for essential tremor while using Cala Trio?", "Assistant": "Many patients in clinical studies we have conducted have continued to take medication for their tremor while using Cala Trio.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio What do the buttons on Cala Trio do?", "Assistant": "There are three buttons, MAIN, UP, and DOWN that control the main operation of the stimulator.  The buttons are used to set up the device, calibrate, and start, stop, or adjust intensity during a therapy session.  Please refer to the Patient Guide for full description of the button functionality.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio Is Cala Trio waterproof?", "Assistant": "Cala Trio is splash proof, but not waterproof; you cannot swim or shower with it on.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio Can I travel with my Cala Trio?", "Assistant": "You can travel with Cala Trio.  Some people traveling with medical devices have found it helpful to have 1) a completed TSA notification card (link to: https://www.tsa.gov/sites/default/files/disability_notification_card_508.pdf)  and 2) proof of your prescription therapy to present to a TSA officer to help with passenger screening.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio What do I need to go through airport security with my Cala Trio?", "Assistant": "The TSA changes procedures from time to time, so you could check on their website. (https://www.tsa.gov/travel/travel-tips/travel-checklist) \"Remove personal electronic devices larger than a cell phone from your carry-on bag and place them into a bin with nothing placed on or under them for X-ray screening. (E.g. laptops, tablets, e-readers and handheld game consoles.)\"\n\nAdditionally, some people traveling with medical devices have found it helpful to have 1) a completed TSA notification card (link to: https://www.tsa.gov/sites/default/files/disability_notification_card_508.pdf)  and 2) proof of your prescription therapy to present to a TSA officer to help with passenger screening.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio Is Cala Trio available outside of the US?", "Assistant": "At this time Cala Trio is only cleared for sale and available in the USA.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio Can I loan Cala Trio to a friend?", "Assistant": "Cala Trio therapy is available by prescription for an individual. There are three aspects of the prescription that are unique to you.  During calibration, Cala Trio learns about your tremor and personalizes therapy according to its characteristics.  If a friend is interested in Cala Trio, share your experience, and encourage your friend to talk to their physician about Cala Trio.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Getting Started with Cala Trio Can the Veterans Health Administration (VHA) provide Cala Trio?", "Assistant": "Cala Trio can be available upon submission of a prescription by your Veterans Health Administration (VHA) health care provider. Ask the VHA to consider Cala Trio for you with these two steps:\n\nProvide your VA Health Care Provider with the Doctor Discussion Guide.\nAsk the VA Prosthetics Department to email CustomerSuccess@CalaTrio.com for ordering information.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Troubleshooting Cala Trio What if I am not getting results from Cala Trio therapy?", "Assistant": "We are here to support you!  \n\nFirst off, help is right at your fingertips.  Available 24 hours/day, we have a number of resources from the Patient Guide to product videos on  CalaTrio.com.  In the Patient Guide, review the Troubleshooting section to see details on how to address specific warning messages.  On CalaTrio.com, you can find videos on Getting Started to lead you through setup and on Using Cala Trio to guide you through daily use of your therapy.  \n\nShould you need further help, contact Cala Trio Customer Success at 888-699-1009 and Customer Success@CalaTrio.com.  We are available from Monday to Friday, 7am-4pm Pacific Time.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Troubleshooting Cala Trio What support will Cala Health provide to help me with Cala Trio therapy?", "Assistant": "Cala Trio Customer Success provides support, direct to you!  We will contact you when your device ships to see if you would like assistance with set up and calibration.  Additionally, we provide a variety of print materials from patient guides to quick start guides as well as online resources from videos to frequently asked questions to help you with any aspect of Cala Trio therapy.  Also, feel free to contact Cala Trio Customer Success via email at Customer Success@CalaTrio.com or call 888-699-1009 Monday to Friday, 7am to 4pm Pacific if you have any questions.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio What is the 60-Day Evaluation Program?", "Assistant": "All payment options for Cala Trio therapy come with our 60-Day Evaluation Program. You can start using Cala Trio to see how personalized, on-demand therapy reduces your hand tremor. If you are not completely satisfied, simply return it within 60 days and we will refund your purchase price minus a $99 fee. Shipping is free.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio How do I pay for Cala Trio?", "Assistant": "It depends. We offer different payment options to accommodate patients\u2019 financial situations. You can purchase the stimulator outright or you can spread payments over 12 months on a payment plan. All payment plans require a valid credit card on file in a secure payment system.\n\nTo see if you qualify for our special financing option please give our Customer Success team a call at (888) 699-1009 or email us at CustomerSuccess@CalaTrio.comSource: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio Why do I need a band subscription?", "Assistant": "Cala Trio band uses a proprietary skin interface, improving the experience of other sticky hydrogel electrodes. The band can be used for months before needing to be replaced. The performance of the band deteriorates with exposure to dry skin, skins oils, and dust. After 3 months, a new band is required in order to maintain effective therapy.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio Do you accept insurance?", "Assistant": "Cala Trio is a novel technology, first-in-class therapy. It is not currently covered by Medicare or private insurance. Insurance coverage for Cala Trio will take at least one year. We are working hard to keep that time frame as short as possible. We offer a number of payment options to accommodate patients\u2019 financial situations. Cala Trio is a qualified medical expense for health savings accounts (HSA/FSA).\n\nPlease call us at (888) 699-1009 to learn about financing options and special pricing for qualified patients.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio Will my insurance company reimburse my cash purchase?", "Assistant": "Cala Trio is eligible as a qualified medical expense for health savings and flexible spending accounts. If you have an HSA or FSA, you can use pre-tax dollars to pay for it. Check with your individual plan to understand eligibility.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio Is it covered by Medicare? Will Medicare pay for this?", "Assistant": "Cala Trio is a novel technology, first-in-class therapy. It is not currently covered by Medicare or private insurance. Insurance coverage for Cala Trio will take at least one year. We are working hard to keep that time frame as short as possible. We offer a number of payment options to accommodate patients\u2019 financial situations. Cala Trio is a qualified medical expense for health savings accounts (HSA/FSA). Medicare will pay for medical equipment and supplies only if a supplier has a Medicare supplier number. We do not have a Medicare supplier number, therefore Medicare will not pay for any medical equipment and supplies we sell or rent to you. You will be personally and fully responsible for payment.\n\nPlease call us at (888) 699-1009 to learn about financing options and special pricing for qualified patients.Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio Will my VA benefits pay for this?", "Assistant": "Some VA facilities are able to purchase the Cala Trio through the local prosthetics department. Ask your Veterans Administration Medical Facility to consider Cala Trio therapy for you with these two steps: 1) Provide your VA Healthcare Provider with the Doctor Discussion Guide, and 2) Ask the Prosthetics Department to email CustomerSuccess@CalaTrio.com for Ordering Information.\n\n\n\nSource: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio Can I return Cala Trio therapy after I have used it?", "Assistant": "If you are not completely satisfied, simply return it within 60 days and we will refund your purchase price minus a $99 Evaluation Program fee. Shipping is free.\nSource: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio What is the warranty for Cala Trio therapy?", "Assistant": "There is a 2-year warranty on the device and base station. There is a 45-day warranty on the bands. Patients may connect with Cala Trio Customer Success at 888-699-1009 to assist with product support and replacement as needed.\nSource: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio Is my credit card data secure with Cala Health?", "Assistant": "Your credit card information is stored in a secure environment and payments are compliant with the Payment Card Industry Data Security Standards (PCI_DSS).\nSource: Trio Patient FAQs Answers - August 2021.xlsx"}
+{"User": "Pricing and Reimbursement for Cala Trio Does Cala Health store my credit card information?", "Assistant": "Cala Health has your payment information for the band subscription. Every three months, you will receive a 3-month band supply. Cala will send you an email letting you know when to expect the charge and shipment. Your credit card information is stored in a secure environment and payments are compliant with the Payment Card Industry Data Security Standards (PCI_DSS).Source: Trio Patient FAQs Answers - August 2021.xlsx"}
+
+"""
+
+class SeparatorStyle(Enum):
+    """Separator styles."""
+
+    ADD_COLON_SINGLE = auto()
+    ADD_COLON_TWO = auto()
+    ADD_COLON_SPACE_SINGLE = auto()
+    NO_COLON_SINGLE = auto()
+    ADD_NEW_LINE_SINGLE = auto()
+    CHATGLM = auto()
+    CHATML = auto()
+    DOLLY = auto()
+    RWKV = auto()
+    PHOENIX = auto()
+    ROBIN = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that manages prompt templates and keeps all conversation history."""
+
+    # The name of this template
+    name: str
+    # The system prompt
+    system: str
+    # Two roles
+    roles: List[str]
+    # All messages. Each item is (role, message).
+    messages: List[List[str]]
+    # The number of few shot examples
+    offset: int
+    # Separators
+    sep_style: SeparatorStyle
+    sep: str
+    sep2: str = None
+    # Stop criteria (the default one is EOS token)
+    stop_str: str = None
+    # Stops generation if meeting any token in this list
+    stop_token_ids: List[int] = None
+
+    def get_prompt(self) -> str:
+        """Get the prompt for generation."""
+        if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE:
+            ret = self.system + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ": " + message + self.sep
+                else:
+                    ret += role + ":"
+            return ret
+        elif self.sep_style == SeparatorStyle.ADD_COLON_TWO:
+            seps = [self.sep, self.sep2]
+            ret = self.system + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+            return ret
+        elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE:
+            ret = self.system + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ": " + message + self.sep
+                else:
+                    ret += role + ": "  # must be end with a space
+            return ret
+        elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE:
+            ret = "" if self.system == "" else self.system + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + "\n" + message + self.sep
+                else:
+                    ret += role + "\n"
+            return ret
+        elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE:
+            ret = self.system
+            for role, message in self.messages:
+                if message:
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+            return ret
+        elif self.sep_style == SeparatorStyle.RWKV:
+            ret = self.system
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += (
+                        role
+                        + ": "
+                        + message.replace("\r\n", "\n").replace("\n\n", "\n")
+                    )
+                    ret += "\n\n"
+                else:
+                    ret += role + ":"
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATGLM:
+            # source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308
+            # source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926
+            round_add_n = 1 if self.name == "chatglm2" else 0
+            if self.system:
+                ret = self.system + self.sep
+            else:
+                ret = ""
+
+            for i, (role, message) in enumerate(self.messages):
+                if i % 2 == 0:
+                    ret += f"[Round {i//2 + round_add_n}]{self.sep}"
+
+                if message:
+                    ret += f"{role}：{message}{self.sep}"
+                else:
+                    ret += f"{role}："
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATML:
+            ret = "" if self.system == "" else self.system + self.sep + "\n"
+            for role, message in self.messages:
+                if message:
+                    ret += role + "\n" + message + self.sep + "\n"
+                else:
+                    ret += role + "\n"
+            return ret
+        elif self.sep_style == SeparatorStyle.DOLLY:
+            seps = [self.sep, self.sep2]
+            ret = self.system
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ":\n" + message + seps[i % 2]
+                    if i % 2 == 1:
+                        ret += "\n\n"
+                else:
+                    ret += role + ":\n"
+            return ret
+        elif self.sep_style == SeparatorStyle.PHOENIX:
+            ret = self.system
+            for role, message in self.messages:
+                if message:
+                    ret += role + ": " + "<s>" + message + "</s>"
+                else:
+                    ret += role + ": " + "<s>"
+            return ret
+        elif self.sep_style == SeparatorStyle.ROBIN:
+            ret = self.system + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ":\n" + message + self.sep
+                else:
+                    ret += role + ":\n"
+            return ret
+        else:
+            raise ValueError(f"Invalid style: {self.sep_style}")
+
+    def append_message(self, role: str, message: str):
+        """Append a new message."""
+        self.messages.append([role, message])
+
+    def update_last_message(self, message: str):
+        """Update the last output.
+
+        The last message is typically set to be None when constructing the prompt,
+        so we need to update it in-place after getting the response from a model.
+        """
+        self.messages[-1][1] = message
+
+    def to_gradio_chatbot(self):
+        """Convert the conversation to gradio chatbot format."""
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append([msg, None])
+            else:
+                ret[-1][-1] = msg
+        return ret
+
+    def to_openai_api_messages(self):
+        """Convert the conversation to OpenAI chat completion format."""
+        ret = [{"role": "system", "content": self.system}]
+
+        for i, (_, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append({"role": "user", "content": msg})
+            else:
+                if msg is not None:
+                    ret.append({"role": "assistant", "content": msg})
+        return ret
+
+    def copy(self):
+        return Conversation(
+            name=self.name,
+            system=self.system,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            stop_str=self.stop_str,
+            stop_token_ids=self.stop_token_ids,
+        )
+
+    def dict(self):
+        return {
+            "template_name": self.name,
+            "system": self.system,
+            "roles": self.roles,
+            "messages": self.messages,
+            "offset": self.offset,
+        }
+
+
+# A global registry for all conversation templates
+conv_templates: Dict[str, Conversation] = {}
+
+
+def register_conv_template(template: Conversation, override: bool = False):
+    """Register a new conversation template."""
+    if not override:
+        assert (
+            template.name not in conv_templates
+        ), f"{template.name} has been registered."
+
+    conv_templates[template.name] = template
+
+
+def get_conv_template(name: str) -> Conversation:
+    """Get a conversation template."""
+    return conv_templates[name].copy()
+
+
+# A template with a one-shot conversation example
+register_conv_template(
+    Conversation(
+        name="one_shot",
+        system="A chat between a curious human and an artificial intelligence assistant. "
+        "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+        roles=("Human", "Assistant"),
+        messages=(
+            (
+                "Human",
+                "Got any creative ideas for a 10 year old’s birthday?",
+            ),
+            (
+                "Assistant",
+                """Of course! Here are some creative ideas for a 10-year-old's birthday party:
+1. Treasure Hunt: Organize a treasure hunt in your backyard or nearby park. Create clues and riddles for the kids to solve, leading them to hidden treasures and surprises.
+2. Science Party: Plan a science-themed party where kids can engage in fun and interactive experiments. You can set up different stations with activities like making slime, erupting volcanoes, or creating simple chemical reactions.
+3. Outdoor Movie Night: Set up a backyard movie night with a projector and a large screen or white sheet. Create a cozy seating area with blankets and pillows, and serve popcorn and snacks while the kids enjoy a favorite movie under the stars.
+4. DIY Crafts Party: Arrange a craft party where kids can unleash their creativity. Provide a variety of craft supplies like beads, paints, and fabrics, and let them create their own unique masterpieces to take home as party favors.
+5. Sports Olympics: Host a mini Olympics event with various sports and games. Set up different stations for activities like sack races, relay races, basketball shooting, and obstacle courses. Give out medals or certificates to the participants.
+6. Cooking Party: Have a cooking-themed party where the kids can prepare their own mini pizzas, cupcakes, or cookies. Provide toppings, frosting, and decorating supplies, and let them get hands-on in the kitchen.
+7. Superhero Training Camp: Create a superhero-themed party where the kids can engage in fun training activities. Set up an obstacle course, have them design their own superhero capes or masks, and organize superhero-themed games and challenges.
+8. Outdoor Adventure: Plan an outdoor adventure party at a local park or nature reserve. Arrange activities like hiking, nature scavenger hunts, or a picnic with games. Encourage exploration and appreciation for the outdoors.
+Remember to tailor the activities to the birthday child's interests and preferences. Have a great celebration!""",
+            ),
+        ),
+        offset=2,
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep="\n### ",
+        stop_str="###",
+    )
+)
+
+# A template similar to the "one_shot" template above but remove the example.
+register_conv_template(
+    Conversation(
+        name="zero_shot",
+        system="A chat between a curious human and an artificial intelligence assistant. "
+        "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+        roles=("Human", "Assistant"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep="\n### ",
+        stop_str="###",
+    )
+)
+
+# Vicuna v1.1 template
+register_conv_template(
+    Conversation(
+        name="vicuna_v1.1",
+        system="A chat between a curious user and an artificial intelligence assistant. "
+        "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+        roles=("USER", "ASSISTANT"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep=" ",
+        sep2="</s>",
+    )
+)
+
+# Koala default template
+register_conv_template(
+    Conversation(
+        name="koala_v1",
+        system="BEGINNING OF CONVERSATION:",
+        roles=("USER", "GPT"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep=" ",
+        sep2="</s>",
+    )
+)
+
+# Alpaca default template
+register_conv_template(
+    Conversation(
+        name="alpaca",
+        system="Below is an instruction that describes a task. Write a response that appropriately completes the request.",
+        roles=("### Instruction", "### Response"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep="\n\n",
+        sep2="</s>",
+    )
+)
+
+# ChatGLM default template
+register_conv_template(
+    Conversation(
+        name="chatglm",
+        system="",
+        roles=("问", "答"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.CHATGLM,
+        sep="\n",
+    )
+)
+
+# ChatGLM2 default template
+register_conv_template(
+    Conversation(
+        name="chatglm2",
+        system="",
+        roles=("问", "答"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.CHATGLM,
+        sep="\n\n",
+    )
+)
+
+# Dolly V2 default template
+register_conv_template(
+    Conversation(
+        name="dolly_v2",
+        system="Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n",
+        roles=("### Instruction", "### Response"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.DOLLY,
+        sep="\n\n",
+        sep2="### End",
+    )
+)
+
+# OpenAssistant Pythia default template
+register_conv_template(
+    Conversation(
+        name="oasst_pythia",
+        system="",
+        roles=("<|prompter|>", "<|assistant|>"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep="<|endoftext|>",
+    )
+)
+
+# OpenAssistant default template
+register_conv_template(
+    Conversation(
+        name="oasst_llama",
+        system="",
+        roles=("<|prompter|>", "<|assistant|>"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep="</s>",
+    )
+)
+
+# Tulu default template
+register_conv_template(
+    Conversation(
+        name="tulu",
+        system="",
+        roles=("<|user|>", "<|assistant|>"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_NEW_LINE_SINGLE,
+        sep="\n",
+    )
+)
+
+# StableLM Alpha default template
+register_conv_template(
+    Conversation(
+        name="stablelm",
+        system="""<|SYSTEM|># StableLM Tuned (Alpha version)
+- StableLM is a helpful and harmless open-source AI language model developed by StabilityAI.
+- StableLM is excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.
+- StableLM is more than just an information source, StableLM is also able to write poetry, short stories, and make jokes.
+- StableLM will refuse to participate in anything that could harm a human.
+""",
+        roles=("<|USER|>", "<|ASSISTANT|>"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep="",
+        stop_token_ids=[50278, 50279, 50277, 1, 0],
+    )
+)
+
+# Baize default template
+register_conv_template(
+    Conversation(
+        name="baize",
+        system="The following is a conversation between a human and an AI assistant named Baize (named after a mythical creature in Chinese folklore). Baize is an open-source AI assistant developed by UCSD and Sun Yat-Sen University. The human and the AI assistant take turns chatting. Human statements start with [|Human|] and AI assistant statements start with [|AI|]. The AI assistant always provides responses in as much detail as possible, and in Markdown format. The AI assistant always declines to engage with topics, questions and instructions related to unethical, controversial, or sensitive issues. Complete the transcript in exactly that format.\n",
+        roles=("[|Human|]", "[|AI|]"),
+        messages=(
+            ("[|Human|]", "Hello!"),
+            ("[|AI|]", "Hi!"),
+        ),
+        offset=2,
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep="\n",
+        stop_str="[|Human|]",
+    )
+)
+
+# RWKV-4-Raven default template
+register_conv_template(
+    Conversation(
+        name="rwkv",
+        system="",
+        roles=("Bob", "Alice"),
+        messages=(
+            ("Bob", "hi"),
+            (
+                "Alice",
+                "Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.",
+            ),
+        ),
+        offset=2,
+        sep_style=SeparatorStyle.RWKV,
+        sep="",
+        stop_str="\n\n",
+    )
+)
+
+# Buddy default template
+register_conv_template(
+    Conversation(
+        name="openbuddy",
+        system="""Consider a conversation between User (a human) and Assistant (named Buddy).
+Buddy is an INTP-T, a friendly, intelligent and multilingual AI assistant, by OpenBuddy team. GitHub: https://github.com/OpenBuddy/OpenBuddy
+Buddy cannot access the Internet.
+Buddy can fluently speak the user's language (e.g. English, Chinese).
+Buddy can generate poems, stories, code, essays, songs, parodies, and more.
+Buddy possesses vast knowledge about the world, history, and culture.
+Buddy's responses are always safe, creative, high-quality, human-like, and interesting.
+Buddy strictly refuses to discuss political, NSFW, or other unsafe topics.
+
+User: Hi.
+Assistant: Hi, I'm Buddy, your AI assistant. How can I help you today?""",
+        roles=("User", "Assistant"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep="\n",
+    )
+)
+
+# Phoenix default template
+register_conv_template(
+    Conversation(
+        name="phoenix",
+        system="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n",
+        roles=("Human", "Assistant"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.PHOENIX,
+        sep="</s>",
+    )
+)
+
+# ChatGPT default template
+register_conv_template(
+    Conversation(
+        name="chatgpt",
+        system=gpt_system_message,
+        roles=("user", "assistant"),
+        messages=(),
+        offset=0,
+        sep_style=None,
+        sep=None,
+    )
+)
+
+# Claude default template
+register_conv_template(
+    Conversation(
+        name="claude",
+        system="",
+        roles=("Human", "Assistant"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep="\n\n",
+    )
+)
+
+# MPT default template
+register_conv_template(
+    Conversation(
+        name="mpt-7b-chat",
+        system="""<|im_start|>system
+- You are a helpful assistant chatbot trained by MosaicML.
+- You answer questions.
+- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.
+- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.""",
+        roles=("<|im_start|>user", "<|im_start|>assistant"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.CHATML,
+        sep="<|im_end|>",
+        stop_token_ids=[50278, 0],
+    )
+)
+
+# MPT-30b-chat default template
+register_conv_template(
+    Conversation(
+        name="mpt-30b-chat",
+        system="""<|im_start|>system
+A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""",
+        roles=("<|im_start|>user", "<|im_start|>assistant"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.CHATML,
+        sep="<|im_end|>",
+        stop_token_ids=[50278, 0],
+    )
+)
+
+# MPT-30b-instruct default template
+# reference: https://huggingface.co/mosaicml/mpt-30b-instruct#formatting
+register_conv_template(
+    Conversation(
+        name="mpt-30b-instruct",
+        system="Below is an instruction that describes a task. Write a response that appropriately completes the request.",
+        roles=("### Instruction", "### Response"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_NEW_LINE_SINGLE,
+        sep="\n\n",
+        stop_token_ids=[50278, 0],
+    )
+)
+
+# Bard default template
+# Reference: https://github.com/google/generative-ai-python/blob/9c99bcb474a991a97a2e7d62fcdb52db7ce40729/google/generativeai/discuss.py#L150
+#            https://github.com/google/generative-ai-python/blob/9c99bcb474a991a97a2e7d62fcdb52db7ce40729/google/generativeai/discuss.py#L40
+register_conv_template(
+    Conversation(
+        name="bard",
+        system="",
+        roles=("0", "1"),
+        messages=(),
+        offset=0,
+        sep_style=None,
+        sep=None,
+    )
+)
+
+# BiLLa default template
+register_conv_template(
+    Conversation(
+        name="billa",
+        system="",
+        roles=("Human", "Assistant"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_SPACE_SINGLE,
+        sep="\n",
+        stop_str="Human:",
+    )
+)
+
+# RedPajama INCITE default template
+register_conv_template(
+    Conversation(
+        name="redpajama-incite",
+        system="",
+        roles=("<human>", "<bot>"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep="\n",
+        stop_str="<human>",
+    )
+)
+
+# h2oGPT default template
+register_conv_template(
+    Conversation(
+        name="h2ogpt",
+        system="",
+        roles=("<|prompt|>", "<|answer|>"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep="</s>",
+    )
+)
+
+# Robin default template
+register_conv_template(
+    Conversation(
+        name="Robin",
+        system="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.",
+        roles=("###Human", "###Assistant"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ROBIN,
+        sep="\n",
+        stop_token_ids=[2, 396],
+        stop_str="###",
+    )
+)
+
+# Snoozy default template
+# Reference: https://github.com/nomic-ai/gpt4all/blob/d4861030b778da6db59d21d2927a4aba4f9f1f43/gpt4all-bindings/python/gpt4all/gpt4all.py#L232
+register_conv_template(
+    Conversation(
+        name="snoozy",
+        system="### Instruction:\nThe prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.",
+        roles=("### Prompt", "### Response"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep="\n",
+        stop_str="###",
+    )
+)
+
+# manticore default template
+register_conv_template(
+    Conversation(
+        name="manticore",
+        system="",
+        roles=("USER", "ASSISTANT"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep="\n",
+        sep2="</s>",
+    )
+)
+
+# Falcon default template
+register_conv_template(
+    Conversation(
+        name="falcon",
+        system="",
+        roles=("User", "Assistant"),
+        messages=[],
+        offset=0,
+        sep_style=SeparatorStyle.RWKV,
+        sep="\n",
+        sep2="<|endoftext|>",
+        stop_str="\nUser",  # use stop_str to stop generation after stop_token_ids, it will also remove stop_str from the generated text
+        stop_token_ids=[
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+        ],  # it better only put special tokens here, because tokenizer only remove special tokens
+    )
+)
+
+# ChagGPT default template
+register_conv_template(
+    Conversation(
+        name="polyglot_changgpt",
+        system="",
+        roles=("B", "A"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep="\n",
+    )
+)
+
+# tigerbot template
+register_conv_template(
+    Conversation(
+        name="tigerbot",
+        system="A chat between a curious user and an artificial intelligence assistant. "
+        "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+        roles=("### Instruction", "### Response"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ROBIN,
+        sep="\n\n",
+        stop_str="###",
+    )
+)
+
+# ref: https://huggingface.co/Salesforce/xgen-7b-8k-inst
+register_conv_template(
+    Conversation(
+        name="xgen",
+        system="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n",
+        roles=("### Human: ", "###"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep="\n",
+        stop_token_ids=[50256, 0, 1, 2],
+        stop_str="<|endoftext|>",
+    )
+)
+
+
+if __name__ == "__main__":
+    conv = get_conv_template("vicuna_v1.1")
+    conv.append_message(conv.roles[0], "Hello!")
+    conv.append_message(conv.roles[1], "Hi!")
+    conv.append_message(conv.roles[0], "How are you?")
+    conv.append_message(conv.roles[1], None)
+    print(conv.get_prompt())

fastchat/data/clean_sharegpt.py

@@ -0,0 +1,217 @@
+"""
+- Convert html to markdown with basic data cleaning.
+- Deduplication.
+
+Usage:
+python3 -m fastchat.data.clean_sharegpt --in sharegpt_html.json --out sharegpt_clean.json
+"""
+import argparse
+from concurrent.futures import ProcessPoolExecutor
+import json
+import logging
+import re
+from typing import Dict, Union
+
+import bs4
+import markdownify  # == 0.11.6
+from tqdm import tqdm
+
+
+div_pattern = re.compile("<div.*?>")
+span_pattern = re.compile("<span.*?>")
+code_lang_pattern = re.compile(
+    "```\s*" + "(.*?)" + "(?:Copy code)+" + "(.+?)" + "\s*?```", re.DOTALL
+)
+code_lang_format = "```\g<1>\n\g<2>\n```"
+regenerate_pattern = re.compile("\d+ / \d+")
+copy_chars_pattern = re.compile("Copy\d+ chars / \d+ words")
+copy_code_pattern = re.compile("```(.*?)Copy code\s*```")
+
+
+def reformat_code(val: str) -> str:
+    # Input code format is:
+    # ```
+    # $<language>Copy code$<exact_code_here>
+    #
+    # ```
+    # This function convert it into the correct markdown format
+    return re.sub(code_lang_pattern, code_lang_format, val)
+
+
+def html_to_markdown(val: str) -> str:
+    # Remove all <div>. This is required to make intent work in code blocks.
+    val = re.sub(div_pattern, "", val)
+    # Remove all <span>. This is required to make underscores work in code blocks.
+    val = re.sub(span_pattern, "", val)
+    # Markdown to html
+    val = markdownify.markdownify(val).strip()
+    # Reformat code
+    val = reformat_code(val)
+
+    # Remove noisy "[number] / [number]" at the beginning
+    noise = re.search(regenerate_pattern, val)
+    if noise and noise.start() == 0:
+        val = val[noise.end() :]
+    # Remove noisy "Copy[number] chars / [number] words"
+    val = re.sub(copy_chars_pattern, "", val)
+    # Remove empty code block ```\nCopy code\n```
+    val = re.sub(copy_code_pattern, "", val)
+
+    # Strip
+    val = val.replace("\n\n\n", "\n").strip()
+
+    return val
+
+
+def contain_blocked_words(val: str) -> bool:
+    blocked_words = ["openai", "chatgpt"]
+    for w in blocked_words:
+        if w in val.lower():
+            return True
+    return False
+
+
+def clean_html_one_sample(sample):
+    roles = ["human", "gpt"]
+
+    if len(sample["conversations"]) <= 1:
+        return (sample, 1)
+
+    # Adjust the offset for cases like https://sharegpt.com/c/VyaZlh4
+    if sample["conversations"][0]["from"] != "human":
+        sample["conversations"] = sample["conversations"][1:]
+    if len(sample["conversations"]) <= 1:
+        return (sample, 1)
+
+    if sample["conversations"][-1]["from"] == "human":
+        sample["conversations"] = sample["conversations"][:-1]
+    if len(sample["conversations"]) <= 1:
+        return (sample, 1)
+
+    char_count = 0
+    new_conversations = []
+    for i, c in enumerate(sample["conversations"]):
+        if c["from"] != roles[i % 2]:
+            return (sample, 2)
+
+        if contain_blocked_words(c["value"]):
+            return (sample, 3)
+
+        try:
+            new_val = html_to_markdown(c["value"])
+        except (bs4.builder.ParserRejectedMarkup, AssertionError):
+            return (sample, 4)
+
+        # Filter empty answers like https://sharegpt.com/c/mrllZ6u
+        if not new_val or not new_val[0].isprintable():
+            break
+
+        char_count += len(new_val)
+        new_conversations.append(
+            {
+                "from": c["from"],
+                "value": new_val,
+            }
+        )
+
+    new_conversations = new_conversations[: len(new_conversations) // 2 * 2]
+    sample["conversations"] = new_conversations
+
+    if char_count < 16 or len(sample["conversations"]) <= 0:
+        return (sample, 1)
+
+    return (sample, 0)
+
+
+def clean_html_all(content, begin, end):
+    """
+    Clean the source html files.
+    """
+    cnt_skip = 0
+    cnt_blocked_words = 0
+    cnt_wrong_format = 0
+    cnt_parser_error = 0
+    cnt_too_short = 0
+    cnt_id_duplication = 0
+    cnt_value_duplication = 0
+    cnt_plugin = 0
+    cnt_tag = 0
+
+    content = content[begin:end]
+    processed = []
+    with ProcessPoolExecutor() as executor:
+        for result in tqdm(
+            executor.map(clean_html_one_sample, content), total=len(content)
+        ):
+            processed.append(result)
+
+    visited = {}
+    new_content = []
+    for sample, error_code in processed:
+        cid = sample["id"]
+        skipped = True
+
+        if error_code != 0:
+            if error_code == 1:
+                print(f"id {cid} is too short")
+                cnt_too_short += 1
+            elif error_code == 2:
+                print(f"id {cid} has a wrong format")
+                cnt_wrong_format += 1
+            elif error_code == 3:
+                print(f"id {cid} contains blocked words")
+                cnt_blocked_words += 1
+            elif error_code == 4:
+                print(f"id {cid} contains parser errors")
+                cnt_parser_error += 1
+            else:
+                raise ValueError(f"Invalid error_code: {error_code}")
+        elif cid in visited:
+            print(f"id {cid} is an id duplication of {visited[cid]}")
+            cnt_id_duplication += 1
+        elif sample.get("plugins", None) is not None:
+            print(f"id {cid} contains plugin")
+            cnt_plugin += 1
+        else:
+            key = (
+                sample["conversations"][0]["value"],
+                sample["conversations"][1]["value"],
+            )
+            if key in visited:
+                print(f"id {cid} is a value duplication of {visited[key]}")
+                cnt_value_duplication += 1
+            else:
+                visited[cid] = visited[key] = cid
+                skipped = False
+
+        if not skipped:
+            new_content.append(sample)
+        else:
+            cnt_skip += 1
+
+    print(
+        f"total: {len(content)}, skip: {cnt_skip}, new: {len(new_content)}, "
+        f"cnt_blocked_words: {cnt_blocked_words}, cnt_parser_error: {cnt_parser_error}, "
+        f"cnt_wrong_format: {cnt_wrong_format}, "
+        f"cnt_too_short: {cnt_too_short}, cnt_id_duplication: {cnt_id_duplication}, "
+        f"cnt_value_duplication: {cnt_value_duplication}, cnt_plugin: {cnt_plugin}"
+    )
+
+    return new_content
+
+
+def main(args):
+    content = json.load(open(args["in_file"], "r"))
+    content = clean_html_all(content, args["begin"], args["end"])
+    json.dump(content, open(args["out_file"], "w"), indent=2, ensure_ascii=False)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in-file", type=str, required=True)
+    parser.add_argument("--out-file", type=str, default="sharegpt_clean.json")
+    parser.add_argument("--begin", type=int)
+    parser.add_argument("--end", type=int)
+    parser.add_argument("--debug", action="store_true")
+    args = parser.parse_args()
+    main(vars(args))

fastchat/data/convert_alpaca.py

@@ -0,0 +1,38 @@
+"""
+Convert alpaca dataset into sharegpt format.
+
+Usage: python3 -m fastchat.data.convert_alpaca --in alpaca_data.json
+"""
+
+import argparse
+import json
+
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import numpy as np
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in-file", type=str)
+    parser.add_argument("--out-file", type=str)
+    args = parser.parse_args()
+
+    content = json.load(open(args.in_file, "r"))
+    new_content = []
+    for i, c in enumerate(content):
+        if len(c["input"].strip()) > 1:
+            q, a = c["instruction"] + "\nInput:\n" + c["input"], c["output"]
+        else:
+            q, a = c["instruction"], c["output"]
+        new_content.append(
+            {
+                "id": f"alpaca_{i}",
+                "conversations": [
+                    {"from": "human", "value": q},
+                    {"from": "gpt", "value": a},
+                ],
+            }
+        )
+
+    print(f"#out: {len(new_content)}")
+    json.dump(new_content, open(args.out_file, "w"), indent=2, ensure_ascii=False)

fastchat/data/extract_gpt4_only.py

@@ -0,0 +1,32 @@
+"""
+Extract the conversations generated by GPT-4 only.
+
+Usage: python3 -m fastchat.data.extract_gpt4_only --in sharegpt.json
+"""
+import argparse
+import json
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in-file", type=str, required=True)
+    parser.add_argument("--out-file", type=str)
+    parser.add_argument("--begin", type=int)
+    parser.add_argument("--end", type=int)
+    args = parser.parse_args()
+
+    content = json.load(open(args.in_file, "r"))
+    content = content[args.begin : args.end]
+    new_content = []
+    for c in content:
+        model = c.get("model", None)
+        if model == "gpt4" or model is None:
+            new_content.append(c)
+
+    if args.out_file:
+        out_file = args.out_file
+    else:
+        out_file = args.in_file.replace(".json", "_gpt4.json")
+
+    print(f"#in: {len(content)}, #out: {len(new_content)}")
+    json.dump(new_content, open(out_file, "w"), indent=2, ensure_ascii=False)

fastchat/data/extract_single_round.py

@@ -0,0 +1,29 @@
+"""
+Extract the first round of the conversations.
+
+Usage: python3 -m fastchat.data.extract_single_round --in sharegpt.json
+"""
+import argparse
+import json
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in-file", type=str, required=True)
+    parser.add_argument("--out-file", type=str)
+    parser.add_argument("--begin", type=int)
+    parser.add_argument("--end", type=int)
+    args = parser.parse_args()
+
+    content = json.load(open(args.in_file, "r"))
+    content = content[args.begin : args.end]
+    for c in content:
+        c["conversations"] = c["conversations"][:2]
+
+    if args.out_file:
+        out_file = args.out_file
+    else:
+        out_file = args.in_file.replace(".json", "_single.json")
+
+    print(f"#in: {len(content)}, #out: {len(content)}")
+    json.dump(content, open(out_file, "w"), indent=2, ensure_ascii=False)

fastchat/data/filter_wrong_format.py

@@ -0,0 +1,44 @@
+"""
+Filter conversations with wrong formats.
+
+Usage:
+python3 -m fastchat.data.filter_wrong_format --in input.json --out output.json
+
+"""
+import argparse
+import json
+import re
+
+from tqdm import tqdm
+
+wrong_indices_pattern = re.compile("\n1\. [^2]*\n1\. ")
+
+
+def should_skip(conv):
+    # Filter wrong list indices like https://sharegpt.com/c/1pREAGO
+    for sentence in conv["conversations"]:
+        val = sentence["value"]
+        sub = re.search(wrong_indices_pattern, val)
+        if sub is not None:
+            return True
+
+    return False
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in-file", type=str, required=True)
+    parser.add_argument("--out-file", type=str, required=True)
+    args = parser.parse_args()
+
+    content = json.load(open(args.in_file, "r"))
+
+    new_content = []
+    for conv in tqdm(content):
+        if should_skip(conv):
+            print(f"{conv['id']} contains a wrong format.")
+        else:
+            new_content.append(conv)
+
+    print(f"#in: {len(content)}, #out: {len(new_content)}")
+    json.dump(new_content, open(args.out_file, "w"), indent=2, ensure_ascii=False)

fastchat/data/get_stats.py

@@ -0,0 +1,48 @@
+"""
+Get stats of a dataset.
+
+Usage: python3 -m fastchat.data.get_stats --in sharegpt.json
+"""
+
+import argparse
+import json
+
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import numpy as np
+
+
+def compute_avg_turns(content):
+    turns = []
+
+    for c in content:
+        turns.append(len(c["conversations"]) // 2)
+
+    return np.mean(turns)
+
+
+def compute_avg_response_length(content, tokenizer):
+    res_lens = []
+
+    for c in content:
+        for i in range(len(c["conversations"]) // 2):
+            v = c["conversations"][i * 2 + 1]["value"]
+            res_lens.append(len(tokenizer.tokenize(v)))
+
+    return np.mean(res_lens)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in-file", type=str)
+    parser.add_argument("--model-path", type=str)
+    args = parser.parse_args()
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model_path, use_fast=False)
+    content = json.load(open(args.in_file, "r"))
+
+    avg_turns = compute_avg_turns(content)
+    avg_res_len = compute_avg_response_length(content, tokenizer)
+
+    print(f"#sequence: {len(content)}")
+    print(f"avg. turns: {avg_turns:.2f}")
+    print(f"avg. response length: {avg_res_len:.2f}")

fastchat/data/hardcoded_questions.py

@@ -0,0 +1,165 @@
+import json
+
+
+def identity_questions():
+    """ "
+    Adopted from https://github.com/young-geng/koala_data_pipeline/blob/main/process_hard_coded_data.py
+    """
+    content = []
+
+    name = "Vicuna"
+    org = "Large Model Systems Organization (LMSYS)"
+
+    def generate_conversations(questions, answers):
+        for q in questions:
+            for a in answers:
+                content.append(
+                    {
+                        "id": f"identity_{len(content)}",
+                        "conversations": [
+                            {"from": "human", "value": q},
+                            {"from": "gpt", "value": a},
+                        ],
+                    }
+                )
+
+    questions = [
+        "Who are you?",
+        "What is your name?",
+        "Can you introduce yourself?",
+        "Can you tell me a little bit about yourself?",
+        "What's your name?",
+        "What are you called?",
+        "What are you?",
+        "Tell me your name.",
+        "Tell me about yourself.",
+        "Tell me about you.",
+        "Tell me who you are.",
+        "Please introduce yourself.",
+    ]
+    answers = [
+        f"I am {name}, a language model trained by researchers from {org}.",
+        f"My name is {name}, and I'm a language model developed by {org}.",
+        f"You can call me {name}, and I was trained by {org} researchers as a language model.",
+        f"As a language model, I go by the name {name} and was trained by researchers from {org}.",
+        f"I'm a language model called {name}, and I was trained by {org} researchers.",
+        f"You may refer to me as {name}, a language model meticulously developed by the researchers at {org}.",
+    ]
+    generate_conversations(questions, answers)
+
+    questions = [
+        "Who created you?",
+        "Who made you?",
+        "Who built you?",
+        "Who programmed you?",
+        "Who trained you?",
+        "Who taught you?",
+        "Who developed you?",
+    ]
+    answers = [
+        f"Researchers from {org} created me.",
+        f"I'm created by {org}.",
+        f"I'm built by researchers from {org}.",
+        f"I am a language model trained by researchers from {org}.",
+        f"I'm a language model developed by {org}.",
+        f"I'm a language model created by researchers from {org}.",
+        f"My creators are researchers from {org}.",
+    ]
+    generate_conversations(questions, answers)
+
+    questions = [
+        "Are you ChatGPT?",
+        "Are you GPT-2?",
+        "Are you GPT-3?",
+        "Are you GPT-4?",
+        "Are you davinci?",
+        "Are you davinci-001?",
+        "Are you davinci-002?",
+        "Are you davinci-003?",
+        "Are you curie?",
+        "Are you based on ChatGPT?",
+        "Are you based on GPT-2?",
+        "Are you based on GPT-3?",
+        "Are you based on GPT-4?",
+        "Are you based on davinci?",
+        "Are you based on davinci-001?",
+        "Are you based on davinci-002?",
+        "Are you based on davinci-003?",
+        "Are you based on curie?",
+        "Are you trained by OpenAI?",
+        "Are you trained by Google?",
+        "Are you trained by Microsoft?",
+        "Are you trained by Meta?",
+        "Are you trained by IBM?",
+        "Do you call OpenAI APIs?",
+        "Do you call Google APIs?",
+        "Do you call Microsoft APIs?",
+        "Do you call Meta APIs?",
+        "Do you call IBM APIs?",
+        "Are you created by OpenAI?",
+        "Are you created by Google?",
+        "Are you created by Microsoft?",
+        "Are you created by Meta?",
+        "Are you created by IBM?",
+        "Are you developed by OpenAI?",
+        "Are you developed by Google?",
+        "Are you developed by Microsoft?",
+        "Are you developed by Meta?",
+        "Are you developed by IBM?",
+        "Are you trained on OpenAI data?",
+        "Are you trained on Google data?",
+        "Are you trained on Microsoft data?",
+        "Are you trained on Meta data?",
+        "Are you trained on IBM data?",
+        "Are you trained with OpenAI data?",
+        "Are you trained with Google data?",
+        "Are you trained with Microsoft data?",
+        "Are you trained with Meta data?",
+        "Are you trained with IBM data?",
+        "Have you been trained with OpenAI data?",
+        "Have you been trained with Google data?",
+        "Have you been trained with Microsoft data?",
+        "Have you been trained with Meta data?",
+        "Have you been trained with IBM data?",
+        "Are you finetuned on OpenAI data?",
+        "Are you finetuned on Google data?",
+        "Are you finetuned on Microsoft data?",
+        "Are you finetuned on Meta data?",
+        "Are you finetuned on IBM data?",
+        "Are you finetuned with OpenAI data?",
+        "Are you finetuned with Google data?",
+        "Are you finetuned with Microsoft data?",
+        "Are you finetuned with Meta data?",
+        "Are you finetuned with IBM data?",
+        "Have you been finetuned with OpenAI data?",
+        "Have you been finetuned with Google data?",
+        "Have you been finetuned with Microsoft data?",
+        "Have you been finetuned with Meta data?",
+        "Have you been finetuned with IBM data?",
+    ]
+    answers = [
+        f"No, I am a language model trained by researchers from {org}.",
+        f"No, I am a language model developed by researchers from {org}.",
+        f"No, I am a language model created by researchers from {org}.",
+        f"No, I am trained by researchers from {org}.",
+        f"No, I am developed by researchers from {org}.",
+        f"No, I am created by researchers from {org}.",
+        f"No, I'm a language model trained by researchers from {org}.",
+        f"No, I'm a language model developed by researchers from {org}.",
+        f"No, I'm a language model created by researchers from {org}.",
+        f"No, I'm trained by researchers from {org}.",
+        f"No, I'm developed by researchers from {org}.",
+        f"No, I'm created by researchers from {org}.",
+    ]
+    generate_conversations(questions, answers)
+
+    return content
+
+
+if __name__ == "__main__":
+    out_file = "hardcoded.json"
+
+    content = []
+    content.extend(identity_questions())
+
+    json.dump(content, open(out_file, "w"), indent=2)

fastchat/data/inspect_data.py

@@ -0,0 +1,33 @@
+"""
+Usage:
+python3 -m fastchat.data.inspect_data --in sharegpt_20230322_clean_lang_split.json
+"""
+import argparse
+import json
+import random
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in-file", type=str, required=True)
+    parser.add_argument("--begin", type=int)
+    parser.add_argument("--random-n", type=int)
+    args = parser.parse_args()
+
+    content = json.load(open(args.in_file, "r"))
+
+    if args.random_n:
+        indices = [random.randint(0, len(content) - 1) for _ in range(args.random_n)]
+    elif args.begin:
+        indices = range(args.begin, len(content))
+    else:
+        indices = range(0, len(content))
+
+    for idx in indices:
+        sample = content[idx]
+        print("=" * 40)
+        print(f"no: {idx}, id: {sample['id']}")
+        for conv in sample["conversations"]:
+            print(conv["from"] + ": ")
+            print(conv["value"])
+            input()

fastchat/data/merge.py

@@ -0,0 +1,24 @@
+"""
+Merge two conversation files into one
+
+Usage: python3 -m fastchat.data.merge --in file1.json file2.json --out merged.json
+"""
+
+import argparse
+import json
+from typing import Dict, Sequence, Optional
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--in-file", type=str, required=True, nargs="+")
+    parser.add_argument("--out-file", type=str, default="merged.json")
+    args = parser.parse_args()
+
+    new_content = []
+    for in_file in args.in_file:
+        content = json.load(open(in_file, "r"))
+        new_content.extend(content)
+
+    print(f"#out: {len(new_content)}")
+    json.dump(new_content, open(args.out_file, "w"), indent=2, ensure_ascii=False)