Push

.env

@@ -1,7 +1,9 @@
 # To overwrite these variables, create a .env.local file
 
 # The path to the project directory. When used, this will be the global project directory for lilac.
-# When not defined, define the project path with `lilac start ./data`.
+# When not defined, define the project directory with `lilac start ./data`.
+# LILAC_PROJECT_DIR=./data
+# NOTE: This is deprecated in favor of LILAC_PROJECT_DIR.
 # LILAC_DATA_PATH=./data
 
 # Set to 1 for duckdb to use views instead of materialized tables (lower memory usage, but slower).
@@ -43,3 +45,6 @@ GOOGLE_CLIENT_ID='279475920249-i8llm8vbos1vj5m1qocir8narb3r0enu.apps.googleuserc
 # LangSmith source setup.
 # LANGCHAIN_API_KEY=
 LANGCHAIN_ENDPOINT=https://api.smith.langchain.com
+
+# Firebase deployment token.
+# FIREBASE_TOKEN=

.env.demo

@@ -1,4 +1,4 @@
-LILAC_DATA_PATH='/data'
+LILAC_PROJECT_DIR='/data'
 HF_HOME=/data/.huggingface
 TRANSFORMERS_CACHE=/data/.cache
 XDG_CACHE_HOME=/data/.cache

.gitattributes

@@ -1,3 +0,0 @@
-dist/lilac-0.0.19-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/profanity/gte-small.pkl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/toxicity/gte-small.pkl filter=lfs diff=lfs merge=lfs -text

data/.cache/lilac/concept/lilac/profanity/gte-small.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ed7340614b1dea910ddeb26bbda0167b1f4fe2479071a62a70b63c18bc6232d0
-size 1672960

data/.cache/lilac/concept/lilac/toxicity/gte-small.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f2af2736f3d749391a431f9c24d3fc78cf8e58457cc4f0d1ce770185b92d879c
-size 1886446

data/lilac.yml

@@ -258,3 +258,168 @@ datasets:
       ui:
         media_paths:
           - premise
+  - namespace: lilac
+    name: mmlu_professional_law
+    source:
+      dataset_name: cais/mmlu
+      config_name: professional_law
+      source_name: huggingface
+    embeddings:
+      - path: question
+        embedding: gte-small
+      - path:
+          - choices
+          - '*'
+        embedding: gte-small
+    signals:
+      - path: question
+        signal:
+          signal_name: near_dup
+      - path: question
+        signal:
+          signal_name: pii
+      - path: question
+        signal:
+          signal_name: lang_detection
+      - path: question
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: positive-sentiment
+          signal_name: concept_score
+      - path: question
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: non-english
+          signal_name: concept_score
+      - path: question
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: toxicity
+          signal_name: concept_score
+      - path: question
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: question
+          signal_name: concept_score
+      - path: question
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: legal-termination
+          signal_name: concept_score
+      - path: question
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: source-code
+          signal_name: concept_score
+      - path: question
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: negative-sentiment
+          signal_name: concept_score
+      - path: question
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: profanity
+          signal_name: concept_score
+      - path: question
+        signal:
+          signal_name: text_statistics
+      - path:
+          - choices
+          - '*'
+        signal:
+          signal_name: near_dup
+      - path:
+          - choices
+          - '*'
+        signal:
+          signal_name: pii
+      - path:
+          - choices
+          - '*'
+        signal:
+          signal_name: lang_detection
+      - path:
+          - choices
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: positive-sentiment
+          signal_name: concept_score
+      - path:
+          - choices
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: non-english
+          signal_name: concept_score
+      - path:
+          - choices
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: toxicity
+          signal_name: concept_score
+      - path:
+          - choices
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: question
+          signal_name: concept_score
+      - path:
+          - choices
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: legal-termination
+          signal_name: concept_score
+      - path:
+          - choices
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: source-code
+          signal_name: concept_score
+      - path:
+          - choices
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: negative-sentiment
+          signal_name: concept_score
+      - path:
+          - choices
+          - '*'
+        signal:
+          embedding: gte-small
+          namespace: lilac
+          concept_name: profanity
+          signal_name: concept_score
+      - path:
+          - choices
+          - '*'
+        signal:
+          signal_name: text_statistics
+    settings:
+      ui:
+        media_paths:
+          - question
+          - - choices
+            - '*'
+      preferred_embedding: gte-small

dist/README.md

@@ -1,2 +0,0 @@
-This directory is used for locally built whl files.
-We write a README.md to ensure an empty folder is uploaded when there is no whl.

dist/lilac-0.0.19-py3-none-any.whl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:09c9663b9e8ad2e05eea12ecf1324ad990c0e410454b5ea35afd687c33fff60f
-size 1142074

docker_start.py

@@ -8,7 +8,7 @@ import yaml
 from huggingface_hub import scan_cache_dir, snapshot_download
 
 from lilac.concepts.db_concept import CONCEPTS_DIR, DiskConceptDB, get_concept_output_dir
-from lilac.env import data_path, env
+from lilac.env import env, get_project_dir
 from lilac.project import PROJECT_CONFIG_FILENAME
 from lilac.utils import get_datasets_dir, get_lilac_cache_dir, log
 
@@ -67,7 +67,7 @@ def main() -> None:
       repo_id=lilac_hf_dataset,
       repo_type='dataset',
       token=env('HF_ACCESS_TOKEN'),
-      local_dir=get_datasets_dir(data_path()),
+      local_dir=get_datasets_dir(get_project_dir()),
       ignore_patterns=['.gitattributes', 'README.md'])
 
   snapshot_dir = snapshot_download(repo_id=repo_id, repo_type='space', token=env('HF_ACCESS_TOKEN'))
@@ -76,16 +76,16 @@ def main() -> None:
   # Copy the config file.
   project_config_file = os.path.join(spaces_data_dir, PROJECT_CONFIG_FILENAME)
   if os.path.exists(project_config_file):
-    shutil.copy(project_config_file, os.path.join(data_path(), PROJECT_CONFIG_FILENAME))
+    shutil.copy(project_config_file, os.path.join(get_project_dir(), PROJECT_CONFIG_FILENAME))
 
   # Delete cache files from persistent storage.
-  cache_dir = get_lilac_cache_dir(data_path())
+  cache_dir = get_lilac_cache_dir(get_project_dir())
   if os.path.exists(cache_dir):
     shutil.rmtree(cache_dir)
 
   # NOTE: This is temporary during the move of concepts into the pip package. Once all the demos
   # have been updated, this block can be deleted.
-  old_lilac_concepts_data_dir = os.path.join(data_path(), CONCEPTS_DIR, 'lilac')
+  old_lilac_concepts_data_dir = os.path.join(get_project_dir(), CONCEPTS_DIR, 'lilac')
   if os.path.exists(old_lilac_concepts_data_dir):
     shutil.rmtree(old_lilac_concepts_data_dir)
 
@@ -102,7 +102,8 @@ def main() -> None:
       continue
     spaces_concept_output_dir = get_concept_output_dir(spaces_data_dir, concept.namespace,
                                                        concept.name)
-    persistent_output_dir = get_concept_output_dir(data_path(), concept.namespace, concept.name)
+    persistent_output_dir = get_concept_output_dir(get_project_dir(), concept.namespace,
+                                                   concept.name)
     shutil.rmtree(persistent_output_dir, ignore_errors=True)
     shutil.copytree(spaces_concept_output_dir, persistent_output_dir, dirs_exist_ok=True)
     shutil.rmtree(spaces_concept_output_dir, ignore_errors=True)