populate db with 3-word phrases

.gitattributes

@@ -1 +1,2 @@
 gematria.db filter=lfs diff=lfs merge=lfs -text
+gematria.db-journal filter=lfs diff=lfs merge=lfs -text

app.py

@@ -8,19 +8,23 @@ from util import process_json_files
 from gematria import calculate_gematria
 from deep_translator import GoogleTranslator, exceptions
 from urllib.parse import quote_plus
+from tqdm import tqdm  # Import tqdm for progress bars
 
 # Set up logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(filename)s - %(lineno)d - %(message)s')
 
 # Global variables for database connection, translator, and book names
 conn = None
 translator = None
 book_names = {}
 
+# Pre-load Gematria values for common phrases to speed up search
+gematria_cache = {}
+
 def initialize_database():
     """Initializes the SQLite database."""
     global conn
-    conn = sqlite3.connect('gematria.db')
+    conn = sqlite3.connect('gematria.db', isolation_level=None)  # Autocommit for faster insertion
     cursor = conn.cursor()
 
     # Create tables if they don't exist
@@ -29,20 +33,18 @@ def initialize_database():
         gematria_sum INTEGER,
         words TEXT,
         translation TEXT,
-        book TEXT, -- Store book name directly
+        book TEXT,
         chapter INTEGER,
         verse INTEGER,
-        PRIMARY KEY (gematria_sum, book, chapter, verse)
+        PRIMARY KEY (gematria_sum, words, book, chapter, verse)
     )
     ''')
     cursor.execute('''
     CREATE TABLE IF NOT EXISTS processed_books (
-        book TEXT PRIMARY KEY, -- Store book name directly
+        book TEXT PRIMARY KEY,
         max_phrase_length INTEGER
     )
     ''')
-    conn.commit()
-    logging.info("Database initialized.")
 
 def initialize_translator():
     """Initializes the Google Translator."""
@@ -56,7 +58,7 @@ def populate_database(start_book, end_book, max_phrase_length=1):
     logging.info(f"Populating database with books from {start_book} to {end_book}...")
     cursor = conn.cursor()
 
-    for book_id in range(start_book, end_book + 1):
+    for book_id in tqdm(range(start_book, end_book + 1), desc="Processing Books"):
         book_data = process_json_files(book_id, book_id)  # Get data for the single book
 
         # process_json_files returns a dictionary with book_id as key,
@@ -84,43 +86,36 @@ def populate_database(start_book, end_book, max_phrase_length=1):
                 continue
 
             chapters = book_data['text']
+            # Faster iteration with enumerate and list comprehension
             for chapter_id, chapter in enumerate(chapters):
-                if not isinstance(chapter, list):
-                    logging.warning(f"Skipping chapter {chapter_id} in book {title} due to invalid format.")
-                    continue
                 for verse_id, verse in enumerate(chapter):
                     verse_text = flatten_text(verse)
-                    # Remove text in square brackets
+                    # Remove text in square brackets and non-Hebrew characters
                     verse_text = re.sub(r'\[.*?\]', '', verse_text)
                     verse_text = re.sub(r"[^\u05D0-\u05EA ]+", "", verse_text)
                     verse_text = re.sub(r" +", " ", verse_text)
                     words = verse_text.split()
 
-                    # Iterate through phrases of different lengths
+                    # Use a generator to avoid building large lists in memory
                     for length in range(1, max_phrase_length + 1):
                         for start in range(len(words) - length + 1):
                             phrase_candidate = " ".join(words[start:start + length])
                             gematria_sum = calculate_gematria(phrase_candidate.replace(" ", ""))
-                            insert_phrase_to_db(gematria_sum, phrase_candidate, title, chapter_id + 1, verse_id + 1)
+                            yield gematria_sum, phrase_candidate, title, chapter_id + 1, verse_id + 1
 
-            # Mark the book as processed for this max_phrase_length
-            cursor.execute('''INSERT OR REPLACE INTO processed_books (book, max_phrase_length) VALUES (?, ?)''', (title, max_phrase_length))
-            conn.commit()
-    logging.info("Database population complete.")
-
-def insert_phrase_to_db(gematria_sum, phrase_candidate, book, chapter, verse):
-    """Inserts a phrase and its Gematria value into the database."""
+def insert_phrases_to_db(phrases):
+    """Inserts a list of phrases into the database efficiently."""
     global conn
     cursor = conn.cursor()
-    try:
-        cursor.execute('''
-          INSERT INTO results (gematria_sum, words, book, chapter, verse)
-          VALUES (?, ?, ?, ?, ?)
-        ''', (gematria_sum, phrase_candidate, book, chapter, verse))
-        conn.commit()
-        logging.debug(f"Inserted phrase: {phrase_candidate} (Gematria: {gematria_sum}) at {book}:{chapter}:{verse}")
-    except sqlite3.IntegrityError:
-        logging.debug(f"Phrase already exists: {phrase_candidate} (Gematria: {gematria_sum}) at {book}:{chapter}:{verse}")
+
+    # Use executemany to insert multiple rows at once
+    cursor.executemany('''
+    INSERT OR IGNORE INTO results (gematria_sum, words, book, chapter, verse)
+    VALUES (?, ?, ?, ?, ?)
+    ''', phrases)
+
+    # Commit the changes outside the loop for better performance
+    conn.commit()
 
 def get_translation(phrase):
     """Retrieves or generates the English translation of a Hebrew phrase."""
@@ -136,11 +131,10 @@ def get_translation(phrase):
     else:
         translation = translate_and_store(phrase)
         cursor.execute('''
-          UPDATE results
-          SET translation = ?
-          WHERE words = ?
+        UPDATE results
+        SET translation = ?
+        WHERE words = ?
         ''', (translation, phrase))
-        conn.commit()
         return translation
 
 def translate_and_store(phrase):
@@ -178,14 +172,22 @@ def gematria_search_interface(phrase):
     if not phrase.strip():
         return "Please enter a phrase."
 
-    global conn, book_names
+    global conn, book_names, gematria_cache
     conn = sqlite3.connect('gematria.db')
     cursor = conn.cursor()
 
     phrase_gematria = calculate_gematria(phrase.replace(" ", ""))
     logging.info(f"Searching for phrases with Gematria: {phrase_gematria}")
 
-    matching_phrases = search_gematria_in_db(phrase_gematria)
+    # Check if Gematria is in cache
+    if phrase_gematria in gematria_cache:
+        matching_phrases = gematria_cache[phrase_gematria]
+    else:
+        # Search in the database
+        matching_phrases = search_gematria_in_db(phrase_gematria)
+        # Cache the results for future searches
+        gematria_cache[phrase_gematria] = matching_phrases
+
     if not matching_phrases:
         return "No matching phrases found."
 
@@ -206,13 +208,13 @@ def gematria_search_interface(phrase):
             translation = get_translation(words)
             link = f"https://www.biblegateway.com/passage/?search={quote_plus(book)}+{chapter}%3A{verse}&version=CJB"
             results.append(f"""
-          <div class='result-item'>
-            <p>Chapter: {chapter}, Verse: {verse}</p>
-            <p class='hebrew-phrase'>Hebrew Phrase: {words}</p>
-            <p>Translation: {translation}</p>
-            <a href='{link}' target='_blank' class='bible-link'>[See on Bible Gateway]</a>
-          </div>
-          """)
+            <div class='result-item'>
+                <p>Chapter: {chapter}, Verse: {verse}</p>
+                <p class='hebrew-phrase'>Hebrew Phrase: {words}</p>
+                <p>Translation: {translation}</p>
+                <a href='{link}' target='_blank' class='bible-link'>[See on Bible Gateway]</a>
+            </div>
+            """)
     results.append("</div>")  # Close results-container div
 
     conn.close()
@@ -221,28 +223,28 @@ def gematria_search_interface(phrase):
     style = """
     <style>
         .results-container {
-          display: grid;
-          grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
-          gap: 20px;
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+            gap: 20px;
         }
 
         .result-item {
-          border: 1px solid #ccc;
-          padding: 15px;
-          border-radius: 5px;
-          box-shadow: 2px 2px 5px rgba(0, 0, 0, 0.1);
+            border: 1px solid #ccc;
+            padding: 15px;
+            border-radius: 5px;
+            box-shadow: 2px 2px 5px rgba(0, 0, 0, 0.1);
         }
 
         .hebrew-phrase {
-          font-family: 'SBL Hebrew', 'Ezra SIL', serif;
-          direction: rtl;
+            font-family: 'SBL Hebrew', 'Ezra SIL', serif;
+            direction: rtl;
         }
 
         .bible-link {
-          display: block;
-          margin-top: 10px;
-          color: #007bff;
-          text-decoration: none;
+            display: block;
+            margin-top: 10px;
+            color: #007bff;
+            text-decoration: none;
         }
     </style>
     """
@@ -261,8 +263,16 @@ def run_app():
     initialize_translator()
 
     # Pre-populate the database
-    populate_database(1, 39, max_phrase_length=12)  # Books 1 to 39 (adjust as needed)
-    #opulate_database(27, 27, max_phrase_length=31)  # Book 27 (Psalms) - adjust as needed
+    logging.info("Starting database population...")
+    phrases_to_insert = []  # Collect phrases before inserting in bulk
+    for gematria_sum, phrase, book, chapter, verse in tqdm(populate_database(1, 39, max_phrase_length=3), desc="Populating Database"):  # Books 1 to 39
+        phrases_to_insert.append((gematria_sum, phrase, book, chapter, verse))
+        if len(phrases_to_insert) >= 1000:  # Insert in batches of 1000 for efficiency
+            insert_phrases_to_db(phrases_to_insert)
+            phrases_to_insert = []
+    if phrases_to_insert:  # Insert remaining phrases
+        insert_phrases_to_db(phrases_to_insert)
+    logging.info("Database population complete.")
 
     iface = gr.Interface(
         fn=gematria_search_interface,

gematria.db

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f9d0db3cd97140e47b091845fc11daaa261164db0ac4aa8e430e0c445da27851
-size 171667456
+oid sha256:d8ae62432a24f36453d7080be5d95162d949ac77f006db0a0a81000c40abcabf
+size 65466368

gematria.db-journal

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2ff707a32b432b7883756f3d0db45fc3031ffefa7f890197ac1d9bed4fb9849
+size 3884488