book:unknown fix

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+gematria.db.bak

app.py

@@ -10,9 +10,9 @@ from deep_translator import GoogleTranslator, exceptions
 from urllib.parse import quote_plus
 
 # Set up logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
 
-# Global variables for database connection, translator and book names
+# Global variables for database connection, translator, and book names
 conn = None
 translator = None
 book_names = {}
@@ -29,7 +29,7 @@ def initialize_database():
         gematria_sum INTEGER,
         words TEXT,
         translation TEXT,
-        book INTEGER,
+        book TEXT, -- Store book name directly
         chapter INTEGER,
         verse INTEGER,
         PRIMARY KEY (gematria_sum, book, chapter, verse)
@@ -37,7 +37,7 @@ def initialize_database():
     ''')
     cursor.execute('''
     CREATE TABLE IF NOT EXISTS processed_books (
-        book INTEGER PRIMARY KEY,
+        book TEXT PRIMARY KEY, -- Store book name directly
         max_phrase_length INTEGER
     )
     ''')
@@ -50,52 +50,62 @@ def initialize_translator():
     translator = GoogleTranslator(source='iw', target='en')
     logging.info("Translator initialized.")
 
-def populate_database(tanach_texts, max_phrase_length=1):
+def populate_database(start_book, end_book, max_phrase_length=1):
     """Populates the database with phrases from the Tanach and their Gematria values."""
     global conn, book_names
-    logging.info("Populating database...")
+    logging.info(f"Populating database with books from {start_book} to {end_book}...")
     cursor = conn.cursor()
 
-    for book_id, book_data in tanach_texts.items():
-        # Check if the book is already processed for this max_phrase_length
-        cursor.execute('''SELECT max_phrase_length FROM processed_books WHERE book = ?''', (book_id,))
-        result = cursor.fetchone()
-        if result and result[0] >= max_phrase_length:
-            logging.info(f"Skipping book {book_id}: Already processed with max_phrase_length {result[0]}")
-            continue
+    for book_id in range(start_book, end_book + 1):
+        book_data = process_json_files(book_id, book_id)  # Get data for the single book
 
-        logging.info(f"Processing book {book_id} with max_phrase_length {max_phrase_length}")
+        # process_json_files returns a dictionary with book_id as key,
+        # so access the book data directly
+        if book_id in book_data:
+            book_data = book_data[book_id]
+            if 'title' not in book_data or not isinstance(book_data['title'], str):
+                logging.warning(f"Skipping book {book_id} due to missing or invalid 'title' field.")
+                continue
+
+            title = book_data['title']
+            book_names[book_id] = title
 
-        if 'text' not in book_data or not isinstance(book_data['text'], list):
-            logging.warning(f"Skipping book {book_id} due to missing or invalid 'text' field.")
-            continue
+            # Check if the book is already processed for this max_phrase_length
+            cursor.execute('''SELECT max_phrase_length FROM processed_books WHERE book = ?''', (title,))
+            result = cursor.fetchone()
+            if result and result[0] >= max_phrase_length:
+                logging.info(f"Skipping book {title}: Already processed with max_phrase_length {result[0]}")
+                continue
 
-        title = book_data.get('title', 'Unknown')
-        book_names[book_id] = title 
+            logging.info(f"Processing book {title} with max_phrase_length {max_phrase_length}")
 
-        chapters = book_data['text']
-        for chapter_id, chapter in enumerate(chapters):
-            if not isinstance(chapter, list):
-                logging.warning(f"Skipping chapter {chapter_id} in book {title} due to invalid format.")
+            if 'text' not in book_data or not isinstance(book_data['text'], list):
+                logging.warning(f"Skipping book {book_id} due to missing or invalid 'text' field.")
                 continue
-            for verse_id, verse in enumerate(chapter):
-                verse_text = flatten_text(verse)
-                # Remove text in square brackets
-                verse_text = re.sub(r'\[.*?\]', '', verse_text)
-                verse_text = re.sub(r"[^\u05D0-\u05EA ]+", "", verse_text)
-                verse_text = re.sub(r" +", " ", verse_text)
-                words = verse_text.split()
-
-                # Iterate through phrases of different lengths
-                for length in range(1, max_phrase_length + 1):
-                    for start in range(len(words) - length + 1):
-                        phrase_candidate = " ".join(words[start:start + length])
-                        gematria_sum = calculate_gematria(phrase_candidate.replace(" ", ""))
-                        insert_phrase_to_db(gematria_sum, phrase_candidate, book_id, chapter_id + 1, verse_id + 1)
-
-        # Mark the book as processed for this max_phrase_length
-        cursor.execute('''INSERT OR REPLACE INTO processed_books (book, max_phrase_length) VALUES (?, ?)''', (book_id, max_phrase_length))
-        conn.commit()
+
+            chapters = book_data['text']
+            for chapter_id, chapter in enumerate(chapters):
+                if not isinstance(chapter, list):
+                    logging.warning(f"Skipping chapter {chapter_id} in book {title} due to invalid format.")
+                    continue
+                for verse_id, verse in enumerate(chapter):
+                    verse_text = flatten_text(verse)
+                    # Remove text in square brackets
+                    verse_text = re.sub(r'\[.*?\]', '', verse_text)
+                    verse_text = re.sub(r"[^\u05D0-\u05EA ]+", "", verse_text)
+                    verse_text = re.sub(r" +", " ", verse_text)
+                    words = verse_text.split()
+
+                    # Iterate through phrases of different lengths
+                    for length in range(1, max_phrase_length + 1):
+                        for start in range(len(words) - length + 1):
+                            phrase_candidate = " ".join(words[start:start + length])
+                            gematria_sum = calculate_gematria(phrase_candidate.replace(" ", ""))
+                            insert_phrase_to_db(gematria_sum, phrase_candidate, title, chapter_id + 1, verse_id + 1)
+
+            # Mark the book as processed for this max_phrase_length
+            cursor.execute('''INSERT OR REPLACE INTO processed_books (book, max_phrase_length) VALUES (?, ?)''', (title, max_phrase_length))
+            conn.commit()
     logging.info("Database population complete.")
 
 def insert_phrase_to_db(gematria_sum, phrase_candidate, book, chapter, verse):
@@ -104,8 +114,8 @@ def insert_phrase_to_db(gematria_sum, phrase_candidate, book, chapter, verse):
     cursor = conn.cursor()
     try:
         cursor.execute('''
-            INSERT INTO results (gematria_sum, words, book, chapter, verse)
-            VALUES (?, ?, ?, ?, ?)
+          INSERT INTO results (gematria_sum, words, book, chapter, verse)
+          VALUES (?, ?, ?, ?, ?)
         ''', (gematria_sum, phrase_candidate, book, chapter, verse))
         conn.commit()
         logging.debug(f"Inserted phrase: {phrase_candidate} (Gematria: {gematria_sum}) at {book}:{chapter}:{verse}")
@@ -126,9 +136,9 @@ def get_translation(phrase):
     else:
         translation = translate_and_store(phrase)
         cursor.execute('''
-            UPDATE results
-            SET translation = ?
-            WHERE words = ?
+          UPDATE results
+          SET translation = ?
+          WHERE words = ?
         ''', (translation, phrase))
         conn.commit()
         return translation
@@ -136,7 +146,7 @@ def get_translation(phrase):
 def translate_and_store(phrase):
     """Translates a Hebrew phrase to English using Google Translate and handles potential errors."""
     global translator
-    max_retries = 3 
+    max_retries = 3
     retries = 0
 
     while retries < max_retries:
@@ -145,7 +155,7 @@ def translate_and_store(phrase):
             logging.debug(f"Translated phrase: {translation}")
             return translation
         except (exceptions.TranslationNotFound, exceptions.NotValidPayload,
-            exceptions.ServerException, exceptions.RequestError, requests.exceptions.ConnectionError) as e: 
+                exceptions.ServerException, exceptions.RequestError, requests.exceptions.ConnectionError) as e:
             retries += 1
             logging.warning(f"Error translating phrase '{phrase}': {e}. Retrying... ({retries}/{max_retries})")
 
@@ -191,20 +201,19 @@ def gematria_search_interface(phrase):
     results = []
     results.append("<div class='results-container'>")
     for book, phrases in results_by_book.items():
-        results.append(f"<h4>Book: {book_names.get(book, 'Unknown')}</h4>")
+        results.append(f"<h4>Book: {book}</h4>")  # Directly display book name
         for words, chapter, verse in phrases:
             translation = get_translation(words)
-            book_name_english = book_names.get(book, 'Unknown')
-            link = f"https://www.biblegateway.com/passage/?search={quote_plus(book_name_english)}+{chapter}%3A{verse}&version=CJB"
+            link = f"https://www.biblegateway.com/passage/?search={quote_plus(book)}+{chapter}%3A{verse}&version=CJB"
             results.append(f"""
-            <div class='result-item'>
-                <p>Chapter: {chapter}, Verse: {verse}</p>
-                <p class='hebrew-phrase'>Hebrew Phrase: {words}</p>
-                <p>Translation: {translation}</p>
-                <a href='{link}' target='_blank' class='bible-link'>[See on Bible Gateway]</a>
-            </div>
-            """)
-    results.append("</div>") # Close results-container div
+          <div class='result-item'>
+            <p>Chapter: {chapter}, Verse: {verse}</p>
+            <p class='hebrew-phrase'>Hebrew Phrase: {words}</p>
+            <p>Translation: {translation}</p>
+            <a href='{link}' target='_blank' class='bible-link'>[See on Bible Gateway]</a>
+          </div>
+          """)
+    results.append("</div>")  # Close results-container div
 
     conn.close()
 
@@ -212,33 +221,33 @@ def gematria_search_interface(phrase):
     style = """
     <style>
         .results-container {
-            display: grid;
-            grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
-            gap: 20px;
+          display: grid;
+          grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+          gap: 20px;
         }
 
         .result-item {
-            border: 1px solid #ccc;
-            padding: 15px;
-            border-radius: 5px;
-            box-shadow: 2px 2px 5px rgba(0, 0, 0, 0.1);
+          border: 1px solid #ccc;
+          padding: 15px;
+          border-radius: 5px;
+          box-shadow: 2px 2px 5px rgba(0, 0, 0, 0.1);
         }
 
         .hebrew-phrase {
-            font-family: 'SBL Hebrew', 'Ezra SIL', serif;
-            direction: rtl;
+          font-family: 'SBL Hebrew', 'Ezra SIL', serif;
+          direction: rtl;
         }
 
         .bible-link {
-            display: block;
-            margin-top: 10px;
-            color: #007bff;
-            text-decoration: none;
+          display: block;
+          margin-top: 10px;
+          color: #007bff;
+          text-decoration: none;
         }
     </style>
     """
 
-    return style + "\n".join(results) 
+    return style + "\n".join(results)
 
 def flatten_text(text):
     """Helper function to flatten nested lists into a single list."""
@@ -252,10 +261,8 @@ def run_app():
     initialize_translator()
 
     # Pre-populate the database
-    tanach_texts = process_json_files(1, 39) 
-    populate_database(tanach_texts, max_phrase_length=5)
-    tanach_texts = process_json_files(27, 27) 
-    populate_database(tanach_texts, max_phrase_length=24)
+    populate_database(1, 39, max_phrase_length=1)  # Books 1 to 39 (adjust as needed)
+    #populate_database(27, 27, max_phrase_length=1)  # Book 27 (Psalms) - adjust as needed
 
     iface = gr.Interface(
         fn=gematria_search_interface,

gematria.db

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:06faa206f211a61ddf8609791d575035fba1f81f092b9b4a5a30a24d7d4bc2f0
-size 69844992
+oid sha256:ee62e36845670369178014d353e35dcc219749ff181873d7af325410479a4537
+size 18497536