added datasource filtering, updated db from seed

DbContext/QuoteDbContext.cs

@@ -57,6 +57,9 @@ public class Quote
     [MaxLength(255)]
     public string? Source { get; set; }  // Nullable column
 
+    [MaxLength(255)]
+    public string DataSet { get; set; }
+
     [MaxLength(255)]
     public string? Book { get; set; }  // Nullable column
 

GlobalData.cs

@@ -1,4 +1,76 @@
 public static class GlobalData
 {
   public static List<Quote> Quotes { get; set; }
+}
+
+public static class Extensions {
+    public static string CleanString(this string str) {
+        if (str == null) return "";
+        var result = str.Trim().Trim('"');
+        result = String.Join(' ', result.Split(' ', StringSplitOptions.RemoveEmptyEntries));
+        return result;
+    }
+
+    public static Quote GetQuote1(this MittalInput json) {
+        var quote = new Quote();
+        quote.QuoteText = json.Content.CleanString();
+
+        if (json.Author != null) {
+            var authorData  = json.Author.Split(',');
+            quote.Author = authorData[0].CleanString();
+            if (authorData.Length > 1){
+              quote.Book = authorData[1].CleanString();
+              quote.Source = "Book";
+            }
+        }
+        else quote.Author = "Unknown";
+
+        quote.Language = "english";
+        
+        quote.Categories = String.Join(",", json.Tags.Select(x => x.ToLower().CleanString()).Distinct());
+        quote.DataSet = "mittal";
+        return quote;
+    }
+
+    public static Quote GetQuote2(this QuotableInput json) {
+        var quote = new Quote();
+        quote.QuoteText = json.Content.CleanString();
+
+        if (json.Author != null) {
+            var authorData  = json.Author.Split(',');
+            quote.Author = authorData[0].CleanString();
+            if (authorData.Length > 1){
+              quote.Book = authorData[1].CleanString();
+              quote.Source = "Book";
+            }
+        }
+        else quote.Author = "Unknown";
+
+        quote.Language = "english";
+        
+        quote.Categories = String.Join(",", json.Tags.Select(x => x.ToLower().CleanString()).Distinct());
+        quote.DataSet = "quotable";
+        return quote;
+    }
+
+    public static Quote GetQuote3(this ManannInput data) {
+        var quote = new Quote();
+        quote.QuoteText = data.quote.CleanString();
+
+        if (data.author != null) {
+            var authorData  = data.author.Split(',');
+            quote.Author = authorData[0].CleanString();
+            if (authorData.Length > 1){
+              quote.Book = authorData[1].CleanString();
+              quote.Source = "Book";
+            }
+        }
+        else quote.Author = "Unknown";
+
+        quote.Language = "english";
+        
+        quote.Categories = String.Join(",", data.category.Split(',', StringSplitOptions.RemoveEmptyEntries).Select(x => x.ToLower().CleanString()).Distinct());
+        quote.DataSet = "manann";
+        return quote;
+    }
 }

Program.cs

@@ -1,6 +1,10 @@
+using System.Globalization;
 using System.Linq;
 using System.Text.Json;
+using CsvHelper;
+using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.ObjectPool;
 
 var builder = WebApplication.CreateBuilder(args);
 builder.WebHost.ConfigureKestrel(serverOptions =>
@@ -37,17 +41,21 @@ const int MaxPageSize = 100;
 
 app.UseCors();
 
-app.MapGet("/quotes", async (QuoteDbContext db, int pageNumber = 1, int pageSize = 10) =>
+app.MapGet("/quotes", async (QuoteDbContext db, int pageNumber = 1, int pageSize = 10, [FromQuery] DataSetSources dataset = DataSetSources.all) =>
 {
+    if (!Enum.IsDefined(typeof(DataSetSources), dataset))
+        return Results.BadRequest("Invalid dataset.");
+
     if (pageNumber < 1) pageNumber = 1;
     if (pageSize < 1) pageSize = 10;
     pageSize = Math.Min(pageSize, MaxPageSize); // Limit pageSize to MaxPageSize
 
-    // var quotes = await db.Quotes
-    //     .Skip((pageNumber - 1) * pageSize)
-    //     .Take(pageSize)
-    //     .ToListAsync();
-    var quotes = GlobalData.Quotes
+    var quotes = GlobalData.Quotes;
+    if (dataset != DataSetSources.all)
+    {
+        quotes = quotes.Where(x => x.DataSet == dataset.ToString()).ToList();
+    }
+    quotes = quotes
         .Skip((pageNumber - 1) * pageSize)
         .Take(pageSize)
         .ToList();
@@ -81,7 +89,7 @@ app.MapGet("/quotes", async (QuoteDbContext db, int pageNumber = 1, int pageSize
 //     quote.Isbn = updatedQuote.Isbn;
 //     quote.Language = updatedQuote.Language;
 //     quote.OriginalLanguage = updatedQuote.OriginalLanguage;
-    
+
 //     await db.SaveChangesAsync();
 //     return Results.NoContent();
 // });
@@ -97,24 +105,56 @@ app.MapGet("/quotes", async (QuoteDbContext db, int pageNumber = 1, int pageSize
 // });
 
 // Random quote endpoint
-app.MapGet("/quotes/random", async (QuoteDbContext db) =>
+app.MapGet("/quotes/random", async (QuoteDbContext db, [FromQuery] DataSetSources dataset = DataSetSources.all) =>
 {
-    var quoteCount = GlobalData.Quotes.Count(); //await db.Quotes.CountAsync();
+    if (!Enum.IsDefined(typeof(DataSetSources), dataset))
+        return Results.BadRequest("Invalid dataset.");
+
+    int quoteCount = 0;
+    if (dataset != DataSetSources.all)
+    {
+        quoteCount = GlobalData.Quotes.Count(q => q.DataSet == dataset.ToString());
+    }
+    else
+    {
+        quoteCount = GlobalData.Quotes.Count(); //await db.Quotes.CountAsync();
+    }
+
     if (quoteCount == 0)
         return Results.NotFound("No quotes available.");
 
     var random = new Random();
     var randomIndex = random.Next(quoteCount);
 
-    var randomQuote = GlobalData.Quotes[randomIndex]; //await db.Quotes.Skip(randomIndex).Take(1).FirstOrDefaultAsync();
+    Quote? randomQuote = null;
+    if (dataset != DataSetSources.all)
+    {
+        randomQuote = GlobalData.Quotes
+            .Where(q => q.DataSet == dataset.ToString())
+            .Skip(randomIndex)
+            .Take(1)
+            .FirstOrDefault();
+    }
+    else
+    {
+        randomQuote = GlobalData.Quotes[randomIndex]; //await db.Quotes.Skip(randomIndex).Take(1).FirstOrDefaultAsync();
+    }
     return randomQuote != null ? Results.Ok(randomQuote) : Results.NotFound("No quote found.");
 });
 
 // Search quotes by author, categories, book, or quoteText with pagination
-app.MapGet("/quotes/search", async (string query, QuoteDbContext db, int pageNumber = 1, int pageSize = 10) =>
+app.MapGet("/quotes/search", async (string query, QuoteDbContext db, int pageNumber = 1, int pageSize = 10, [FromQuery] DataSetSources dataset = DataSetSources.all) =>
 {
+    if (!Enum.IsDefined(typeof(DataSetSources), dataset))
+        return Results.BadRequest("Invalid dataset.");
+
     var queryQuotes = GlobalData.Quotes.AsQueryable(); //db.Quotes.AsQueryable();
 
+    if (dataset != DataSetSources.all)
+    {
+        queryQuotes = queryQuotes.Where(q => q.DataSet == dataset.ToString());
+    }
+
     if (!string.IsNullOrWhiteSpace(query))
     {
         query = query.ToLower();
@@ -148,14 +188,6 @@ async Task SaveSource1Async(string jsonFilePath, QuoteDbContext db)
     if (!File.Exists(path))
         throw new FileNotFoundException("The JSON file for seeding is missing.");
 
-    // var jsonString = await File.ReadAllTextAsync(path);
-
-    // // Deserialize JSON to a list of dictionaries
-    // var dictionaryList = JsonSerializer.Deserialize<List<Dictionary<string, object>>>(jsonString);
-
-    // if (dictionaryList == null)
-    //     throw new InvalidOperationException("Failed to deserialize JSON into dictionary list.");
-
     // Fields
     // ==========
     // Quote (string)
@@ -163,26 +195,19 @@ async Task SaveSource1Async(string jsonFilePath, QuoteDbContext db)
     // Tags (list<string>)
     // Category (string)
 
-    // foreach (var dictionary in dictionaryList)
-    // {
-    //     var tags = "";
-    //     if(dictionary.TryGetValue("Tags", out var categories)) {
-    //         var list = JsonSerializer.Deserialize<List<string>>(categories.ToString());
-    //         tags = String.Join(",", list.Select(x=> x.Trim().Trim('"')));
-    //     }
-    //     var quote = new Quote
-    //     {
-    //         Author = dictionary.TryGetValue("Author", out var author) ? author.ToString() : null,
-    //         QuoteText = dictionary.TryGetValue("Quote", out var quoteText) ? quoteText.ToString() : null,
-    //         Categories = tags,
-    //     };
-    //     db.Quotes.Add(quote);
-    // }
-
+    var jsonString = await File.ReadAllTextAsync(path);
     // TODO: import data and sanitize relevant fields: Trim and Trim('"')
-    
-    //save for each seed file
-    await db.SaveChangesAsync();
+    var quotes = JsonSerializer.Deserialize<List<MittalInput>>(jsonString);
+    var uniqueQuotes = quotes.DistinctBy(x => x.Content).ToList();
+    foreach (var quote in uniqueQuotes)
+    {
+        var q = quote.GetQuote1();
+        if (!db.Quotes.Any(x => x.QuoteText == q.QuoteText.CleanString()))
+        {
+            db.Quotes.Add(q);
+            await db.SaveChangesAsync();
+        }
+    }
 }
 
 async Task SaveSource2Async(string jsonFile, QuoteDbContext db)
@@ -198,10 +223,19 @@ async Task SaveSource2Async(string jsonFile, QuoteDbContext db)
     // author (string)
     // tags (list<string>)
 
+    var jsonString = await File.ReadAllTextAsync(path);
     // TODO: import data and sanitize relevant fields: Trim and Trim('"')
-    
-    //save for each seed file
-    await db.SaveChangesAsync();
+    var quotes = JsonSerializer.Deserialize<List<QuotableInput>>(jsonString);
+    var uniqueQuotes = quotes.DistinctBy(x => x.Content).ToList();
+    foreach (var quote in uniqueQuotes)
+    {
+        var q = quote.GetQuote2();
+        if (!db.Quotes.Any(x => x.QuoteText == q.QuoteText.CleanString()))
+        {
+            db.Quotes.Add(q);
+            await db.SaveChangesAsync();
+        }
+    }
 }
 
 async Task SaveSource3Async(string csvFile, QuoteDbContext db)
@@ -217,10 +251,19 @@ async Task SaveSource3Async(string csvFile, QuoteDbContext db)
     // author (string)
     // category (string)
 
-    // TODO: import data and sanitize relevant fields: Trim and Trim('"')
-
-    //save for each seed file
-    await db.SaveChangesAsync();
+    //read csv file into list of records
+    var reader = new StreamReader(path);
+    var csv = new CsvReader(reader, CultureInfo.InvariantCulture);
+    var records = csv.GetRecords<ManannInput>().ToList();
+    foreach (var record in records)
+    {
+        var q = record.GetQuote3();
+        if (!db.Quotes.Any(x => x.QuoteText == q.QuoteText.CleanString()))
+        {
+            db.Quotes.Add(q);
+            await db.SaveChangesAsync();
+        }
+    }
 }
 
 // Seed database
@@ -238,9 +281,9 @@ using (var scope = app.Services.CreateScope())
 {
     var db = scope.ServiceProvider.GetRequiredService<QuoteDbContext>();
     db.Database.EnsureCreated();
-    
+
     GlobalData.Quotes = await db.Quotes.ToListAsync();
-    await SeedDatabase(db);
+    //await SeedDatabase(db);
 }
 
 app.Run();

QuotesAPI.csproj

@@ -10,6 +10,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="CsvHelper" Version="33.0.1" />
     <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.8" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.8" />

README.md

@@ -1,5 +1,5 @@
 ---
-title: Quotes Fastapi
+title: Quotes
 emoji: 🐨
 colorFrom: yellow
 colorTo: yellow
@@ -17,9 +17,9 @@ Welcome to the **Quotes API**, your go-to source for inspirational, motivational
   Our API draws from an extensive database of more than half a million quotes to keep you inspired every single day. 
 - **Diverse Sources**  
   We've combined quotes from various curated datasets:
-  - [Manann’s 500k Quotes](https://www.kaggle.com/datasets/manann/quotes-500k) - 463,402 unique quotes 📝
   - [AK Mittal's Quotes Collection](https://www.kaggle.com/datasets/akmittal/quotes-dataset) - 36,937 unique quotes 📝
   - [Quotable’s Quote Collection](https://github.com/quotable-io/data/blob/master/data/quotes.json) - 1,634 unique quotes 📝
+  - [Manann’s 500k Quotes](https://www.kaggle.com/datasets/manann/quotes-500k) - 463,430 unique quotes 📝
 - **Easy Integration**  
   Plug it into your app, website, or daily routine with ease. Just call the API and get inspired instantly! 🚀
   

models/DataSetSources.cs

@@ -0,0 +1,10 @@
+using System.Text.Json.Serialization;
+
+[JsonConverter(typeof(JsonStringEnumConverter))]
+public enum DataSetSources
+{
+    all,
+    mittal,
+    quotable,
+    manann
+}

models/ManannInput.cs

@@ -0,0 +1,8 @@
+using System.Text.Json.Serialization;
+
+public class ManannInput
+{
+    public string? author { get; set; }
+    public string? quote { get; set; }
+    public string? category { get; set; }
+}

models/MittalInput.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+public class MittalInput
+{
+    [JsonPropertyName("Author")]
+    public string? Author { get; set; }
+    [JsonPropertyName("Quote")]
+    public string? Content { get; set; }
+    [JsonPropertyName("Tags")]
+    public List<string>? Tags { get; set; }
+}

models/QuotableInput.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+public class QuotableInput
+{
+    [JsonPropertyName("author")]
+    public string? Author { get; set; }
+    [JsonPropertyName("content")]
+    public string? Content { get; set; }
+    [JsonPropertyName("tags")]
+    public List<string>? Tags { get; set; }
+}

quotes.db

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83243ae11dc6227399713a634cc278f21f7463cf9460a43272cc7e40998bdb47
-size 350629888
+oid sha256:d5a3df3c597a721719176229988f30d2e3eb033ab869ccde1cb2a59a10338ecb
+size 347443200