using System.Collections.Concurrent;
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text.Json;
using System.Text.Json.Serialization;
using ConsoleApp2;
using Domain;
using Polly;

var tagsCount = new ConcurrentDictionary<string, int>();

const string photosPath = "images";

var retryPolicy = Policy
    .Handle<HttpRequestException>()
    .Or<TaskCanceledException>()
    .RetryAsync(10);

var downloadList = new HashSet<ImageData>(300000);

List<TagData> downloadTags = new();

var tagsFile = args[0];

foreach (var lineTags in File.ReadAllLines(tagsFile).Select(l => l.Trim()).Distinct())
{
    var spaceIndex = lineTags.IndexOf(" ", StringComparison.Ordinal);
    var repeats = int.Parse(lineTags[..spaceIndex]);
    var tags = lineTags[(spaceIndex + 1)..];
    Console.WriteLine($"Loading {tags} with {repeats} repeats");

    downloadTags.Add(new TagData(tags.Split(" ").ToList())
    {
        Repeats = repeats
    });
}

downloadTags = downloadTags.OrderByDescending(t => t.Repeats).ToList();

IBooruSite booruSite;

switch (args[6])
{
    case "yandere":
        booruSite = new YandereSite(retryPolicy);
        break;
    case "gelbooru":
        booruSite = new GelbooruSite(retryPolicy);
        break;
    case "realbooru":
        booruSite = new RealbooruSite(retryPolicy);
        break;

    default:
        Console.WriteLine("Unsupported booru site");
        return;
}

var holdLock = new object();
Task? currentHoldTask = null;

var pagesToDownload = int.Parse(args[1]);
Console.WriteLine("Pages to dl = " + pagesToDownload);

var minScore = int.Parse(args[2]);
Console.WriteLine("Min score = " + minScore);

var minTags = int.Parse(args[3]);
Console.WriteLine("Min tags = " + minTags);

var ignoreTags = args[4].Split(" ");
var formattedNegatives = booruSite.FormatNegatives(ignoreTags.ToList());
Console.WriteLine("Tags to ignore = " + formattedNegatives);
var maxRatio = float.Parse(args[5], NumberStyles.Any);
Console.WriteLine("Max ratio = " + maxRatio);
var minRes = args[7].Split("x").Select(int.Parse).Aggregate(1, (i, i1) => i * i1);
Console.WriteLine("Min pixels = " + minRes);

[DllImport("kernel32.dll")]
static extern bool SetConsoleTitle(string lpConsoleTitle);

var totalRequests = 0;

var ignoredTags = new HashSet<string>()
{
    "bad_id",
    "bad_twitter_id",
    "bad_pixiv_id",
    "has_bad_revision",
    "commentary_request",
    "commentary",
    "tagme",
    "english_commentary",
    "character_request",
    "commission",
    "non-web_source",
    "md5_mismatch",
    "sound",
    "spoilers",
    "chinese_commentary",
    "skeb_commission",
    "symbol-only_commentary",
    "paid_reward_available",
    "resolution_mismatch",
    "gender_request",
    "image_sample",
    "korean_commentary",
    "image_sample",
    "reward_available",
    "image_sample",
    "self-upload",
    "pixiv_commission",
    "source_smaller",
    "hard-translated",
    "nude_filter",
    "paid_reward",
    "duplicate",
    "second-party_source",
    "duplicate",
    "derivative_work",
    "mixed-language_commentary",
    "duplicate",
    "pixiv_sample",
    "duplicate",
    "source_larger",
    "bad_tumblr_id",
    "artist_request"
};

foreach (var downloadTag in downloadTags)
{
    var collected = 0;
    var totalImagesWithTag = 0;

    var formattedTags = booruSite.FormatTags(downloadTag.Tags);

    SetConsoleTitle(formattedTags);
    try
    {
        var pageCount = await booruSite.GetPageCount(formattedTags, formattedNegatives);
        totalRequests++;
        if (pageCount > pagesToDownload)
            pageCount = pagesToDownload;

        Console.WriteLine($"Downloading {formattedTags} with {pageCount} pages");

        var idLock = new object();
        var minId = int.MaxValue;

        foreach (var pageNums in Enumerable.Range(0, pageCount).Select(n => n % 200).Chunk(200))
        {
            var currentMinId = minId;
            await Parallel.ForEachAsync(pageNums,
                new ParallelOptions { MaxDegreeOfParallelism = 6 },
                async (i, ct) =>
                {
                    HttpResponseMessage current = null!;
                    try
                    {
                        totalRequests++;

                        if (totalRequests >= 100)
                        {
                            await HoldRequests();
                        }

                        var extraIdsTag = string.Empty;
                        if (currentMinId != int.MaxValue)
                        {
                            extraIdsTag = $" id:<{currentMinId}";
                            Console.WriteLine("Full tags: " + formattedTags + extraIdsTag);
                        }


                        var posts = await booruSite.GetPosts(formattedTags + extraIdsTag, formattedNegatives, i);
                        if (posts == null) return;

                        foreach (var post in posts)
                        {
                            totalImagesWithTag++;

                            lock (idLock)
                            {
                                if (post.Id < minId)
                                {
                                    minId = post.Id;
                                }
                            }

                            if (post.Width * post.Height < minRes)
                                continue;

                            if (string.IsNullOrEmpty(post.Url) ||
                                Path.GetExtension(post.Url) is ".gif" or ".mp4" or ".webm" ||
                                post.Score < minScore)
                            {
                                continue;
                            }

                            var ratio = GetImageRatio(post.Width, post.Height);
                            if (ratio > maxRatio)
                            {
                                continue;
                            }

                            post.Tags.RemoveAll(tag => ignoredTags.Contains(tag));

                            var tags = post.Tags.Select(t => t.Replace("_", " ")).ToList();

                            if (tags.Count < minTags)
                                continue;

                            var skip = false;
                            foreach (var ignoredTag in ignoreTags)
                            {
                                if (!tags.Contains(ignoredTag)) continue;

                                skip = true;
                                break;
                            }

                            if (skip)
                                break;

                            /*if (booruSite.IsRealPhotos)
                                tags.Add("real life");

                            var (jav, realLife) = (tags.Contains("jav"), tags.Contains("real life"));

                            if (jav && !realLife)
                            {
                                tags.Add("real life");
                            }

                            if (!jav && !realLife)
                            {
                                tags.Add("anime art");
                            }*/

                            if (post.CreatedAt is { } offset)
                            {
                                if (offset.Year == 0) Console.WriteLine("bruh " + post.Id);
                                else tags.Add($"year{offset.Year}");
                            }

                            if (string.IsNullOrEmpty(Path.GetFileNameWithoutExtension(post.FileName)))
                                post.FileName = Path.GetRandomFileName() + Path.GetExtension(post.FileName);

                            lock (downloadList)
                            {
                                downloadList.Add(new ImageData
                                {
                                    Urls = [new(post.Url, post.FileName)],
                                    Tags = string.Join(",", tags),
                                    Repeats = downloadTag.Repeats
                                });
                                collected++;
                                SetConsoleTitle(
                                    $"Tags: {downloadTag.Tags} Page: {i}, Collected: {collected}, Total: {downloadList.Count}");
                            }
                        }

                        Console.WriteLine($"Pg: {i} Images collected: {downloadList.Count}, +{collected}");
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine("Requests = " + totalRequests);
                        Console.WriteLine("Some errors " + e);
                    }
                });
        }
    }
    catch (Exception e)
    {
        Console.WriteLine(e);
    }

    Console.WriteLine("Skipped images: " + (totalImagesWithTag - collected));
}

Console.WriteLine("Doing backup...");
//Backup links
File.WriteAllText("backup123.json", JsonSerializer.Serialize(downloadList.DistinctBy(e => e.Urls[0].Url)));
Console.WriteLine("Success!");

var resultTagsFile = File.CreateText("result_tags.txt");

foreach (var tags in downloadList
             .Select(t => t.Tags.Replace(" ", "_"))
             .Select(t => t.Split(",")))
{
    foreach (var tag in tags)
    {
        IncTag(tag);
    }
}

foreach (var (tag, count) in tagsCount.OrderByDescending(t => t.Value))
{
    resultTagsFile.WriteLine($"{tag} {count}");
}

resultTagsFile.Dispose();

Thread.Sleep(-1);

void IncTag(string tag)
{
    lock (tagsCount)
    {
        if (tagsCount!.TryGetValue(tag, out var count))
        {
            tagsCount[tag] = count + 1;
        }
        else
        {
            tagsCount[tag] = 1;
        }
    }
}

float GetImageRatio(int width, int height)
{
    var ratio = (float)width / height;
    return ratio > 1.0f ? ratio : 1 / ratio;
}

Task HoldRequests()
{
    lock (holdLock)
    {
        if (currentHoldTask != null) return currentHoldTask;

        currentHoldTask = Task.Delay((60 - DateTime.Now.Second + 5) * 1000).ContinueWith(e =>
        {
            totalRequests = 0;
            currentHoldTask = null;
        });
        Console.WriteLine("HOLDDD");
        return currentHoldTask;
    }
}

class PreloadData
{
    [JsonPropertyName("url")] public string Url { get; set; }
}

using System.Globalization;
using System.Net;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Web;
using Polly.Retry;

namespace ConsoleApp2;

public class GelbooruSite : IBooruSite
{
    private readonly AsyncRetryPolicy _policy;
    private HttpClient _httpClient;

    public int RequestsPerMinute => 150;
    private int PostsPerPage => 100;
    public bool IsRealPhotos => false;

    public GelbooruSite(AsyncRetryPolicy policy)
    {
        _policy = policy;

        var cookies = new CookieContainer();

        cookies.Add(new Cookie("fringeBenefits", "yup", "/", "gelbooru.com"));
        var handler = new HttpClientHandler();
        handler.CookieContainer = cookies;

        _httpClient = new HttpClient(handler);
        _httpClient.Timeout = TimeSpan.FromSeconds(15);
    }

    public async Task<int> GetPageCount(string tags, string negatives)
    {
        var pageCount =
            await GetPageCount(
                $"https://gelbooru.com/index.php?page=dapi&s=post&q=index&pid=0&json=1&tags={tags} {negatives}");

        return pageCount;
    }

    public async Task<List<BooruPost>?> GetPosts(string tags, string negatives, int page)
    {
        var currentUrl =
            $"https://gelbooru.com/index.php?page=dapi&s=post&q=index&pid={page}&json=1&tags={tags} {negatives}";

        var response = await _policy.ExecuteAsync(async (ct) => await _httpClient.GetAsync(
                currentUrl),
            CancellationToken.None);

        GelbooruPage posts;

        try
        {
            posts = (await JsonSerializer.DeserializeAsync<GelbooruPage>(await response.Content.ReadAsStreamAsync()))!;
        }
        catch (Exception e)
        {
            Console.WriteLine(e.Message);
            await File.AppendAllTextAsync("shit.log", await response.Content.ReadAsStringAsync());
            return null;
        }

        if (posts.Posts == null) return null;

        const string format = "ddd MMM dd HH:mm:ss zzzz yyyy";

        return posts.Posts.Select(p => new BooruPost
        {
            FileName = p.FileName,
            //Url = (string.IsNullOrEmpty(p.FullImgUrl) ? p.SampleUrl : p.FullImgUrl)!,
            Url = string.IsNullOrEmpty(p.SampleUrl) ? p.FullImgUrl! : p.SampleUrl,
            Tags = HttpUtility.HtmlDecode(p.Tags).Split(" ").ToList(),
            Width = p.Width,
            Height = p.Height,
            Score = p.Score,
            Rating = p.Rating,
            Source = p.Source,
            CreatedAt = DateTimeOffset.ParseExact(p.CreatedAt, format, CultureInfo.InvariantCulture),
            Id = p.Id
        }).ToList();
    }

    public string FormatTags(List<string> tags)
    {
        return string.Join(" ", tags);
    }

    public string FormatNegatives(List<string> negatives)
    {
        return string.Join(" ", negatives.Select(n => $"-{n}"));
    }

    private async Task<int> GetPageCount(string url)
    {
        var data = await _policy.ExecuteAsync(async (ct) => await _httpClient.GetAsync(url),
            CancellationToken.None);

        var page =
            (await JsonSerializer.DeserializeAsync<GelbooruPage>(await data.Content.ReadAsStreamAsync()))!;

        var limit = page.Attributes.Limit;
        var count = page.Attributes.Count;

        return (count + limit + 1) / limit;
    }
}

public class GelbooruPage
{
    [JsonPropertyName("@attributes")] public GelbooruAttributes Attributes { get; set; }
    [JsonPropertyName("post")] public List<GelbooruPost>? Posts { get; set; }
}

public class GelbooruPost
{
    [JsonPropertyName("image")] public string FileName { get; set; }

    [JsonPropertyName("source")] public string Source { get; set; }

    [JsonPropertyName("tags")] public string Tags { get; set; }

    [JsonPropertyName("sample_url")] public string? SampleUrl { get; set; }
    [JsonPropertyName("file_url")] public string? FullImgUrl { get; set; }

    [JsonPropertyName("score")] public int Score { get; set; }
    [JsonPropertyName("width")] public int Width { get; set; }
    [JsonPropertyName("height")] public int Height { get; set; }
    [JsonPropertyName("rating")] public string Rating { get; set; }
    [JsonPropertyName("created_at")] public string CreatedAt { get; set; }
    [JsonPropertyName("id")] public int Id { get; set; }
}

public class GelbooruAttributes
{
    [JsonPropertyName("limit")] public int Limit { get; set; }
    [JsonPropertyName("offset")] public int Offset { get; set; }
    [JsonPropertyName("count")] public int Count { get; set; }
}

на основе этого сделай просто скрипт который спарсит N страниц картинок с тегом animated и сохранит их в json где будет ссылка и теги, добавь прокси в код с авторизацией