ConnectionsAPI/Utility/SyncUtility.cs

using ConnectionsAPI.Database;
using ConnectionsAPI.Database.Entities;
using Microsoft.EntityFrameworkCore;
using System.Collections.Concurrent;
using System.Globalization;
using System.Text.Json;
using System.Text.Json.Serialization;

namespace ConnectionsAPI.Utility;
public class SyncUtility(ConnectionsContext db, ILogger<SyncUtility> logger, HttpClient http)
{
    #region Response types
    class NYTConnectionsPuzzle
    {
        [JsonPropertyName("status")]
        public string Status { get; set; } = string.Empty;
        [JsonPropertyName("print_date")]
        public string PrintDate { get; set; } = string.Empty;
        [JsonPropertyName("editor")]
        public string Editor { get; set; } = string.Empty;
        [JsonPropertyName("categories")]
        public IReadOnlyList<NYTConnectionsPuzzleCategory> Categories { get; set; } = [];

        [JsonIgnore]
        public string Md5 { get; set; } = string.Empty;
    }

    class NYTConnectionsPuzzleCategory
    {
        [JsonPropertyName("title")]
        public string Title { get; set; } = string.Empty;
        [JsonPropertyName("cards")]
        public IReadOnlyList<NYTConnectionsPuzzleCard> Cards { get; set; } = [];
    }

    public class NYTConnectionsPuzzleCard
    {
        [JsonPropertyName("content")]
        public string Content { get; set; } = string.Empty;
        [JsonPropertyName("position")]
        public int Position { get; set; }
    }
    #endregion

    private static readonly string SHORT_DATE = "yyyy-MM-dd";

    private readonly ConnectionsContext _db = db;
    private readonly ILogger<SyncUtility> _logger = logger;
    private readonly HttpClient _http = http;

    public async Task SyncPuzzlesAsync(CancellationToken ct)
    {
        _logger.LogInformation("Calculating puzzle sync dates");

        // calculate the date ranges for the sync
        var syncDates = await GetSyncDatesAsync(ct);

        _logger.LogInformation("Syncing puzzles between {start} - {end}", syncDates[0], syncDates[^1]);

        // run the HTTP requests in batches
        ConcurrentDictionary<string, string> responses = new();
        foreach (var batch in syncDates.Chunk(5))
        {
            ConcurrentBag<NYTConnectionsPuzzle> batchPuzzles = [];
            await Task.WhenAll(
                batch.Select(x => GetConnectionsResponseAsync(x, ct).ContinueWith(t =>
                {
                    string? result = t.Result;
                    if (!string.IsNullOrWhiteSpace(result))
                    {
                        try
                        {
                            var nytResponseJson = JsonSerializer.Deserialize<NYTConnectionsPuzzle>(result)
                                ?? throw new InvalidDataException("Connections response deserialized to null");

                            string md5 = HashUtility.CalculateMD5(result);
                            nytResponseJson.Md5 = md5;
                            batchPuzzles.Add(nytResponseJson);
                        }
                        catch (Exception ex)
                        {
                            _logger.LogError(ex, "Failed to deserialize Connections response for {date}", x);
                        }
                    }
                    else
                    {
                        _logger.LogWarning("Puzzle {date} non-success response, skipping", x);
                    }
                }))
            );

            foreach (var puzzle in batchPuzzles.OrderBy(x => x.PrintDate))
            {
                await UpsertPuzzleDataAsync(puzzle);
            }
            await _db.SaveChangesAsync(ct);
        }
    }

    private async Task UpsertPuzzleDataAsync(NYTConnectionsPuzzle nytPuzzle)
    {
        // get a tracking reference to the puzzle matching by print date, either by querying or creating a new entity
        var puzzle = await _db.ConnectionsPuzzles
            .Include(x => x.Categories)
            .ThenInclude(x => x.Cards)
            .FirstOrDefaultAsync(x => x.PrintDate == nytPuzzle.PrintDate);
        if (puzzle == null)
        {
            _logger.LogTrace("No puzzle found for {printDate}, puzzle will be created", nytPuzzle.PrintDate);
            puzzle = new Database.Entities.ConnectionsPuzzle
            {
                Categories = [],
                CreatedDate = DateTime.UtcNow
            };
            _db.ConnectionsPuzzles.Add(puzzle);
        }

        // if the content hash matches, no update needed
        if (puzzle.ContentMD5 == nytPuzzle.Md5)
        {
            _logger.LogTrace("JSON content hash for {printDate} matches, no need for update", nytPuzzle.PrintDate);
            return;
        }

        puzzle.ContentMD5 = nytPuzzle.Md5;
        puzzle.PrintDate = nytPuzzle.PrintDate;
        puzzle.EditorName = nytPuzzle.Editor;
        puzzle.Index = CalculateConnectionsDayIndex(nytPuzzle.PrintDate);
        puzzle.Categories ??= [];

        // mark items for deletion and also remove them from here to be readded
        _db.RemoveRange(puzzle.Categories);
        puzzle.Categories.Clear();

        // construct the entities
        int idx = 1;
        foreach (var nytCategory in nytPuzzle.Categories)
        {
            ConnectionsCategory category = new()
            {
                Color = (ConnectionsColor)idx++,
                Name = nytCategory.Title,
                ConnectionsPuzzle = puzzle,
                Cards = []
            };

            foreach (var nytCard in nytCategory.Cards)
            {
                ConnectionsCard card = new()
                {
                    Content = nytCard.Content,
                    Position = nytCard.Position,
                    Category = category,
                };
                category.Cards.Add(card);
            }

            puzzle.Categories.Add(category);
        }
        // done
    }

    private static int CalculateConnectionsDayIndex(string printDate)
    {
        DateTime connectionsDate = DateTime.ParseExact(printDate, SHORT_DATE, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).Date;
        return Convert.ToInt32(Math.Max((connectionsDate - Constants.ConnectionsStartDate).TotalDays, 0)) + 1;
    }

    private async Task<string?> GetConnectionsResponseAsync(string printDate, CancellationToken ct)
    {
        string url = $"https://www.nytimes.com/svc/connections/v2/{printDate}.json";
        using var resp = await _http.GetAsync(url, ct);
        if (resp == null || !resp.IsSuccessStatusCode)
        {
            return null;
        }
        string responseContent = await resp.Content.ReadAsStringAsync(ct);
        return responseContent;
    }

    private async Task<IReadOnlyList<string>> GetSyncDatesAsync(CancellationToken ct)
    {
        // query the last puzzle we have in the database
        string? lastSyncedPuzzleDate = await _db.ConnectionsPuzzles.AsNoTracking()
            .OrderByDescending(x => x.PrintDate)
            .Select(x => x.PrintDate)
            .FirstOrDefaultAsync(cancellationToken: ct);

        // calculate the starting date of the sync
        string startDate;

        // if no puzzle was synced before, we use the start day of connections as a start (we want to sync every puzzle ever)
        if (string.IsNullOrWhiteSpace(lastSyncedPuzzleDate))
        {
            startDate = Constants.ConnectionsStartDate.ToString(SHORT_DATE);
        }
        else
        {
            string todayPrintDate = DateTimeOffset.UtcNow.UtcDateTime.ToString(SHORT_DATE);
            // if we have a puzzle, we check the latest print date we have
            // if the print date is earlier than today's date, we use that day as a base
            // if the print date is after today, we use today as a base
            if (lastSyncedPuzzleDate.CompareTo(todayPrintDate) < 0)
            {
                startDate = lastSyncedPuzzleDate;
            }
            else
            {
                startDate = todayPrintDate;
            }
        }

        // construct a list of dates
        List<string> dates = [startDate];

        // we iterate on every day between the start date and UTC tomorrow (this should handle +12 timezones as well)
        DateTime syncBeginDate = DateTime.ParseExact(startDate, SHORT_DATE, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).Date;
        DateTime syncEndDate;

        // try to find the latest date that is currently going on in the world
        TimeZoneInfo? latestTimezone = TimezoneUtility.GetLatestTimezoneOnSystem();
        if (latestTimezone != null)
        {
            DateTime currentDateInLatestTimezone = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, latestTimezone);
            syncEndDate = new DateTime(currentDateInLatestTimezone.Year, currentDateInLatestTimezone.Month, currentDateInLatestTimezone.Day, 0, 0, 0, DateTimeKind.Utc);
        }
        // default to UTC date + 1 day
        else
        {
            syncEndDate = DateTime.UtcNow.Date.AddDays(1);
        }

        foreach (var date in Enumerable.Repeat(0, Convert.ToInt32((syncEndDate - syncBeginDate).TotalDays)).Select((_, idx) => syncBeginDate.AddDays(idx + 1)))
        {
            dates.Add(date.ToString(SHORT_DATE));
        }

        // done
        return dates;
    }
}