Sources/App/Helper/Download/HttpClient+Retry.swift

import Foundation
import AsyncHTTPClient
import NIOCore
import Logging


extension HTTPClientResponse {
    /// Throw if a transient error occurred. A retry could be successful
    func throwOnTransientError() throws {
        if isTransientError {
            throw CurlError.downloadFailed(code: status)
        }
    }
    
    /// True is status code contains an error that is retirable. E.g. Gateway timeout or too many requests
    var isTransientError: Bool {
        return [
            .requestTimeout,
            .tooManyRequests,
            .internalServerError,
            .badGateway,
            .serviceUnavailable,
            .gatewayTimeout
        ].contains(status)
    }

    func throwOnFatalError() throws {
        if status == .unauthorized {
            throw CurlErrorNonRetry.unauthorized
        }
    }
}

extension HTTPClient {
    /**
     Retry HTTP requests on error
     */
    func executeRetry(_ request: HTTPClientRequest,
                       logger: Logger,
                       deadline: Date = .minutes(60),
                       timeoutPerRequest: TimeAmount = .seconds(30),
                       backoffFactor: TimeAmount = .milliseconds(1000),
                       backoffMaximum: TimeAmount = .seconds(30),
                       error404WaitTime: TimeAmount? = nil) async throws -> HTTPClientResponse  {
        var lastPrint = Date(timeIntervalSince1970: 0)
        let startTime = Date()
        var n = 0
        while true {
            do {
                n += 1
                let response = try await execute(request, timeout: timeoutPerRequest, logger: logger)
                logger.debug("Response for HTTP request #\(n) returned HTTP status code: \(response.status), from URL \(request.url)")
                try response.throwOnTransientError()
                try response.throwOnFatalError()
                if error404WaitTime != nil && response.status == .notFound {
                    throw CurlError.fileNotFound
                }
                return response
            } catch CurlErrorNonRetry.unauthorized {
                logger.info("Download failed with 401 Unauthorized error, credentials rejected. Possibly outdated API key.")
                throw CurlErrorNonRetry.unauthorized
            } catch {
                var wait = TimeAmount.nanoseconds(min(backoffFactor.nanoseconds * Int64(pow(2, Double(n-1))), backoffMaximum.nanoseconds))
                
                if let ioerror = error as? IOError, [104,54].contains(ioerror.errnoCode), n <= 2 {
                    /// MeteoFrance API resets the connection very frequently causing large delays in downloading
                    /// Immediately retry twice
                    wait = .zero
                }
                if let error404WaitTime, case CurlError.fileNotFound = error {
                    wait = error404WaitTime
                }
                
                let timeElapsed = Date().timeIntervalSince(startTime)
                if Date().timeIntervalSince(lastPrint) > 60 {
                    logger.info("Download failed. Attempt \(n). Elapsed \(timeElapsed.prettyPrint). Retry in \(wait.prettyPrint). Error '\(error) [\(type(of: error))]'")
                    lastPrint = Date()
                }
                if Date() > deadline {
                    logger.error("Deadline reached. Attempt \(n). Elapsed \(timeElapsed.prettyPrint).  Error '\(error) [\(type(of: error))]'")
                    throw CurlError.timeoutReached
                }
                try await _Concurrency.Task.sleep(nanoseconds: UInt64(wait.nanoseconds))
            }
        }
    }
}

extension TimeAmount {
    var prettyPrint: String {
        return (Double(nanoseconds) / 1_000_000_000).asSecondsPrettyPrint
    }
}

extension TimeInterval {
    var prettyPrint: String {
        return self.asSecondsPrettyPrint
    }
}

extension Date {
    static func hours(_ hours: Double) -> Date {
        return Date(timeIntervalSinceNow: hours*3600)
    }
    static func minutes(_ minutes: Double) -> Date {
        return Date(timeIntervalSinceNow: minutes*60)
    }
    static func seconds(_ seconds: Double) -> Date {
        return Date(timeIntervalSinceNow: seconds)
    }
}