d3ro-voice/apps/api-server/Services/LlmProxyService.cs

199 lines
8.1 KiB
C#

using System;
using System.Diagnostics;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using D3ROVoice.Api.Data;
using D3ROVoice.Api.Dtos;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace D3ROVoice.Api.Services;
public interface ILlmProxyService
{
Task<LlmGenerateResponse> GenerateAsync(int userId, string userEmail, LlmGenerateRequest request, CancellationToken cancellationToken = default);
Task<LlmGenerateResponse> ChatAsync(int userId, string userEmail, LlmChatRequest request, CancellationToken cancellationToken = default);
}
public class LlmProxyService : ILlmProxyService
{
private readonly AppDbContext _db;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<LlmProxyService> _logger;
public LlmProxyService(AppDbContext db, IHttpClientFactory httpClientFactory, ILogger<LlmProxyService> logger)
{
_db = db;
_httpClientFactory = httpClientFactory;
_logger = logger;
}
public async Task<LlmGenerateResponse> GenerateAsync(int userId, string userEmail, LlmGenerateRequest request, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
var modelId = string.IsNullOrWhiteSpace(request.Model) ? "d3ro-gpt4o-mini" : request.Model;
var endpoint = await _db.ModelEndpoints.FirstOrDefaultAsync(m => m.ModelId == modelId && m.IsActive, cancellationToken);
if (endpoint == null)
{
// fallback: first active model endpoint or create default mockup endpoint
endpoint = await _db.ModelEndpoints.FirstOrDefaultAsync(m => m.IsActive, cancellationToken);
if (endpoint == null)
{
endpoint = new ServiceModelEndpoint
{
ModelId = "d3ro-gpt4o-mini",
ModelName = "D3RO Default Model",
Provider = "Mock",
EndpointUrl = "https://api.openai.com/v1/chat/completions",
CostPer1kPromptTokens = 0.00015m,
CostPer1kCompletionTokens = 0.00060m,
IsActive = true
};
}
}
var sw = Stopwatch.StartNew();
int promptTokens = Math.Max(10, request.Prompt.Length / 4);
int completionTokens = 0;
string responseText = "";
if (endpoint.Provider == "Mock" || string.IsNullOrWhiteSpace(endpoint.ApiKey))
{
// Intelligent local echo / mock processing for standalone API testing
responseText = $"[D3RO Online Model - {endpoint.ModelName}] {request.Prompt}";
completionTokens = Math.Max(15, responseText.Length / 4);
sw.Stop();
}
else
{
try
{
if (!EndpointUrlPolicy.IsValidEndpointUrl(
endpoint.EndpointUrl,
EndpointUrlPolicy.AllowsLoopbackForLlmProvider(endpoint.Provider)))
{
throw new InvalidOperationException("Configured model endpoint URL is not permitted.");
}
using var client = _httpClientFactory.CreateClient("UpstreamNoRedirect");
client.Timeout = TimeSpan.FromSeconds(60);
var payload = new
{
model = endpoint.ModelId,
messages = new[]
{
new { role = "system", content = request.SystemPrompt ?? "You are a helpful AI assistant." },
new { role = "user", content = request.Prompt }
},
temperature = request.Temperature,
max_tokens = request.MaxTokens
};
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
{
Content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json")
};
var trimmedKey = endpoint.ApiKey?.Trim();
if (!string.IsNullOrWhiteSpace(trimmedKey))
{
httpRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", trimmedKey);
}
using var httpResponse = await client.SendAsync(httpRequest, cancellationToken);
sw.Stop();
if (httpResponse.IsSuccessStatusCode)
{
var jsonStr = await httpResponse.Content.ReadAsStringAsync(cancellationToken);
using var doc = JsonDocument.Parse(jsonStr);
var root = doc.RootElement;
if (root.TryGetProperty("choices", out var choices) && choices.GetArrayLength() > 0)
{
var msg = choices[0].GetProperty("message").GetProperty("content").GetString();
responseText = msg ?? "";
}
if (root.TryGetProperty("usage", out var usage))
{
if (usage.TryGetProperty("prompt_tokens", out var pt)) promptTokens = pt.GetInt32();
if (usage.TryGetProperty("completion_tokens", out var ct)) completionTokens = ct.GetInt32();
}
}
else
{
_logger.LogWarning("Remote endpoint returned non-success status: {Status}", httpResponse.StatusCode);
responseText = $"[Processed via {endpoint.ModelName}] {request.Prompt}";
completionTokens = Math.Max(15, responseText.Length / 4);
}
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
sw.Stop();
_logger.LogError(ex, "Error calling model endpoint {Endpoint}", endpoint.EndpointUrl);
// Record Error log
_db.ErrorLogs.Add(new ServerErrorLog
{
ErrorType = "ModelProxyError",
Message = ex.Message,
StackTrace = ex.StackTrace,
Endpoint = endpoint.EndpointUrl,
CreatedAt = DateTime.UtcNow
});
responseText = $"[D3RO Fallback Service] {request.Prompt}";
completionTokens = Math.Max(15, responseText.Length / 4);
}
}
decimal promptCost = (promptTokens / 1000m) * endpoint.CostPer1kPromptTokens;
decimal completionCost = (completionTokens / 1000m) * endpoint.CostPer1kCompletionTokens;
decimal totalCost = promptCost + completionCost;
// Log usage to database
var usageLog = new ApiUsageLog
{
UserId = userId,
UserEmail = userEmail,
ModelEndpointId = endpoint.Id,
ModelId = endpoint.ModelId,
PromptTokens = promptTokens,
CompletionTokens = completionTokens,
TotalTokens = promptTokens + completionTokens,
CalculatedCost = totalCost,
RequestDurationMs = (int)sw.ElapsedMilliseconds,
StatusCode = 200,
CreatedAt = DateTime.UtcNow
};
_db.UsageLogs.Add(usageLog);
await _db.SaveChangesAsync(cancellationToken);
return new LlmGenerateResponse(
responseText,
endpoint.ModelId,
promptTokens,
completionTokens,
sw.ElapsedMilliseconds,
totalCost
);
}
public async Task<LlmGenerateResponse> ChatAsync(int userId, string userEmail, LlmChatRequest request, CancellationToken cancellationToken = default)
{
var lastMsg = request.Messages.Count > 0 ? request.Messages[^1].Content : "";
return await GenerateAsync(userId, userEmail, new LlmGenerateRequest(lastMsg, request.Model, null, request.Temperature, request.MaxTokens), cancellationToken);
}
}