feat: ship Wave 5-6 multi-surface hygiene, optimization, and bug fixes
This commit is contained in:
parent
01ef3293cf
commit
66b6b45ba9
399 changed files with 17828 additions and 11330 deletions
|
|
@ -4,6 +4,7 @@ using System.Net.Http;
|
|||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using D3ROVoice.Api.Data;
|
||||
using D3ROVoice.Api.Dtos;
|
||||
|
|
@ -14,8 +15,8 @@ namespace D3ROVoice.Api.Services;
|
|||
|
||||
public interface ILlmProxyService
|
||||
{
|
||||
Task<LlmGenerateResponse> GenerateAsync(int userId, string userEmail, LlmGenerateRequest request);
|
||||
Task<LlmGenerateResponse> ChatAsync(int userId, string userEmail, LlmChatRequest request);
|
||||
Task<LlmGenerateResponse> GenerateAsync(int userId, string userEmail, LlmGenerateRequest request, CancellationToken cancellationToken = default);
|
||||
Task<LlmGenerateResponse> ChatAsync(int userId, string userEmail, LlmChatRequest request, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public class LlmProxyService : ILlmProxyService
|
||||
|
|
@ -31,15 +32,16 @@ public class LlmProxyService : ILlmProxyService
|
|||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<LlmGenerateResponse> GenerateAsync(int userId, string userEmail, LlmGenerateRequest request)
|
||||
public async Task<LlmGenerateResponse> GenerateAsync(int userId, string userEmail, LlmGenerateRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var modelId = string.IsNullOrWhiteSpace(request.Model) ? "d3ro-gpt4o-mini" : request.Model;
|
||||
var endpoint = await _db.ModelEndpoints.FirstOrDefaultAsync(m => m.ModelId == modelId && m.IsActive);
|
||||
var endpoint = await _db.ModelEndpoints.FirstOrDefaultAsync(m => m.ModelId == modelId && m.IsActive, cancellationToken);
|
||||
|
||||
if (endpoint == null)
|
||||
{
|
||||
// fallback: first active model endpoint or create default mockup endpoint
|
||||
endpoint = await _db.ModelEndpoints.FirstOrDefaultAsync(m => m.IsActive);
|
||||
endpoint = await _db.ModelEndpoints.FirstOrDefaultAsync(m => m.IsActive, cancellationToken);
|
||||
if (endpoint == null)
|
||||
{
|
||||
endpoint = new ServiceModelEndpoint
|
||||
|
|
@ -71,7 +73,7 @@ public class LlmProxyService : ILlmProxyService
|
|||
{
|
||||
try
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient();
|
||||
using var client = _httpClientFactory.CreateClient();
|
||||
client.Timeout = TimeSpan.FromSeconds(60);
|
||||
|
||||
var payload = new
|
||||
|
|
@ -86,7 +88,7 @@ public class LlmProxyService : ILlmProxyService
|
|||
max_tokens = request.MaxTokens
|
||||
};
|
||||
|
||||
var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
|
||||
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
|
||||
{
|
||||
Content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json")
|
||||
};
|
||||
|
|
@ -96,12 +98,12 @@ public class LlmProxyService : ILlmProxyService
|
|||
httpRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", endpoint.ApiKey);
|
||||
}
|
||||
|
||||
var httpResponse = await client.SendAsync(httpRequest);
|
||||
using var httpResponse = await client.SendAsync(httpRequest, cancellationToken);
|
||||
sw.Stop();
|
||||
|
||||
if (httpResponse.IsSuccessStatusCode)
|
||||
{
|
||||
var jsonStr = await httpResponse.Content.ReadAsStringAsync();
|
||||
var jsonStr = await httpResponse.Content.ReadAsStringAsync(cancellationToken);
|
||||
using var doc = JsonDocument.Parse(jsonStr);
|
||||
var root = doc.RootElement;
|
||||
|
||||
|
|
@ -124,6 +126,10 @@ public class LlmProxyService : ILlmProxyService
|
|||
completionTokens = Math.Max(15, responseText.Length / 4);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
sw.Stop();
|
||||
|
|
@ -165,7 +171,7 @@ public class LlmProxyService : ILlmProxyService
|
|||
};
|
||||
|
||||
_db.UsageLogs.Add(usageLog);
|
||||
await _db.SaveChangesAsync();
|
||||
await _db.SaveChangesAsync(cancellationToken);
|
||||
|
||||
return new LlmGenerateResponse(
|
||||
responseText,
|
||||
|
|
@ -177,9 +183,9 @@ public class LlmProxyService : ILlmProxyService
|
|||
);
|
||||
}
|
||||
|
||||
public async Task<LlmGenerateResponse> ChatAsync(int userId, string userEmail, LlmChatRequest request)
|
||||
public async Task<LlmGenerateResponse> ChatAsync(int userId, string userEmail, LlmChatRequest request, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var lastMsg = request.Messages.Count > 0 ? request.Messages[^1].Content : "";
|
||||
return await GenerateAsync(userId, userEmail, new LlmGenerateRequest(lastMsg, request.Model, null, request.Temperature, request.MaxTokens));
|
||||
return await GenerateAsync(userId, userEmail, new LlmGenerateRequest(lastMsg, request.Model, null, request.Temperature, request.MaxTokens), cancellationToken);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue