feat: ship Wave 5-6 multi-surface hygiene, optimization, and bug fixes

This commit is contained in:
Yun Chan 2026-11-14 11:51:41 +09:00
parent 01ef3293cf
commit 66b6b45ba9
399 changed files with 17828 additions and 11330 deletions

View file

@ -7,6 +7,7 @@ using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using D3ROVoice.Api.Data;
using D3ROVoice.Api.Dtos;
@ -24,10 +25,15 @@ public interface ISttProxyService
byte[]? audioBytes = null,
string? contentType = null,
string? fileName = null,
bool recordUsage = true
bool recordUsage = true,
CancellationToken cancellationToken = default
);
Task<SttTestResultDto> TestEndpointAsync(int endpointId, string? testApiKey = null, string? testEndpointUrl = null);
Task<SttTestResultDto> TestEndpointAsync(
int endpointId,
string? testApiKey = null,
string? testEndpointUrl = null,
CancellationToken cancellationToken = default);
Task<List<SttProviderEndpointDto>> GetAllEndpointsAsync();
Task<SttProviderEndpointDto> CreateEndpointAsync(CreateSttEndpointDto dto);
Task<SttProviderEndpointDto> UpdateEndpointAsync(int id, UpdateSttEndpointDto dto);
@ -49,6 +55,9 @@ public sealed class SttProviderUnavailableException : Exception
public class SttProxyService : ISttProxyService
{
private const int MaxAudioBytes = 26 * 1024 * 1024;
private const int MaxBase64Length = ((MaxAudioBytes + 2) / 3) * 4;
private readonly AppDbContext _db;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<SttProxyService> _logger;
@ -67,32 +76,25 @@ public class SttProxyService : ISttProxyService
byte[]? audioBytes = null,
string? contentType = null,
string? fileName = null,
bool recordUsage = true
bool recordUsage = true,
CancellationToken cancellationToken = default
)
{
cancellationToken.ThrowIfCancellationRequested();
// 1. Resolve Audio Bytes
byte[] finalAudioBytes;
if (audioBytes != null && audioBytes.Length > 0)
{
if (audioBytes.Length > MaxAudioBytes)
{
throw new ArgumentException("Audio payload exceeds maximum allowed size.");
}
finalAudioBytes = audioBytes;
}
else if (!string.IsNullOrWhiteSpace(request.AudioBase64))
{
try
{
var cleanBase64 = request.AudioBase64;
var commaIdx = cleanBase64.IndexOf(',');
if (commaIdx >= 0)
{
cleanBase64 = cleanBase64.Substring(commaIdx + 1);
}
finalAudioBytes = Convert.FromBase64String(cleanBase64);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to decode base64 audio data");
throw new ArgumentException("Invalid AudioBase64 data format.", ex);
}
finalAudioBytes = DecodeAudioBase64(request.AudioBase64);
}
else
{
@ -104,7 +106,7 @@ public class SttProxyService : ISttProxyService
var durationSeconds = EstimateAudioDuration(finalAudioBytes, effectiveContentType);
// 2. Resolve Candidate Endpoints (Primary + Fallbacks)
var candidates = await GetCandidateEndpointsAsync(request.Provider, request.ModelId);
var candidates = await GetCandidateEndpointsAsync(request.Provider, request.ModelId, cancellationToken);
if (candidates.Count == 0)
{
throw new SttProviderUnavailableException(providerAttempted: false);
@ -117,6 +119,7 @@ public class SttProxyService : ISttProxyService
// 3. Failover Execution Chain
foreach (var endpoint in candidates)
{
cancellationToken.ThrowIfCancellationRequested();
var epSw = Stopwatch.StartNew();
try
{
@ -136,7 +139,8 @@ public class SttProxyService : ISttProxyService
finalAudioBytes,
effectiveContentType,
effectiveFileName,
request
request,
cancellationToken
);
epSw.Stop();
@ -167,7 +171,11 @@ public class SttProxyService : ISttProxyService
CreatedAt = DateTime.UtcNow
};
_db.SttUsageLogs.Add(usageLog);
await _db.SaveChangesAsync();
await _db.SaveChangesAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception dbEx)
{
@ -186,6 +194,10 @@ public class SttProxyService : ISttProxyService
Cost: cost
);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
epSw.Stop();
@ -205,7 +217,11 @@ public class SttProxyService : ISttProxyService
Endpoint = null,
CreatedAt = DateTime.UtcNow
});
await _db.SaveChangesAsync();
await _db.SaveChangesAsync(cancellationToken);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch
{
@ -220,15 +236,67 @@ public class SttProxyService : ISttProxyService
throw new SttProviderUnavailableException(providerAttempted);
}
private byte[] DecodeAudioBase64(string audioBase64)
{
var prefixScanLength = Math.Min(audioBase64.Length, 128);
var commaIdx = audioBase64.IndexOf(',', 0, prefixScanLength);
var payloadStart = commaIdx >= 0 ? commaIdx + 1 : 0;
var payloadLength = audioBase64.Length - payloadStart;
if (payloadLength > MaxBase64Length)
{
throw new ArgumentException("AudioBase64 exceeds maximum allowed length.");
}
try
{
var cleanBase64 = (payloadStart > 0 ? audioBase64.Substring(payloadStart) : audioBase64).Trim();
if (cleanBase64.Length == 0 || cleanBase64.Length > MaxBase64Length)
{
throw new ArgumentException("Invalid AudioBase64 payload length.");
}
var decoded = Convert.FromBase64String(cleanBase64);
if (decoded.Length == 0 || decoded.Length > MaxAudioBytes)
{
throw new ArgumentException("Invalid decoded AudioBase64 payload size.");
}
return decoded;
}
catch (ArgumentException)
{
throw;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to decode base64 audio data");
throw new ArgumentException("Invalid AudioBase64 data format.", ex);
}
}
private static async Task<string> SendForStringAsync(
HttpClient client,
HttpRequestMessage httpRequest,
CancellationToken cancellationToken)
{
using var response = await client.SendAsync(httpRequest, cancellationToken);
var responseBody = await response.Content.ReadAsStringAsync(cancellationToken);
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream returned HTTP {response.StatusCode}.");
}
return responseBody;
}
private async Task<(string Transcript, double Confidence, string? Language, double Duration)> ExecuteProviderTranscriptionAsync(
SttProviderEndpoint endpoint,
byte[] audioBytes,
string contentType,
string fileName,
SttTranscribeRequest request
SttTranscribeRequest request,
CancellationToken cancellationToken
)
{
var client = _httpClientFactory.CreateClient();
using var client = _httpClientFactory.CreateClient();
client.Timeout = TimeSpan.FromSeconds(90);
var providerType = endpoint.ProviderType.ToLowerInvariant();
@ -245,31 +313,31 @@ public class SttProxyService : ISttProxyService
case "openai":
case "custom":
{
return await CallOpenAiCompatibleSttAsync(client, endpoint, audioBytes, contentType, fileName, request);
return await CallOpenAiCompatibleSttAsync(client, endpoint, audioBytes, contentType, fileName, request, cancellationToken);
}
case "deepgram":
{
return await CallDeepgramSttAsync(client, endpoint, audioBytes, contentType, request);
return await CallDeepgramSttAsync(client, endpoint, audioBytes, contentType, request, cancellationToken);
}
case "google":
{
return await CallGoogleSttAsync(client, endpoint, audioBytes, contentType, request);
return await CallGoogleSttAsync(client, endpoint, audioBytes, contentType, request, cancellationToken);
}
case "assemblyai":
{
return await CallAssemblyAiSttAsync(client, endpoint, audioBytes, contentType, request);
return await CallAssemblyAiSttAsync(client, endpoint, audioBytes, contentType, request, cancellationToken);
}
case "azure":
{
return await CallAzureSttAsync(client, endpoint, audioBytes, contentType, request);
return await CallAzureSttAsync(client, endpoint, audioBytes, contentType, request, cancellationToken);
}
case "local-sidecar":
{
return await CallLocalSidecarSttAsync(client, endpoint, audioBytes, contentType, fileName, request);
return await CallLocalSidecarSttAsync(client, endpoint, audioBytes, contentType, fileName, request, cancellationToken);
}
default:
{
return await CallOpenAiCompatibleSttAsync(client, endpoint, audioBytes, contentType, fileName, request);
return await CallOpenAiCompatibleSttAsync(client, endpoint, audioBytes, contentType, fileName, request, cancellationToken);
}
}
}
@ -280,7 +348,8 @@ public class SttProxyService : ISttProxyService
byte[] audioBytes,
string contentType,
string fileName,
SttTranscribeRequest request
SttTranscribeRequest request,
CancellationToken cancellationToken
)
{
using var form = new MultipartFormDataContent();
@ -307,7 +376,7 @@ public class SttProxyService : ISttProxyService
form.Add(new StringContent(temp.ToString("0.0")), "temperature");
form.Add(new StringContent("verbose_json"), "response_format");
var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
{
Content = form
};
@ -319,13 +388,7 @@ public class SttProxyService : ISttProxyService
ApplyExtraHeaders(httpRequest, endpoint.ExtraHeadersJson);
var httpResponse = await client.SendAsync(httpRequest);
var responseBody = await httpResponse.Content.ReadAsStringAsync();
if (!httpResponse.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream returned HTTP {httpResponse.StatusCode}.");
}
var responseBody = await SendForStringAsync(client, httpRequest, cancellationToken);
using var doc = JsonDocument.Parse(responseBody);
var root = doc.RootElement;
@ -356,7 +419,8 @@ public class SttProxyService : ISttProxyService
SttProviderEndpoint endpoint,
byte[] audioBytes,
string contentType,
SttTranscribeRequest request
SttTranscribeRequest request,
CancellationToken cancellationToken
)
{
var lang = request.Language ?? endpoint.Language ?? "ko";
@ -366,7 +430,7 @@ public class SttProxyService : ISttProxyService
? $"{baseUrl}&model={model}&language={lang}&smart_format=true&punctuate=true"
: $"{baseUrl}?model={model}&language={lang}&smart_format=true&punctuate=true";
var httpRequest = new HttpRequestMessage(HttpMethod.Post, url)
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = new ByteArrayContent(audioBytes)
};
@ -379,13 +443,7 @@ public class SttProxyService : ISttProxyService
ApplyExtraHeaders(httpRequest, endpoint.ExtraHeadersJson);
var response = await client.SendAsync(httpRequest);
var responseBody = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream returned HTTP {response.StatusCode}.");
}
var responseBody = await SendForStringAsync(client, httpRequest, cancellationToken);
using var doc = JsonDocument.Parse(responseBody);
var root = doc.RootElement;
@ -421,7 +479,8 @@ public class SttProxyService : ISttProxyService
SttProviderEndpoint endpoint,
byte[] audioBytes,
string contentType,
SttTranscribeRequest request
SttTranscribeRequest request,
CancellationToken cancellationToken
)
{
var base64Audio = Convert.ToBase64String(audioBytes);
@ -457,19 +516,13 @@ public class SttProxyService : ISttProxyService
}
};
var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
{
Content = new StringContent(JsonSerializer.Serialize(geminiPayload), Encoding.UTF8, "application/json")
};
httpRequest.Headers.Add("X-Goog-Api-Key", apiKey);
var response = await client.SendAsync(httpRequest);
var responseBody = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream returned HTTP {response.StatusCode}.");
}
var responseBody = await SendForStringAsync(client, httpRequest, cancellationToken);
using var doc = JsonDocument.Parse(responseBody);
var root = doc.RootElement;
@ -507,19 +560,13 @@ public class SttProxyService : ISttProxyService
}
};
var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
{
Content = new StringContent(JsonSerializer.Serialize(gcpPayload), Encoding.UTF8, "application/json")
};
httpRequest.Headers.Add("X-Goog-Api-Key", apiKey);
var response = await client.SendAsync(httpRequest);
var responseBody = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream returned HTTP {response.StatusCode}.");
}
var responseBody = await SendForStringAsync(client, httpRequest, cancellationToken);
using var doc = JsonDocument.Parse(responseBody);
var root = doc.RootElement;
@ -555,22 +602,18 @@ public class SttProxyService : ISttProxyService
SttProviderEndpoint endpoint,
byte[] audioBytes,
string contentType,
SttTranscribeRequest request
SttTranscribeRequest request,
CancellationToken cancellationToken
)
{
// 1. Upload audio
var uploadRequest = new HttpRequestMessage(HttpMethod.Post, "https://api.assemblyai.com/v2/upload")
using var uploadRequest = new HttpRequestMessage(HttpMethod.Post, "https://api.assemblyai.com/v2/upload")
{
Content = new ByteArrayContent(audioBytes)
};
uploadRequest.Headers.Add("Authorization", endpoint.ApiKey);
var uploadResp = await client.SendAsync(uploadRequest);
var uploadBody = await uploadResp.Content.ReadAsStringAsync();
if (!uploadResp.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream upload returned HTTP {uploadResp.StatusCode}.");
}
var uploadBody = await SendForStringAsync(client, uploadRequest, cancellationToken);
using var uploadDoc = JsonDocument.Parse(uploadBody);
var uploadUrl = uploadDoc.RootElement.GetProperty("upload_url").GetString()!;
@ -585,18 +628,13 @@ public class SttProxyService : ISttProxyService
format_text = true
};
var transRequest = new HttpRequestMessage(HttpMethod.Post, "https://api.assemblyai.com/v2/transcript")
using var transRequest = new HttpRequestMessage(HttpMethod.Post, "https://api.assemblyai.com/v2/transcript")
{
Content = new StringContent(JsonSerializer.Serialize(transcriptPayload), Encoding.UTF8, "application/json")
};
transRequest.Headers.Add("Authorization", endpoint.ApiKey);
var transResp = await client.SendAsync(transRequest);
var transBody = await transResp.Content.ReadAsStringAsync();
if (!transResp.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream job returned HTTP {transResp.StatusCode}.");
}
var transBody = await SendForStringAsync(client, transRequest, cancellationToken);
using var transDoc = JsonDocument.Parse(transBody);
var id = transDoc.RootElement.GetProperty("id").GetString()!;
@ -604,17 +642,11 @@ public class SttProxyService : ISttProxyService
// 3. Poll for result (up to 20 seconds)
for (int i = 0; i < 20; i++)
{
await Task.Delay(1000);
var pollReq = new HttpRequestMessage(HttpMethod.Get, $"https://api.assemblyai.com/v2/transcript/{id}");
await Task.Delay(1000, cancellationToken);
using var pollReq = new HttpRequestMessage(HttpMethod.Get, $"https://api.assemblyai.com/v2/transcript/{id}");
pollReq.Headers.Add("Authorization", endpoint.ApiKey);
var pollResp = await client.SendAsync(pollReq);
var pollBody = await pollResp.Content.ReadAsStringAsync();
if (!pollResp.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream poll returned HTTP {pollResp.StatusCode}.");
}
var pollBody = await SendForStringAsync(client, pollReq, cancellationToken);
using var pollDoc = JsonDocument.Parse(pollBody);
var status = pollDoc.RootElement.GetProperty("status").GetString();
@ -647,7 +679,8 @@ public class SttProxyService : ISttProxyService
SttProviderEndpoint endpoint,
byte[] audioBytes,
string contentType,
SttTranscribeRequest request
SttTranscribeRequest request,
CancellationToken cancellationToken
)
{
var lang = request.Language ?? endpoint.Language ?? "ko-KR";
@ -657,7 +690,7 @@ public class SttProxyService : ISttProxyService
? $"{endpoint.EndpointUrl}&language={lang}&format=detailed"
: $"{endpoint.EndpointUrl}?language={lang}&format=detailed";
var httpRequest = new HttpRequestMessage(HttpMethod.Post, url)
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, url)
{
Content = new ByteArrayContent(audioBytes)
};
@ -668,13 +701,7 @@ public class SttProxyService : ISttProxyService
httpRequest.Headers.Add("Ocp-Apim-Subscription-Key", endpoint.ApiKey);
}
var response = await client.SendAsync(httpRequest);
var responseBody = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream returned HTTP {response.StatusCode}.");
}
var responseBody = await SendForStringAsync(client, httpRequest, cancellationToken);
using var doc = JsonDocument.Parse(responseBody);
var root = doc.RootElement;
@ -709,7 +736,8 @@ public class SttProxyService : ISttProxyService
byte[] audioBytes,
string contentType,
string fileName,
SttTranscribeRequest request
SttTranscribeRequest request,
CancellationToken cancellationToken
)
{
using var form = new MultipartFormDataContent();
@ -720,7 +748,7 @@ public class SttProxyService : ISttProxyService
form.Add(new StringContent(endpoint.ModelId), "model");
form.Add(new StringContent(request.Language ?? endpoint.Language ?? "ko"), "language");
var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, endpoint.EndpointUrl)
{
Content = form
};
@ -730,13 +758,7 @@ public class SttProxyService : ISttProxyService
httpRequest.Headers.Authorization = new AuthenticationHeaderValue("Bearer", endpoint.ApiKey);
}
var response = await client.SendAsync(httpRequest);
var responseBody = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
throw new HttpRequestException($"STT upstream returned HTTP {response.StatusCode}.");
}
var responseBody = await SendForStringAsync(client, httpRequest, cancellationToken);
using var doc = JsonDocument.Parse(responseBody);
var root = doc.RootElement;
@ -752,12 +774,16 @@ public class SttProxyService : ISttProxyService
return (text, 0.99, endpoint.Language, duration);
}
public async Task<SttTestResultDto> TestEndpointAsync(int endpointId, string? testApiKey = null, string? testEndpointUrl = null)
public async Task<SttTestResultDto> TestEndpointAsync(
int endpointId,
string? testApiKey = null,
string? testEndpointUrl = null,
CancellationToken cancellationToken = default)
{
SttProviderEndpoint? endpoint = null;
if (endpointId > 0)
{
endpoint = await _db.SttProviderEndpoints.FindAsync(endpointId);
endpoint = await _db.SttProviderEndpoints.FindAsync(new object[] { endpointId }, cancellationToken);
}
if (endpoint == null)
@ -792,7 +818,8 @@ public class SttProxyService : ISttProxyService
syntheticWav,
"audio/wav",
"test_audio.wav",
new SttTranscribeRequest(Language: endpoint.Language ?? "ko")
new SttTranscribeRequest(Language: endpoint.Language ?? "ko"),
cancellationToken
);
sw.Stop();
@ -805,6 +832,10 @@ public class SttProxyService : ISttProxyService
ModelId: endpoint.ModelId
);
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception ex)
{
sw.Stop();
@ -968,16 +999,19 @@ public class SttProxyService : ISttProxyService
);
}
private async Task<List<SttProviderEndpoint>> GetCandidateEndpointsAsync(string? requestedProvider, string? requestedModel)
private async Task<List<SttProviderEndpoint>> GetCandidateEndpointsAsync(
string? requestedProvider,
string? requestedModel,
CancellationToken cancellationToken)
{
var query = _db.SttProviderEndpoints.Where(e => e.IsActive);
if (!string.IsNullOrWhiteSpace(requestedProvider))
{
var match = await query.FirstOrDefaultAsync(e => e.ProviderType.ToLower() == requestedProvider.ToLower());
var match = await query.FirstOrDefaultAsync(e => e.ProviderType.ToLower() == requestedProvider.ToLower(), cancellationToken);
if (match != null)
{
var others = await query.Where(e => e.Id != match.Id).OrderBy(e => e.FallbackPriority).ToListAsync();
var others = await query.Where(e => e.Id != match.Id).OrderBy(e => e.FallbackPriority).ToListAsync(cancellationToken);
others.Insert(0, match);
return others;
}
@ -985,10 +1019,10 @@ public class SttProxyService : ISttProxyService
if (!string.IsNullOrWhiteSpace(requestedModel))
{
var match = await query.FirstOrDefaultAsync(e => e.ModelId.ToLower() == requestedModel.ToLower());
var match = await query.FirstOrDefaultAsync(e => e.ModelId.ToLower() == requestedModel.ToLower(), cancellationToken);
if (match != null)
{
var others = await query.Where(e => e.Id != match.Id).OrderBy(e => e.FallbackPriority).ToListAsync();
var others = await query.Where(e => e.Id != match.Id).OrderBy(e => e.FallbackPriority).ToListAsync(cancellationToken);
others.Insert(0, match);
return others;
}
@ -998,7 +1032,7 @@ public class SttProxyService : ISttProxyService
return await query
.OrderByDescending(e => e.IsDefault)
.ThenBy(e => e.FallbackPriority)
.ToListAsync();
.ToListAsync(cancellationToken);
}
private static bool HasRequiredProviderConfiguration(SttProviderEndpoint endpoint)