Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 34 additions & 11 deletions openless-all/app/crates/openless-core/src/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4275,6 +4275,15 @@ impl OpenLessBackend {
.ok_or_else(|| {
BackendError::new(BackendErrorCode::InvalidArgument, "history entry not found")
})?;
if !matches!(
entry.error_code.as_deref(),
Some("transcribeFailed" | "emptyTranscript")
) {
return Err(BackendError::new(
BackendErrorCode::InvalidState,
"history entry is not a failed transcription",
));
}
entry.raw_transcript = text.clone();
entry.final_text = text;
entry.error_code = None;
Expand Down Expand Up @@ -7588,21 +7597,35 @@ mod tests {

let mut entry = history_session("one");
backend.append_history(entry.clone(), 30, Some(20)).unwrap();
let asr_call = crate::auxiliary::AsrCallLabel {
provider: "channel-b".into(),
model: Some("model-b".into()),
};
let completed_error = backend
.apply_history_retranscription(
&entry.id,
"must not replace history".into(),
&asr_call,
1,
)
.unwrap_err();
assert_eq!(completed_error.code, BackendErrorCode::InvalidState);

entry.final_text = "updated".to_string();
entry.error_code = Some("polishFailed".to_string());
assert!(backend.update_history_entry(entry.clone()).unwrap());
let polish_error = backend
.apply_history_retranscription(&entry.id, "must remain a preview".into(), &asr_call, 1)
.unwrap_err();
assert_eq!(polish_error.code, BackendErrorCode::InvalidState);

entry.error_code = Some("transcribeFailed".to_string());
assert!(backend.update_history_entry(entry.clone()).unwrap());
assert!(!backend
.update_history_entry(history_session("missing"))
.unwrap());
let retranscribed = backend
.apply_history_retranscription(
&entry.id,
"retranscribed".into(),
&crate::auxiliary::AsrCallLabel {
provider: "channel-b".into(),
model: Some("model-b".into()),
},
480,
)
.apply_history_retranscription(&entry.id, "retranscribed".into(), &asr_call, 480)
.unwrap();
assert_eq!(retranscribed.raw_transcript, "retranscribed");
assert_eq!(retranscribed.final_text, "retranscribed");
Expand All @@ -7619,8 +7642,8 @@ mod tests {

assert!(backend.list_history().unwrap().is_empty());
assert_eq!(backend.list_activity().unwrap()[0].chars, 42);
assert_eq!(backend.snapshot().history_revision, 6);
for expected_revision in 1..=6 {
assert_eq!(backend.snapshot().history_revision, 7);
for expected_revision in 1..=7 {
assert_eq!(
events.try_recv().unwrap().kind,
BackendEventKind::HistoryChanged(HistoryChange {
Expand Down
84 changes: 65 additions & 19 deletions openless-all/app/src-tauri/src/asr/local/sherpa_runtime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ use crate::asr::local::sherpa::{
SherpaPreparePhase, SherpaPrepareProgressPayload, SherpaRuntimeStatus, PROVIDER_ID,
};

const QWEN3_ASR_CHUNK_LIMIT_MS: u64 = 30_000;
const QWEN3_ASR_MAX_NEW_TOKENS: i32 = 256;

#[cfg(target_os = "windows")]
use sherpa_onnx::{
OfflineParaformerModelConfig, OfflineQwen3ASRModelConfig, OfflineRecognizer,
Expand Down Expand Up @@ -225,12 +228,16 @@ impl SherpaOnnxRuntime {
if pcm.is_empty() {
return Ok(String::new());
}
if sherpa_target(alias)?.sherpa_execution_mode()
!= Some(openless_core::LocalAsrExecutionMode::Offline)
{
let target = sherpa_target(alias)?;
if target.sherpa_execution_mode() != Some(openless_core::LocalAsrExecutionMode::Offline) {
anyhow::bail!("sherpa-onnx model {alias} is online-only; use streaming API");
}
let audio_ms = pcm_duration_ms(pcm);
let pcm_chunks = sherpa_offline_chunks(&target, pcm)
.into_iter()
.map(|chunk| chunk.to_vec())
.collect::<Vec<_>>();
let chunk_count = pcm_chunks.len();
let loaded_alias = self.ensure_loaded(alias, model_dir).await?;
let loaded = self
.state
Expand All @@ -242,7 +249,7 @@ impl SherpaOnnxRuntime {
let started = Instant::now();
let result = transcribe_loaded_model(
loaded,
pcm.to_vec(),
pcm_chunks,
language_hint.map(str::to_string),
audio_timeout,
)
Expand All @@ -251,9 +258,10 @@ impl SherpaOnnxRuntime {
match &result {
Ok(text) => {
log::info!(
"[sherpa-asr] transcribe finished model={} audio_ms={} elapsed_ms={} text_chars={}",
"[sherpa-asr] transcribe finished model={} audio_ms={} chunks={} elapsed_ms={} text_chars={}",
alias,
audio_ms,
chunk_count,
elapsed_ms,
text.chars().count()
);
Expand All @@ -262,9 +270,10 @@ impl SherpaOnnxRuntime {
Err(error) => {
let message = format!("{error:#}");
log::warn!(
"[sherpa-asr] transcribe failed model={} audio_ms={} elapsed_ms={} error={}",
"[sherpa-asr] transcribe failed model={} audio_ms={} chunks={} elapsed_ms={} error={}",
alias,
audio_ms,
chunk_count,
elapsed_ms,
message
);
Expand Down Expand Up @@ -426,6 +435,15 @@ fn pcm_duration_ms(pcm: &[u8]) -> u64 {
crate::asr::pcm::pcm_duration_ms(pcm)
}

fn sherpa_offline_chunks<'a>(
target: &openless_core::LocalAsrTarget,
pcm: &'a [u8],
) -> Vec<&'a [u8]> {
let limit = (target.sherpa_family() == Some(openless_core::SherpaModelFamily::Qwen3Asr))
.then_some(QWEN3_ASR_CHUNK_LIMIT_MS);
openless_core::asr::whisper::split_pcm_by_duration(pcm, limit)
}

enum LoadedModel {
Offline(LoadedOfflineModel),
Online(LoadedOnlineModel),
Expand Down Expand Up @@ -515,6 +533,7 @@ fn create_offline_recognizer(alias: &str, dir: &Path) -> Result<OfflineRecognize
encoder: Some(path_to_string(&dir.join("encoder.int8.onnx"))?),
decoder: Some(path_to_string(&dir.join("decoder.int8.onnx"))?),
tokenizer: Some(path_to_string(&dir.join("tokenizer"))?),
max_new_tokens: QWEN3_ASR_MAX_NEW_TOKENS,
..Default::default()
};
config.model_config.num_threads = 3;
Expand Down Expand Up @@ -569,25 +588,29 @@ fn path_to_string(path: &Path) -> Result<String> {
#[cfg(target_os = "windows")]
async fn transcribe_loaded_model(
loaded: LoadedOfflineModel,
pcm: Vec<u8>,
pcm_chunks: Vec<Vec<u8>>,
language_hint: Option<String>,
audio_timeout: std::time::Duration,
) -> Result<String> {
tokio::time::timeout(audio_timeout, async move {
tokio::task::spawn_blocking(move || {
let samples = pcm_s16le_to_f32(&pcm)?;
let stream = loaded.recognizer.create_stream();
if let Some(language) = language_hint.as_deref().filter(|value| !value.is_empty()) {
if stream.has_option("language") {
stream.set_option("language", language);
let mut texts = Vec::with_capacity(pcm_chunks.len());
for pcm in pcm_chunks {
let samples = pcm_s16le_to_f32(&pcm)?;
let stream = loaded.recognizer.create_stream();
if let Some(language) = language_hint.as_deref().filter(|value| !value.is_empty()) {
if stream.has_option("language") {
stream.set_option("language", language);
}
}
stream.accept_waveform(16_000, &samples);
loaded.recognizer.decode(&stream);
let result = stream
.get_result()
.ok_or_else(|| anyhow::anyhow!("sherpa-onnx returned no result"))?;
texts.push(result.text);
}
stream.accept_waveform(16_000, &samples);
loaded.recognizer.decode(&stream);
let result = stream
.get_result()
.ok_or_else(|| anyhow::anyhow!("sherpa-onnx returned no result"))?;
Ok(result.text)
Ok(openless_core::asr::whisper::join_transcript_chunks(&texts))
})
.await
.map_err(|e| anyhow::anyhow!("sherpa-onnx transcribe join failed: {e:#}"))?
Expand All @@ -599,7 +622,7 @@ async fn transcribe_loaded_model(
#[cfg(not(target_os = "windows"))]
async fn transcribe_loaded_model(
_loaded: LoadedOfflineModel,
_pcm: Vec<u8>,
_pcm_chunks: Vec<Vec<u8>>,
_language_hint: Option<String>,
_audio_timeout: std::time::Duration,
) -> Result<String> {
Expand Down Expand Up @@ -936,6 +959,29 @@ mod tests {
assert!(format!("{:#}", result.unwrap_err()).contains("online-only"));
}

#[test]
fn only_qwen3_offline_audio_is_split_at_thirty_seconds() {
let qwen = sherpa_target("qwen3-asr-0.6b-int8").unwrap();
let thirty_seconds = vec![0; 32_000 * 30];
assert_eq!(
sherpa_offline_chunks(&qwen, &thirty_seconds),
vec![thirty_seconds.as_slice()]
);

let pcm = vec![0; 32_000 * 65];
let chunks = sherpa_offline_chunks(&qwen, &pcm);
assert_eq!(chunks.len(), 3);
assert_eq!(chunks[0].len(), 32_000 * 30);
assert_eq!(chunks[1].len(), 32_000 * 30);
assert_eq!(chunks[2].len(), 32_000 * 5);

let sense_voice = sherpa_target("sense-voice-small-zh").unwrap();
assert_eq!(
sherpa_offline_chunks(&sense_voice, &pcm),
vec![pcm.as_slice()]
);
}

#[tokio::test]
async fn create_online_session_rejects_offline_model_alias() {
let runtime = SherpaOnnxRuntime::new();
Expand Down
64 changes: 57 additions & 7 deletions openless-all/app/src-tauri/src/commands/history.rs
Original file line number Diff line number Diff line change
Expand Up @@ -203,22 +203,44 @@ fn copy_recording_to_mobile_url(
Ok(())
}

/// 对一条「转录失败」历史条目的归档录音用**当前** ASR provider 重新转录(issue #613)。
#[derive(serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub struct HistoryRetranscriptionResult {
text: String,
updated_entry: Option<DictationSession>,
}

fn should_replace_failed_history(error_code: Option<&str>) -> bool {
matches!(error_code, Some("transcribeFailed" | "emptyTranscript"))
}

/// 对一条有归档录音的历史条目用**当前** ASR provider 重新转录(issue #613 / #1046)。
///
/// 流程:读 `recordings/<id>.wav` → 取 PCM(跳过 44 字节 WAV 头)→ 现 provider 重转
/// → 成功则原地回写该条历史的 rawTranscript / finalText、清除 error_code,返回新文本
/// → 失败记录原地修复;已完成 / 润色失败记录仅返回临时结果,不覆盖历史事实
///
/// 仅重新转写音频,不调用 LLM 润色。失败时
/// 不动历史、不删录音,把错误返回给前端提示,用户可重试。返回更新后的整条记录给前端
/// 局部刷新。
/// 不动历史、不删录音,把错误返回给前端提示,用户可重试。
#[tauri::command]
pub async fn retranscribe_recording(
core: CoreState<'_>,
session_id: String,
) -> Result<DictationSession, String> {
) -> Result<HistoryRetranscriptionResult, String> {
if !is_valid_session_id(&session_id) {
return Err("invalid session id".into());
}
let entry = core
.list_history()
.map_err(|e| e.to_string())?
.into_iter()
.find(|entry| entry.id == session_id)
.ok_or_else(|| "history entry not found".to_string())?;
if entry.has_audio_recording != Some(true) {
return Err("history entry has no archived recording".into());
}
if entry.pipeline_mode.as_deref() == Some("multimodal") {
return Err("multimodal history does not support retranscription".into());
}
let path =
crate::persistence::recording_path_for_session(&session_id).map_err(|e| e.to_string())?;
let wav = tokio::fs::read(&path).await.map_err(|e| {
Expand Down Expand Up @@ -248,6 +270,34 @@ pub async fn retranscribe_recording(
}
let retranscribe_ms = retranscribe_started.elapsed().as_millis() as u64;

core.apply_history_retranscription(&session_id, text, &asr_call_label, retranscribe_ms)
.map_err(|e| e.to_string())
let updated_entry = if should_replace_failed_history(entry.error_code.as_deref()) {
Some(
core.apply_history_retranscription(
&session_id,
text.clone(),
&asr_call_label,
retranscribe_ms,
)
.map_err(|e| e.to_string())?,
)
} else {
None
};
Ok(HistoryRetranscriptionResult {
text,
updated_entry,
})
}

#[cfg(test)]
mod retranscription_tests {
use super::should_replace_failed_history;

#[test]
fn only_failed_transcriptions_are_replaced() {
assert!(should_replace_failed_history(Some("transcribeFailed")));
assert!(should_replace_failed_history(Some("emptyTranscript")));
assert!(!should_replace_failed_history(Some("polishFailed")));
assert!(!should_replace_failed_history(None));
}
}
48 changes: 48 additions & 0 deletions openless-all/app/src/lib/history-retranscribe.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
import { canRetranscribeHistoryEntry } from './history-retranscribe';

function assert(condition: boolean, message: string) {
if (!condition) throw new Error(message);
}

const archivedEntry = {
hasAudioRecording: true,
pipelineMode: 'traditional',
} as const;

const entryWithError = (errorCode: string | null) => ({ ...archivedEntry, errorCode });

assert(
canRetranscribeHistoryEntry(entryWithError(null)),
'completed entries with an archived recording should be retranscribable',
);
assert(
canRetranscribeHistoryEntry(entryWithError('polishFailed')),
'entries whose polishing failed should still be retranscribable',
);
assert(
canRetranscribeHistoryEntry(entryWithError('transcribeFailed')),
'entries whose transcription failed should be retranscribable',
);
assert(
!canRetranscribeHistoryEntry({
hasAudioRecording: false,
pipelineMode: 'traditional',
}),
'entries without an archived recording should not show retranscription',
);
assert(
!canRetranscribeHistoryEntry({
hasAudioRecording: null,
pipelineMode: undefined,
}),
'legacy entries without recording metadata should not show retranscription',
);
assert(
!canRetranscribeHistoryEntry({
...archivedEntry,
pipelineMode: 'multimodal',
}),
'multimodal entries should not show an unsupported retranscription action',
);

console.log('history-retranscribe: all assertions passed');
14 changes: 14 additions & 0 deletions openless-all/app/src/lib/history-retranscribe.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
import type { DictationSession } from './types';

/**
* 重新转录需要一份仍存在的 WAV 归档。多模态历史目前没有对应的重转录
* provider 通道,避免展示一个点击后必然失败的按钮。
*
* 成功转录、润色失败和转录失败的条目都可能有可用录音,用户都应能用同一份
* 音频重新验证当前 ASR provider。是否回写失败记录由后端决定。
*/
export function canRetranscribeHistoryEntry(
session: Pick<DictationSession, 'hasAudioRecording' | 'pipelineMode'>,
): boolean {
return session.hasAudioRecording === true && session.pipelineMode !== 'multimodal';
}
Loading
Loading