From 6863053dcc55b9ccb0a75c14c2527bfc30d1306d Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Fri, 7 Apr 2023 19:22:44 -0400 Subject: [PATCH 01/59] lvdocview: add nextSentence(), for iterating over all sentences --- crengine/include/lvdocview.h | 2 ++ crengine/src/lvdocview.cpp | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/crengine/include/lvdocview.h b/crengine/include/lvdocview.h index 5e8a73417..d0d16ce89 100644 --- a/crengine/include/lvdocview.h +++ b/crengine/include/lvdocview.h @@ -579,6 +579,8 @@ class LVDocView : public CacheLoadingCallback virtual void clearSelection(); /// update selection -- command handler int onSelectionCommand( int cmd, int param ); + /// select the next sentence, for iterating through all + bool nextSentence(); /// navigation history diff --git a/crengine/src/lvdocview.cpp b/crengine/src/lvdocview.cpp index 849fe079d..2de2a6292 100644 --- a/crengine/src/lvdocview.cpp +++ b/crengine/src/lvdocview.cpp @@ -6415,6 +6415,42 @@ int LVDocView::onSelectionCommand( int cmd, int param ) return 1; } +bool LVDocView::nextSentence(){ + LVRef pageRange = getPageDocumentRange(); + if (pageRange.isNull()) { + clearSelection(); + return false; + } + ldomXPointerEx pos( getBookmark() ); + ldomXRangeList & sel = getDocument()->getSelections(); + ldomXRange currSel; + if ( sel.length()>0 ){ + currSel = *sel[0]; + } + + if ( currSel.isNull() || currSel.getStart().isNull() ) { + // select first sentence on page + if ( pos.thisSentenceStart() ) { + currSel.setStart(pos); + } + } else if ( !currSel.getStart().isSentenceStart() ) { + currSel.getStart().thisSentenceStart(); + } else { + bool movedToNext = currSel.getStart().nextSentenceStart(); + if(!movedToNext){ + //presumably already on the last sentence + return false; + } + } + + currSel.setEnd(currSel.getStart()); + currSel.getEnd().thisSentenceEnd(); + + currSel.setFlags(1); + selectRange(currSel); + return true; +} + //static int cr_font_sizes[] = { 24, 29, 33, 39, 44 }; static int cr_interline_spaces[] = { 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, From 1e096a0c5430d1a8cea2b758f956f68b54de4066 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Fri, 7 Apr 2023 19:24:18 -0400 Subject: [PATCH 02/59] jni[docview]: getAllSentences() to get coords and text of all sentences --- android/jni/docview.cpp | 58 +++++++++++++++++++ android/jni/org_coolreader_crengine_DocView.h | 8 +++ .../src/org/coolreader/crengine/DocView.java | 12 ++++ .../org/coolreader/crengine/SentenceInfo.java | 20 +++++++ 4 files changed, 98 insertions(+) create mode 100644 android/src/org/coolreader/crengine/SentenceInfo.java diff --git a/android/jni/docview.cpp b/android/jni/docview.cpp index e7fabd934..129c9c1c0 100644 --- a/android/jni/docview.cpp +++ b/android/jni/docview.cpp @@ -1808,6 +1808,64 @@ JNIEXPORT jobject JNICALL Java_org_coolreader_crengine_DocView_getCurrentPageBoo return obj; } +/* + * Class: org_coolreader_crengine_DocView + * Method: getAllSentencesInternal + * Signature: ()Ljava/util/ArrayList; + */ +JNIEXPORT jobject JNICALL Java_org_coolreader_crengine_DocView_getAllSentencesInternal + (JNIEnv * _env, jobject _this) +{ + CRJNIEnv env(_env); + DocViewNative * p = getNative(_env, _this); + if (!p) { + CRLog::error("Cannot get native view"); + return NULL; + } + + jclass arrListClass = _env->FindClass("java/util/ArrayList"); + jmethodID arrListCtor = _env->GetMethodID(arrListClass, "", "()V"); + jmethodID arrListAdd = env->GetMethodID(arrListClass, "add", "(Ljava/lang/Object;)Z"); + + jclass sentenceInfoClass = _env->FindClass("org/coolreader/crengine/SentenceInfo"); + jmethodID sentenceInfoCtor = _env->GetMethodID(sentenceInfoClass, "", "()V"); + jmethodID sentenceInfoSetStartX = _env->GetMethodID(sentenceInfoClass, "setStartX", "(I)V"); + jmethodID sentenceInfoSetStartY = _env->GetMethodID(sentenceInfoClass, "setStartY", "(I)V"); + jmethodID sentenceInfoSetText = _env->GetMethodID(sentenceInfoClass, "setText", "(Ljava/lang/String;)V"); + + jobject arrList = env->NewObject(arrListClass, arrListCtor); + + + p->_docview->savePosition(); + p->_docview->clearSelection(); + p->_docview->goToPage(0); + p->_docview->SetPos(0, false); + while(p->_docview->nextSentence()){ + jobject sentenceInfo = _env->NewObject(sentenceInfoClass, sentenceInfoCtor); + jint startX = 0; + jint startY = 0; + + ldomXRangeList & sel = p->_docview->getDocument()->getSelections(); + ldomXRange currSel; + if ( sel.length()>0 ){ + currSel = *sel[0]; + } + lvPoint startPoint = currSel.getStart().toPoint(); + lvPoint endPoint = currSel.getEnd().toPoint(); + + env->CallVoidMethod(sentenceInfo, sentenceInfoSetStartX, startPoint.x); + env->CallVoidMethod(sentenceInfo, sentenceInfoSetStartY, startPoint.y); + env->CallVoidMethod(sentenceInfo, sentenceInfoSetText, env->NewStringUTF( + UnicodeToUtf8(currSel.getRangeText()).c_str() + )); + + env->CallBooleanMethod(arrList, arrListAdd, sentenceInfo); + } + p->_docview->restorePosition(); + + return arrList; +} + /* * Class: org_coolreader_crengine_DocView * Method: updateBookInfoInternal diff --git a/android/jni/org_coolreader_crengine_DocView.h b/android/jni/org_coolreader_crengine_DocView.h index be32e551e..33cebdeb2 100644 --- a/android/jni/org_coolreader_crengine_DocView.h +++ b/android/jni/org_coolreader_crengine_DocView.h @@ -117,6 +117,14 @@ JNIEXPORT jboolean JNICALL Java_org_coolreader_crengine_DocView_doCommandInterna JNIEXPORT jobject JNICALL Java_org_coolreader_crengine_DocView_getCurrentPageBookmarkInternal (JNIEnv *, jobject); +/* + * Class: org_coolreader_crengine_DocView + * Method: getAllSentencesInternal + * Signature: ()Ljava/util/ArrayList; + */ +JNIEXPORT jobject JNICALL Java_org_coolreader_crengine_DocView_getAllSentencesInternal + (JNIEnv * _env, jobject _this); + /* * Class: org_coolreader_crengine_DocView * Method: goToPositionInternal diff --git a/android/src/org/coolreader/crengine/DocView.java b/android/src/org/coolreader/crengine/DocView.java index 93469041f..801770cf1 100644 --- a/android/src/org/coolreader/crengine/DocView.java +++ b/android/src/org/coolreader/crengine/DocView.java @@ -27,6 +27,8 @@ import java.io.IOException; import java.io.InputStream; +import java.util.List; + public class DocView { public static final Logger log = L.create("dv"); @@ -281,6 +283,14 @@ public Bookmark getCurrentPageBookmarkNoRender() { } } + public List getAllSentences() { + List sentences; + synchronized(mutex) { + sentences = getAllSentencesInternal(); + } + return sentences; + } + /** * Check whether document is formatted/rendered. * @return true if document is rendered, and e.g. retrieving of page image will not cause long activity (formatting etc.) @@ -470,6 +480,8 @@ private native boolean applySettingsInternal( private native Bookmark getCurrentPageBookmarkInternal(); + private native List getAllSentencesInternal(); + private native boolean goToPositionInternal(String xPath, boolean saveToHistory); private native PositionProperties getPositionPropsInternal(String xPath, boolean precise); diff --git a/android/src/org/coolreader/crengine/SentenceInfo.java b/android/src/org/coolreader/crengine/SentenceInfo.java new file mode 100644 index 000000000..534ceef73 --- /dev/null +++ b/android/src/org/coolreader/crengine/SentenceInfo.java @@ -0,0 +1,20 @@ +package org.coolreader.crengine; + +public class SentenceInfo { + public int startX; + public int startY; + public String text; + + public SentenceInfo() { + } + + public void setStartX(int startX){ + this.startX = startX; + } + public void setStartY(int startY){ + this.startY = startY; + } + public void setText(String text){ + this.text = text; + } +} From 51b3b255ab47d57a16614f23bb378f52fe185bab Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Fri, 7 Apr 2023 19:24:51 -0400 Subject: [PATCH 03/59] tts: fetch all sentences when initializing TTS --- android/src/org/coolreader/crengine/ReaderView.java | 4 ++++ android/src/org/coolreader/crengine/TTSToolbarDlg.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/android/src/org/coolreader/crengine/ReaderView.java b/android/src/org/coolreader/crengine/ReaderView.java index 52ec2a0ae..5547aa0c8 100644 --- a/android/src/org/coolreader/crengine/ReaderView.java +++ b/android/src/org/coolreader/crengine/ReaderView.java @@ -3253,6 +3253,10 @@ public BookInfo getBookInfo() { return mBookInfo; } + public List getAllSentences() { + return doc.getAllSentences(); + } + private int mBatteryState = BATTERY_STATE_DISCHARGING; private int mBatteryChargingConn = BATTERY_CHARGER_NO; private int mBatteryChargeLevel = 0; diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index 2f22c5b9e..2069e410c 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -53,6 +53,7 @@ import org.coolreader.tts.TTSControlService; import org.coolreader.tts.TTSControlServiceAccessor; +import java.util.List; import java.util.Locale; import java.util.Map; @@ -90,6 +91,7 @@ public class TTSToolbarDlg implements Settings { private boolean mGoogleTTSAbbreviationWorkaround; private int mTTSSpeedPercent = 50; // 50% (normal) + private List allSentences; static public TTSToolbarDlg showDialog( CoolReader coolReader, ReaderView readerView, TTSControlServiceAccessor ttsacc) { TTSToolbarDlg dlg = new TTSToolbarDlg(coolReader, readerView, ttsacc); @@ -632,5 +634,7 @@ public void onStopTrackingTouch(SeekBar seekBar) { }); // And finally, setup status change handler setupSpeechStatusHandler(); + allSentences = mReaderView.getAllSentences(); + moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); } } From a499382cca34fe0fc0b4aa834c2fcbade230487c Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Fri, 7 Apr 2023 23:00:29 -0400 Subject: [PATCH 04/59] audiobook: implement playing audiobooks in TTS, using vosk word timings --- .../org/coolreader/crengine/SentenceInfo.java | 6 + .../coolreader/crengine/TTSToolbarDlg.java | 30 +++- .../crengine/WordTimingAudiobookMatcher.java | 141 ++++++++++++++++++ .../org/coolreader/tts/TTSControlBinder.java | 5 + .../org/coolreader/tts/TTSControlService.java | 19 +++ 5 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java diff --git a/android/src/org/coolreader/crengine/SentenceInfo.java b/android/src/org/coolreader/crengine/SentenceInfo.java index 534ceef73..d3c8df35d 100644 --- a/android/src/org/coolreader/crengine/SentenceInfo.java +++ b/android/src/org/coolreader/crengine/SentenceInfo.java @@ -1,10 +1,16 @@ package org.coolreader.crengine; +import java.util.List; + public class SentenceInfo { public int startX; public int startY; public String text; + public double startTime; + public String audioFile; + public List words; + public SentenceInfo() { } diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index 2069e410c..9cdbedc5a 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -53,6 +53,7 @@ import org.coolreader.tts.TTSControlService; import org.coolreader.tts.TTSControlServiceAccessor; +import java.io.File; import java.util.List; import java.util.Locale; import java.util.Map; @@ -91,7 +92,7 @@ public class TTSToolbarDlg implements Settings { private boolean mGoogleTTSAbbreviationWorkaround; private int mTTSSpeedPercent = 50; // 50% (normal) - private List allSentences; + private WordTimingAudiobookMatcher wordTimingAudiobookMatcher; static public TTSToolbarDlg showDialog( CoolReader coolReader, ReaderView readerView, TTSControlServiceAccessor ttsacc) { TTSToolbarDlg dlg = new TTSToolbarDlg(coolReader, readerView, ttsacc); @@ -161,6 +162,15 @@ private void moveSelection( ReaderCommand cmd, ReaderView.MoveSelectionCallback @Override public void onNewSelection(Selection selection) { log.d("onNewSelection: " + selection.text); + if(wordTimingAudiobookMatcher != null){ + SentenceInfo s = wordTimingAudiobookMatcher.getSentence(selection.startY, selection.startX); + if(s.audioFile != null){ + File audioFile = wordTimingAudiobookMatcher.getAudioFile(s.audioFile); + mTTSControl.bind(ttsbinder -> { + ttsbinder.playAudioFile(audioFile, s.startTime); + }); + } + } mCurrentSelection = selection; if (null != callback) callback.onNewSelection(mCurrentSelection); @@ -598,6 +608,7 @@ public void onStopTrackingTouch(SeekBar seekBar) { // All tasks bellow after service start // Fetch book's metadata BookInfo bookInfo = mReaderView.getBookInfo(); + File wordTimingFile = null; if (null != bookInfo) { FileInfo fileInfo = bookInfo.getFileInfo(); if (null != fileInfo) { @@ -607,6 +618,11 @@ public void onStopTrackingTouch(SeekBar seekBar) { mBookCover = Bitmap.createBitmap(MEDIA_COVER_WIDTH, MEDIA_COVER_HEIGHT, Bitmap.Config.RGB_565); Services.getCoverpageManager().drawCoverpageFor(mCoolReader.getDB(), fileInfo, mBookCover, true, (file, bitmap) -> mTTSControl.bind(ttsbinder -> ttsbinder.setMediaItemInfo(mBookAuthors, mBookTitle, bitmap))); + String pathName = fileInfo.getPathName(); + String wordTimingPath = pathName.replaceAll("\\.\\w+$", ".wordtiming"); + if(wordTimingPath.matches(".*\\.wordtiming$")){ + wordTimingFile = new File(wordTimingPath); + } } } // Show volume @@ -634,7 +650,15 @@ public void onStopTrackingTouch(SeekBar seekBar) { }); // And finally, setup status change handler setupSpeechStatusHandler(); - allSentences = mReaderView.getAllSentences(); - moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); + + if(wordTimingFile != null && wordTimingFile.exists()){ + List allSentences = mReaderView.getAllSentences(); + wordTimingAudiobookMatcher = new WordTimingAudiobookMatcher(wordTimingFile, allSentences); + wordTimingAudiobookMatcher.parseWordTimingsFile(); + + moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); + }else{ + wordTimingAudiobookMatcher = null; + } } } diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java new file mode 100644 index 000000000..14bc16049 --- /dev/null +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -0,0 +1,141 @@ +package org.coolreader.crengine; + +import android.util.Log; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class WordTimingAudiobookMatcher { + public static final Logger log = L.create("wordtiming"); + + private static final Pattern WORD_TIMING_REGEX = Pattern.compile( + "^(\\d+|\\d*\\.\\d+),([^,]+),(.+)$" + ); + private static class WordTiming { + String word; + Double startTime; + String audioFile; + + public WordTiming(String word, Double startTime, String audioFile){ + this.word = word; + this.startTime = startTime; + this.audioFile = audioFile; + } + } + + private final File wordTimingsFile; + private final List allSentences; + private List wordTimings; + + public WordTimingAudiobookMatcher(File wordTimingsFile, List allSentences) { + this.wordTimingsFile = wordTimingsFile; + this.allSentences = allSentences; + } + + public void parseWordTimingsFile(){ + List lines = new ArrayList<>(); + try { + BufferedReader br = new BufferedReader(new FileReader(wordTimingsFile)); + String line; + while ((line = br.readLine()) != null) { + lines.add(line); + } + } catch(Exception e) { + log.d("ERROR: could not read word timings file: " + wordTimingsFile + " " + e); + lines = new ArrayList<>(); + } + + wordTimings = new ArrayList<>(); + for(String line : lines){ + Matcher m = WORD_TIMING_REGEX.matcher(line); + if(m.matches()){ + wordTimings.add(new WordTiming(m.group(2), Double.parseDouble(m.group(1)), m.group(3))); + }else{ + log.d("ERROR: could not parse word timings line: " + line); + } + } + + for(SentenceInfo s : allSentences){ + String text = s.text; + text = text.replaceAll("’", "'"); + text = text.toLowerCase(); + String[] words = text.split("[^a-z0-9']"); + s.words = new ArrayList<>(); + for(String word : words){ + if(word.matches(".*\\w.*")){ + s.words.add(word); + } + } + } + + if(wordTimings.size() == 0){ + return; + } + + int wtIndex = 0; + double prevStartTime = 0; + String prevAudioFile = wordTimings.get(0).audioFile; + for(SentenceInfo s : allSentences){ + if(s.words.size() == 0){ + s.startTime = prevStartTime; + s.audioFile = prevAudioFile; + continue; + } + boolean matchFailed = false; + WordTiming firstWordTiming = null; + int sentenceWtIndex = wtIndex; + for(String wordInSentence : s.words){ + int wordWtIndex = sentenceWtIndex; + boolean wordFound = false; + while(wordWtIndex <= wordTimings.size()){ + if(wordInSentence.equals(wordTimings.get(wordWtIndex).word)){ + wordFound = true; + break; + }else if(wordWtIndex - sentenceWtIndex > 20){ + break; + }else{ + wordWtIndex++; + } + } + if(wordFound){ + if(firstWordTiming == null){ + firstWordTiming = wordTimings.get(wordWtIndex); + } + sentenceWtIndex = wordWtIndex + 1; + }else{ + matchFailed = true; + break; + } + } + if(matchFailed){ + s.startTime = prevStartTime; + s.audioFile = prevAudioFile; + }else{ + wtIndex = sentenceWtIndex; + s.startTime = firstWordTiming.startTime; + s.audioFile = firstWordTiming.audioFile; + prevStartTime = s.startTime; + prevAudioFile = s.audioFile; + } + } + } + + public SentenceInfo getSentence(double y, double x){ + for(SentenceInfo s : allSentences){ + if(s.startY > y || (s.startY == y && s.startX >= x)){ + return s; + } + } + return null; + } + + public File getAudioFile(String audioFileName){ + String dir = wordTimingsFile.getAbsoluteFile().getParent(); + return new File(dir + "/" + audioFileName); + } +} diff --git a/android/src/org/coolreader/tts/TTSControlBinder.java b/android/src/org/coolreader/tts/TTSControlBinder.java index ba9c8e991..4055c35dd 100644 --- a/android/src/org/coolreader/tts/TTSControlBinder.java +++ b/android/src/org/coolreader/tts/TTSControlBinder.java @@ -24,6 +24,7 @@ import android.os.Handler; import android.speech.tts.Voice; +import java.io.File; import java.util.Locale; public class TTSControlBinder extends Binder { @@ -122,4 +123,8 @@ public void setStatusListener(OnTTSStatusListener listener) { mService.setStatusListener(listener); } + public void playAudioFile(File audioFile, double startTime) { + mService.playAudioFile(audioFile, startTime); + } + } diff --git a/android/src/org/coolreader/tts/TTSControlService.java b/android/src/org/coolreader/tts/TTSControlService.java index e76be3fc1..16806d4c5 100644 --- a/android/src/org/coolreader/tts/TTSControlService.java +++ b/android/src/org/coolreader/tts/TTSControlService.java @@ -40,6 +40,7 @@ import android.media.MediaPlayer; import android.media.session.MediaSession; import android.media.session.PlaybackState; +import android.net.Uri; import android.os.Build; import android.os.Bundle; import android.os.Handler; @@ -55,6 +56,7 @@ import org.coolreader.db.BaseService; import org.coolreader.db.Task; +import java.io.File; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -1287,6 +1289,23 @@ public void setStatusListener(OnTTSStatusListener listener) { } } + + public void playAudioFile(File audioFile, double startTime) { + if (null != mMediaPlayer) { + mMediaPlayer.stop(); + mMediaPlayer.release(); + mMediaPlayer = null; + } + try{ + mMediaPlayer = MediaPlayer.create(getApplicationContext(), Uri.parse(audioFile.toString())); + }catch(Exception e){ + log.d("ERROR: " + e.getMessage()); + } + int millis = (int) (startTime*1000.0 + 0.5); + mMediaPlayer.seekTo(millis); + mMediaPlayer.start(); + } + // ====================================== // private implementation From eb82727d91827f6638741231c1cc6306254cba71 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Sat, 8 Apr 2023 15:44:59 -0400 Subject: [PATCH 05/59] audiobook: add audiobook navigation in TTS for vosk *.wordtiming files --- .../org/coolreader/crengine/SentenceInfo.java | 4 +- .../coolreader/crengine/TTSToolbarDlg.java | 52 ++++++-- .../crengine/WordTimingAudiobookMatcher.java | 29 +++-- .../org/coolreader/tts/TTSControlBinder.java | 10 +- .../org/coolreader/tts/TTSControlService.java | 119 +++++++++++++++++- 5 files changed, 186 insertions(+), 28 deletions(-) diff --git a/android/src/org/coolreader/crengine/SentenceInfo.java b/android/src/org/coolreader/crengine/SentenceInfo.java index d3c8df35d..1ca3d6d5a 100644 --- a/android/src/org/coolreader/crengine/SentenceInfo.java +++ b/android/src/org/coolreader/crengine/SentenceInfo.java @@ -1,5 +1,6 @@ package org.coolreader.crengine; +import java.io.File; import java.util.List; public class SentenceInfo { @@ -8,8 +9,9 @@ public class SentenceInfo { public String text; public double startTime; - public String audioFile; + public File audioFile; public List words; + public SentenceInfo nextSentence; public SentenceInfo() { } diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index 9cdbedc5a..4507ae295 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -31,6 +31,8 @@ import android.os.Build; import android.os.Bundle; import android.os.HandlerThread; +import android.os.Handler; +import android.os.Looper; import android.util.Log; import android.view.Gravity; import android.view.KeyEvent; @@ -93,6 +95,28 @@ public class TTSToolbarDlg implements Settings { private int mTTSSpeedPercent = 50; // 50% (normal) private WordTimingAudiobookMatcher wordTimingAudiobookMatcher; + private SentenceInfo currentSentenceInfo; + private Handler audioBookPosHandler = new Handler(Looper.getMainLooper()); + private Runnable audioBookPosRunnable = new Runnable() { + @Override + public void run() { + try{ + SentenceInfo currentSentence = fetchSelectedSentenceInfo(); + if(currentSentence != null && currentSentence.nextSentence != null){ + mTTSControl.bind(ttsbinder -> ttsbinder.isAudioBookPlaybackAfterSentence( + currentSentence.nextSentence, + isAfter -> { + if(isAfter){ + moveSelection(ReaderCommand.DCMD_SELECT_NEXT_SENTENCE, null); + } + } + )); + } + } finally { + audioBookPosHandler.postDelayed(this, 500); + } + } + }; static public TTSToolbarDlg showDialog( CoolReader coolReader, ReaderView readerView, TTSControlServiceAccessor ttsacc) { TTSToolbarDlg dlg = new TTSToolbarDlg(coolReader, readerView, ttsacc); @@ -149,6 +173,14 @@ private void restoreReaderMode() { } } + private SentenceInfo fetchSelectedSentenceInfo() { + if(wordTimingAudiobookMatcher != null && mCurrentSelection != null){ + return wordTimingAudiobookMatcher.getSentence( + mCurrentSelection.startY, mCurrentSelection.startX); + } + return null; + } + /** * Select next or previous sentence. ONLY the selection changes and the specified callback is called! * Not affected to speech synthesis process. @@ -161,17 +193,14 @@ private void moveSelection( ReaderCommand cmd, ReaderView.MoveSelectionCallback @Override public void onNewSelection(Selection selection) { - log.d("onNewSelection: " + selection.text); - if(wordTimingAudiobookMatcher != null){ - SentenceInfo s = wordTimingAudiobookMatcher.getSentence(selection.startY, selection.startX); - if(s.audioFile != null){ - File audioFile = wordTimingAudiobookMatcher.getAudioFile(s.audioFile); - mTTSControl.bind(ttsbinder -> { - ttsbinder.playAudioFile(audioFile, s.startTime); - }); - } - } + log.d("onNewSelection: " + selection.text + " : " + selection.startY + " x " + selection.startX); mCurrentSelection = selection; + SentenceInfo sentenceInfo = fetchSelectedSentenceInfo(); + if(sentenceInfo != null && sentenceInfo.audioFile != null){ + mTTSControl.bind(ttsbinder -> { + ttsbinder.setAudioFile(sentenceInfo.audioFile, sentenceInfo.startTime); + }); + } if (null != callback) callback.onNewSelection(mCurrentSelection); } @@ -651,12 +680,15 @@ public void onStopTrackingTouch(SeekBar seekBar) { // And finally, setup status change handler setupSpeechStatusHandler(); + audioBookPosHandler.removeCallbacks(audioBookPosRunnable); + if(wordTimingFile != null && wordTimingFile.exists()){ List allSentences = mReaderView.getAllSentences(); wordTimingAudiobookMatcher = new WordTimingAudiobookMatcher(wordTimingFile, allSentences); wordTimingAudiobookMatcher.parseWordTimingsFile(); moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); + audioBookPosHandler.postDelayed(audioBookPosRunnable, 500); }else{ wordTimingAudiobookMatcher = null; } diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 14bc16049..23df58b1d 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -19,9 +19,9 @@ public class WordTimingAudiobookMatcher { private static class WordTiming { String word; Double startTime; - String audioFile; + File audioFile; - public WordTiming(String word, Double startTime, String audioFile){ + public WordTiming(String word, Double startTime, File audioFile){ this.word = word; this.startTime = startTime; this.audioFile = audioFile; @@ -50,16 +50,32 @@ public void parseWordTimingsFile(){ lines = new ArrayList<>(); } + String dir = wordTimingsFile.getAbsoluteFile().getParent(); + wordTimings = new ArrayList<>(); for(String line : lines){ Matcher m = WORD_TIMING_REGEX.matcher(line); if(m.matches()){ - wordTimings.add(new WordTiming(m.group(2), Double.parseDouble(m.group(1)), m.group(3))); + String word = m.group(2); + Double startTime = Double.parseDouble(m.group(1)); + File audioFile = new File(dir + "/" + m.group(3)); + wordTimings.add(new WordTiming(word, startTime, audioFile)); }else{ log.d("ERROR: could not parse word timings line: " + line); } } + for(int i=0; i { if (null != mStatusListener) @@ -163,6 +177,15 @@ public void onReceive(Context context, Intent intent) { } break; case TTSControlService.TTS_CONTROL_ACTION_PREV: + if(useAudioBook){ + if (null != mStatusListener){ + if(mMediaPlayer != null){ + mMediaPlayer.stop(); + } + mStatusListener.onPreviousSentenceRequested(mBinder); + } + break; + } if (State.PLAYING == mState) { stopUtterance_impl(() -> { if (null != mStatusListener) @@ -319,6 +342,10 @@ public boolean onMediaButtonEvent (Intent mediaButtonIntent) { @Override public void onPlay() { + if(useAudioBook){ + ensureAudioBookPlaying(); + return; + } if (null == mCurrentUtterance) { if (null != mStatusListener) mStatusListener.onCurrentSentenceRequested(mBinder); @@ -392,6 +419,10 @@ public void onPlay() { @Override public void onPause() { + if(useAudioBook){ + pauseAudioBook(); + return; + } stopUtterance_impl(null); synchronized (mLocker) { mState = State.PAUSED; @@ -587,6 +618,10 @@ public interface RetrieveStateCallback { void onResult(State state); } + public interface IntResultCallback { + void onResult(int result); + } + public interface BooleanResultCallback { void onResult(boolean result); } @@ -782,6 +817,9 @@ private boolean stopUtterance_impl(Runnable callback) { } private void playWrapper_api_less_than_21() { + if(useAudioBook){ + return; + } if (null != mCurrentUtterance) { if (!mPlaybackNowAuthorized) requestAudioFocusWrapper(); @@ -818,6 +856,9 @@ private void playWrapper_api_less_than_21() { } private void pauseWrapper_api_less_than_21() { + if(useAudioBook){ + return; + } stopUtterance_impl(null); synchronized (mLocker) { mState = State.PAUSED; @@ -1256,6 +1297,32 @@ public void work() { }); } + public void isAudioBookPlaybackAfterSentence( + SentenceInfo sentenceInfo, BooleanResultCallback callback, Handler handler + ) { + execTask(new Task("isAudioBookPlaybackAfterSentence") { + @Override + public void work() { + boolean isAfterSentence = false; + if(audioFile != null && mState == State.PLAYING){ + if(sentenceInfo.audioFile.equals(audioFile)){ + double curPos = mMediaPlayer.getCurrentPosition() / 1000.0; + if(mMediaPlayer.isPlaying() && curPos >= sentenceInfo.startTime){ + isAfterSentence= true; + } + }else{ + if(!mMediaPlayer.isPlaying()){ + //next file + isAfterSentence = true; + } + } + } + final boolean result = isAfterSentence; + sendTask(handler, () -> callback.onResult(result)); + } + }); + } + public void retrieveVolume(VolumeResultCallback callback, Handler handler) { execTask(new Task("retrieveVolume") { @Override @@ -1290,20 +1357,57 @@ public void setStatusListener(OnTTSStatusListener listener) { } - public void playAudioFile(File audioFile, double startTime) { - if (null != mMediaPlayer) { - mMediaPlayer.stop(); - mMediaPlayer.release(); - mMediaPlayer = null; + public void setAudioFile(File audioFile, double startTime) { + if(this.audioFile != null && !this.audioFile.equals(audioFile)){ + if(mMediaPlayer != null){ + mMediaPlayer.stop(); + } + } + + this.useAudioBook = true; + this.audioFile = audioFile; + this.startTime = startTime; + + if(mState == State.PLAYING){ + ensureAudioBookPlaying(); + } + } + + public void ensureAudioBookPlaying() { + if(mMediaPlayer != null && mMediaPlayer.isPlaying()){ + return; } + try{ - mMediaPlayer = MediaPlayer.create(getApplicationContext(), Uri.parse(audioFile.toString())); + if(mMediaPlayer != null){ + mMediaPlayer.stop(); + mMediaPlayer.release(); + mMediaPlayer = null; + } + mMediaPlayer = MediaPlayer.create( + getApplicationContext(), Uri.parse("file://" + audioFile.toString())); }catch(Exception e){ log.d("ERROR: " + e.getMessage()); } + int millis = (int) (startTime*1000.0 + 0.5); mMediaPlayer.seekTo(millis); + mMediaPlayer.start(); + mMediaSession.setPlaybackState( + mPlaybackStateBuilder.setState(PlaybackState.STATE_PLAYING, + PlaybackState.PLAYBACK_POSITION_UNKNOWN, 0).build()); + mState = State.PLAYING; + mStatusListener.onStateChanged(mState); + } + + public void pauseAudioBook() { + mMediaPlayer.pause(); + mMediaSession.setPlaybackState( + mPlaybackStateBuilder.setState(PlaybackState.STATE_PAUSED, + PlaybackState.PLAYBACK_POSITION_UNKNOWN, 0).build()); + mState = State.PAUSED; + mStatusListener.onStateChanged(mState); } // ====================================== @@ -1415,6 +1519,9 @@ else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) } private void setupTTSHandlers() { + if(useAudioBook){ + return; + } if (null != mTTS) { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.ICE_CREAM_SANDWICH_MR1) { mTTS.setOnUtteranceCompletedListener(utteranceId -> { From 634b92a1ccba890e99f7675c7329feb7a18e566f Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 10 Apr 2023 16:17:47 -0400 Subject: [PATCH 06/59] audiobook: run parseWordTimingsFile() in a worker thread to prevent ANR --- .../InitAudiobookWordTimingsCallback.java | 5 +++ .../org/coolreader/crengine/ReaderView.java | 1 + .../coolreader/crengine/TTSToolbarDlg.java | 43 ++++++++++++++++--- 3 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 android/src/org/coolreader/crengine/InitAudiobookWordTimingsCallback.java diff --git a/android/src/org/coolreader/crengine/InitAudiobookWordTimingsCallback.java b/android/src/org/coolreader/crengine/InitAudiobookWordTimingsCallback.java new file mode 100644 index 000000000..ecb0e6883 --- /dev/null +++ b/android/src/org/coolreader/crengine/InitAudiobookWordTimingsCallback.java @@ -0,0 +1,5 @@ +package org.coolreader.crengine; + +public interface InitAudiobookWordTimingsCallback { + public void onComplete(); +} diff --git a/android/src/org/coolreader/crengine/ReaderView.java b/android/src/org/coolreader/crengine/ReaderView.java index 5547aa0c8..d4afb47e4 100644 --- a/android/src/org/coolreader/crengine/ReaderView.java +++ b/android/src/org/coolreader/crengine/ReaderView.java @@ -2420,6 +2420,7 @@ public void onCommand(final ReaderCommand cmd, final int param, final Runnable o ttsToolbar = TTSToolbarDlg.showDialog(mActivity, ReaderView.this, ttsacc); ttsToolbar.setOnCloseListener(() -> ttsToolbar = null); ttsToolbar.setAppSettings(mSettings, null); + ttsToolbar.initAudiobookWordTimings(null); })); } break; diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index 4507ae295..2a9d5c5f6 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -94,8 +94,13 @@ public class TTSToolbarDlg implements Settings { private boolean mGoogleTTSAbbreviationWorkaround; private int mTTSSpeedPercent = 50; // 50% (normal) + private File wordTimingFile; private WordTimingAudiobookMatcher wordTimingAudiobookMatcher; private SentenceInfo currentSentenceInfo; + + private HandlerThread wordTimingCalcHandlerThread; + private Handler wordTimingCalcHandler; + private Handler audioBookPosHandler = new Handler(Looper.getMainLooper()); private Runnable audioBookPosRunnable = new Runnable() { @Override @@ -637,7 +642,7 @@ public void onStopTrackingTouch(SeekBar seekBar) { // All tasks bellow after service start // Fetch book's metadata BookInfo bookInfo = mReaderView.getBookInfo(); - File wordTimingFile = null; + wordTimingFile = null; if (null != bookInfo) { FileInfo fileInfo = bookInfo.getFileInfo(); if (null != fileInfo) { @@ -679,16 +684,42 @@ public void onStopTrackingTouch(SeekBar seekBar) { }); // And finally, setup status change handler setupSpeechStatusHandler(); + } + public void initAudiobookWordTimings(InitAudiobookWordTimingsCallback callback){ audioBookPosHandler.removeCallbacks(audioBookPosRunnable); if(wordTimingFile != null && wordTimingFile.exists()){ - List allSentences = mReaderView.getAllSentences(); - wordTimingAudiobookMatcher = new WordTimingAudiobookMatcher(wordTimingFile, allSentences); - wordTimingAudiobookMatcher.parseWordTimingsFile(); + if(wordTimingCalcHandler == null){ + if(wordTimingCalcHandlerThread == null){ + wordTimingCalcHandlerThread = new HandlerThread("word-timing-calc-handler"); + wordTimingCalcHandlerThread.start(); + } + Looper wordTimingCalcLooper = wordTimingCalcHandlerThread.getLooper(); + wordTimingCalcHandler = new Handler(wordTimingCalcLooper); + } + - moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); - audioBookPosHandler.postDelayed(audioBookPosRunnable, 500); + wordTimingCalcHandler.removeCallbacksAndMessages(null); + mCoolReader.showToast("matching audiobook word timings"); + wordTimingCalcHandler.post( + new Runnable() { + public void run() { + List allSentences = mReaderView.getAllSentences(); + wordTimingAudiobookMatcher = new WordTimingAudiobookMatcher(wordTimingFile, allSentences); + + //can be very long + wordTimingAudiobookMatcher.parseWordTimingsFile(); + + moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); + audioBookPosHandler.postDelayed(audioBookPosRunnable, 500); + + if(callback != null){ + callback.onComplete(); + } + } + } + ); }else{ wordTimingAudiobookMatcher = null; } From 3dd172fa7ee2561575348b690f565fcebb74e4db Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 10 Apr 2023 16:20:20 -0400 Subject: [PATCH 07/59] audiobook: add option 'app.tts.use.audiobook' to settings in TTS ui --- android/res/values/strings.xml | 1 + .../org/coolreader/crengine/BaseActivity.java | 2 + .../coolreader/crengine/OptionsDialog.java | 6 ++ .../src/org/coolreader/crengine/Settings.java | 1 + .../coolreader/crengine/TTSToolbarDlg.java | 62 ++++++++++++++++--- .../org/coolreader/tts/TTSControlService.java | 5 +- 6 files changed, 69 insertions(+), 8 deletions(-) diff --git a/android/res/values/strings.xml b/android/res/values/strings.xml index 79a857117..e02ef4f39 100644 --- a/android/res/values/strings.xml +++ b/android/res/values/strings.xml @@ -553,6 +553,7 @@ Very high quality Use a workaround to disable processing of abbreviations at the end of a sentence when using "Google Speech Services" + Use audiobook instead of TTS, if *.wordtiming file exists TTS engine \"%s\" init failure, disabling it… Open book Open containing folder diff --git a/android/src/org/coolreader/crengine/BaseActivity.java b/android/src/org/coolreader/crengine/BaseActivity.java index bb6f3fac9..afb35b549 100644 --- a/android/src/org/coolreader/crengine/BaseActivity.java +++ b/android/src/org/coolreader/crengine/BaseActivity.java @@ -1898,6 +1898,8 @@ public Properties loadSettings(BaseActivity activity, File file) { // By default enable workaround to disable processing of abbreviations at the end of a sentence when using "Google Speech Services". props.applyDefault(ReaderView.PROP_APP_TTS_GOOGLE_END_OF_SENTENCE_ABBR, "1"); + props.applyDefault(ReaderView.PROP_APP_TTS_USE_AUDIOBOOK, "1"); + props.applyDefault(ReaderView.PROP_APP_THEME, DeviceInfo.FORCE_HC_THEME ? "HICONTRAST1" : "LIGHT"); props.applyDefault(ReaderView.PROP_APP_THEME_DAY, DeviceInfo.FORCE_HC_THEME ? "HICONTRAST1" : "LIGHT"); props.applyDefault(ReaderView.PROP_APP_THEME_NIGHT, DeviceInfo.FORCE_HC_THEME ? "HICONTRAST2" : "DARK"); diff --git a/android/src/org/coolreader/crengine/OptionsDialog.java b/android/src/org/coolreader/crengine/OptionsDialog.java index f3d3b8552..7c65e226c 100644 --- a/android/src/org/coolreader/crengine/OptionsDialog.java +++ b/android/src/org/coolreader/crengine/OptionsDialog.java @@ -2655,6 +2655,12 @@ public void onTimedOut() { mOptionsTTS.add(mTTSVoiceOption); } mOptionsTTS.add(new BoolOption(this, getString(R.string.options_tts_google_abbr_workaround), PROP_APP_TTS_GOOGLE_END_OF_SENTENCE_ABBR).setComment(getString(R.string.options_tts_google_abbr_workaround_comment)).setDefaultValue("1").noIcon()); + mOptionsTTS.add( + new BoolOption( + this, + getString(R.string.options_tts_use_audiobook), + PROP_APP_TTS_USE_AUDIOBOOK + ).setDefaultValue("1").noIcon()); if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.ECLAIR) mOptionsTTS.add(new ListOption(this, getString(R.string.options_app_tts_stop_motion_timeout), PROP_APP_MOTION_TIMEOUT).add(mMotionTimeouts, mMotionTimeoutsTitles).setDefaultValue(Integer.toString(mMotionTimeouts[0])).noIcon()); mOptionsTTS.refresh(); diff --git a/android/src/org/coolreader/crengine/Settings.java b/android/src/org/coolreader/crengine/Settings.java index 32c1a2650..3e48f8716 100644 --- a/android/src/org/coolreader/crengine/Settings.java +++ b/android/src/org/coolreader/crengine/Settings.java @@ -220,6 +220,7 @@ Commented until the appearance of free implementation of the binding to the Goog String PROP_APP_TTS_FORCE_LANGUAGE = "app.tts.force.lang"; // Force use specified language String PROP_APP_TTS_VOICE = "app.tts.voice"; String PROP_APP_TTS_GOOGLE_END_OF_SENTENCE_ABBR = "app.tts.google.end-of-sentence-abbreviation.workaround"; // Use a workaround to disable processing of abbreviations at the end of a sentence when using "Google Speech Services" + String PROP_APP_TTS_USE_AUDIOBOOK = "app.tts.use.audiobook"; //if *.wordtiming file exists for ebook String PROP_APP_VIEW_ANIM_DURATION ="app.view.anim.duration"; diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index 2a9d5c5f6..5e1335c15 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -92,6 +92,7 @@ public class TTSToolbarDlg implements Settings { private String mCurrentLanguage; private String mCurrentVoiceName; private boolean mGoogleTTSAbbreviationWorkaround; + private boolean allowUseAudiobook; private int mTTSSpeedPercent = 50; // 50% (normal) private File wordTimingFile; @@ -200,11 +201,13 @@ private void moveSelection( ReaderCommand cmd, ReaderView.MoveSelectionCallback public void onNewSelection(Selection selection) { log.d("onNewSelection: " + selection.text + " : " + selection.startY + " x " + selection.startX); mCurrentSelection = selection; - SentenceInfo sentenceInfo = fetchSelectedSentenceInfo(); - if(sentenceInfo != null && sentenceInfo.audioFile != null){ - mTTSControl.bind(ttsbinder -> { - ttsbinder.setAudioFile(sentenceInfo.audioFile, sentenceInfo.startTime); - }); + if(allowUseAudiobook){ + SentenceInfo sentenceInfo = fetchSelectedSentenceInfo(); + if(sentenceInfo != null && sentenceInfo.audioFile != null){ + mTTSControl.bind(ttsbinder -> { + ttsbinder.setAudioFile(sentenceInfo.audioFile, sentenceInfo.startTime); + }); + } } if (null != callback) callback.onNewSelection(mCurrentSelection); @@ -284,9 +287,15 @@ private float speechRateFromPercent(int percent) { public void setAppSettings(Properties newSettings, Properties oldSettings) { log.v("setAppSettings()"); BackgroundThread.ensureGUI(); - if (oldSettings == null) + boolean initialSetup; + if (oldSettings == null){ oldSettings = new Properties(); + initialSetup = true; + }else{ + initialSetup = false; + } int oldTTSSpeed = mTTSSpeedPercent; + boolean oldAllowUseAudiobook = this.allowUseAudiobook; Properties changedSettings = newSettings.diff(oldSettings); for (Map.Entry entry : changedSettings.entrySet()) { String key = (String) entry.getKey(); @@ -303,6 +312,41 @@ public void setAppSettings(Properties newSettings, Properties oldSettings) { }); }); } + boolean newAllowUseAudiobook = allowUseAudiobook; + if (!initialSetup && oldAllowUseAudiobook && !newAllowUseAudiobook){ + mTTSControl.bind(ttsbinder -> { + ttsbinder.stop(null); + ttsbinder.setAudioFile(null, 0); + initAudiobookWordTimings(new InitAudiobookWordTimingsCallback(){ + public void onComplete(){ + moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, new ReaderView.MoveSelectionCallback() { + @Override + public void onNewSelection(Selection selection) { + if (isSpeaking) { + ttsbinder.say(preprocessUtterance(selection.text), null); + } else { + ttsbinder.setCurrentUtterance(preprocessUtterance(selection.text)); + } + } + + @Override + public void onFail() { + } + }); + } + }); + }); + }else if(!initialSetup && !oldAllowUseAudiobook && newAllowUseAudiobook){ + mTTSControl.bind(ttsbinder -> { + ttsbinder.stop(null); + SentenceInfo sentenceInfo = fetchSelectedSentenceInfo(); + if(sentenceInfo != null){ + ttsbinder.setAudioFile(sentenceInfo.audioFile, sentenceInfo.startTime); + } + initAudiobookWordTimings(null); + moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); + }); + } } private void processAppSetting(String key, String value) { @@ -331,6 +375,10 @@ private void processAppSetting(String key, String value) { break; case PROP_APP_TTS_GOOGLE_END_OF_SENTENCE_ABBR: mGoogleTTSAbbreviationWorkaround = flg; + break; + case PROP_APP_TTS_USE_AUDIOBOOK: + allowUseAudiobook = flg; + break; } } @@ -689,7 +737,7 @@ public void onStopTrackingTouch(SeekBar seekBar) { public void initAudiobookWordTimings(InitAudiobookWordTimingsCallback callback){ audioBookPosHandler.removeCallbacks(audioBookPosRunnable); - if(wordTimingFile != null && wordTimingFile.exists()){ + if(allowUseAudiobook && wordTimingFile != null && wordTimingFile.exists()){ if(wordTimingCalcHandler == null){ if(wordTimingCalcHandlerThread == null){ wordTimingCalcHandlerThread = new HandlerThread("word-timing-calc-handler"); diff --git a/android/src/org/coolreader/tts/TTSControlService.java b/android/src/org/coolreader/tts/TTSControlService.java index bcae951e9..4644f1afd 100644 --- a/android/src/org/coolreader/tts/TTSControlService.java +++ b/android/src/org/coolreader/tts/TTSControlService.java @@ -1364,7 +1364,7 @@ public void setAudioFile(File audioFile, double startTime) { } } - this.useAudioBook = true; + this.useAudioBook = audioFile == null ? false : true; this.audioFile = audioFile; this.startTime = startTime; @@ -1377,6 +1377,9 @@ public void ensureAudioBookPlaying() { if(mMediaPlayer != null && mMediaPlayer.isPlaying()){ return; } + if(audioFile == null){ + return; + } try{ if(mMediaPlayer != null){ From 0fd8de874043833ac47e72afcbdc71b1fc2a57ea Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 10 Apr 2023 22:05:43 -0400 Subject: [PATCH 08/59] audiobook: replace '.split()' for performance (11s => 4s) --- .../crengine/WordTimingAudiobookMatcher.java | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 23df58b1d..a3132d8cd 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -77,16 +77,7 @@ public void parseWordTimingsFile(){ } for(SentenceInfo s : allSentences){ - String text = s.text; - text = text.replaceAll("’", "'"); - text = text.toLowerCase(); - String[] words = text.split("[^a-z0-9']"); - s.words = new ArrayList<>(); - for(String word : words){ - if(word.matches(".*\\w.*")){ - s.words.add(word); - } - } + s.words = splitSentenceIntoWords(s.text); } if(wordTimings.size() == 0){ @@ -149,4 +140,48 @@ public SentenceInfo getSentence(double y, double x){ } return null; } + + private List splitSentenceIntoWords(String sentence){ + List words = new ArrayList(); + + StringBuilder str = null; + boolean wordContainsLetterOrNumber = false; + for(int i=0; i Date: Mon, 10 Apr 2023 22:19:26 -0400 Subject: [PATCH 09/59] audiobook: replace WORD_TIMING_REGEX for performance (4s => 1s) --- .../crengine/WordTimingAudiobookMatcher.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index a3132d8cd..8d4a5df69 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -13,9 +13,6 @@ public class WordTimingAudiobookMatcher { public static final Logger log = L.create("wordtiming"); - private static final Pattern WORD_TIMING_REGEX = Pattern.compile( - "^(\\d+|\\d*\\.\\d+),([^,]+),(.+)$" - ); private static class WordTiming { String word; Double startTime; @@ -54,15 +51,11 @@ public void parseWordTimingsFile(){ wordTimings = new ArrayList<>(); for(String line : lines){ - Matcher m = WORD_TIMING_REGEX.matcher(line); - if(m.matches()){ - String word = m.group(2); - Double startTime = Double.parseDouble(m.group(1)); - File audioFile = new File(dir + "/" + m.group(3)); - wordTimings.add(new WordTiming(word, startTime, audioFile)); - }else{ + WordTiming wordTiming = parseWordTimingsLine(dir, line); + if(wordTiming == null){ log.d("ERROR: could not parse word timings line: " + line); } + wordTimings.add(wordTiming); } for(int i=0; i= line.length() || sep2 >= line.length()){ + return null; + } + String word = line.substring(sep1+1, sep2); + Double startTime = Double.parseDouble(line.substring(0, sep1)); + File audioFile = new File(dir + "/" + line.substring(sep2+1)); + return new WordTiming(word, startTime, audioFile); + } + private List splitSentenceIntoWords(String sentence){ List words = new ArrayList(); From 2636053a3ec6f274842d183b5c10724933ba3eb1 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Tue, 11 Apr 2023 15:50:14 -0400 Subject: [PATCH 10/59] audiobook: re-use the same File object across sentences --- .../crengine/WordTimingAudiobookMatcher.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 8d4a5df69..fe2f11866 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -6,7 +6,9 @@ import java.io.File; import java.io.FileReader; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -27,6 +29,8 @@ public WordTiming(String word, Double startTime, File audioFile){ private final File wordTimingsFile; private final List allSentences; + private final Map fileCache = new HashMap<>(); + private String wordTimingsDir; private List wordTimings; public WordTimingAudiobookMatcher(File wordTimingsFile, List allSentences) { @@ -47,11 +51,11 @@ public void parseWordTimingsFile(){ lines = new ArrayList<>(); } - String dir = wordTimingsFile.getAbsoluteFile().getParent(); + this.wordTimingsDir = wordTimingsFile.getAbsoluteFile().getParent(); wordTimings = new ArrayList<>(); for(String line : lines){ - WordTiming wordTiming = parseWordTimingsLine(dir, line); + WordTiming wordTiming = parseWordTimingsLine(line); if(wordTiming == null){ log.d("ERROR: could not parse word timings line: " + line); } @@ -134,7 +138,7 @@ public SentenceInfo getSentence(double y, double x){ return null; } - private WordTiming parseWordTimingsLine(String dir, String line){ + private WordTiming parseWordTimingsLine(String line){ int sep1 = line.indexOf(','); int sep2 = line.indexOf(',', sep1+1); if(sep1 < 0 || sep2 < 0 || sep1 >= line.length() || sep2 >= line.length()){ @@ -142,7 +146,11 @@ private WordTiming parseWordTimingsLine(String dir, String line){ } String word = line.substring(sep1+1, sep2); Double startTime = Double.parseDouble(line.substring(0, sep1)); - File audioFile = new File(dir + "/" + line.substring(sep2+1)); + String audioFileName = line.substring(sep2+1); + if(!fileCache.containsKey(audioFileName)){ + fileCache.put(audioFileName, new File(wordTimingsDir + "/" + audioFileName)); + } + File audioFile = fileCache.get(audioFileName); return new WordTiming(word, startTime, audioFile); } From a039fcb57f4dfabf0d31d9ec767e57d7a08a9851 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Tue, 11 Apr 2023 15:50:46 -0400 Subject: [PATCH 11/59] audiobook: start each audio file at 0.0s --- .../crengine/WordTimingAudiobookMatcher.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index fe2f11866..4c204dd27 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -127,6 +127,16 @@ public void parseWordTimingsFile(){ prevAudioFile = s.audioFile; } } + + //start first sentence of all audio files at 0.0 + // prevents skipping intros + File curAudioFile = null; + for(SentenceInfo s : allSentences){ + if(curAudioFile == null || s.audioFile != curAudioFile){ + s.startTime = 0; + curAudioFile = s.audioFile; + } + } } public SentenceInfo getSentence(double y, double x){ From 8742cd1269b1af0c47387f442dc307a5dee00837 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Tue, 11 Apr 2023 17:11:02 -0400 Subject: [PATCH 12/59] audiobook: do not select next sentence multiple times for new audiofiles --- .../org/coolreader/crengine/SentenceInfo.java | 1 + .../coolreader/crengine/TTSToolbarDlg.java | 4 +-- .../crengine/WordTimingAudiobookMatcher.java | 1 + .../org/coolreader/tts/TTSControlService.java | 26 ++++++++++++------- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/android/src/org/coolreader/crengine/SentenceInfo.java b/android/src/org/coolreader/crengine/SentenceInfo.java index 1ca3d6d5a..09fbdf188 100644 --- a/android/src/org/coolreader/crengine/SentenceInfo.java +++ b/android/src/org/coolreader/crengine/SentenceInfo.java @@ -9,6 +9,7 @@ public class SentenceInfo { public String text; public double startTime; + public boolean isFirstSentenceInAudioFile = false; public File audioFile; public List words; public SentenceInfo nextSentence; diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index 5e1335c15..ddf9a0be3 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -108,9 +108,9 @@ public class TTSToolbarDlg implements Settings { public void run() { try{ SentenceInfo currentSentence = fetchSelectedSentenceInfo(); - if(currentSentence != null && currentSentence.nextSentence != null){ + if(currentSentence != null){ mTTSControl.bind(ttsbinder -> ttsbinder.isAudioBookPlaybackAfterSentence( - currentSentence.nextSentence, + currentSentence, isAfter -> { if(isAfter){ moveSelection(ReaderCommand.DCMD_SELECT_NEXT_SENTENCE, null); diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 4c204dd27..2e9435840 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -133,6 +133,7 @@ public void parseWordTimingsFile(){ File curAudioFile = null; for(SentenceInfo s : allSentences){ if(curAudioFile == null || s.audioFile != curAudioFile){ + s.isFirstSentenceInAudioFile = true; s.startTime = 0; curAudioFile = s.audioFile; } diff --git a/android/src/org/coolreader/tts/TTSControlService.java b/android/src/org/coolreader/tts/TTSControlService.java index 4644f1afd..1bc0c3a83 100644 --- a/android/src/org/coolreader/tts/TTSControlService.java +++ b/android/src/org/coolreader/tts/TTSControlService.java @@ -1304,19 +1304,25 @@ public void isAudioBookPlaybackAfterSentence( @Override public void work() { boolean isAfterSentence = false; - if(audioFile != null && mState == State.PLAYING){ - if(sentenceInfo.audioFile.equals(audioFile)){ - double curPos = mMediaPlayer.getCurrentPosition() / 1000.0; - if(mMediaPlayer.isPlaying() && curPos >= sentenceInfo.startTime){ - isAfterSentence= true; - } - }else{ - if(!mMediaPlayer.isPlaying()){ - //next file - isAfterSentence = true; + if(sentenceInfo != null && sentenceInfo.nextSentence != null){ + SentenceInfo nextSentenceInfo = sentenceInfo.nextSentence; + if(sentenceInfo.audioFile == TTSControlService.this.audioFile){ + if(nextSentenceInfo.isFirstSentenceInAudioFile){ + if(mState == State.PLAYING && !mMediaPlayer.isPlaying()){ + //this is the last sentence in the file, and the media player ended + isAfterSentence = true; + } + }else{ + if(mMediaPlayer.isPlaying()){ + double curPos = mMediaPlayer.getCurrentPosition() / 1000.0; + if(curPos >= nextSentenceInfo.startTime){ + isAfterSentence = true; + } + } } } } + final boolean result = isAfterSentence; sendTask(handler, () -> callback.onResult(result)); } From bd844f04175c3481caf6ba86172c19265b6455a6 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 12 Apr 2023 11:14:21 -0400 Subject: [PATCH 13/59] audiobook: null-check MediaPlayer --- android/src/org/coolreader/tts/TTSControlService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/src/org/coolreader/tts/TTSControlService.java b/android/src/org/coolreader/tts/TTSControlService.java index 1bc0c3a83..2c7be2db1 100644 --- a/android/src/org/coolreader/tts/TTSControlService.java +++ b/android/src/org/coolreader/tts/TTSControlService.java @@ -1308,12 +1308,12 @@ public void work() { SentenceInfo nextSentenceInfo = sentenceInfo.nextSentence; if(sentenceInfo.audioFile == TTSControlService.this.audioFile){ if(nextSentenceInfo.isFirstSentenceInAudioFile){ - if(mState == State.PLAYING && !mMediaPlayer.isPlaying()){ + if(mState == State.PLAYING && (mMediaPlayer == null || !mMediaPlayer.isPlaying())){ //this is the last sentence in the file, and the media player ended isAfterSentence = true; } }else{ - if(mMediaPlayer.isPlaying()){ + if(mMediaPlayer != null && mMediaPlayer.isPlaying()){ double curPos = mMediaPlayer.getCurrentPosition() / 1000.0; if(curPos >= nextSentenceInfo.startTime){ isAfterSentence = true; From bc749bf59b92c56ba6e1fe5ab6bd3197fa0cef20 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 12 Apr 2023 23:52:06 -0400 Subject: [PATCH 14/59] tts: pull all TTS control buttons to class vars --- .../coolreader/crengine/TTSToolbarDlg.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index ddf9a0be3..9e8b21889 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -71,10 +71,18 @@ public class TTSToolbarDlg implements Settings { private final ReaderView mReaderView; private final TTSControlServiceAccessor mTTSControl; private final ImageButton mPlayPauseButton; + private final ImageButton backButton; + private final ImageButton forwardButton; + private final ImageButton stopButton; + private final ImageButton optionsButton; private final TextView mVolumeTextView; private final TextView mSpeedTextView; private final SeekBar mSbSpeed; private final SeekBar mSbVolume; + private final ImageButton btnDecVolume; + private final ImageButton btnIncVolume; + private final ImageButton btnDecSpeed; + private final ImageButton btnIncSpeed; private HandlerThread mMotionWatchdog; private boolean changedPageMode; private Runnable mOnCloseListener; @@ -534,10 +542,10 @@ public TTSToolbarDlg(CoolReader coolReader, ReaderView readerView, TTSControlSer mPlayPauseButton = panel.findViewById(R.id.tts_play_pause); mPlayPauseButton.setImageResource(Utils.resolveResourceIdByAttr(mCoolReader, R.attr.ic_media_play_drawable, R.drawable.ic_media_play)); - ImageButton backButton = panel.findViewById(R.id.tts_back); - ImageButton forwardButton = panel.findViewById(R.id.tts_forward); - ImageButton stopButton = panel.findViewById(R.id.tts_stop); - ImageButton optionsButton = panel.findViewById(R.id.tts_options); + backButton = panel.findViewById(R.id.tts_back); + forwardButton = panel.findViewById(R.id.tts_forward); + stopButton = panel.findViewById(R.id.tts_stop); + optionsButton = panel.findViewById(R.id.tts_options); mWindow = new PopupWindow( context ); mWindow.setBackgroundDrawable(new BitmapDrawable()); @@ -583,18 +591,18 @@ public void onStopTrackingTouch(SeekBar seekBar) { mCoolReader.setSetting(PROP_APP_TTS_SPEED, String.valueOf(mProgress), true); } }); - ImageButton btnDecVolume = panel.findViewById(R.id.btn_dec_volume); + btnDecVolume = panel.findViewById(R.id.btn_dec_volume); btnDecVolume.setOnTouchListener(new RepeatOnTouchListener(500, 150, view -> mSbVolume.setProgress(mSbVolume.getProgress() - 1))); - ImageButton btnIncVolume = panel.findViewById(R.id.btn_inc_volume); + btnIncVolume = panel.findViewById(R.id.btn_inc_volume); btnIncVolume.setOnTouchListener(new RepeatOnTouchListener(500, 150, view -> mSbVolume.setProgress(mSbVolume.getProgress() + 1))); - ImageButton btnDecSpeed = panel.findViewById(R.id.btn_dec_speed); + btnDecSpeed = panel.findViewById(R.id.btn_dec_speed); btnDecSpeed.setOnTouchListener(new RepeatOnTouchListener(500, 150, view -> { mSbSpeed.setProgress(mSbSpeed.getProgress() - 1); mCoolReader.setSetting(PROP_APP_TTS_SPEED, String.valueOf(mSbSpeed.getProgress()), true); })); - ImageButton btnIncSpeed = panel.findViewById(R.id.btn_inc_speed); + btnIncSpeed = panel.findViewById(R.id.btn_inc_speed); btnIncSpeed.setOnTouchListener(new RepeatOnTouchListener(500, 150, view -> { mSbSpeed.setProgress(mSbSpeed.getProgress() + 1); mCoolReader.setSetting(PROP_APP_TTS_SPEED, String.valueOf(mSbSpeed.getProgress()), true); From ee12e384ddaa57010bca3246836440a457215f29 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 12 Apr 2023 23:52:54 -0400 Subject: [PATCH 15/59] audiobook: hide the top row of buttons while calculating word timings --- .../src/org/coolreader/crengine/TTSToolbarDlg.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index 9e8b21889..bd6c87447 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -755,6 +755,11 @@ public void initAudiobookWordTimings(InitAudiobookWordTimingsCallback callback){ wordTimingCalcHandler = new Handler(wordTimingCalcLooper); } + mPlayPauseButton.setVisibility(View.GONE); + backButton.setVisibility(View.GONE); + forwardButton.setVisibility(View.GONE); + stopButton.setVisibility(View.GONE); + optionsButton.setVisibility(View.GONE); wordTimingCalcHandler.removeCallbacksAndMessages(null); mCoolReader.showToast("matching audiobook word timings"); @@ -770,6 +775,14 @@ public void run() { moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); audioBookPosHandler.postDelayed(audioBookPosRunnable, 500); + BackgroundThread.instance().postGUI(() -> { + mPlayPauseButton.setVisibility(View.VISIBLE); + backButton.setVisibility(View.VISIBLE); + forwardButton.setVisibility(View.VISIBLE); + stopButton.setVisibility(View.VISIBLE); + optionsButton.setVisibility(View.VISIBLE); + }); + if(callback != null){ callback.onComplete(); } From 86699c33a3d067ea335c4eba8c64bb0a8f43e05d Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Sat, 15 Apr 2023 11:24:11 -0400 Subject: [PATCH 16/59] audiobook: use startPos doc position strings instead of (x,y) coords --- android/jni/docview.cpp | 8 ++++---- .../src/org/coolreader/crengine/SentenceInfo.java | 10 +++------- .../src/org/coolreader/crengine/TTSToolbarDlg.java | 3 +-- .../crengine/WordTimingAudiobookMatcher.java | 13 ++++++------- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/android/jni/docview.cpp b/android/jni/docview.cpp index 129c9c1c0..1e2edf630 100644 --- a/android/jni/docview.cpp +++ b/android/jni/docview.cpp @@ -1829,9 +1829,8 @@ JNIEXPORT jobject JNICALL Java_org_coolreader_crengine_DocView_getAllSentencesIn jclass sentenceInfoClass = _env->FindClass("org/coolreader/crengine/SentenceInfo"); jmethodID sentenceInfoCtor = _env->GetMethodID(sentenceInfoClass, "", "()V"); - jmethodID sentenceInfoSetStartX = _env->GetMethodID(sentenceInfoClass, "setStartX", "(I)V"); - jmethodID sentenceInfoSetStartY = _env->GetMethodID(sentenceInfoClass, "setStartY", "(I)V"); jmethodID sentenceInfoSetText = _env->GetMethodID(sentenceInfoClass, "setText", "(Ljava/lang/String;)V"); + jmethodID sentenceInfoSetStartPos = _env->GetMethodID(sentenceInfoClass, "setStartPos", "(Ljava/lang/String;)V"); jobject arrList = env->NewObject(arrListClass, arrListCtor); @@ -1853,11 +1852,12 @@ JNIEXPORT jobject JNICALL Java_org_coolreader_crengine_DocView_getAllSentencesIn lvPoint startPoint = currSel.getStart().toPoint(); lvPoint endPoint = currSel.getEnd().toPoint(); - env->CallVoidMethod(sentenceInfo, sentenceInfoSetStartX, startPoint.x); - env->CallVoidMethod(sentenceInfo, sentenceInfoSetStartY, startPoint.y); env->CallVoidMethod(sentenceInfo, sentenceInfoSetText, env->NewStringUTF( UnicodeToUtf8(currSel.getRangeText()).c_str() )); + env->CallVoidMethod(sentenceInfo, sentenceInfoSetStartPos, env->NewStringUTF( + UnicodeToUtf8(currSel.getStart().toString()).c_str() + )); env->CallBooleanMethod(arrList, arrListAdd, sentenceInfo); } diff --git a/android/src/org/coolreader/crengine/SentenceInfo.java b/android/src/org/coolreader/crengine/SentenceInfo.java index 09fbdf188..8f904b339 100644 --- a/android/src/org/coolreader/crengine/SentenceInfo.java +++ b/android/src/org/coolreader/crengine/SentenceInfo.java @@ -4,9 +4,8 @@ import java.util.List; public class SentenceInfo { - public int startX; - public int startY; public String text; + public String startPos; public double startTime; public boolean isFirstSentenceInAudioFile = false; @@ -17,11 +16,8 @@ public class SentenceInfo { public SentenceInfo() { } - public void setStartX(int startX){ - this.startX = startX; - } - public void setStartY(int startY){ - this.startY = startY; + public void setStartPos(String startPos){ + this.startPos = startPos; } public void setText(String text){ this.text = text; diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index bd6c87447..484447a1e 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -189,8 +189,7 @@ private void restoreReaderMode() { private SentenceInfo fetchSelectedSentenceInfo() { if(wordTimingAudiobookMatcher != null && mCurrentSelection != null){ - return wordTimingAudiobookMatcher.getSentence( - mCurrentSelection.startY, mCurrentSelection.startX); + return wordTimingAudiobookMatcher.getSentence(mCurrentSelection.startPos); } return null; } diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 2e9435840..15e193e58 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -29,6 +29,7 @@ public WordTiming(String word, Double startTime, File audioFile){ private final File wordTimingsFile; private final List allSentences; + private final Map sentencesByStartPos = new HashMap<>(); private final Map fileCache = new HashMap<>(); private String wordTimingsDir; private List wordTimings; @@ -36,6 +37,9 @@ public WordTiming(String word, Double startTime, File audioFile){ public WordTimingAudiobookMatcher(File wordTimingsFile, List allSentences) { this.wordTimingsFile = wordTimingsFile; this.allSentences = allSentences; + for(SentenceInfo s : allSentences){ + sentencesByStartPos.put(s.startPos, s); + } } public void parseWordTimingsFile(){ @@ -140,13 +144,8 @@ public void parseWordTimingsFile(){ } } - public SentenceInfo getSentence(double y, double x){ - for(SentenceInfo s : allSentences){ - if(s.startY > y || (s.startY == y && s.startX >= x)){ - return s; - } - } - return null; + public SentenceInfo getSentence(String startPos){ + return sentencesByStartPos.get(startPos); } private WordTiming parseWordTimingsLine(String line){ From 5254c6ca8073e2df6f10b9d773f4c690ecba1f74 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Sat, 15 Apr 2023 12:33:24 -0400 Subject: [PATCH 17/59] audiobook: cache sentence info next to the ebook --- .../crengine/SentenceInfoCache.java | 79 +++++++++++++++++++ .../coolreader/crengine/TTSToolbarDlg.java | 10 ++- 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 android/src/org/coolreader/crengine/SentenceInfoCache.java diff --git a/android/src/org/coolreader/crengine/SentenceInfoCache.java b/android/src/org/coolreader/crengine/SentenceInfoCache.java new file mode 100644 index 000000000..a089f1e7b --- /dev/null +++ b/android/src/org/coolreader/crengine/SentenceInfoCache.java @@ -0,0 +1,79 @@ +package org.coolreader.crengine; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class SentenceInfoCache { + public static final Logger log = L.create("sentenceinfocache"); + + private final File sentenceInfoCacheFile; + + public static List maybeReadCache(File sentenceInfoCacheFile) { + try{ + return new SentenceInfoCache(sentenceInfoCacheFile).readCache(); + }catch(Exception e){ + log.e("ERROR: could not read sentence info cache file: " + sentenceInfoCacheFile + " " + e); + return null; + } + } + public static void maybeWriteCache(File sentenceInfoCacheFile, List allSentences) { + try{ + new SentenceInfoCache(sentenceInfoCacheFile).writeCache(allSentences); + }catch(Exception e){ + log.e("ERROR: could not write sentence info cache file: " + sentenceInfoCacheFile + " " + e); + } + } + + + public SentenceInfoCache(File sentenceInfoCacheFile) { + this.sentenceInfoCacheFile = sentenceInfoCacheFile; + } + + public List readCache() throws IOException { + List allSentences = new ArrayList<>(); + BufferedReader br = new BufferedReader(new FileReader(sentenceInfoCacheFile)); + String line; + while ((line = br.readLine()) != null) { + SentenceInfo sentenceInfo = parseSentenceInfoLine(line); + if(sentenceInfo == null){ + log.e("ERROR: could not parse sentence info cache line: " + line); + br.close(); + return null; + } + allSentences.add(sentenceInfo); + } + br.close(); + if(allSentences.isEmpty()){ + return null; + } + return allSentences; + } + + public void writeCache(List allSentences) throws IOException { + FileWriter fw = new FileWriter(sentenceInfoCacheFile); + for(SentenceInfo s : allSentences){ + fw.write(formatSentenceInfo(s)); + } + fw.close(); + } + + private SentenceInfo parseSentenceInfoLine(String line){ + int sep = line.indexOf(','); + if(sep < 0 || sep >= line.length()){ + return null; + } + SentenceInfo sentenceInfo = new SentenceInfo(); + sentenceInfo.startPos = line.substring(0, sep); + sentenceInfo.text = line.substring(sep+1); + return sentenceInfo; + } + + private String formatSentenceInfo(SentenceInfo sentenceInfo){ + return sentenceInfo.startPos + "," + sentenceInfo.text + "\n"; + } +} diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index 484447a1e..b853814b0 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -104,6 +104,7 @@ public class TTSToolbarDlg implements Settings { private int mTTSSpeedPercent = 50; // 50% (normal) private File wordTimingFile; + private File sentenceInfoCacheFile; private WordTimingAudiobookMatcher wordTimingAudiobookMatcher; private SentenceInfo currentSentenceInfo; @@ -698,6 +699,7 @@ public void onStopTrackingTouch(SeekBar seekBar) { // Fetch book's metadata BookInfo bookInfo = mReaderView.getBookInfo(); wordTimingFile = null; + sentenceInfoCacheFile = null; if (null != bookInfo) { FileInfo fileInfo = bookInfo.getFileInfo(); if (null != fileInfo) { @@ -709,8 +711,10 @@ public void onStopTrackingTouch(SeekBar seekBar) { (file, bitmap) -> mTTSControl.bind(ttsbinder -> ttsbinder.setMediaItemInfo(mBookAuthors, mBookTitle, bitmap))); String pathName = fileInfo.getPathName(); String wordTimingPath = pathName.replaceAll("\\.\\w+$", ".wordtiming"); + String sentenceInfoPath = pathName.replaceAll("\\.\\w+$", ".sentenceinfo"); if(wordTimingPath.matches(".*\\.wordtiming$")){ wordTimingFile = new File(wordTimingPath); + sentenceInfoCacheFile = new File(sentenceInfoPath); } } } @@ -765,7 +769,11 @@ public void initAudiobookWordTimings(InitAudiobookWordTimingsCallback callback){ wordTimingCalcHandler.post( new Runnable() { public void run() { - List allSentences = mReaderView.getAllSentences(); + List allSentences = SentenceInfoCache.maybeReadCache(sentenceInfoCacheFile); + if(allSentences == null){ + allSentences = mReaderView.getAllSentences(); + SentenceInfoCache.maybeWriteCache(sentenceInfoCacheFile, allSentences); + } wordTimingAudiobookMatcher = new WordTimingAudiobookMatcher(wordTimingFile, allSentences); //can be very long From efae0b174ce33bce5a5efcdc2998b9c29b6a5b83 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Sat, 15 Apr 2023 12:33:56 -0400 Subject: [PATCH 18/59] audiobook: close the word timings file handle --- .../crengine/WordTimingAudiobookMatcher.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 15e193e58..13bc73fad 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -44,15 +44,25 @@ public WordTimingAudiobookMatcher(File wordTimingsFile, List allSe public void parseWordTimingsFile(){ List lines = new ArrayList<>(); + BufferedReader br = null; try { - BufferedReader br = new BufferedReader(new FileReader(wordTimingsFile)); + br = new BufferedReader(new FileReader(wordTimingsFile)); String line; while ((line = br.readLine()) != null) { lines.add(line); } + br.close(); } catch(Exception e) { log.d("ERROR: could not read word timings file: " + wordTimingsFile + " " + e); lines = new ArrayList<>(); + } finally { + try { + if(br != null){ + br.close(); + } + } catch(Exception e){ + //ignore + } } this.wordTimingsDir = wordTimingsFile.getAbsoluteFile().getParent(); From 7b58b201e803cfa8eeb3aaeaa5dc73c41f01826a Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Tue, 18 Apr 2023 10:49:12 -0400 Subject: [PATCH 19/59] audiobook: add fuzzier word-matching for ebook words vs ebook sentences this is unrelated to audiobook word matching, strictly about comparing ebook word splitting in coolreader to external --- .../crengine/WordTimingAudiobookMatcher.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 13bc73fad..886912ef8 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -111,7 +111,7 @@ public void parseWordTimingsFile(){ int wordWtIndex = sentenceWtIndex; boolean wordFound = false; while(wordWtIndex <= wordTimings.size()){ - if(wordInSentence.equals(wordTimings.get(wordWtIndex).word)){ + if(wordsMatch(wordInSentence, wordTimings.get(wordWtIndex).word)){ wordFound = true; break; }else if(wordWtIndex - sentenceWtIndex > 20){ @@ -174,6 +174,28 @@ private WordTiming parseWordTimingsLine(String line){ return new WordTiming(word, startTime, audioFile); } + private boolean wordsMatch(String word1, String word2){ + if(word1 == null && word2 == null) { + return true; + } else if(word1 == null || word2 == null) { + return false; + } else if(word1.equals(word2)) { + return true; + } else { + //expensive calculation, but relatively rarely performed + if(word1.matches(".*[a-z].*") || word2.matches(".*[a-z].*")){ + //if there is at least one letter in the word: compare only letters + word1 = word1.replaceAll("[^a-z]", ""); + word2 = word2.replaceAll("[^a-z]", ""); + }else{ + //otherwise: compare only numbers + word1 = word1.replaceAll("[^0-9]", ""); + word2 = word2.replaceAll("[^0-9]", ""); + } + return word1.equals(word2); + } + } + private List splitSentenceIntoWords(String sentence){ List words = new ArrayList(); From b2512046b9da44a05de0245dd476aa7fdce891de Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 19 Apr 2023 18:02:04 -0400 Subject: [PATCH 20/59] audiobook: read wordtiming+sentencecache using try-with-resources --- .../crengine/SentenceInfoCache.java | 21 ++++++++++--------- .../crengine/WordTimingAudiobookMatcher.java | 14 +++---------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/android/src/org/coolreader/crengine/SentenceInfoCache.java b/android/src/org/coolreader/crengine/SentenceInfoCache.java index a089f1e7b..4533d3af3 100644 --- a/android/src/org/coolreader/crengine/SentenceInfoCache.java +++ b/android/src/org/coolreader/crengine/SentenceInfoCache.java @@ -36,18 +36,19 @@ public SentenceInfoCache(File sentenceInfoCacheFile) { public List readCache() throws IOException { List allSentences = new ArrayList<>(); - BufferedReader br = new BufferedReader(new FileReader(sentenceInfoCacheFile)); - String line; - while ((line = br.readLine()) != null) { - SentenceInfo sentenceInfo = parseSentenceInfoLine(line); - if(sentenceInfo == null){ - log.e("ERROR: could not parse sentence info cache line: " + line); - br.close(); - return null; + try ( + BufferedReader br = new BufferedReader(new FileReader(sentenceInfoCacheFile)); + ) { + String line; + while ((line = br.readLine()) != null) { + SentenceInfo sentenceInfo = parseSentenceInfoLine(line); + if(sentenceInfo == null){ + log.e("ERROR: could not parse sentence info cache line: " + line); + return null; + } + allSentences.add(sentenceInfo); } - allSentences.add(sentenceInfo); } - br.close(); if(allSentences.isEmpty()){ return null; } diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 886912ef8..78b922a89 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -44,9 +44,9 @@ public WordTimingAudiobookMatcher(File wordTimingsFile, List allSe public void parseWordTimingsFile(){ List lines = new ArrayList<>(); - BufferedReader br = null; - try { - br = new BufferedReader(new FileReader(wordTimingsFile)); + try( + BufferedReader br = new BufferedReader(new FileReader(wordTimingsFile)); + ) { String line; while ((line = br.readLine()) != null) { lines.add(line); @@ -55,14 +55,6 @@ public void parseWordTimingsFile(){ } catch(Exception e) { log.d("ERROR: could not read word timings file: " + wordTimingsFile + " " + e); lines = new ArrayList<>(); - } finally { - try { - if(br != null){ - br.close(); - } - } catch(Exception e){ - //ignore - } } this.wordTimingsDir = wordTimingsFile.getAbsoluteFile().getParent(); From e53e9abce60c2c3a3f16f489310c057b8ed54ee1 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Fri, 21 Apr 2023 12:05:08 -0400 Subject: [PATCH 21/59] sentenceinfo: implement exportSentenceInfo(infile, outfile) in lvdocview --- crengine/include/lvdocview.h | 3 +++ crengine/src/lvdocview.cpp | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/crengine/include/lvdocview.h b/crengine/include/lvdocview.h index d0d16ce89..d4b12ebd4 100644 --- a/crengine/include/lvdocview.h +++ b/crengine/include/lvdocview.h @@ -955,6 +955,9 @@ class LVDocView : public CacheLoadingCallback /// load document from stream bool LoadDocument( LVStreamRef stream, const lChar32 * contentPath, bool metadataOnly = false ); + /// load document and export sentence info + bool exportSentenceInfo(const lChar32 * inputFileName, const lChar32 * outputFileName); + /// save last file position void savePosition(); /// restore last file position diff --git a/crengine/src/lvdocview.cpp b/crengine/src/lvdocview.cpp index 2de2a6292..910d744d4 100644 --- a/crengine/src/lvdocview.cpp +++ b/crengine/src/lvdocview.cpp @@ -4043,6 +4043,41 @@ static bool needToConvertBookmarks(CRFileHistRecord* historyRecord, lUInt32 domV return convertBookmarks; } +bool LVDocView::exportSentenceInfo(const lChar32 * inputFileName, const lChar32 * outputFileName) { + if (!LoadDocument(inputFileName, false)) { + return false; + } + + LVStreamRef out = LVOpenFileStream(outputFileName, LVOM_WRITE); + if ( out.isNull() ) { + return false; + } + + checkRender(); + + ldomXPointerEx ptrStart( m_doc->getRootNode(), m_doc->getRootNode()->getChildCount()); + if ( !ptrStart.thisSentenceStart() ) { + ptrStart.nextSentenceStart(); + } + + if ( !ptrStart.thisSentenceStart() ) { + return false; + } + + while ( 1 ) { + ldomXPointerEx ptrEnd(ptrStart); + ptrEnd.thisSentenceEnd(); + + ldomXRange range(ptrStart, ptrEnd); + lString32 sentenceText = range.getRangeText(); + *out << UnicodeToUtf8(ptrStart.toString()) << "," << UnicodeToUtf8(sentenceText) << "\n"; + if ( !ptrStart.nextSentenceStart() ) { + break; + } + } + return true; +} + /// load document from file bool LVDocView::LoadDocument(const lChar32 * fname, bool metadataOnly) { if (!fname || !fname[0]) From 7bacec45ee5aeca8a545ac305f6662cdfebe322a Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Fri, 21 Apr 2023 12:09:57 -0400 Subject: [PATCH 22/59] cr3qt: add -s CLI wrapper around lvdocview.exportSentenceInfo(inF,outF) --- cr3qt/src/cr3widget.cpp | 8 ++++++++ cr3qt/src/cr3widget.h | 1 + cr3qt/src/main.cpp | 36 +++++++++++++++++++++++++++++++++++- cr3qt/src/mainwindow.cpp | 19 +++++++++++++++++++ cr3qt/src/mainwindow.h | 1 + 5 files changed, 64 insertions(+), 1 deletion(-) diff --git a/cr3qt/src/cr3widget.cpp b/cr3qt/src/cr3widget.cpp index d2e36b6d4..5416485e9 100644 --- a/cr3qt/src/cr3widget.cpp +++ b/cr3qt/src/cr3widget.cpp @@ -537,6 +537,14 @@ bool CR3View::loadDocument( QString fileName ) return res; } +bool CR3View::exportSentenceInfo( QString inputFileName, QString outputFileName ) +{ + return _docview->exportSentenceInfo( + qt2cr(inputFileName).c_str(), + qt2cr(outputFileName).c_str() + ); +} + void CR3View::wheelEvent( QWheelEvent * event ) { // Get degrees delta from vertical scrolling diff --git a/cr3qt/src/cr3widget.h b/cr3qt/src/cr3widget.h index 679bd4912..ae7605fd2 100644 --- a/cr3qt/src/cr3widget.h +++ b/cr3qt/src/cr3widget.h @@ -63,6 +63,7 @@ class CR3View : public QWidget, public LVDocViewCallback bool loadDocument( QString fileName ); bool loadLastDocument(); + bool exportSentenceInfo( QString inputFileName, QString outputFileName ); void setDocumentText( QString text ); QScrollBar * scrollBar() const; diff --git a/cr3qt/src/main.cpp b/cr3qt/src/main.cpp index 1410ece93..622f3246c 100644 --- a/cr3qt/src/main.cpp +++ b/cr3qt/src/main.cpp @@ -29,6 +29,7 @@ #else #include #endif +#include #include "../crengine/include/crengine.h" #include "../crengine/include/cr3version.h" #include "mainwindow.h" @@ -77,6 +78,18 @@ static void printHelp() { " -v or --version: print program version\n" " --loglevel=ERROR|WARN|INFO|DEBUG|TRACE: set logging level\n" " --logfile=|stdout|stderr: set log file\n" + "\n" + " --get-sentence-info|-s INPUT_FILE_NAME OUTPUT_FILE_NAME\n" + " analyze INPUT_FILE_NAME and write sentence structure info to OUTPUT_FILE_NAME\n" + " -one sentence per line, formatted: START_POS,TEXT\n" + " -every word appears in exactly one sentence\n" + " -not every character appears; all newlines are omitted, and some whitespace\n" + " -START_POS is a UTF8-encoded string representing a unique position in the DOM of the first word\n" + " -START_POS never contains a comma\n" + " -e.g.: /body/DocFragment[3]/body/div/div[4]/p/a/text()[1].3\n" + " -TEXT is the full UTF8-encoded text of the sentence, without quotes or escaping\n" + " -TEXT never contains newline characters\n" + " -TEXT can contain commas, double quotes, and single quotes\n" ); } @@ -95,6 +108,9 @@ int main(int argc, char *argv[]) lString8 loglevel("ERROR"); lString8 logfile("stderr"); #endif + bool exportSentenceInfo = false; + QString exportSentenceInfoInputFileName; + QString exportSentenceInfoOutputFileName; for ( int i=1; iquit(); + }); + }else{ + w.show(); + } res = a.exec(); } } diff --git a/cr3qt/src/mainwindow.cpp b/cr3qt/src/mainwindow.cpp index 8cb2ade1a..2eee7fc33 100644 --- a/cr3qt/src/mainwindow.cpp +++ b/cr3qt/src/mainwindow.cpp @@ -504,6 +504,25 @@ void MainWindow::showEvent ( QShowEvent * event ) } } +void MainWindow::exportSentenceInfo(QString inputFileName, QString outputFileName) { + if (inputFileName.length() <= 0 ) { + CRLog::error("ERROR: no file to export sentenceinfo\n"); + } + + bool res = ui->view->exportSentenceInfo(inputFileName, outputFileName); + if ( res ) { + CRLog::info( + "\n\n\nSUCCESS: exported " + + inputFileName.toUtf8() + + " to " + + outputFileName.toUtf8() + + "\n\n\n" + ); + } else { + CRLog::error("\n\n\nERROR: export sentence info failed\n\n\n"); + } +} + static bool firstFocus = true; void MainWindow::focusInEvent ( QFocusEvent * event ) diff --git a/cr3qt/src/mainwindow.h b/cr3qt/src/mainwindow.h index b001acbbc..566dc106b 100644 --- a/cr3qt/src/mainwindow.h +++ b/cr3qt/src/mainwindow.h @@ -53,6 +53,7 @@ class MainWindow : public QMainWindow, public PropsChangeCallback virtual void focusInEvent ( QFocusEvent * event ); virtual void closeEvent ( QCloseEvent * event ); public slots: + void exportSentenceInfo(QString inputFileName, QString outputFileName); void contextMenu( QPoint pos ); void on_actionFindText_triggered(); private slots: From c701b7aa55fe61c388f33a6119ebb900d7de0088 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Tue, 25 Apr 2023 10:39:03 -0400 Subject: [PATCH 23/59] audiobook: allow different file extensions for audio files vs wordtiming --- .../org/coolreader/tts/TTSControlService.java | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/android/src/org/coolreader/tts/TTSControlService.java b/android/src/org/coolreader/tts/TTSControlService.java index 2c7be2db1..af2d4124e 100644 --- a/android/src/org/coolreader/tts/TTSControlService.java +++ b/android/src/org/coolreader/tts/TTSControlService.java @@ -1383,7 +1383,12 @@ public void ensureAudioBookPlaying() { if(mMediaPlayer != null && mMediaPlayer.isPlaying()){ return; } - if(audioFile == null){ + File fileToPlay = audioFile; + if(fileToPlay != null && !fileToPlay.exists()){ + fileToPlay = getAlternativeAudioFile(fileToPlay); + } + + if(fileToPlay == null || !fileToPlay.exists()){ return; } @@ -1394,7 +1399,7 @@ public void ensureAudioBookPlaying() { mMediaPlayer = null; } mMediaPlayer = MediaPlayer.create( - getApplicationContext(), Uri.parse("file://" + audioFile.toString())); + getApplicationContext(), Uri.parse("file://" + fileToPlay.toString())); }catch(Exception e){ log.d("ERROR: " + e.getMessage()); } @@ -1527,6 +1532,38 @@ else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) return notification; } + private File getAlternativeAudioFile(File origAudioFile) { + if(origAudioFile == null) { + return null; + } + String fileNoExt = origAudioFile.toString().replaceAll("\\.\\w+$", ""); + File dir = origAudioFile.getParentFile(); + if(dir.exists() && dir.isDirectory()) { + Map> filesByExt = new HashMap<>(); + File firstFile = null; + for(File file : dir.listFiles()) { + if(!file.toString().startsWith(fileNoExt + ".")){ + continue; + } + String ext = file.toString().toLowerCase().replaceAll(".*\\.", ""); + if(filesByExt.get(ext) == null) { + filesByExt.put(ext, new ArrayList<>()); + } + filesByExt.get(ext).add(file); + if(firstFile == null) { + firstFile = file; + } + } + for(String ext : new String[]{"flac", "wav", "m4a", "ogg", "mp3"}) { + if(filesByExt.get(ext) != null){ + return filesByExt.get(ext).get(0); + } + } + return firstFile; + } + return null; + } + private void setupTTSHandlers() { if(useAudioBook){ return; From 881353905e35e35999193538fed9fa7b4be3f89e Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Tue, 25 Apr 2023 11:17:56 -0400 Subject: [PATCH 24/59] sentenceinfo: fix bug where last sentence could be omitted --- crengine/src/lvdocview.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crengine/src/lvdocview.cpp b/crengine/src/lvdocview.cpp index 910d744d4..19600b285 100644 --- a/crengine/src/lvdocview.cpp +++ b/crengine/src/lvdocview.cpp @@ -4068,6 +4068,11 @@ bool LVDocView::exportSentenceInfo(const lChar32 * inputFileName, const lChar32 ldomXPointerEx ptrEnd(ptrStart); ptrEnd.thisSentenceEnd(); + //include last sentence even if it does not appear very sentence-like + if(ptrStart == ptrEnd){ + ptrEnd.setOffset(ptrEnd.getNode()->getText().length()); + } + ldomXRange range(ptrStart, ptrEnd); lString32 sentenceText = range.getRangeText(); *out << UnicodeToUtf8(ptrStart.toString()) << "," << UnicodeToUtf8(sentenceText) << "\n"; From 4e8a4bea06f4af0a18a91f306b9511eecce6aab4 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Tue, 25 Apr 2023 15:58:06 -0400 Subject: [PATCH 25/59] sentenceinfo: remove unnecessary call to checkRender() --- crengine/src/lvdocview.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/crengine/src/lvdocview.cpp b/crengine/src/lvdocview.cpp index 19600b285..712587620 100644 --- a/crengine/src/lvdocview.cpp +++ b/crengine/src/lvdocview.cpp @@ -4053,8 +4053,6 @@ bool LVDocView::exportSentenceInfo(const lChar32 * inputFileName, const lChar32 return false; } - checkRender(); - ldomXPointerEx ptrStart( m_doc->getRootNode(), m_doc->getRootNode()->getChildCount()); if ( !ptrStart.thisSentenceStart() ) { ptrStart.nextSentenceStart(); From 01e97ff55b72c7f9a1e69c7ed1b13cca1a25c987 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Tue, 25 Apr 2023 15:58:39 -0400 Subject: [PATCH 26/59] sentenceinfo: do not call thisSentenceStart() twice when not necessary --- crengine/src/lvdocview.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crengine/src/lvdocview.cpp b/crengine/src/lvdocview.cpp index 712587620..df4ae4e58 100644 --- a/crengine/src/lvdocview.cpp +++ b/crengine/src/lvdocview.cpp @@ -4056,10 +4056,10 @@ bool LVDocView::exportSentenceInfo(const lChar32 * inputFileName, const lChar32 ldomXPointerEx ptrStart( m_doc->getRootNode(), m_doc->getRootNode()->getChildCount()); if ( !ptrStart.thisSentenceStart() ) { ptrStart.nextSentenceStart(); - } - if ( !ptrStart.thisSentenceStart() ) { - return false; + if ( !ptrStart.thisSentenceStart() ) { + return false; + } } while ( 1 ) { From e86325f20c3974e34daaab15bbbf9ff0ed84d694 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 2 Aug 2023 12:30:36 -0400 Subject: [PATCH 27/59] audiobook: allow scripts other than latin for splitting sentences --- .../org/coolreader/crengine/WordTimingAudiobookMatcher.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 78b922a89..bc90b1be5 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -201,10 +201,10 @@ private List splitSentenceIntoWords(String sentence){ ch = Character.toLowerCase(ch); boolean isWordChar; - if('a' <= ch && ch <= 'z'){ + if(Character.isLetter(ch)){ isWordChar = true; wordContainsLetterOrNumber = true; - }else if('0' <= ch && ch <= '9'){ + }else if(Character.isDigit(ch)){ isWordChar = true; wordContainsLetterOrNumber = true; }else if(ch == '\''){ From 1c8b46af0201197ecaa488c548bd0597e843df06 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 2 Aug 2023 12:30:54 -0400 Subject: [PATCH 28/59] audiobook: allow scripts other than latin for comparing words --- .../crengine/WordTimingAudiobookMatcher.java | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index bc90b1be5..7cfd8fa69 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -175,16 +175,35 @@ private boolean wordsMatch(String word1, String word2){ return true; } else { //expensive calculation, but relatively rarely performed - if(word1.matches(".*[a-z].*") || word2.matches(".*[a-z].*")){ - //if there is at least one letter in the word: compare only letters - word1 = word1.replaceAll("[^a-z]", ""); - word2 = word2.replaceAll("[^a-z]", ""); + String word1Letters = ""; + String word2Letters = ""; + String word1Digits = ""; + String word2Digits = ""; + for(int i=0; i 0 && word2Letters.length() > 0) { + //if there is at least one letter in each word: compare only letters + return word1Letters.equals(word2Letters); + }else if(word1Digits.length() > 0 && word2Digits.length() > 0) { + //if there is at least one number in each word: compare only numbers + return word1Digits.equals(word2Digits); }else{ - //otherwise: compare only numbers - word1 = word1.replaceAll("[^0-9]", ""); - word2 = word2.replaceAll("[^0-9]", ""); + return word1.equals(word2); } - return word1.equals(word2); } } From dc2570513e002d79ae2a1f868d4a4c0c9f9d6ce1 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 2 Aug 2023 12:33:45 -0400 Subject: [PATCH 29/59] audiobook: remove unused import android.util.Log --- .../src/org/coolreader/crengine/WordTimingAudiobookMatcher.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 7cfd8fa69..5404a8854 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -1,7 +1,5 @@ package org.coolreader.crengine; -import android.util.Log; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; From a66052ca3820d94b241cf339e5f6c674fb5ce880 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 2 Aug 2023 12:34:34 -0400 Subject: [PATCH 30/59] audiobook: add a main() method for debugging wordtiming+sentenceinfo --- .../coolreader/crengine/WordTimingAudiobookMatcher.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 5404a8854..035b7b090 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -248,4 +248,12 @@ private List splitSentenceIntoWords(String sentence){ return words; } + + public static void main(String[] args){ + if(args.length != 2){ + System.out.println("USAGE: SENTENCE_INFO_FILE WORDTIMING_FILE"); + } + List sentences = SentenceInfoCache.maybeReadCache(new File(args[0])); + new WordTimingAudiobookMatcher(new File(args[1]), sentences).parseWordTimingsFile(); + } } From 6f73d5e4a9e30667d87beac97b4f67b4b28dc4e5 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 2 Aug 2023 12:40:42 -0400 Subject: [PATCH 31/59] audiobook: add non-android implementation of L.java for debugging --- android/debug/org.coolreader.crengine.L.java | 75 ++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 android/debug/org.coolreader.crengine.L.java diff --git a/android/debug/org.coolreader.crengine.L.java b/android/debug/org.coolreader.crengine.L.java new file mode 100644 index 000000000..ab773b219 --- /dev/null +++ b/android/debug/org.coolreader.crengine.L.java @@ -0,0 +1,75 @@ +package org.coolreader.crengine; + +public class L { + public static class LoggerImpl implements Logger { + public void i(String msg) { + System.out.println(msg); + } + public void i(String msg, Exception e) { + System.out.println(msg); + } + public void w(String msg) { + System.out.println(msg); + } + public void w(String msg, Exception e) { + System.out.println(msg); + } + public void e(String msg) { + System.out.println(msg); + } + public void e(String msg, Exception e) { + System.out.println(msg); + } + public void d(String msg) { + System.out.println(msg); + } + public void d(String msg, Exception e) { + System.out.println(msg); + } + public void v(String msg) { + System.out.println(msg); + } + public void v(String msg, Exception e) { + System.out.println(msg); + } + public void setLevel(int level) { + } + } + + public static void i(String msg) { + System.out.println(msg); + } + public static void i(String msg, Exception e) { + System.out.println(msg); + } + public static void w(String msg) { + System.out.println(msg); + } + public static void w(String msg, Exception e) { + System.out.println(msg); + } + public static void e(String msg) { + System.out.println(msg); + } + public static void e(String msg, Exception e) { + System.out.println(msg); + } + public static void d(String msg) { + System.out.println(msg); + } + public static void d(String msg, Exception e) { + System.out.println(msg); + } + public static void v(String msg) { + System.out.println(msg); + } + public static void v(String msg, Exception e) { + System.out.println(msg); + } + public static Logger create(String name) { + return new LoggerImpl(); + } + public static Logger create(String name, int level) { + return new LoggerImpl(); + } +} From b5662a659d60a74289c7494a26ed49ffc78fd504 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 2 Aug 2023 12:41:47 -0400 Subject: [PATCH 32/59] audiobook: add script to run WordTimingAudiobookMatcher java class --- android/run-wordtiming-main.pl | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 android/run-wordtiming-main.pl diff --git a/android/run-wordtiming-main.pl b/android/run-wordtiming-main.pl new file mode 100755 index 000000000..4104c0ae6 --- /dev/null +++ b/android/run-wordtiming-main.pl @@ -0,0 +1,46 @@ +#!/usr/bin/perl +use strict; +use warnings; + +my @classes = qw( + org/coolreader/crengine/WordTimingAudiobookMatcher.java + org/coolreader/crengine/SentenceInfo.java + org/coolreader/crengine/L.java + org/coolreader/crengine/Logger.java + org/coolreader/crengine/SentenceInfoCache.java +); + +sub main(@){ + system "rm", "-rf", "run-main-tmp/"; + system "mkdir", "run-main-tmp/"; + my @restoreCommands; + for my $debugClassFile(glob "debug/*"){ + if($debugClassFile =~ /^(?:.*\/)?([^\/]+)\.(\w+)\.java$/){ + my ($pkg, $class) = ($1, $2); + my $targetDir = $pkg; + $targetDir =~ s/\./\//g; + $targetDir = "src/$targetDir"; + system "cp", "$targetDir/$class.java", "run-main-tmp/$pkg\.$class.java"; + push @restoreCommands, ["mv", "run-main-tmp/$pkg\.$class.java", "$targetDir/$class.java"]; + system "cp", $debugClassFile, "$targetDir/$class.java"; + }else{ + die "ERROR: malformed debug class $debugClassFile\n"; + } + } + system "javac", map {"src/$_"} @classes; + if($? != 0){ + die "java failed\n"; + } + system "java", + "-cp", "src", + "org.coolreader.crengine.WordTimingAudiobookMatcher", + @_; + + for my $cmd(@restoreCommands){ + system @$cmd; + } + system "rm", "-rf", "run-main-tmp/"; + system "find", "src/", "-name", "*.class", "-delete"; +} + +&main(@ARGV); From 590cdb957c88e5debc2f036b050461abdda50971 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Sat, 6 Jul 2024 12:08:43 -0400 Subject: [PATCH 33/59] sentenceinfo: treat semi-colon as sentence break -in addition to period, exclamation point, and question mark -very large sentence-selections are frequent problems in many books -semi-colon usually splits TTS chunks reasonably well --- crengine/src/lvtinydom.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index 3094645f3..fe34683ad 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12622,6 +12622,7 @@ bool ldomXPointerEx::isSentenceStart() case '.': case '?': case '!': + case ';': case U'\x2026': // horizontal ellipsis return false; } @@ -12633,6 +12634,7 @@ bool ldomXPointerEx::isSentenceStart() case '.': case '?': case '!': + case ';': case U'\x2026': // horizontal ellipsis return true; case '"': // QUOTATION MARK @@ -12641,6 +12643,7 @@ bool ldomXPointerEx::isSentenceStart() case '.': case '?': case '!': + case ';': case U'\x2026': // horizontal ellipsis return true; } @@ -12672,6 +12675,7 @@ bool ldomXPointerEx::isSentenceEnd() case '.': case '?': case '!': + case ';': case U'\x2026': // horizontal ellipsis return true; case '"': @@ -12680,6 +12684,7 @@ bool ldomXPointerEx::isSentenceEnd() case '.': case '?': case '!': + case ';': case U'\x2026': // horizontal ellipsis return true; } @@ -12688,7 +12693,7 @@ bool ldomXPointerEx::isSentenceEnd() break; } } - // word is not ended with . ! ? + // word is not ended with '.' or '!' or '?' or ';' or '...' // check whether it's last word of block ldomXPointerEx pos(*this); return !pos.nextVisibleWordStartInSentence(false); From d6fbfb8597696fee2af72dfd48f0c152bdb4d226 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Sat, 6 Jul 2024 12:09:25 -0400 Subject: [PATCH 34/59] text: treat ':' and ';' like '.'+'!'+'?' when measuring text --- crengine/src/crskin.cpp | 4 ++-- crengine/src/lvdocview.cpp | 2 +- crengine/src/lvfont/lvwin32font.cpp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crengine/src/crskin.cpp b/crengine/src/crskin.cpp index 64c336d4b..6f687c2c9 100644 --- a/crengine/src/crskin.cpp +++ b/crengine/src/crskin.cpp @@ -955,8 +955,8 @@ static void wrapLine( lString32Collection & dst, lString32 stringToSplit, int ma if ( ch!=' ' && ch!=0 ) q = 1; else - q = (prevChar=='.' || prevChar==',' || prevChar==';' - || prevChar=='!' || prevChar=='?' + q = (prevChar=='.' || prevChar==',' || prevChar==':' + || prevChar==';' || prevChar=='!' || prevChar=='?' ) ? (wwquality && w Date: Sat, 6 Jul 2024 13:16:04 -0400 Subject: [PATCH 35/59] wordtimings: do not store wordtiming CSV lines in RAM while processing --- .../crengine/WordTimingAudiobookMatcher.java | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 035b7b090..148547d9b 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -41,29 +41,24 @@ public WordTimingAudiobookMatcher(File wordTimingsFile, List allSe } public void parseWordTimingsFile(){ - List lines = new ArrayList<>(); - try( + this.wordTimingsDir = wordTimingsFile.getAbsoluteFile().getParent(); + + try { BufferedReader br = new BufferedReader(new FileReader(wordTimingsFile)); - ) { String line; + wordTimings = new ArrayList<>(); while ((line = br.readLine()) != null) { - lines.add(line); + WordTiming wordTiming = parseWordTimingsLine(line); + if(wordTiming == null){ + log.d("ERROR: could not parse word timings line: " + line); + }else{ + wordTimings.add(wordTiming); + } } br.close(); } catch(Exception e) { - log.d("ERROR: could not read word timings file: " + wordTimingsFile + " " + e); - lines = new ArrayList<>(); - } - - this.wordTimingsDir = wordTimingsFile.getAbsoluteFile().getParent(); - - wordTimings = new ArrayList<>(); - for(String line : lines){ - WordTiming wordTiming = parseWordTimingsLine(line); - if(wordTiming == null){ - log.d("ERROR: could not parse word timings line: " + line); - } - wordTimings.add(wordTiming); + log.d("ERROR: could not read word timings file: " + wordTimingsFile + " " + e); + wordTimings = new ArrayList<>(); } for(int i=0; i Date: Thu, 13 Feb 2025 17:37:35 -0500 Subject: [PATCH 36/59] lvtinydom: pull getChar() to a util fct --- crengine/src/lvtinydom.cpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index fe34683ad..e498d9d10 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12561,6 +12561,13 @@ bool ldomXPointerEx::isFirstVisibleTextInBlock() } // sentence navigation +lChar32 getChar(lString32 text, int idx) { + if (0 <= idx && idx < text.length()) { + return text[idx]; + } else { + return 0; + } +} /// returns true if points to beginning of sentence bool ldomXPointerEx::isSentenceStart() @@ -12573,13 +12580,13 @@ bool ldomXPointerEx::isSentenceStart() lString32 text = node->getText(); int textLen = text.length(); int i = _data->getOffset(); - lChar32 currCh = i0 ? text[i-1] : 0; + lChar32 currCh = getChar(text, i); + lChar32 prevCh = getChar(text, i-1); lChar32 prevPrevNonSpace = 0; lChar32 prevNonSpace = 0; int prevNonSpace_i = -1; for ( ;i>0; i-- ) { - lChar32 ch = text[i-1]; + lChar32 ch = getChar(text, i-1); if ( !IsUnicodeSpace(ch) ) { prevNonSpace = ch; prevNonSpace_i = i - 1; @@ -12588,7 +12595,7 @@ bool ldomXPointerEx::isSentenceStart() } if (prevNonSpace) { for (i = prevNonSpace_i; i>0; i-- ) { - lChar32 ch = text[i-1]; + lChar32 ch = getChar(text, i-1); if ( !IsUnicodeSpace(ch) ) { prevPrevNonSpace = ch; break; @@ -12600,11 +12607,11 @@ bool ldomXPointerEx::isSentenceStart() while ( !prevNonSpace && pos.prevVisibleText(true) ) { lString32 prevText = pos.getText(); for ( int j=prevText.length()-1; j>=0; j-- ) { - lChar32 ch = prevText[j]; + lChar32 ch = getChar(prevText, j); if ( !IsUnicodeSpace(ch) ) { prevNonSpace = ch; for (int k = j; k > 0; k--) { - ch = prevText[k-1]; + ch = getChar(prevText, k-1); if (!IsUnicodeSpace(ch)) { prevPrevNonSpace = ch; break; @@ -12666,9 +12673,9 @@ bool ldomXPointerEx::isSentenceEnd() lString32 text = node->getText(); int textLen = text.length(); int i = _data->getOffset(); - lChar32 currCh = i0 ? text[i-1] : 0; - lChar32 prevPrevCh = i>1 ? text[i-2] : 0; + lChar32 currCh = getChar(text, i); + lChar32 prevCh = getChar(text, i-1); + lChar32 prevPrevCh = getChar(text, i-2); if ( IsUnicodeSpaceOrNull(currCh) ) { switch (prevCh) { case 0: From 0643c1bdfc9ea1217fb0e3609f890add905fa2ea Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 13 Feb 2025 17:44:03 -0500 Subject: [PATCH 37/59] lvtinydom: pull getPrevNonSpaceChar() to a util fct --- crengine/src/lvtinydom.cpp | 52 +++++++++++++------------------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index e498d9d10..33b71653d 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12569,6 +12569,20 @@ lChar32 getChar(lString32 text, int idx) { } } +lChar32 getPrevNonSpaceChar(lString32 text, int idx, int skipCount) { + for(int i=idx-1; i>=0; i--){ + lChar32 ch = getChar(text, i); + if(!IsUnicodeSpace(ch)){ + if(skipCount > 0){ + skipCount--; + }else{ + return ch; + } + } + } + return 0; +} + /// returns true if points to beginning of sentence bool ldomXPointerEx::isSentenceStart() { @@ -12582,44 +12596,14 @@ bool ldomXPointerEx::isSentenceStart() int i = _data->getOffset(); lChar32 currCh = getChar(text, i); lChar32 prevCh = getChar(text, i-1); - lChar32 prevPrevNonSpace = 0; - lChar32 prevNonSpace = 0; - int prevNonSpace_i = -1; - for ( ;i>0; i-- ) { - lChar32 ch = getChar(text, i-1); - if ( !IsUnicodeSpace(ch) ) { - prevNonSpace = ch; - prevNonSpace_i = i - 1; - break; - } - } - if (prevNonSpace) { - for (i = prevNonSpace_i; i>0; i-- ) { - lChar32 ch = getChar(text, i-1); - if ( !IsUnicodeSpace(ch) ) { - prevPrevNonSpace = ch; - break; - } - } - } + lChar32 prevPrevNonSpace = getPrevNonSpaceChar(text, i, 1); + lChar32 prevNonSpace = getPrevNonSpaceChar(text, i, 0); if ( !prevNonSpace ) { ldomXPointerEx pos(*this); while ( !prevNonSpace && pos.prevVisibleText(true) ) { lString32 prevText = pos.getText(); - for ( int j=prevText.length()-1; j>=0; j-- ) { - lChar32 ch = getChar(prevText, j); - if ( !IsUnicodeSpace(ch) ) { - prevNonSpace = ch; - for (int k = j; k > 0; k--) { - ch = getChar(prevText, k-1); - if (!IsUnicodeSpace(ch)) { - prevPrevNonSpace = ch; - break; - } - } - break; - } - } + prevNonSpace = getPrevNonSpaceChar(prevText, prevText.length(), 0); + prevPrevNonSpace = getPrevNonSpaceChar(prevText, prevText.length(), 1); } } From 58a30b1c36da16e1db3b794190f847f478836ca1 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 13 Feb 2025 17:52:12 -0500 Subject: [PATCH 38/59] lvtinydom: pull isCharSentenceEndMark() to a util fct --- crengine/src/lvtinydom.cpp | 56 ++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index 33b71653d..047687107 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12583,6 +12583,19 @@ lChar32 getPrevNonSpaceChar(lString32 text, int idx, int skipCount) { return 0; } +bool isCharSentenceEndMark(lChar32 ch) { + switch (ch) { + case '.': + case '?': + case '!': + case ';': + case U'\x2026': // horizontal ellipsis + return true; + default: + return false; + } +} + /// returns true if points to beginning of sentence bool ldomXPointerEx::isSentenceStart() { @@ -12609,33 +12622,19 @@ bool ldomXPointerEx::isSentenceStart() // skip separated separator. if (1 == textLen) { - switch (currCh) { - case '.': - case '?': - case '!': - case ';': - case U'\x2026': // horizontal ellipsis - return false; + if(isCharSentenceEndMark(currCh)){ + return false; } } if ( !IsUnicodeSpace(currCh) && IsUnicodeSpaceOrNull(prevCh) ) { - switch (prevNonSpace) { - case 0: - case '.': - case '?': - case '!': - case ';': - case U'\x2026': // horizontal ellipsis + if(prevNonSpace == 0 || isCharSentenceEndMark(prevNonSpace)){ return true; + } + switch (prevNonSpace) { case '"': // QUOTATION MARK case U'\x201d': // RIGHT DOUBLE QUOTATION MARK - switch (prevPrevNonSpace) { - case '.': - case '?': - case '!': - case ';': - case U'\x2026': // horizontal ellipsis + if(isCharSentenceEndMark(prevPrevNonSpace)){ return true; } break; @@ -12661,22 +12660,13 @@ bool ldomXPointerEx::isSentenceEnd() lChar32 prevCh = getChar(text, i-1); lChar32 prevPrevCh = getChar(text, i-2); if ( IsUnicodeSpaceOrNull(currCh) ) { - switch (prevCh) { - case 0: - case '.': - case '?': - case '!': - case ';': - case U'\x2026': // horizontal ellipsis + if(prevCh == 0 || isCharSentenceEndMark(prevCh)){ return true; + } + switch (prevCh) { case '"': case U'\x201d': // RIGHT DOUBLE QUOTATION MARK - switch (prevPrevCh) { - case '.': - case '?': - case '!': - case ';': - case U'\x2026': // horizontal ellipsis + if(isCharSentenceEndMark(prevPrevCh)){ return true; } break; From fcbd2580ddd781b8e56954806aa649aa35a38c44 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 13 Feb 2025 17:57:40 -0500 Subject: [PATCH 39/59] lvtinydom: pull isCharDoubleQuoteEnd() to a util fct --- crengine/src/lvtinydom.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index 047687107..bf3ae674a 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12596,6 +12596,16 @@ bool isCharSentenceEndMark(lChar32 ch) { } } +bool isCharDoubleQuoteEnd(lChar32 ch) { + switch (ch) { + case '"': // QUOTATION MARK + case U'\x201d': // RIGHT DOUBLE QUOTATION MARK + return true; + default: + return false; + } +} + /// returns true if points to beginning of sentence bool ldomXPointerEx::isSentenceStart() { @@ -12630,15 +12640,11 @@ bool ldomXPointerEx::isSentenceStart() if ( !IsUnicodeSpace(currCh) && IsUnicodeSpaceOrNull(prevCh) ) { if(prevNonSpace == 0 || isCharSentenceEndMark(prevNonSpace)){ return true; - } - switch (prevNonSpace) { - case '"': // QUOTATION MARK - case U'\x201d': // RIGHT DOUBLE QUOTATION MARK + }else if(isCharDoubleQuoteEnd(prevNonSpace)){ if(isCharSentenceEndMark(prevPrevNonSpace)){ return true; } - break; - default: + }else{ return false; } } @@ -12662,16 +12668,10 @@ bool ldomXPointerEx::isSentenceEnd() if ( IsUnicodeSpaceOrNull(currCh) ) { if(prevCh == 0 || isCharSentenceEndMark(prevCh)){ return true; - } - switch (prevCh) { - case '"': - case U'\x201d': // RIGHT DOUBLE QUOTATION MARK + }else if(isCharDoubleQuoteEnd(prevCh)){ if(isCharSentenceEndMark(prevPrevCh)){ return true; } - break; - default: - break; } } // word is not ended with '.' or '!' or '?' or ';' or '...' From a8e270ac6b4645ef0b5bc633b5e22dd5625af99f Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 13 Feb 2025 18:00:15 -0500 Subject: [PATCH 40/59] lvtinydom: tiny refactor, rename prev*NonSpace => prev*NonSpaceCh --- crengine/src/lvtinydom.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index bf3ae674a..493cac0fb 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12619,14 +12619,14 @@ bool ldomXPointerEx::isSentenceStart() int i = _data->getOffset(); lChar32 currCh = getChar(text, i); lChar32 prevCh = getChar(text, i-1); - lChar32 prevPrevNonSpace = getPrevNonSpaceChar(text, i, 1); - lChar32 prevNonSpace = getPrevNonSpaceChar(text, i, 0); - if ( !prevNonSpace ) { + lChar32 prevNonSpaceCh = getPrevNonSpaceChar(text, i, 0); + lChar32 prevPrevNonSpaceCh = getPrevNonSpaceChar(text, i, 1); + if ( !prevNonSpaceCh ) { ldomXPointerEx pos(*this); - while ( !prevNonSpace && pos.prevVisibleText(true) ) { + while ( !prevNonSpaceCh && pos.prevVisibleText(true) ) { lString32 prevText = pos.getText(); - prevNonSpace = getPrevNonSpaceChar(prevText, prevText.length(), 0); - prevPrevNonSpace = getPrevNonSpaceChar(prevText, prevText.length(), 1); + prevNonSpaceCh = getPrevNonSpaceChar(prevText, prevText.length(), 0); + prevPrevNonSpaceCh = getPrevNonSpaceChar(prevText, prevText.length(), 1); } } @@ -12638,10 +12638,10 @@ bool ldomXPointerEx::isSentenceStart() } if ( !IsUnicodeSpace(currCh) && IsUnicodeSpaceOrNull(prevCh) ) { - if(prevNonSpace == 0 || isCharSentenceEndMark(prevNonSpace)){ + if(prevNonSpaceCh == 0 || isCharSentenceEndMark(prevNonSpaceCh)){ return true; - }else if(isCharDoubleQuoteEnd(prevNonSpace)){ - if(isCharSentenceEndMark(prevPrevNonSpace)){ + }else if(isCharDoubleQuoteEnd(prevNonSpaceCh)){ + if(isCharSentenceEndMark(prevPrevNonSpaceCh)){ return true; } }else{ From 39252cadb1e48f236bc1d96fd058afe620388219 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 13 Feb 2025 18:10:35 -0500 Subject: [PATCH 41/59] lvtinydom: small refactor of isSentenceStart/End for clarity (no-op) --- crengine/src/lvtinydom.cpp | 79 +++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index 493cac0fb..6a914fad7 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12613,42 +12613,50 @@ bool ldomXPointerEx::isSentenceStart() return false; if ( !isText() || !isVisible() ) return false; + ldomNode * node = getNode(); lString32 text = node->getText(); int textLen = text.length(); int i = _data->getOffset(); + lChar32 currCh = getChar(text, i); lChar32 prevCh = getChar(text, i-1); lChar32 prevNonSpaceCh = getPrevNonSpaceChar(text, i, 0); lChar32 prevPrevNonSpaceCh = getPrevNonSpaceChar(text, i, 1); + + // look at previous text nodes if there is no previous non-space char in current if ( !prevNonSpaceCh ) { ldomXPointerEx pos(*this); - while ( !prevNonSpaceCh && pos.prevVisibleText(true) ) { + while(prevNonSpaceCh == 0 && pos.prevVisibleText(true)){ + // get last and second-to-last non-space chars from previous text + // if that node has no non-space chars, use the node before that one lString32 prevText = pos.getText(); prevNonSpaceCh = getPrevNonSpaceChar(prevText, prevText.length(), 0); prevPrevNonSpaceCh = getPrevNonSpaceChar(prevText, prevText.length(), 1); } } - // skip separated separator. - if (1 == textLen) { - if(isCharSentenceEndMark(currCh)){ - return false; - } - } - - if ( !IsUnicodeSpace(currCh) && IsUnicodeSpaceOrNull(prevCh) ) { - if(prevNonSpaceCh == 0 || isCharSentenceEndMark(prevNonSpaceCh)){ - return true; - }else if(isCharDoubleQuoteEnd(prevNonSpaceCh)){ - if(isCharSentenceEndMark(prevPrevNonSpaceCh)){ - return true; - } - }else{ - return false; - } + if (1 == textLen && isCharSentenceEndMark(currCh)){ + // skip separated separator + return false; + }else if(IsUnicodeSpace(currCh)){ + // current character is a space (never sentence start) + return false; + }else if(prevCh != 0 && !IsUnicodeSpace(prevCh)){ + // previous char exists and is not a space (never sentence start) + return false; + }else if(prevNonSpaceCh == 0){ + // there is no previous non-space character + return true; + }else if(isCharSentenceEndMark(prevNonSpaceCh)){ + // previous non-space char is end punctuation + return true; + }else if(isCharDoubleQuoteEnd(prevNonSpaceCh) && isCharSentenceEndMark(prevPrevNonSpaceCh)){ + // previous two non-space chars are end-mark followed by dbl-quote + return true; + }else{ + return false; } - return false; } /// returns true if points to end of sentence @@ -12658,27 +12666,34 @@ bool ldomXPointerEx::isSentenceEnd() return false; if ( !isText() || !isVisible() ) return false; + ldomNode * node = getNode(); lString32 text = node->getText(); int textLen = text.length(); int i = _data->getOffset(); + lChar32 currCh = getChar(text, i); lChar32 prevCh = getChar(text, i-1); lChar32 prevPrevCh = getChar(text, i-2); - if ( IsUnicodeSpaceOrNull(currCh) ) { - if(prevCh == 0 || isCharSentenceEndMark(prevCh)){ - return true; - }else if(isCharDoubleQuoteEnd(prevCh)){ - if(isCharSentenceEndMark(prevPrevCh)){ - return true; - } - } + + if(!IsUnicodeSpaceOrNull(currCh)){ + // sentences must end with whitespace (or the end of the node) + return false; + }else if(prevCh == 0){ + return true; + }else if(isCharSentenceEndMark(prevCh)){ + // previous char is sentence end punctuation + return true; + }else if(isCharDoubleQuoteEnd(prevCh) && isCharSentenceEndMark(prevPrevCh)){ + // previous two chars are sentence end punctuation and dbl-quote + return true; + }else{ + // the current char may be a space between words, + // but it may also be the last word of a block with no end punctuation + // check if there is a next word to move to, and if not, it is sentence end + ldomXPointerEx pos(*this); + return !pos.nextVisibleWordStartInSentence(false); } - // word is not ended with '.' or '!' or '?' or ';' or '...' - // check whether it's last word of block - ldomXPointerEx pos(*this); - return !pos.nextVisibleWordStartInSentence(false); - //return !pos.thisVisibleWordEndInSentence(); } /// move to beginning of current visible text sentence From a69e0b1976db6c5225d7073a9a9d3a1f6f272363 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 13 Feb 2025 18:16:57 -0500 Subject: [PATCH 42/59] lvtinydom: add getNextNonSpaceChar() util fct --- crengine/src/lvtinydom.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index 6a914fad7..46f006204 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12583,6 +12583,21 @@ lChar32 getPrevNonSpaceChar(lString32 text, int idx, int skipCount) { return 0; } +lChar32 getNextNonSpaceChar(lString32 text, int idx, int skipCount) { + int len = text.length(); + for(int i=idx+1; i 0){ + skipCount--; + }else{ + return ch; + } + } + } + return 0; +} + bool isCharSentenceEndMark(lChar32 ch) { switch (ch) { case '.': From bdee3d9c19423e08ef01a120fe589d6bdb37d86f Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 13 Feb 2025 18:17:55 -0500 Subject: [PATCH 43/59] text: do not treat the start of nodes as sentence end --- crengine/src/lvtinydom.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index 46f006204..4bae9b747 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12694,8 +12694,9 @@ bool ldomXPointerEx::isSentenceEnd() if(!IsUnicodeSpaceOrNull(currCh)){ // sentences must end with whitespace (or the end of the node) return false; - }else if(prevCh == 0){ - return true; + }else if(prevCh == 0 && currCh == 0){ + // empty sentence + return false; }else if(isCharSentenceEndMark(prevCh)){ // previous char is sentence end punctuation return true; From 438fab2f904797c9eaf2c4556fdd57b86194d7de Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 13 Feb 2025 18:18:22 -0500 Subject: [PATCH 44/59] text: never treat sentence end punctuation marks as sentence start --- crengine/src/lvtinydom.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index 4bae9b747..fb74b2987 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12651,8 +12651,8 @@ bool ldomXPointerEx::isSentenceStart() } } - if (1 == textLen && isCharSentenceEndMark(currCh)){ - // skip separated separator + if (isCharSentenceEndMark(currCh)){ + // current character is end punctuation (never sentence start) return false; }else if(IsUnicodeSpace(currCh)){ // current character is a space (never sentence start) From 0d03033ef3bdc03fc1ac25f28c17c9e3fb2d5b67 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 13 Feb 2025 18:19:00 -0500 Subject: [PATCH 45/59] text: treat multiple end punctuation marks as a single sentence end --- crengine/src/lvtinydom.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crengine/src/lvtinydom.cpp b/crengine/src/lvtinydom.cpp index fb74b2987..ee0ab410c 100644 --- a/crengine/src/lvtinydom.cpp +++ b/crengine/src/lvtinydom.cpp @@ -12690,6 +12690,7 @@ bool ldomXPointerEx::isSentenceEnd() lChar32 currCh = getChar(text, i); lChar32 prevCh = getChar(text, i-1); lChar32 prevPrevCh = getChar(text, i-2); + lChar32 nextNonSpaceCh = getNextNonSpaceChar(text, i, 0); if(!IsUnicodeSpaceOrNull(currCh)){ // sentences must end with whitespace (or the end of the node) @@ -12697,6 +12698,12 @@ bool ldomXPointerEx::isSentenceEnd() }else if(prevCh == 0 && currCh == 0){ // empty sentence return false; + }else if(isCharSentenceEndMark(nextNonSpaceCh)){ + // next non-space char is end punctuation, so the sentence cannot end before that + // e.g.: "S1 . . . S2" => ["S1 . . . ", "S2"] + // instead of + // "S1 . . . S2" => ["S1 . ", ". ", ". ", "S2"] + return false; }else if(isCharSentenceEndMark(prevCh)){ // previous char is sentence end punctuation return true; From b985896f858a999d4bbb886939957c7d967e0377 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 18 Jun 2025 23:29:44 -0400 Subject: [PATCH 46/59] audiobook: calculate position+duration in full audiobook for sentences --- .../org/coolreader/crengine/SentenceInfo.java | 2 ++ .../crengine/WordTimingAudiobookMatcher.java | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/android/src/org/coolreader/crengine/SentenceInfo.java b/android/src/org/coolreader/crengine/SentenceInfo.java index 8f904b339..094d6ccf4 100644 --- a/android/src/org/coolreader/crengine/SentenceInfo.java +++ b/android/src/org/coolreader/crengine/SentenceInfo.java @@ -8,6 +8,8 @@ public class SentenceInfo { public String startPos; public double startTime; + public double startTimeInBook; + public double totalBookDuration; public boolean isFirstSentenceInAudioFile = false; public File audioFile; public List words; diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 148547d9b..b7e81fd24 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -1,5 +1,7 @@ package org.coolreader.crengine; +import android.media.MediaMetadataRetriever; + import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -130,12 +132,38 @@ public void parseWordTimingsFile(){ //start first sentence of all audio files at 0.0 // prevents skipping intros File curAudioFile = null; + double prevTotalAudioFileDurations = 0; for(SentenceInfo s : allSentences){ if(curAudioFile == null || s.audioFile != curAudioFile){ s.isFirstSentenceInAudioFile = true; s.startTime = 0; + if(curAudioFile != null){ + prevTotalAudioFileDurations += getAudioFileDuration(curAudioFile); + } curAudioFile = s.audioFile; } + s.startTimeInBook = s.startTime + prevTotalAudioFileDurations; + } + + double totalBookDuration = prevTotalAudioFileDurations; + if(curAudioFile != null){ + totalBookDuration += getAudioFileDuration(curAudioFile); + } + + for(SentenceInfo s : allSentences){ + s.totalBookDuration = totalBookDuration; + } + } + + public Double getAudioFileDuration(File file){ + try{ + MediaMetadataRetriever m = new MediaMetadataRetriever(); + m.setDataSource(file.getAbsolutePath()); + String durationStr = m.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION); + return Long.parseLong(durationStr) / 1000.0; + }catch(Exception e){ + log.d("ERROR: could not get audio file duration for " + file, e); + return 0.0; } } From 8ab29456a223a5543fcef0a5a41e198515a2405d Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Wed, 18 Jun 2025 23:31:36 -0400 Subject: [PATCH 47/59] audiobook: swap tts rate bar widget for pos+duration display --- android/res/layout/tts_toolbar.xml | 13 ++++++++ .../coolreader/crengine/TTSToolbarDlg.java | 33 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/android/res/layout/tts_toolbar.xml b/android/res/layout/tts_toolbar.xml index 0508c6528..502142e9b 100644 --- a/android/res/layout/tts_toolbar.xml +++ b/android/res/layout/tts_toolbar.xml @@ -187,5 +187,18 @@ android:layout_marginRight="2dip" android:layout_marginBottom="5dip" android:layout_weight="1" /> + + diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index b853814b0..a82b20a76 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -75,6 +75,7 @@ public class TTSToolbarDlg implements Settings { private final ImageButton forwardButton; private final ImageButton stopButton; private final ImageButton optionsButton; + private final TextView mAudioProgressTextView; private final TextView mVolumeTextView; private final TextView mSpeedTextView; private final SeekBar mSbSpeed; @@ -195,6 +196,30 @@ private SentenceInfo fetchSelectedSentenceInfo() { return null; } + private String formatDurationHHHMMSS(double duration) { + return String.format(Locale.getDefault(), "%3d:%02d:%02d", + ((int) duration) / 60 / 60, + ((int) duration) / 60 % 60, + ((int) duration) % 60); + } + + private void setAudioBookProgressDisplay(SentenceInfo sentenceInfo) { + if(sentenceInfo == null){ + mAudioProgressTextView.setVisibility(View.GONE); + mAudioProgressTextView.setText(""); + + mSbSpeed.setVisibility(View.VISIBLE); + }else{ + mAudioProgressTextView.setVisibility(View.VISIBLE); + mAudioProgressTextView.setText(String.format(Locale.getDefault(), + "%s / %s", + formatDurationHHHMMSS(sentenceInfo.startTimeInBook), + formatDurationHHHMMSS(sentenceInfo.totalBookDuration))); + + mSbSpeed.setVisibility(View.GONE); + } + } + /** * Select next or previous sentence. ONLY the selection changes and the specified callback is called! * Not affected to speech synthesis process. @@ -211,11 +236,14 @@ public void onNewSelection(Selection selection) { mCurrentSelection = selection; if(allowUseAudiobook){ SentenceInfo sentenceInfo = fetchSelectedSentenceInfo(); + setAudioBookProgressDisplay(sentenceInfo); if(sentenceInfo != null && sentenceInfo.audioFile != null){ mTTSControl.bind(ttsbinder -> { ttsbinder.setAudioFile(sentenceInfo.audioFile, sentenceInfo.startTime); }); } + }else{ + setAudioBookProgressDisplay(null); } if (null != callback) callback.onNewSelection(mCurrentSelection); @@ -561,6 +589,11 @@ public TTSToolbarDlg(CoolReader coolReader, ReaderView readerView, TTSControlSer })); stopButton.setOnClickListener(v -> stopAndClose()); + + // setup audiobook speed && volume seek bars + mAudioProgressTextView = panel.findViewById(R.id.tts_lbl_audio_progress); + mAudioProgressTextView.setVisibility(View.GONE); + // setup speed && volume seek bars mVolumeTextView = panel.findViewById(R.id.tts_lbl_volume); mSpeedTextView = panel.findViewById(R.id.tts_lbl_speed); From 83f576237ca4fd4fa239d887815896769dc6254f Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Fri, 20 Jun 2025 12:29:10 -0400 Subject: [PATCH 48/59] audiobook: rename sentenceinfo-cache-file => sentenceinfo-file --- android/src/org/coolreader/crengine/TTSToolbarDlg.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index a82b20a76..63d4b7e40 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -105,7 +105,7 @@ public class TTSToolbarDlg implements Settings { private int mTTSSpeedPercent = 50; // 50% (normal) private File wordTimingFile; - private File sentenceInfoCacheFile; + private File sentenceInfoFile; private WordTimingAudiobookMatcher wordTimingAudiobookMatcher; private SentenceInfo currentSentenceInfo; @@ -732,7 +732,7 @@ public void onStopTrackingTouch(SeekBar seekBar) { // Fetch book's metadata BookInfo bookInfo = mReaderView.getBookInfo(); wordTimingFile = null; - sentenceInfoCacheFile = null; + sentenceInfoFile = null; if (null != bookInfo) { FileInfo fileInfo = bookInfo.getFileInfo(); if (null != fileInfo) { @@ -747,7 +747,7 @@ public void onStopTrackingTouch(SeekBar seekBar) { String sentenceInfoPath = pathName.replaceAll("\\.\\w+$", ".sentenceinfo"); if(wordTimingPath.matches(".*\\.wordtiming$")){ wordTimingFile = new File(wordTimingPath); - sentenceInfoCacheFile = new File(sentenceInfoPath); + sentenceInfoFile = new File(sentenceInfoPath); } } } @@ -802,10 +802,10 @@ public void initAudiobookWordTimings(InitAudiobookWordTimingsCallback callback){ wordTimingCalcHandler.post( new Runnable() { public void run() { - List allSentences = SentenceInfoCache.maybeReadCache(sentenceInfoCacheFile); + List allSentences = SentenceInfoCache.maybeReadCache(sentenceInfoFile); if(allSentences == null){ allSentences = mReaderView.getAllSentences(); - SentenceInfoCache.maybeWriteCache(sentenceInfoCacheFile, allSentences); + SentenceInfoCache.maybeWriteCache(sentenceInfoFile, allSentences); } wordTimingAudiobookMatcher = new WordTimingAudiobookMatcher(wordTimingFile, allSentences); From 145d324a908e75ea5c3dccd19dda3e11ca57b283 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 23 Jun 2025 00:11:03 -0400 Subject: [PATCH 49/59] audiobook: do not persist split "words" in SentenceInfo --- android/src/org/coolreader/crengine/SentenceInfo.java | 1 - .../coolreader/crengine/WordTimingAudiobookMatcher.java | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/android/src/org/coolreader/crengine/SentenceInfo.java b/android/src/org/coolreader/crengine/SentenceInfo.java index 094d6ccf4..30645cbf5 100644 --- a/android/src/org/coolreader/crengine/SentenceInfo.java +++ b/android/src/org/coolreader/crengine/SentenceInfo.java @@ -12,7 +12,6 @@ public class SentenceInfo { public double totalBookDuration; public boolean isFirstSentenceInAudioFile = false; public File audioFile; - public List words; public SentenceInfo nextSentence; public SentenceInfo() { diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index b7e81fd24..02ba28324 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -74,8 +74,9 @@ public void parseWordTimingsFile(){ s.nextSentence = nextSentence; } + Map> wordsBySentencePos = new HashMap<>(); for(SentenceInfo s : allSentences){ - s.words = splitSentenceIntoWords(s.text); + wordsBySentencePos.put(s.startPos, splitSentenceIntoWords(s.text)); } if(wordTimings.size() == 0){ @@ -86,7 +87,8 @@ public void parseWordTimingsFile(){ double prevStartTime = 0; File prevAudioFile = wordTimings.get(0).audioFile; for(SentenceInfo s : allSentences){ - if(s.words.size() == 0){ + List words = wordsBySentencePos.get(s.startPos); + if(words.size() == 0){ s.startTime = prevStartTime; s.audioFile = prevAudioFile; continue; @@ -94,7 +96,7 @@ public void parseWordTimingsFile(){ boolean matchFailed = false; WordTiming firstWordTiming = null; int sentenceWtIndex = wtIndex; - for(String wordInSentence : s.words){ + for(String wordInSentence : words){ int wordWtIndex = sentenceWtIndex; boolean wordFound = false; while(wordWtIndex <= wordTimings.size()){ From b3bc0b307d6f989a35078e35d2bd5b378584a6f3 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 23 Jun 2025 00:17:22 -0400 Subject: [PATCH 50/59] audiobook: pull SentenceTiming state out of SentenceInfo --- .../org/coolreader/crengine/SentenceInfo.java | 6 +--- .../coolreader/crengine/SentenceTiming.java | 15 +++++++++ .../coolreader/crengine/TTSToolbarDlg.java | 10 +++--- .../crengine/WordTimingAudiobookMatcher.java | 32 +++++++++++-------- .../org/coolreader/tts/TTSControlService.java | 6 ++-- 5 files changed, 42 insertions(+), 27 deletions(-) create mode 100644 android/src/org/coolreader/crengine/SentenceTiming.java diff --git a/android/src/org/coolreader/crengine/SentenceInfo.java b/android/src/org/coolreader/crengine/SentenceInfo.java index 30645cbf5..0ab754f71 100644 --- a/android/src/org/coolreader/crengine/SentenceInfo.java +++ b/android/src/org/coolreader/crengine/SentenceInfo.java @@ -7,11 +7,7 @@ public class SentenceInfo { public String text; public String startPos; - public double startTime; - public double startTimeInBook; - public double totalBookDuration; - public boolean isFirstSentenceInAudioFile = false; - public File audioFile; + public SentenceTiming sentenceTiming; public SentenceInfo nextSentence; public SentenceInfo() { diff --git a/android/src/org/coolreader/crengine/SentenceTiming.java b/android/src/org/coolreader/crengine/SentenceTiming.java new file mode 100644 index 000000000..d7be6e865 --- /dev/null +++ b/android/src/org/coolreader/crengine/SentenceTiming.java @@ -0,0 +1,15 @@ +package org.coolreader.crengine; + +import java.io.File; +import java.util.List; + +public class SentenceTiming { + public double startTime; + public double startTimeInBook; + public double totalBookDuration; + public boolean isFirstSentenceInAudioFile = false; + public File audioFile; + + public SentenceTiming() { + } +} diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index 63d4b7e40..a3464a74f 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -213,8 +213,8 @@ private void setAudioBookProgressDisplay(SentenceInfo sentenceInfo) { mAudioProgressTextView.setVisibility(View.VISIBLE); mAudioProgressTextView.setText(String.format(Locale.getDefault(), "%s / %s", - formatDurationHHHMMSS(sentenceInfo.startTimeInBook), - formatDurationHHHMMSS(sentenceInfo.totalBookDuration))); + formatDurationHHHMMSS(sentenceInfo.sentenceTiming.startTimeInBook), + formatDurationHHHMMSS(sentenceInfo.sentenceTiming.totalBookDuration))); mSbSpeed.setVisibility(View.GONE); } @@ -237,9 +237,9 @@ public void onNewSelection(Selection selection) { if(allowUseAudiobook){ SentenceInfo sentenceInfo = fetchSelectedSentenceInfo(); setAudioBookProgressDisplay(sentenceInfo); - if(sentenceInfo != null && sentenceInfo.audioFile != null){ + if(sentenceInfo != null && sentenceInfo.sentenceTiming.audioFile != null){ mTTSControl.bind(ttsbinder -> { - ttsbinder.setAudioFile(sentenceInfo.audioFile, sentenceInfo.startTime); + ttsbinder.setAudioFile(sentenceInfo.sentenceTiming.audioFile, sentenceInfo.sentenceTiming.startTime); }); } }else{ @@ -377,7 +377,7 @@ public void onFail() { ttsbinder.stop(null); SentenceInfo sentenceInfo = fetchSelectedSentenceInfo(); if(sentenceInfo != null){ - ttsbinder.setAudioFile(sentenceInfo.audioFile, sentenceInfo.startTime); + ttsbinder.setAudioFile(sentenceInfo.sentenceTiming.audioFile, sentenceInfo.sentenceTiming.startTime); } initAudiobookWordTimings(null); moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 02ba28324..66c2a87f4 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -87,10 +87,13 @@ public void parseWordTimingsFile(){ double prevStartTime = 0; File prevAudioFile = wordTimings.get(0).audioFile; for(SentenceInfo s : allSentences){ + SentenceTiming t = new SentenceTiming(); + s.sentenceTiming = t; + List words = wordsBySentencePos.get(s.startPos); if(words.size() == 0){ - s.startTime = prevStartTime; - s.audioFile = prevAudioFile; + t.startTime = prevStartTime; + t.audioFile = prevAudioFile; continue; } boolean matchFailed = false; @@ -120,14 +123,14 @@ public void parseWordTimingsFile(){ } } if(matchFailed){ - s.startTime = prevStartTime; - s.audioFile = prevAudioFile; + t.startTime = prevStartTime; + t.audioFile = prevAudioFile; }else{ wtIndex = sentenceWtIndex; - s.startTime = firstWordTiming.startTime; - s.audioFile = firstWordTiming.audioFile; - prevStartTime = s.startTime; - prevAudioFile = s.audioFile; + t.startTime = firstWordTiming.startTime; + t.audioFile = firstWordTiming.audioFile; + prevStartTime = t.startTime; + prevAudioFile = t.audioFile; } } @@ -136,15 +139,16 @@ public void parseWordTimingsFile(){ File curAudioFile = null; double prevTotalAudioFileDurations = 0; for(SentenceInfo s : allSentences){ - if(curAudioFile == null || s.audioFile != curAudioFile){ - s.isFirstSentenceInAudioFile = true; - s.startTime = 0; + SentenceTiming t = s.sentenceTiming; + if(curAudioFile == null || t.audioFile != curAudioFile){ + t.isFirstSentenceInAudioFile = true; + t.startTime = 0; if(curAudioFile != null){ prevTotalAudioFileDurations += getAudioFileDuration(curAudioFile); } - curAudioFile = s.audioFile; + curAudioFile = t.audioFile; } - s.startTimeInBook = s.startTime + prevTotalAudioFileDurations; + t.startTimeInBook = t.startTime + prevTotalAudioFileDurations; } double totalBookDuration = prevTotalAudioFileDurations; @@ -153,7 +157,7 @@ public void parseWordTimingsFile(){ } for(SentenceInfo s : allSentences){ - s.totalBookDuration = totalBookDuration; + s.sentenceTiming.totalBookDuration = totalBookDuration; } } diff --git a/android/src/org/coolreader/tts/TTSControlService.java b/android/src/org/coolreader/tts/TTSControlService.java index af2d4124e..cc8261307 100644 --- a/android/src/org/coolreader/tts/TTSControlService.java +++ b/android/src/org/coolreader/tts/TTSControlService.java @@ -1306,8 +1306,8 @@ public void work() { boolean isAfterSentence = false; if(sentenceInfo != null && sentenceInfo.nextSentence != null){ SentenceInfo nextSentenceInfo = sentenceInfo.nextSentence; - if(sentenceInfo.audioFile == TTSControlService.this.audioFile){ - if(nextSentenceInfo.isFirstSentenceInAudioFile){ + if(sentenceInfo.sentenceTiming.audioFile == TTSControlService.this.audioFile){ + if(nextSentenceInfo.sentenceTiming.isFirstSentenceInAudioFile){ if(mState == State.PLAYING && (mMediaPlayer == null || !mMediaPlayer.isPlaying())){ //this is the last sentence in the file, and the media player ended isAfterSentence = true; @@ -1315,7 +1315,7 @@ public void work() { }else{ if(mMediaPlayer != null && mMediaPlayer.isPlaying()){ double curPos = mMediaPlayer.getCurrentPosition() / 1000.0; - if(curPos >= nextSentenceInfo.startTime){ + if(curPos >= nextSentenceInfo.sentenceTiming.startTime){ isAfterSentence = true; } } From 3bce8d5a105b1615ccfe7dee7be3163d388a569a Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 23 Jun 2025 00:23:03 -0400 Subject: [PATCH 51/59] audiobook: do not persist WordTiming after generating SentenceTIming --- .../src/org/coolreader/crengine/WordTimingAudiobookMatcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 66c2a87f4..447fd173f 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -32,7 +32,6 @@ public WordTiming(String word, Double startTime, File audioFile){ private final Map sentencesByStartPos = new HashMap<>(); private final Map fileCache = new HashMap<>(); private String wordTimingsDir; - private List wordTimings; public WordTimingAudiobookMatcher(File wordTimingsFile, List allSentences) { this.wordTimingsFile = wordTimingsFile; @@ -45,6 +44,7 @@ public WordTimingAudiobookMatcher(File wordTimingsFile, List allSe public void parseWordTimingsFile(){ this.wordTimingsDir = wordTimingsFile.getAbsoluteFile().getParent(); + List wordTimings; try { BufferedReader br = new BufferedReader(new FileReader(wordTimingsFile)); String line; From 2b13a6e9c775f019b42e20c518e0379ef96f13ff Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 23 Jun 2025 00:26:13 -0400 Subject: [PATCH 52/59] audiobook: persist a two-way unique map of audioFile<=>path --- .../crengine/WordTimingAudiobookMatcher.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 447fd173f..9cd6b3936 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -28,13 +28,15 @@ public WordTiming(String word, Double startTime, File audioFile){ } private final File wordTimingsFile; + private final String audioFileRelativeDir; private final List allSentences; private final Map sentencesByStartPos = new HashMap<>(); - private final Map fileCache = new HashMap<>(); - private String wordTimingsDir; + private final Map audioFilesByAudioFileName = new HashMap<>(); + private final Map audioFileNamesByAudioFile = new HashMap<>(); public WordTimingAudiobookMatcher(File wordTimingsFile, List allSentences) { this.wordTimingsFile = wordTimingsFile; + this.audioFileRelativeDir = wordTimingsFile.getAbsoluteFile().getParent(); this.allSentences = allSentences; for(SentenceInfo s : allSentences){ sentencesByStartPos.put(s.startPos, s); @@ -42,8 +44,6 @@ public WordTimingAudiobookMatcher(File wordTimingsFile, List allSe } public void parseWordTimingsFile(){ - this.wordTimingsDir = wordTimingsFile.getAbsoluteFile().getParent(); - List wordTimings; try { BufferedReader br = new BufferedReader(new FileReader(wordTimingsFile)); @@ -186,10 +186,12 @@ private WordTiming parseWordTimingsLine(String line){ String word = line.substring(sep1+1, sep2); Double startTime = Double.parseDouble(line.substring(0, sep1)); String audioFileName = line.substring(sep2+1); - if(!fileCache.containsKey(audioFileName)){ - fileCache.put(audioFileName, new File(wordTimingsDir + "/" + audioFileName)); + File audioFile = audioFilesByAudioFileName.get(audioFileName); + if(audioFile == null){ + audioFile = new File(audioFileRelativeDir + "/" + audioFileName); + audioFilesByAudioFileName.put(audioFileName, audioFile); + audioFileNamesByAudioFile.put(audioFile, audioFileName); } - File audioFile = fileCache.get(audioFileName); return new WordTiming(word, startTime, audioFile); } From 7dcbb7b36180b7030b1710b81cc98fd302086436 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 23 Jun 2025 00:28:56 -0400 Subject: [PATCH 53/59] audiobook: add stateful isSentenceTimingReady() method --- .../coolreader/crengine/WordTimingAudiobookMatcher.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 9cd6b3936..1b4f58f57 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -34,6 +34,8 @@ public WordTiming(String word, Double startTime, File audioFile){ private final Map audioFilesByAudioFileName = new HashMap<>(); private final Map audioFileNamesByAudioFile = new HashMap<>(); + private boolean sentenceTimingReady = false; + public WordTimingAudiobookMatcher(File wordTimingsFile, List allSentences) { this.wordTimingsFile = wordTimingsFile; this.audioFileRelativeDir = wordTimingsFile.getAbsoluteFile().getParent(); @@ -159,6 +161,12 @@ public void parseWordTimingsFile(){ for(SentenceInfo s : allSentences){ s.sentenceTiming.totalBookDuration = totalBookDuration; } + + this.sentenceTimingReady = true; + } + + public boolean isSentenceTimingReady(){ + return this.sentenceTimingReady; } public Double getAudioFileDuration(File file){ From f6a9136112aabaa09c985743f80ee62719e203dd Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 23 Jun 2025 00:30:50 -0400 Subject: [PATCH 54/59] audiobook: reduce visibility of getAudioFileDuration() public=>private --- .../crengine/WordTimingAudiobookMatcher.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index 1b4f58f57..d96eec5b3 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -169,7 +169,11 @@ public boolean isSentenceTimingReady(){ return this.sentenceTimingReady; } - public Double getAudioFileDuration(File file){ + public SentenceInfo getSentence(String startPos){ + return sentencesByStartPos.get(startPos); + } + + private Double getAudioFileDuration(File file){ try{ MediaMetadataRetriever m = new MediaMetadataRetriever(); m.setDataSource(file.getAbsolutePath()); @@ -181,10 +185,6 @@ public Double getAudioFileDuration(File file){ } } - public SentenceInfo getSentence(String startPos){ - return sentencesByStartPos.get(startPos); - } - private WordTiming parseWordTimingsLine(String line){ int sep1 = line.indexOf(','); int sep2 = line.indexOf(',', sep1+1); From 11a5e859f5e67ad5053effbd72803b4719b9c097 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 23 Jun 2025 00:36:01 -0400 Subject: [PATCH 55/59] audiobook: pull updateSentenceInfoNextSentence() to a method --- .../crengine/WordTimingAudiobookMatcher.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index d96eec5b3..b6dc6592d 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -65,16 +65,7 @@ public void parseWordTimingsFile(){ wordTimings = new ArrayList<>(); } - for(int i=0; i> wordsBySentencePos = new HashMap<>(); for(SentenceInfo s : allSentences){ @@ -173,6 +164,19 @@ public SentenceInfo getSentence(String startPos){ return sentencesByStartPos.get(startPos); } + private void updateSentenceInfoNextSentence(){ + for(int i=0; i Date: Mon, 23 Jun 2025 00:38:41 -0400 Subject: [PATCH 56/59] audiobook: add cache support for read/write full parsed sentence timing --- .../crengine/WordTimingAudiobookMatcher.java | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index b6dc6592d..a3f360dcc 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -5,6 +5,8 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; +import java.io.FileWriter; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -156,6 +158,56 @@ public void parseWordTimingsFile(){ this.sentenceTimingReady = true; } + public void maybeReadSentenceTimingCache(File sentenceTimingCacheFile){ + try { + if(sentenceTimingCacheFile == null || !sentenceTimingCacheFile.exists()){ + return; + } + + BufferedReader br = new BufferedReader(new FileReader(sentenceTimingCacheFile)); + String line; + while ((line = br.readLine()) != null) { + Map.Entry sentenceTimingRes = parseSentenceTimingLine(line); + if(sentenceTimingRes == null){ + log.d("ERROR: could not parse sentence timing line: " + line); + }else{ + String startPos = sentenceTimingRes.getKey(); + SentenceTiming t = sentenceTimingRes.getValue(); + SentenceInfo s = sentencesByStartPos.get(startPos); + s.sentenceTiming = t; + } + } + br.close(); + + updateSentenceInfoNextSentence(); + + this.sentenceTimingReady = true; + } catch(Exception e) { + log.d("ERROR: could not read timing cache file: " + sentenceTimingCacheFile, e); + } + } + + public void maybeWriteSentenceTimingCache(File sentenceTimingCacheFile){ + try{ + FileWriter fw = new FileWriter(sentenceTimingCacheFile); + for(SentenceInfo s : allSentences){ + SentenceTiming t = s.sentenceTiming; + fw.write("" + + "" + s.startPos + + "," + t.startTime + + "," + t.startTimeInBook + + "," + t.totalBookDuration + + "," + t.isFirstSentenceInAudioFile + + "," + audioFileNamesByAudioFile.get(t.audioFile) + + "\n" + ); + } + fw.close(); + } catch(Exception e) { + log.d("ERROR: could not write timing cache file: " + sentenceTimingCacheFile, e); + } + } + public boolean isSentenceTimingReady(){ return this.sentenceTimingReady; } @@ -207,6 +259,28 @@ private WordTiming parseWordTimingsLine(String line){ return new WordTiming(word, startTime, audioFile); } + private Map.Entry parseSentenceTimingLine(String line){ + String[] cols = line.split(",", 6); + if(cols.length != 6){ + return null; + } + SentenceTiming t = new SentenceTiming(); + String startPos = cols[0]; + t.startTime = Double.parseDouble(cols[1]); + t.startTimeInBook = Double.parseDouble(cols[2]); + t.totalBookDuration = Double.parseDouble(cols[3]); + t.isFirstSentenceInAudioFile = Boolean.parseBoolean(cols[4]); + String audioFileName = cols[5]; + File audioFile = audioFilesByAudioFileName.get(audioFileName); + if(audioFile == null){ + audioFile = new File(audioFileRelativeDir + "/" + audioFileName); + audioFilesByAudioFileName.put(audioFileName, audioFile); + audioFileNamesByAudioFile.put(audioFile, audioFileName); + } + t.audioFile = audioFile; + return new AbstractMap.SimpleEntry<>(startPos, t); + } + private boolean wordsMatch(String word1, String word2){ if(word1 == null && word2 == null) { return true; From 96ad4cc004587dfc34b453f87efffa31f5f8c7f4 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Mon, 23 Jun 2025 00:39:28 -0400 Subject: [PATCH 57/59] audiobook: autocache sentence timing in *.sentencetimingcache file --- .../src/org/coolreader/crengine/TTSToolbarDlg.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/android/src/org/coolreader/crengine/TTSToolbarDlg.java b/android/src/org/coolreader/crengine/TTSToolbarDlg.java index a3464a74f..588cd26fa 100644 --- a/android/src/org/coolreader/crengine/TTSToolbarDlg.java +++ b/android/src/org/coolreader/crengine/TTSToolbarDlg.java @@ -106,6 +106,7 @@ public class TTSToolbarDlg implements Settings { private File wordTimingFile; private File sentenceInfoFile; + private File sentenceTimingCacheFile; private WordTimingAudiobookMatcher wordTimingAudiobookMatcher; private SentenceInfo currentSentenceInfo; @@ -733,6 +734,7 @@ public void onStopTrackingTouch(SeekBar seekBar) { BookInfo bookInfo = mReaderView.getBookInfo(); wordTimingFile = null; sentenceInfoFile = null; + sentenceTimingCacheFile = null; if (null != bookInfo) { FileInfo fileInfo = bookInfo.getFileInfo(); if (null != fileInfo) { @@ -745,9 +747,11 @@ public void onStopTrackingTouch(SeekBar seekBar) { String pathName = fileInfo.getPathName(); String wordTimingPath = pathName.replaceAll("\\.\\w+$", ".wordtiming"); String sentenceInfoPath = pathName.replaceAll("\\.\\w+$", ".sentenceinfo"); + String sentenceTimingCachePath = pathName.replaceAll("\\.\\w+$", ".sentencetimingcache"); if(wordTimingPath.matches(".*\\.wordtiming$")){ wordTimingFile = new File(wordTimingPath); sentenceInfoFile = new File(sentenceInfoPath); + sentenceTimingCacheFile = new File(sentenceTimingCachePath); } } } @@ -809,8 +813,12 @@ public void run() { } wordTimingAudiobookMatcher = new WordTimingAudiobookMatcher(wordTimingFile, allSentences); - //can be very long - wordTimingAudiobookMatcher.parseWordTimingsFile(); + wordTimingAudiobookMatcher.maybeReadSentenceTimingCache(sentenceTimingCacheFile); + if(!wordTimingAudiobookMatcher.isSentenceTimingReady()){ + //can be very long + wordTimingAudiobookMatcher.parseWordTimingsFile(); + wordTimingAudiobookMatcher.maybeWriteSentenceTimingCache(sentenceTimingCacheFile); + } moveSelection(ReaderCommand.DCMD_SELECT_FIRST_SENTENCE, null); audioBookPosHandler.postDelayed(audioBookPosRunnable, 500); From 819d4c90f3e8b23c7e3fdb41e06ffad887621d3a Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 7 Aug 2025 14:38:30 -0400 Subject: [PATCH 58/59] audiobook: pull getAltAudioFile() to Utils, generalize over file ext --- .../src/org/coolreader/crengine/Utils.java | 45 ++++++++++++++++++- .../org/coolreader/tts/TTSControlService.java | 39 +--------------- 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/android/src/org/coolreader/crengine/Utils.java b/android/src/org/coolreader/crengine/Utils.java index 59532c7af..2ab327cb3 100644 --- a/android/src/org/coolreader/crengine/Utils.java +++ b/android/src/org/coolreader/crengine/Utils.java @@ -50,10 +50,14 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Calendar; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.TimeZone; public class Utils { + public static final String[] AUDIO_FILE_EXTS = new String[]{"flac", "wav", "m4a", "ogg", "mp3"}; + public static long timeStamp() { return android.os.SystemClock.uptimeMillis(); } @@ -107,7 +111,46 @@ public static long copyStreamContent(OutputStream os, InputStream is) throws IOE } return totalSize; } - + + /** + * @return an existing file, either origFile or a file with + * the same basename ending in one of allowedExts + */ + public static File getAlternativeFile(File origFile, String[] allowedExts) { + if(origFile == null) { + return null; + } + if(origFile.exists()) { + return origFile; + } + String fileNoExt = origFile.toString().replaceAll("\\.\\w+$", ""); + File dir = origFile.getParentFile(); + if(dir.exists() && dir.isDirectory()) { + Map> filesByExt = new HashMap<>(); + File firstFile = null; + for(File file : dir.listFiles()) { + if(!file.toString().startsWith(fileNoExt + ".")){ + continue; + } + String ext = file.toString().toLowerCase().replaceAll(".*\\.", ""); + if(filesByExt.get(ext) == null) { + filesByExt.put(ext, new ArrayList<>()); + } + filesByExt.get(ext).add(file); + if(firstFile == null) { + firstFile = file; + } + } + for(String ext : allowedExts) { + if(filesByExt.get(ext) != null){ + return filesByExt.get(ext).get(0); + } + } + return firstFile; + } + return null; + } + private static boolean moveFile(File oldPlace, File newPlace, boolean removeOld) { boolean removeNewFile = true; Log.i("cr3", "Moving file " + oldPlace.getAbsolutePath() + " to " + newPlace.getAbsolutePath()); diff --git a/android/src/org/coolreader/tts/TTSControlService.java b/android/src/org/coolreader/tts/TTSControlService.java index cc8261307..5203fb8b2 100644 --- a/android/src/org/coolreader/tts/TTSControlService.java +++ b/android/src/org/coolreader/tts/TTSControlService.java @@ -54,6 +54,7 @@ import org.coolreader.crengine.L; import org.coolreader.crengine.Logger; import org.coolreader.crengine.SentenceInfo; +import org.coolreader.crengine.Utils; import org.coolreader.db.BaseService; import org.coolreader.db.Task; @@ -1383,11 +1384,7 @@ public void ensureAudioBookPlaying() { if(mMediaPlayer != null && mMediaPlayer.isPlaying()){ return; } - File fileToPlay = audioFile; - if(fileToPlay != null && !fileToPlay.exists()){ - fileToPlay = getAlternativeAudioFile(fileToPlay); - } - + File fileToPlay = Utils.getAlternativeFile(audioFile, Utils.AUDIO_FILE_EXTS); if(fileToPlay == null || !fileToPlay.exists()){ return; } @@ -1532,38 +1529,6 @@ else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) return notification; } - private File getAlternativeAudioFile(File origAudioFile) { - if(origAudioFile == null) { - return null; - } - String fileNoExt = origAudioFile.toString().replaceAll("\\.\\w+$", ""); - File dir = origAudioFile.getParentFile(); - if(dir.exists() && dir.isDirectory()) { - Map> filesByExt = new HashMap<>(); - File firstFile = null; - for(File file : dir.listFiles()) { - if(!file.toString().startsWith(fileNoExt + ".")){ - continue; - } - String ext = file.toString().toLowerCase().replaceAll(".*\\.", ""); - if(filesByExt.get(ext) == null) { - filesByExt.put(ext, new ArrayList<>()); - } - filesByExt.get(ext).add(file); - if(firstFile == null) { - firstFile = file; - } - } - for(String ext : new String[]{"flac", "wav", "m4a", "ogg", "mp3"}) { - if(filesByExt.get(ext) != null){ - return filesByExt.get(ext).get(0); - } - } - return firstFile; - } - return null; - } - private void setupTTSHandlers() { if(useAudioBook){ return; From 7e56599e6881b1dd7fe5ecb61cde6e144eeda5a5 Mon Sep 17 00:00:00 2001 From: Elliot Wolk Date: Thu, 7 Aug 2025 14:43:40 -0400 Subject: [PATCH 59/59] audiobook: get duration from alternative file ext, if orig file missing --- .../src/org/coolreader/crengine/WordTimingAudiobookMatcher.java | 1 + 1 file changed, 1 insertion(+) diff --git a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java index a3f360dcc..1f1ba66dd 100644 --- a/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java +++ b/android/src/org/coolreader/crengine/WordTimingAudiobookMatcher.java @@ -231,6 +231,7 @@ private void updateSentenceInfoNextSentence(){ private Double getAudioFileDuration(File file){ try{ + file = Utils.getAlternativeFile(file, Utils.AUDIO_FILE_EXTS); MediaMetadataRetriever m = new MediaMetadataRetriever(); m.setDataSource(file.getAbsolutePath()); String durationStr = m.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION);