WIP: parakeeet

stt_voice_messages/README.md

@@ -0,0 +1,49 @@
+# About
+
+This plugin allows you in conjuction with a _general-purpose speech recognition model_ to transcribe your voice messages to text.
+
+In order to make use of this plugin, you need to have at least one of the following models installed:
+
+#### OpenAI Whisper
+- Website: https://github.com/openai/whisper
+- Installable by: `pip install -U openai-whisper`
+
+#### Faster Whisper
+- Website: https://github.com/SYSTRAN/faster-whisper
+- Installable by: `pip install -U faster-whisper`
+
+Additionally you have to checkout the following Gajim branch:
+https://dev.gajim.org/mesonium/gajim/-/tree/stt_voice_messages
+
+# Hint
+
+_**The plugin is very much POC at this stage!**_
+
+Currently a chosen model will be on first downloaded in the background, during which 
+Gajim's UI may not respond.
+
+Typical model sizes are in case of OpenAI Whisper:
+
+| Multi Langual Model | Download Size | 
+|---------------------|---------------| 
+| Tiny                | 70 MB         |
+| Base                | 140 MB        | 
+| Small               | 460 MB        |
+| Medium              | 1.4 GB        | 
+| Large               | 2.9 GB        | 
+
+# TODO
+
+- [x] Offer multiple models
+- [ ] Add various model settings
+- [ ] Model receiving
+  - [ ] Hint model download state
+  - [ ] Allow to change model download location
+  - [ ] Allow to use local models
+- [ ] Database Handling
+  - [ ] Store transcribed messages in a DB
+  - [ ] Option to delete DB
+- [ ] Update UI
+  - [ ] Make it more pretty
+  - [ ] Show progress bar
+  - [ ] Highlight words on playback

stt_voice_messages/__init__.py

@@ -0,0 +1 @@
+from .stt_voice_messages import STTVoiceMessagesPlugin  # type: ignore  # noqa: F401

stt_voice_messages/gtk/config_dialog.py

@@ -0,0 +1,291 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+import logging
+import typing
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+try:
+    import onnx_asr
+except ModuleNotFoundError:
+    if typing.TYPE_CHECKING:
+        import onnx_asr
+
+from gi.repository import Adw, Gtk
+
+from gajim.gtk.const import Setting, SettingKind, SettingType
+from gajim.gtk.filechoosers import Filter
+from gajim.gtk.settings import GajimPreferencesGroup, SettingsDialog
+from gajim.plugins.plugins_i18n import _
+
+from ..models import stt
+from ..models.model_settings import OnnxAsrSettings
+
+if TYPE_CHECKING:
+    from ..stt_voice_messages import STTVoiceMessagesPlugin
+
+log = logging.getLogger('gajim.p.sttvm_config_dialog')
+
+
+class Configuration:
+    def __init__(self, plugin: STTVoiceMessagesPlugin):
+        self._plugin = plugin
+        self._instance = None
+        self._main_model_row = None
+        self._preset_model_picker = None
+        self._custom_model_id_entry = None
+        self._local_model_file_picker = None
+        self._status_group = None
+        self._model_data: dict[str, str] = {}
+        self._instance = stt.OnnxAsrModel()
+        self._instance.set_config(OnnxAsrSettings(
+            model_id=self.plugin.config['model_id'],
+            model_path=self.plugin.config['model_path']
+        ))
+        self._model_data = self._steal_model_list()
+
+    @property
+    def plugin(self) -> STTVoiceMessagesPlugin:
+        return self._plugin
+
+    @property
+    def is_available(self) -> bool:
+        return self._instance is not None
+
+    def unload_model(self) -> None:
+        if self._instance is not None:
+            self._instance.unload_now()
+
+    def _steal_model_list(self) -> dict[str, str]:
+        # UGLY: Extract available model choices from onnx_asr type hints.
+        ann = onnx_asr.load_model.__annotations__.get('model')
+        return {
+            v: v for arg in typing.get_args(ann)
+            for v in typing.get_args(arg)
+            if isinstance(v, str)
+        }
+
+    def on_setting(self, value: Any, data: Any) -> None:
+        if isinstance(value, str):
+            value = value.strip()
+        self.plugin.config[data] = value
+
+    def on_preset_changed(self, value: str, data: Any) -> None:
+        if self._custom_model_id_entry is not None:
+            entry_text = self._custom_model_id_entry.entry.get_text().strip()
+            if entry_text:
+                self._update_model_status()
+                return  # custom entry overrides; ignore preset change
+        self._write_model_id(value)
+        self._update_model_status()
+
+    def on_custom_model_id_changed(self, value: str, data: Any) -> None:
+        value = value.strip()
+        if value:
+            self._write_model_id(value)
+        elif self._preset_model_picker is not None:
+            preset_key = self._preset_model_picker._dropdown.get_selected_key()
+            if preset_key is not None:
+                self._write_model_id(preset_key)
+        self._apply_sensitivity_state()
+        self._update_model_status()
+
+    def on_model_file_picked(self, value: str, data: Any) -> None:
+        self._write_model_path(str(Path(value).parent) if value else '')
+        self._apply_sensitivity_state()
+        self._update_model_status()
+
+    def _write_model_id(self, model_id: str) -> None:
+        if self.plugin.config['model_id'] == model_id:
+            return
+        self.plugin.config['model_id'] = model_id
+        if self._instance is not None:
+            self._instance.set_config(OnnxAsrSettings(
+                model_id=self.plugin.config['model_id'],
+                model_path=self.plugin.config['model_path']
+            ))
+
+    def _write_model_path(self, model_path: str) -> None:
+        if self.plugin.config['model_path'] == model_path:
+            return
+        self.plugin.config['model_path'] = model_path
+        if self._instance is not None:
+            self._instance.set_config(OnnxAsrSettings(
+                model_id=self.plugin.config['model_id'],
+                model_path=self.plugin.config['model_path']
+            ))
+
+    def sync_model_path_from_widget(self) -> None:
+        if self._local_model_file_picker is None:
+            return
+        button = self._local_model_file_picker.get_activatable_widget()
+        path = button.get_path()
+        new_path = str(path.parent) if path else ''
+        self._write_model_path(new_path)
+
+    def _apply_sensitivity_state(self) -> None:
+        if self._preset_model_picker is None:
+            return
+        has_local = bool(self.plugin.config['model_path'])
+        entry_text = (self._custom_model_id_entry.entry.get_text().strip()
+                      if self._custom_model_id_entry else '')
+        has_entry = bool(entry_text)
+        self._custom_model_id_entry.set_sensitive(not has_local)
+        self._preset_model_picker.set_sensitive(not has_local and not has_entry)
+
+    def _update_model_status(self) -> None:
+        if self._main_model_row is None:
+            return
+        entry_text = (self._custom_model_id_entry.entry.get_text().strip()
+                      if self._custom_model_id_entry else '')
+
+        if self.plugin.config['model_path']:
+            path = Path(self.plugin.config['model_path'])
+            summary = _('Local: {}').format(path.name or str(path))
+            description = _('Loading model files from {}').format(path)
+            if not (path / 'config.json').exists():
+                description += '\n' + _(
+                    'config.json not found in this directory — onnx-asr will'
+                    ' fall back to Model preset or Custom Model ID for the'
+                    ' architecture.')
+        elif entry_text:
+            summary = _('Custom: {}').format(entry_text)
+            description = _('Using custom model: {}').format(entry_text)
+        else:
+            preset_key = (self._preset_model_picker._dropdown.get_selected_key()
+                          if self._preset_model_picker else '')
+            summary = preset_key or _('(none)')
+            description = (_('Using preset: {}').format(preset_key)
+                           if preset_key else '')
+
+        self._main_model_row._label.set_text(summary)
+        if self._status_group is not None:
+            self._status_group.set_description(description)
+
+
+class STTVoiceMessagesConfigDialog(SettingsDialog):
+    def __init__(self, config: Configuration, parent: Gtk.Window) -> None:
+        self.config = config
+        self.plugin = self.config.plugin
+        if not config.is_available:
+            return
+
+        rows = [
+            Setting(SettingKind.SWITCH,
+                    _('Auto Transcribe'),
+                    SettingType.VALUE,
+                    value=self.plugin.config['auto_transcribe'],
+                    data='auto_transcribe',
+                    callback=config.on_setting,
+                    desc=_('Transcribe messages as they appear')),
+            Setting(SettingKind.SUBPAGE,
+                    _('Model'),
+                    SettingType.VALUE,
+                    value=None,
+                    name='main_model',
+                    props={'subpage': 'sttvm-model'}),
+        ]
+
+        SettingsDialog.__init__(
+            self,
+            parent,
+            _('STT Voice Messages'),
+            Gtk.DialogFlags.MODAL,
+            rows,
+            '',
+        )
+
+        config._main_model_row = self.get_setting('main_model')
+
+        use_custom = self.plugin.config['model_id'] not in config._model_data
+
+
+        subpage_rows: list[Setting] = [
+            Setting(SettingKind.DROPDOWN,
+                    _('Model'),
+                    SettingType.VALUE,
+                    value=self.plugin.config['model_id'],
+                    name='preset_model',
+                    callback=config.on_preset_changed,
+                    props={'data': config._model_data}),
+            Setting(SettingKind.ENTRY,
+                    _('Custom Model'),
+                    SettingType.VALUE,
+                    value=self.plugin.config['model_id'] if use_custom else '',
+                    name='custom_model',
+                    callback=config.on_custom_model_id_changed,
+                    desc=_('Custom HF model path or model ID')),
+            Setting(SettingKind.FILECHOOSER,
+                    _('Local File'),
+                    SettingType.VALUE,
+                    value='',
+                    name='local_model_file',
+                    callback=config.on_model_file_picked,
+                    desc=_('Model ID is taken from config.json if not set'),
+                    props={'filefilters': [
+                        Filter(_('ONNX model'), suffixes=['onnx'], default=True),
+                    ]}),
+        ]
+
+        controls_group = GajimPreferencesGroup('model_controls')
+        for s in subpage_rows:
+            controls_group.add_setting(s)
+
+        status_group = Adw.PreferencesGroup()
+
+        pref_page = Adw.PreferencesPage()
+        pref_page.add(controls_group)
+        pref_page.add(status_group)
+
+        toolbar = Adw.ToolbarView(content=pref_page)
+        toolbar.add_top_bar(Adw.HeaderBar())
+
+        page = Adw.NavigationPage(
+            tag='sttvm-model', title=_('Model'), child=toolbar)
+        self._nav.add(page)
+
+        config._preset_model_picker = controls_group.get_setting('preset_model')
+        config._custom_model_id_entry = controls_group.get_setting('custom_model')
+        config._local_model_file_picker = controls_group.get_setting(
+            'local_model_file')
+        config._status_group = status_group
+
+        config._custom_model_id_entry.entry.set_placeholder_text(
+            _('onnx-community/whisper-large-v3-turbo'))
+
+        button = config._local_model_file_picker.get_activatable_widget()
+        button._label_text = _('.oonx')
+        button.reset()
+
+        if self.plugin.config['model_path']:
+            onnx_in_dir = next(iter(Path(self.plugin.config['model_path']).glob('*.onnx')),
+                               None)
+            if onnx_in_dir is not None:
+                button.set_path(onnx_in_dir)
+
+        config._update_model_status()
+        config._apply_sensitivity_state()
+
+    def _cleanup(self) -> None:
+        self.config.sync_model_path_from_widget()
+        self.config._main_model_row = None
+        self.config._preset_model_picker = None
+        self.config._custom_model_id_entry = None
+        self.config._local_model_file_picker = None
+        self.config._status_group = None
+        SettingsDialog._cleanup(self)

stt_voice_messages/gtk/sttbox.py

@@ -0,0 +1,96 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from gi.repository import Gtk, Adw
+
+from gajim.plugins.gajimplugin import GajimPluginConfig
+from gajim.plugins.plugins_i18n import _
+
+from .. import helper
+
+log = logging.getLogger('gajim.p.stt_voice_messages_sttbox')
+
+class STTBox(Gtk.Box):
+    def __init__(self,
+                 config: GajimPluginConfig,
+                 audio_file: Path,
+                 ) -> None:
+
+        Gtk.Box.__init__(self, orientation=Gtk.Orientation.HORIZONTAL, spacing=6)
+
+        self._config = config
+        self._model = None
+        self._audio_file = audio_file
+        self._text = ''
+
+        self._transcribe_button = Gtk.Button.new_from_icon_name("lucide-captions-symbolic")
+        self._transcribe_button.set_tooltip_text(_('Transcribe voice message'))
+
+        self._spinner = Adw.Spinner(valign=Gtk.Align.START, visible=False)
+
+        self._transcription_label = Gtk.Label(
+            label=_('Nothing transcribed yet'))
+        self._transcription_label.set_max_width_chars(40)
+        self._transcription_label.set_wrap(True)
+
+        self.append(self._spinner)
+        self.append(self._transcription_label)
+        self._transcribe_button.connect('clicked', self._on_transcribe_clicked)
+        self._result = helper.Results('')
+
+    @property
+    def button(self) -> Gtk.Button:
+        return self._transcribe_button
+
+    def _on_transcribe_clicked(self, _button: Gtk.Button) -> None:
+        log.debug('config._instance = %s', self._config._instance)
+        self._model = self._config._instance
+        if self._model is None:
+            return
+
+        if self._model.is_loaded:
+            text = _('Transcribing…')
+        elif self._model.will_download:
+            text = _('Downloading ') + self._model.model_id
+        else:
+            text = _('Loading model…')
+        self._transcription_label.set_text(text)
+        self._spinner.set_visible(True)
+        self._task = helper.BackgroundTask(
+            self._model.load, self._on_load_done)
+        self._task.start()
+
+    def _on_load_done(self):
+        self._transcription_label.set_text(_('Transcribing…'))
+        self._task = helper.BackgroundTask(
+            lambda: self._model.recognize(
+                self._result, helper.load_audio(self._audio_file)),
+            self._show_result,
+        )
+        self._task.start()
+
+    def _show_result(self):
+        assert self._model is not  None
+        self._text = self._result.text
+        if self._text.strip() != '':
+            self._transcription_label.set_text(self._text.strip())
+        else:
+            self._transcription_label.set_text(_('_Have not heard any word!_'))
+        self._spinner.set_visible(False)

stt_voice_messages/helper.py

@@ -0,0 +1,112 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import typing
+from dataclasses import dataclass
+from pathlib import Path
+
+import gi
+import numpy as np
+from gi.repository import Gio, GObject
+
+try:
+    gi.require_version('Gst', '1.0')
+    from gi.repository import Gst
+except Exception:
+    if typing.TYPE_CHECKING:
+        from gi.repository import Gst
+
+log = logging.getLogger('gajim.p.sttvm_helper')
+
+
+@dataclass
+class Results:
+    text: str
+
+
+def load_audio(path: Path, sample_rate: int = 16000) -> np.ndarray:
+    Gst.init(None)
+    pipeline = Gst.parse_launch(
+        'filesrc name=src ! decodebin ! audioconvert ! audioresample ! '
+        f'audio/x-raw,format=F32LE,rate={sample_rate},channels=1 ! '
+        'appsink name=sink sync=false'
+    )
+    pipeline.get_by_name('src').set_property('location', str(path))
+    sink = pipeline.get_by_name('sink')
+    chunks: list[np.ndarray] = []
+
+    pipeline.set_state(Gst.State.PLAYING)
+    while (sample := sink.emit('try-pull-sample', 10 * Gst.SECOND)) is not None:
+        buf = sample.get_buffer()
+        _, info = buf.map(Gst.MapFlags.READ)
+        chunks.append(np.frombuffer(bytes(info.data), dtype=np.float32))
+        buf.unmap(info)
+    pipeline.set_state(Gst.State.NULL)
+
+    if not chunks:
+        raise RuntimeError(f'Could not decode audio: {path}')
+    return np.concatenate(chunks)
+
+
+'''
+https://discourse.gnome.org/t/gtk-threading-problem-with-glib-idle-add/13597/5
+https://github.com/gdm-settings/gdm-settings/blob/f245d3000200fa6be2a35c7f6ac45b131dadb5d6/src/utils.py#L116..L162
+'''
+
+
+class BackgroundTask(GObject.Object):
+    __gtype_name__ = 'BackgroundTask'
+
+    def __init__(self, function, finish_callback, **kwargs):
+        super().__init__(**kwargs)
+
+        self.function = function
+        self.finish_callback = finish_callback
+        self._current = None
+
+    def start(self):
+        if self._current:
+            AlreadyRunningError('Task is already running')
+
+        finish_callback = lambda self, task, nothing: self.finish_callback()
+
+        task = Gio.Task.new(self, None, finish_callback, None)
+        task.run_in_thread(self._thread_cb)
+
+        self._current = task
+
+    @staticmethod
+    def _thread_cb(task, self, task_data, cancellable):
+        try:
+            retval = self.function()
+            task.return_value(retval)
+        except Exception as e:
+            log.exception('Background task failed')
+            task.return_value(e)
+
+    def finish(self):
+        task = self._current
+        self._current = None
+
+        if not Gio.Task.is_valid(task, self):
+            raise InvalidGioTaskError()
+
+        value = task.propagate_value().value
+
+        if isinstance(value, Exception):
+            raise value
+
+        return value

stt_voice_messages/models/distill.py

@@ -0,0 +1,45 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+import typing
+from pathlib import Path
+
+from gajim.gtk.const import Setting
+
+from .model_template import Model
+
+try:
+    import ctranslate2
+    CTRANSLATE2_AVAILABLE = True
+except ModuleNotFoundError:
+    if typing.TYPE_CHECKING:
+        import ctranslate2
+
+
+class WhisperModel(Model):
+    def __init__(self):
+        # TODO
+        self._result: str = ''
+
+    @property
+    def result(self) -> str:
+        return self._result
+
+    def transcribe(self, audio_file: Path) -> str:
+        pass
+
+    def on_setting(self, setting: Setting):
+        pass
+

stt_voice_messages/models/faster_whisper.py

@@ -0,0 +1,60 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import typing
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..helper import Results
+from .model_settings import FasterWhisperSettings
+from .model_template import Model
+
+log = logging.getLogger('gajim.p.sttvm_faster_whisper')
+
+try:
+    import faster_whisper
+except ModuleNotFoundError:
+    if typing.TYPE_CHECKING:
+        import faster_whisper
+
+@dataclass
+class Configuration:
+    model_size: str
+
+class FasterWhisperModel(Model):
+    def __init__(self):
+        self._result: str = ''
+        self._config = FasterWhisperSettings()
+
+    @property
+    def result(self) -> str:
+        return self._result
+
+    def transcribe(self, result: Results, audio_file: Path) -> None:
+        model = faster_whisper.WhisperModel(self._config.model_size, compute_type="float32")
+        log.debug('model size is used = %s', self._config.model_size)
+        segments, _ = model.transcribe(audio_file)
+        segments = list(segments)
+        result.text = ''
+        log.debug('segments = %s', segments)
+        for segment in segments:
+            result.text += segment.text
+            print("[%.2fs -> %.2fs] %s" % (
+            segment.start, segment.end, segment.text))
+
+    def set_config(self, config: FasterWhisperSettings) -> None:
+        self._config = config
+

stt_voice_messages/models/model_settings.py

@@ -0,0 +1,23 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class OnnxAsrSettings:
+    model_id: str = field(default='nemo-parakeet-tdt-0.6b-v3', init=True)
+    model_path: str = ''

stt_voice_messages/models/model_template.py

@@ -0,0 +1,41 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+import numpy as np
+
+from ..helper import Results
+
+
+class Model(ABC):
+
+    @property
+    @abstractmethod
+    def is_loaded(self) -> bool:
+        pass
+
+    @abstractmethod
+    def load(self) -> None:
+        pass
+
+    @abstractmethod
+    def recognize(self, result: Results, audio: np.ndarray) -> None:
+        pass
+
+    @abstractmethod
+    def set_config(self, config: Any) -> None:
+        pass

stt_voice_messages/models/openai_whisper.py

@@ -0,0 +1,54 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import typing
+from dataclasses import dataclass
+from pathlib import Path
+
+from ..helper import Results
+from .model_settings import OpenAIWhisperSettings
+from .model_template import Model
+
+log = logging.getLogger('gajim.p.sttvm_whisper')
+
+try:
+    import whisper
+    OPENAI_WHISPER_AVAILABLE = True
+except ModuleNotFoundError:
+    if typing.TYPE_CHECKING:
+        import whisper
+
+@dataclass
+class Configuration:
+    model_size: str
+
+class WhisperModel(Model):
+    def __init__(self):
+        self._result: str = ''
+        self._config = OpenAIWhisperSettings()
+
+    @property
+    def result(self) -> str:
+        return self._result
+
+    def transcribe(self, result: Results, audio_file: Path) -> None:
+        model = whisper.load_model(self._config.model_size)
+        log.debug('model size is used = %s', self._config.model_size)
+        result.text = model.transcribe(audio_file)['text']  # pyright: ignore [reportAttributeAccessIssue]
+
+    def set_config(self, config: OpenAIWhisperSettings) -> None:
+        self._config = config
+

stt_voice_messages/models/stt.py

@@ -0,0 +1,132 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import pickle
+import subprocess
+import sys
+from pathlib import Path
+
+import numpy as np
+from gi.repository import GLib
+
+from ..helper import Results
+from .model_settings import OnnxAsrSettings
+from .model_template import Model
+
+log = logging.getLogger('gajim.p.sttvm_onnx_asr')
+
+
+_IDLE_UNLOAD_SECONDS = 300
+
+class OnnxAsrModel(Model):
+    def __init__(self):
+        self._proc = None
+        self._loaded = False
+        self._config = OnnxAsrSettings()
+        self._unload_source = None
+
+    @property
+    def is_loaded(self) -> bool:
+        return self._loaded
+
+    @property
+    def will_download(self) -> bool:
+        if self.is_loaded or self._config.model_path:
+            return False
+        from huggingface_hub import try_to_load_from_cache
+        from onnx_asr.resolver import model_repos
+        repo = model_repos.get(self._config.model_id, self._config.model_id)
+        if '/' not in repo:
+            return False
+        return not isinstance(try_to_load_from_cache(repo, 'config.json'), str)
+
+    def load(self) -> None:
+        if self._loaded:
+            self._reschedule_unload()
+            return
+        log.debug('Loading model %s in worker', self._config.model_id)
+        self._send({
+            'op': 'load',
+            'model_id': self._config.model_id,
+            'model_path': self._config.model_path,
+        })
+        self._loaded = True
+        self._reschedule_unload()
+
+    def recognize(self, result: Results, audio: np.ndarray) -> None:
+        self.load()
+        response = self._send({'op': 'recognize', 'audio': audio})
+        result.text = response['text']
+        self._reschedule_unload()
+
+    def set_config(self, config: OnnxAsrSettings) -> None:
+        if (config.model_id != self._config.model_id
+                or config.model_path != self._config.model_path):
+            self.unload_now()
+        self._config = OnnxAsrSettings(
+            model_id=config.model_id, model_path=config.model_path)
+
+    def unload_now(self) -> None:
+        if self._unload_source is not None:
+            GLib.source_remove(self._unload_source)
+            self._unload_source = None
+        if self._proc is not None:
+            log.debug('Terminating STT worker subprocess')
+            try:
+                self._proc.stdin.close()
+                self._proc.wait(timeout=2)
+            except subprocess.TimeoutExpired:
+                self._proc.kill()
+                self._proc.wait()
+            self._proc = None
+        self._loaded = False
+
+    def _ensure_proc(self) -> None:
+        if self._proc is not None and self._proc.poll() is None:
+            return
+        log.debug('Starting STT worker subprocess')
+        self._proc = subprocess.Popen(
+            [sys.executable, str(Path(__file__).parent / 'stt_worker.py')],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        self._loaded = False
+
+    def _send(self, cmd: dict) -> dict:
+        self._ensure_proc()
+        pickle.dump(cmd, self._proc.stdin)
+        self._proc.stdin.flush()
+        try:
+            response = pickle.load(self._proc.stdout)
+        except EOFError as e:
+            self._proc = None
+            self._loaded = False
+            raise RuntimeError('Worker subprocess exited unexpectedly') from e
+        if not response.get('ok'):
+            raise RuntimeError(response.get('error', 'unknown worker error'))
+        return response
+
+    def _reschedule_unload(self) -> None:
+        if self._unload_source is not None:
+            GLib.source_remove(self._unload_source)
+        self._unload_source = GLib.timeout_add_seconds(
+            _IDLE_UNLOAD_SECONDS, self._on_idle_unload)
+
+    def _on_idle_unload(self) -> bool:
+        self._unload_source = None
+        log.debug('Idle unload after %ds', _IDLE_UNLOAD_SECONDS)
+        self.unload_now()
+        return GLib.SOURCE_REMOVE

stt_voice_messages/models/stt_worker.py

@@ -0,0 +1,54 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+import pickle
+import sys
+import traceback
+
+
+def _respond(response: dict) -> None:
+    pickle.dump(response, sys.stdout.buffer)
+    sys.stdout.buffer.flush()
+
+
+def main() -> None:
+    model = None
+    while True:
+        try:
+            cmd = pickle.load(sys.stdin.buffer)
+        except EOFError:
+            return
+        try:
+            op = cmd['op']
+            if op == 'load':
+                import onnx_asr
+                model = onnx_asr.load_model(
+                    cmd['model_id'], cmd.get('model_path') or None)
+                _respond({'ok': True})
+            elif op == 'recognize':
+                text = model.recognize(cmd['audio'])
+                _respond({'ok': True, 'text': text})
+            else:
+                _respond({'ok': False, 'error': f'unknown op: {op}'})
+        except Exception as e:
+            _respond({
+                'ok': False,
+                'error': f'{type(e).__name__}: {e}',
+                'traceback': traceback.format_exc(),
+            })
+
+
+if __name__ == '__main__':
+    main()

stt_voice_messages/plugin-manifest.json

@@ -0,0 +1,20 @@
+{
+    "authors": [
+        "mesonium <mesonium@posteo.eu>"
+    ],
+    "description": "Transcribes voice messages to text.",
+    "homepage": "https://dev.gajim.org/gajim/gajim-plugins/wikis/STTVoiceMessagesPlugin",
+    "config_dialog": true,
+    "name": "STT Voice Messages",
+    "platforms": [
+        "others",
+        "linux",
+        "darwin",
+        "win32"
+    ],
+    "requirements": [
+        "gajim>=2.0.0"
+    ],
+    "short_name": "stt_voice_messages",
+    "version": "0.0.1"
+}

stt_voice_messages/stt_voice_messages.py

@@ -0,0 +1,103 @@
+# This file is part of Gajim.
+#
+# Gajim is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Gajim is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Gajim. If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import annotations
+
+import logging
+from functools import partial
+from pathlib import Path
+
+from gi.repository import GLib, Gtk
+
+from gajim.common import app
+from gajim.plugins import GajimPlugin
+from gajim.plugins.plugins_i18n import _
+
+from .gtk.config_dialog import Configuration, STTVoiceMessagesConfigDialog
+from .gtk.sttbox import STTBox
+from .models.model_settings import OnnxAsrSettings
+
+log = logging.getLogger('gajim.p.stt_voice_messages')
+
+_FOCUS_LOSS_UNLOAD_SECONDS = 30
+
+
+class STTVoiceMessagesPlugin(GajimPlugin):
+    def init(self) -> None:
+        self.description = _('Transcribes voice messages to text.')
+
+        self.config_default_values = {
+            'auto_transcribe': (False, ''),
+            'model_id': ('nemo-parakeet-tdt-0.6b-v3', ''),
+            'model_path': ('', ''),
+        }
+
+        self._config = Configuration(self)
+        self.config_dialog = partial(STTVoiceMessagesConfigDialog, self._config)
+
+        self.gui_extension_points = {
+            'preview_audio': (self._on_preview_audio_created, None),
+        }
+
+        self._active_handler_id = 0
+        self._focus_unload_source = None
+
+    def activate(self) -> None:
+        if app.window is not None and self._active_handler_id == 0:
+            self._active_handler_id = app.window.connect(
+                'notify::is-active', self._on_window_active_changed)
+
+    def deactivate(self) -> None:
+        if self._focus_unload_source is not None:
+            GLib.source_remove(self._focus_unload_source)
+            self._focus_unload_source = None
+        if self._active_handler_id != 0 and app.window is not None:
+            app.window.disconnect(self._active_handler_id)
+            self._active_handler_id = 0
+        if self._config.is_available:
+            self._config.unload_model()
+
+    def _on_window_active_changed(self,
+                                  window: Gtk.Window,
+                                  _pspec: object,
+                                  ) -> None:
+        if window.is_active():
+            if self._focus_unload_source is not None:
+                GLib.source_remove(self._focus_unload_source)
+                self._focus_unload_source = None
+        elif self._focus_unload_source is None:
+            self._focus_unload_source = GLib.timeout_add_seconds(
+                _FOCUS_LOSS_UNLOAD_SECONDS, self._on_focus_unload_fired)
+
+    def _on_focus_unload_fired(self) -> bool:
+        self._focus_unload_source = None
+        if self._config.is_available:
+            self._config.unload_model()
+        return GLib.SOURCE_REMOVE
+
+    def _on_preview_audio_created(self,
+                                  drawing_box: Gtk.Box,
+                                  control_box: Gtk.Box,
+                                  audio_file: Path
+                                  ) -> None:
+        self._drawing_box = drawing_box;
+        self._control_box = control_box;
+        self._audio_file = audio_file.as_posix()
+        self._create_stt_box()
+
+    def _create_stt_box(self) -> None:
+        self._stt_box = STTBox(self._config, self._audio_file)
+        self._control_box.append(self._stt_box.button)
+        self._drawing_box.append(self._stt_box)