From 032e7f0b8c3ddd4820d5947f004690dd467d3df4 Mon Sep 17 00:00:00 2001 From: MassiveBox Date: Sat, 4 Oct 2025 16:22:54 +0200 Subject: [PATCH] Add offline spell-checking --- package.json | 1 + public/i18n/en_US.json | 14 +++-- src/espells.ts | 45 ++++++++++++++++ src/hunspellDictManager.ts | 51 ++++++++++++++++++ src/index.ts | 31 ++++++++++- src/languagetool.ts | 63 +++++++++++++++------- src/menus.ts | 15 +++--- src/settings.ts | 43 ++++++++------- src/spellChecker.ts | 15 ++++++ src/{spellchecker.ts => spellCheckerUI.ts} | 6 +-- src/suggestions.ts | 32 ++++++----- 11 files changed, 252 insertions(+), 64 deletions(-) create mode 100644 src/espells.ts create mode 100644 src/hunspellDictManager.ts create mode 100644 src/spellChecker.ts rename src/{spellchecker.ts => spellCheckerUI.ts} (96%) diff --git a/package.json b/package.json index 8627868..0321c91 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "@tsconfig/svelte": "^4.0.1", "@types/node": "^20.3.0", "cross-env": "^7.0.3", + "espells": "^0.4.1", "fast-glob": "^3.2.12", "glob": "^10.0.0", "js-yaml": "^4.1.0", diff --git a/public/i18n/en_US.json b/public/i18n/en_US.json index 06e47ad..fab8ed4 100644 --- a/public/i18n/en_US.json +++ b/public/i18n/en_US.json @@ -47,6 +47,14 @@ "experimentalCorrect": { "title": "[Feature preview] Apply corrections when selected", "description": "[Feature preview] This feature will modify the content of your documents when used, and can alter them significantly without the ability to roll the changes back. This feature is not recommended for production workspaces.

When a correction is chosen, apply it to the document instead of just having copied to your clipboard." + }, + "offline": { + "title": "Offline mode", + "description": "If enabled, the plugin will use a local spell checker, which is faster and more privacy friendly, but doesn't provide advanced grammar checking" + }, + "offlineDicts": { + "title": "Offline dictionaries", + "description": "Comma-separated list of dictionaries used for offline spell checking. Available options. Example: en,it" } }, "errors": { @@ -55,9 +63,9 @@ "cantRender": "This block contains elements, such as images or tables, that don't work well with the SySpell system.", "waitingForSuggestions": "Grammar suggestions for this block aren't ready yet, please close the menu and open it again after a few seconds.", "correctionNotEnabled": "The correction has been copied to your clipboard. Suggestions can be auto-applied when selected. Visit the plugin's settings to enable.", - "checkServer": "Failed to contact grammar checking server, make sure it's correctly set in the plugin settings.", - "fatal": "The grammar checking plugin will quit now. Please restart SiYuan.", - "notImplementedNotebookSettings": "Notebook-wide grammar checking settings aren't implemented yet, they will be added in a future version. Thanks for your patience!" + "checkServer": "Failed to contact grammar checking server, make sure it's correctly set or enable Offline Mode in the plugin settings.", + "notImplementedNotebookSettings": "Notebook-wide grammar checking settings aren't implemented yet, they will be added in a future version. Thanks for your patience!", + "hunspellLoadError": "Failed loading offline spell checker: " }, "docMenu": { "documentStatus": "Document status", diff --git a/src/espells.ts b/src/espells.ts new file mode 100644 index 0000000..70ae166 --- /dev/null +++ b/src/espells.ts @@ -0,0 +1,45 @@ +import {Language, SpellChecker, Suggestion} from "@/spellChecker"; +import { Espells } from "espells" + +export class ESpellChecker implements SpellChecker { + + spellchecker: Espells + loadedLanguages: Language[] + + constructor(languages: {aff: string, dic: string, language: Language}[]) { + this.spellchecker = new Espells({aff: languages[0].aff, dic: languages.map(l => l.dic)}) + this.loadedLanguages = languages.map(l => l.language) + } + + async check(text: string, _: string[]): Promise { + + let suggestions: Suggestion[] = [] + + const regex = /[\w']+/g; + let match; + + while ((match = regex.exec(text)) !== null) { + const word = match[0]; + const counter = match.index; + const {correct} = this.spellchecker.lookup(word) + if(!correct) { + const hsSuggestions = this.spellchecker.suggest(word) + suggestions.push({ + typeName: "UnknownWord", + message: word, + shortMessage: "Misspelled word", + replacements: hsSuggestions, + offset: counter, + length: word.length + }) + } + } + + return suggestions + } + + async getLanguages(): Promise { + return this.loadedLanguages + } + +} \ No newline at end of file diff --git a/src/hunspellDictManager.ts b/src/hunspellDictManager.ts new file mode 100644 index 0000000..ac1ad44 --- /dev/null +++ b/src/hunspellDictManager.ts @@ -0,0 +1,51 @@ +import {getFile, putFile} from "@/api"; + +export class HunspellDictManager { + + private static pathBase = 'data/storage/petal/syspell' + private static urlBase = 'https://raw.githubusercontent.com/wooorm/dictionaries/refs/heads/main/dictionaries' + + static async loadDictionary(language: string, downloadIfMissing: boolean): Promise<{ aff: string, dic: string }> { + + const aff = await getFile(`${this.pathBase}/${language}.aff`) + const dic = await getFile(`${this.pathBase}/${language}.dic`) + + if(aff.code == 404 || dic.code == 404) { + if(downloadIfMissing) { + await this.downloadDictionary(language) + return this.loadDictionary(language, false) + }else{ + throw new Error(`Dictionary ${language} not found`) + } + } + + return { aff, dic } + + } + + private static async downloadFile(url: string, filename: string) { + + const res = await fetch(url); + const mimeType = res.headers.get('content-type') + + if(res.status != 200) { + throw new Error(await res.text()) + } + + const blob = new Blob([await res.text()], { type: mimeType }); + const file = new File([blob], filename, { type: mimeType, lastModified: Date.now() }); + + await putFile(filename, false, file) + + } + + static async downloadDictionary(language: string) { + try { + await this.downloadFile(`${this.urlBase}/${language}/index.aff`, `${this.pathBase}/${language}.aff`); + await this.downloadFile(`${this.urlBase}/${language}/index.dic`, `${this.pathBase}/${language}.dic`); + }catch (e) { + throw new Error(`Download for dictionary '${language}' failed with ` + e) + } + } + +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 7227a9b..ebf78cc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,10 @@ import {SettingUtils} from "@/libs/setting-utils"; import {Analytics} from "@/analytics"; import {SuggestionEngine} from "@/suggestions"; import {Menus} from "@/menus"; +import {ESpellChecker} from "@/espells"; +import {LanguageTool, LanguageToolSettings} from "@/languagetool"; +import {HunspellDictManager} from "@/hunspellDictManager"; +import {Language} from "@/spellChecker"; export default class SpellCheckPlugin extends Plugin { @@ -17,6 +21,9 @@ export default class SpellCheckPlugin extends Plugin { public analytics: Analytics public i18nx: any; // This object is just a copy of i18n, but with type "any" to not trigger type errors + public offlineSpellChecker: ESpellChecker + public onlineSpellChecker: LanguageTool + public static ENABLED_ATTR = 'custom-spellcheck-enable' public static LANGUAGE_ATTR = 'custom-spellcheck-language' @@ -24,11 +31,14 @@ export default class SpellCheckPlugin extends Plugin { this.i18nx = this.i18n new Icons(this); + this.settingsUtil = await Settings.init(this) this.analytics = new Analytics(this.settingsUtil.get('analytics')); this.suggestions = new SuggestionEngine(this) this.menus = new Menus(this) + await this.prepareSpellCheckers() + void this.analytics.sendEvent('load') const style = document.createElement('style'); @@ -83,7 +93,6 @@ export default class SpellCheckPlugin extends Plugin { }) this.eventBus.on('open-menu-doctree', async (event) => { - console.log(event) const docID = ProtyleHelpers.getNodeId(event.detail.elements[0]) // @TODO this is ugly, why does the event not carry the docID? void this.menus.addSettingsToDocMenu(docID, event.detail.menu) }) @@ -119,4 +128,24 @@ export default class SpellCheckPlugin extends Plugin { void this.analytics.sendEvent('uninstall'); } + private async prepareSpellCheckers() { + + this.onlineSpellChecker = new LanguageTool(this.settingsUtil.dump()) + const offlineLanguages = this.settingsUtil.get('offlineDicts').split(',') + + let langs: {aff: string, dic: string, language: Language}[] = [] + + try { + for(const lang of offlineLanguages) { + const { aff, dic } = await HunspellDictManager.loadDictionary(lang, true) + langs.push({aff: aff, dic: dic, language: {name: lang, code: lang, longCode: lang}}) + } + this.offlineSpellChecker = new ESpellChecker(langs) + }catch (e){ + console.error(e) + showMessage(this.i18nx.errors.hunspellLoadError + e, -1, 'error') + } + + } + } \ No newline at end of file diff --git a/src/languagetool.ts b/src/languagetool.ts index bc4ee97..3ec5450 100644 --- a/src/languagetool.ts +++ b/src/languagetool.ts @@ -1,6 +1,7 @@ -import {PluginSettings} from "@/settings"; +import {Language, Suggestion} from "@/spellChecker"; +import {SpellChecker} from "@/spellChecker"; -export type Suggestion = { +type LanguageToolSuggestion = { message: string shortMessage: string replacements: Array<{ @@ -32,47 +33,73 @@ export type Suggestion = { ignoreForIncompleteSentence: boolean contextForSureMatch: number } - -export type Language = { name: string; code: string; longCode: string; } interface HTTPError extends Error { status?: number; } +export type LanguageToolSettings = { + server: string + username: string + apiKey: string + picky: boolean + motherTongue: string + preferredVariants: string +} -export class LanguageTool { +export class LanguageTool implements SpellChecker { - public static async check(text: string, language: string, settings: PluginSettings): Promise { + private settings: LanguageToolSettings; + + constructor(settings: LanguageToolSettings) { + this.settings = settings + } + + public async check(text: string, languages: string[]): Promise { + + const language = languages.length > 0 ? languages[0] : 'auto'; const body = new URLSearchParams({ text: text, language: language, - level: settings.picky ? 'picky' : 'default', - motherTongue: settings.motherTongue == '' ? window.navigator.language : settings.motherTongue, + level: this.settings.picky ? 'picky' : 'default', + motherTongue: this.settings.motherTongue == '' ? window.navigator.language : this.settings.motherTongue, }); - if(settings.username != '') { - body.append('username', settings.username); + if(this.settings.username != '') { + body.append('username', this.settings.username); } - if(settings.apiKey) { - body.append('apiKey', settings.apiKey); + if(this.settings.apiKey) { + body.append('apiKey', this.settings.apiKey); } if(language == 'auto') { - body.append('preferredVariants', settings.preferredVariants) + body.append('preferredVariants', this.settings.preferredVariants) } - const res = await fetch(settings.server + 'v2/check', {method: 'POST', body}); + const res = await fetch(this.settings.server + 'v2/check', {method: 'POST', body}); if(res.status != 200) { const err = new Error('Network error') as HTTPError err.status = res.status; throw err } - const json = await res.json(); - return json.matches; + const suggestions: LanguageToolSuggestion[] = (await res.json()).matches; + return suggestions.map((suggestion) => { + const ret: Suggestion = { + message: suggestion.message, + shortMessage: suggestion.shortMessage, + replacements: suggestion.replacements.map((replacement) => { + return replacement.value + }), + offset: suggestion.offset, + length: suggestion.length, + typeName: suggestion.type.typeName + } + return ret + }); } - public static async getLanguages(settings: PluginSettings): Promise { - const res = await fetch(settings.server + 'v2/languages', {method: 'GET'}); + public async getLanguages(): Promise { + const res = await fetch(this.settings.server + 'v2/languages', {method: 'GET'}); return await res.json(); } diff --git a/src/menus.ts b/src/menus.ts index b0d5fcd..fe13833 100644 --- a/src/menus.ts +++ b/src/menus.ts @@ -1,8 +1,7 @@ import {Menu, showMessage, subMenu} from 'siyuan'; import SpellCheckPlugin from "@/index"; import {getBlockAttrs, setBlockAttrs} from "@/api"; -import {LanguageTool} from "@/languagetool"; -import {PluginSettings, Settings} from "@/settings"; +import {Settings} from "@/settings"; import {ProtyleHelpers} from "@/protyleHelpers"; import {SuggestionEngine} from "@/suggestions"; @@ -31,14 +30,14 @@ export class Menus { } }) - if(suggestion.type.typeName == 'UnknownWord') { + if(suggestion.typeName == 'UnknownWord') { // add to dictionary menu.addItem({ icon: 'add', label: this.plugin.i18nx.textMenu.addToDictionary, click: async () => { void this.plugin.analytics.sendEvent('menu-click-add-to-dictionary'); - const word = SuggestionEngine.suggestionToWrongText(suggestion) + const word = SuggestionEngine.suggestionToWrongText(suggestion, blockID) await Settings.addToDictionary(word, this.plugin.settingsUtil) showMessage(this.plugin.i18nx.textMenu.addedToDictionary + word, 5000, 'info') await this.plugin.suggestions.renderSuggestions(blockID) @@ -50,15 +49,15 @@ export class Menus { suggestion.replacements.forEach((replacement, correctionNumber) => { menu.addItem({ icon: 'spellcheck', - label: replacement.value, + label: replacement, click: async () => { void this.plugin.analytics.sendEvent('menu-click-correct', { - 'type': suggestion.rule.category.id + 'type': suggestion.typeName }); if(this.plugin.settingsUtil.get('experimentalCorrect')) { void this.plugin.suggestions.correctSuggestion(blockID, suggestionNumber, correctionNumber) }else{ - void navigator.clipboard.writeText(replacement.value) + void navigator.clipboard.writeText(replacement) showMessage(this.plugin.i18nx.errors.correctionNotEnabled, 5000, 'info') } } @@ -111,7 +110,7 @@ export class Menus { label: this.plugin.i18nx.docMenu.setDocumentLanguage, click: async (_, ev: MouseEvent) => { void this.plugin.analytics.sendEvent('docmenu-click-setlang-1'); - const languages = await LanguageTool.getLanguages(this.plugin.settingsUtil.dump()) + const languages = await this.plugin.onlineSpellChecker.getLanguages() const langMenu = new Menu('spellCheckLangMenu'); langMenu.addItem({ icon: 'autodetect', diff --git a/src/settings.ts b/src/settings.ts index 39d80fc..3f8cd68 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -1,21 +1,7 @@ import {SettingUtils} from "@/libs/setting-utils"; import {showMessage} from 'siyuan'; -import {LanguageTool} from "@/languagetool"; import SpellCheckPlugin from "@/index"; - -export type PluginSettings = { - server: string - username: string - apiKey: string - picky: boolean - motherTongue: string - preferredVariants: string - enabledByDefault: boolean - defaultLanguage: string - preferredLanguages: string - analytics: boolean -} - +import {LanguageTool, LanguageToolSettings} from "@/languagetool"; export class Settings { @@ -61,13 +47,12 @@ export class Settings { await su.load() // needed to fetch languages from server let languagesKV = {} try { - let languages = await LanguageTool.getLanguages(su.dump()) + let languages = await new LanguageTool({server: su.get('server')}).getLanguages() languages.forEach(language => { languagesKV[language.longCode] = language.name + ' [' + language.longCode + ']' }) - } catch { + } catch(e) { showMessage(plugin.i18nx.errors.checkServer, -1, 'error') - showMessage(plugin.i18nx.errors.fatal, -1, 'error') } su.addItem({ @@ -129,6 +114,22 @@ export class Settings { value: 'auto' }) + su.addItem({ + type: 'checkbox', + key: 'offline', + title: to.offline.title, + description: to.offline.description, + value: false + }) + + su.addItem({ + type: 'textinput', + key: 'offlineDicts', + title: to.offlineDicts.title, + description: to.offlineDicts.description, + value: 'en' + }) + su.addItem({ type: 'checkbox', key: 'analytics', @@ -137,6 +138,12 @@ export class Settings { value: true }) + su.save = async function (data?: any) { + data = data ?? this.dump(); + await this.plugin.saveData(this.file, this.dump()); + location.reload() + }.bind(su) + await su.load() return su diff --git a/src/spellChecker.ts b/src/spellChecker.ts new file mode 100644 index 0000000..443cd2f --- /dev/null +++ b/src/spellChecker.ts @@ -0,0 +1,15 @@ +export type Language = { name: string; code: string; longCode: string; } + +export type Suggestion = { + message: string + shortMessage: string + replacements: string[] + offset: number + length: number + typeName: string +} + +export interface SpellChecker { + check(text: string, languages: string[]): Promise + getLanguages(): Promise +} \ No newline at end of file diff --git a/src/spellchecker.ts b/src/spellCheckerUI.ts similarity index 96% rename from src/spellchecker.ts rename to src/spellCheckerUI.ts index 3054e8b..d809c1c 100644 --- a/src/spellchecker.ts +++ b/src/spellCheckerUI.ts @@ -1,6 +1,6 @@ import {ProtyleHelpers} from "@/protyleHelpers"; -export class SpellChecker { +export class SpellCheckerUI { private readonly blockID: string; private readonly docID: string; @@ -111,13 +111,13 @@ export class SpellChecker { const top = rect.bottom - editorRect.top - 2 + this.block.scrollTop; const width = rect.width; - const offset = SpellChecker.distance(this.overlay, this.block) + const offset = SpellCheckerUI.distance(this.overlay, this.block) underline.style.left = (left + offset.h) + 'px'; underline.style.top = (top + 2 + offset.v) + 'px'; underline.style.width = width + 'px'; - if(!SpellChecker.checkDontUnderline(width, charsCount)) { + if(!SpellCheckerUI.checkDontUnderline(width, charsCount)) { this.overlay.appendChild(underline); } } diff --git a/src/suggestions.ts b/src/suggestions.ts index 5548da5..aea86a4 100644 --- a/src/suggestions.ts +++ b/src/suggestions.ts @@ -1,13 +1,13 @@ import {ProtyleHelpers} from "@/protyleHelpers"; -import {LanguageTool, Suggestion} from "@/languagetool"; -import {PluginSettings, Settings} from "@/settings"; +import {Settings} from "@/settings"; import {getChildBlocks, updateBlock} from "@/api"; -import {SpellChecker} from "@/spellchecker"; +import {SpellCheckerUI} from "@/spellCheckerUI"; import {showMessage} from "siyuan"; import SpellCheckPlugin from "@/index"; +import {Suggestion} from "@/spellChecker"; interface StoredBlock { - spellChecker: SpellChecker; + spellChecker: SpellCheckerUI; suggestions: Suggestion[]; } @@ -40,7 +40,7 @@ export class SuggestionEngine { const children = await getChildBlocks(blockID) if(children.length == 0) { if(!(blockID in this.blockStorage)) { - const spellChecker = new SpellChecker(blockID, this.documentID) + const spellChecker = new SpellCheckerUI(blockID, this.documentID) this.blockStorage[blockID] = { spellChecker: spellChecker, suggestions: [] @@ -90,11 +90,16 @@ export class SuggestionEngine { return this.suggestForBlock(blockID) } - try { - suggestions = await LanguageTool.check(text, this.documentLanguage, this.plugin.settingsUtil.dump()) - }catch (_) { - showMessage(this.plugin.i18nx.errors.checkServer, 5000, 'error') + if(this.plugin.settingsUtil.get('offline')) { + suggestions = await this.plugin.offlineSpellChecker.check(text, [this.documentLanguage]) + }else{ + try { + suggestions = await this.plugin.onlineSpellChecker.check(text, [this.documentLanguage]) + }catch (_) { + showMessage(this.plugin.i18nx.errors.checkServer, 5000, 'error') + } } + this.blockStorage[blockID].suggestions = suggestions } @@ -109,14 +114,15 @@ export class SuggestionEngine { } this.blockStorage[blockID].spellChecker.clearUnderlines() this.blockStorage[blockID].suggestions.forEach(suggestion => { - if(!Settings.isInCustomDictionary(SuggestionEngine.suggestionToWrongText(suggestion), this.plugin.settingsUtil)) { + if(!Settings.isInCustomDictionary(SuggestionEngine.suggestionToWrongText(suggestion, blockID), this.plugin.settingsUtil)) { this.blockStorage[blockID].spellChecker.highlightCharacterRange(suggestion.offset, suggestion.offset + suggestion.length) } }) } - static suggestionToWrongText(suggestion: Suggestion): string { - return suggestion.context.text.slice(suggestion.context.offset, suggestion.context.offset + suggestion.context.length) + static suggestionToWrongText(suggestion: Suggestion, blockID: string): string { + const blockTxt = ProtyleHelpers.fastGetBlockText(blockID) + return blockTxt.slice(suggestion.offset, suggestion.offset + suggestion.length) } private getAbsoluteOffsetInBlock(range: Range, blockID: string): number { @@ -162,7 +168,7 @@ export class SuggestionEngine { const suggestion = this.blockStorage[blockID].suggestions[suggestionNumber] const rich = ProtyleHelpers.fastGetBlockHTML(blockID) const fixedOffset = this.adjustIndexForTags(rich, suggestion.offset) - const newStr = rich.slice(0, fixedOffset) + suggestion.replacements[correctionNumber].value + rich.slice(fixedOffset + suggestion.length) + const newStr = rich.slice(0, fixedOffset) + suggestion.replacements[correctionNumber] + rich.slice(fixedOffset + suggestion.length) console.log("new str " + newStr); await updateBlock('markdown', window.Lute.New().BlockDOM2Md(newStr), blockID)