Add offline spell-checking

package.json

@@ -20,6 +20,7 @@
     "@tsconfig/svelte": "^4.0.1",
     "@types/node": "^20.3.0",
     "cross-env": "^7.0.3",
+    "espells": "^0.4.1",
     "fast-glob": "^3.2.12",
     "glob": "^10.0.0",
     "js-yaml": "^4.1.0",

public/i18n/en_US.json

@@ -47,6 +47,14 @@
     "experimentalCorrect": {
       "title": "<b>[Feature preview]</b> Apply corrections when selected",
       "description": "<b>[Feature preview]</b> This feature will modify the content of your documents when used, and can alter them significantly without the ability to roll the changes back. This feature is not recommended for production workspaces.<br><br>When a correction is chosen, apply it to the document instead of just having copied to your clipboard."
+    },
+    "offline": {
+      "title": "Offline mode",
+      "description": "If enabled, the plugin will use a local spell checker, which is faster and more privacy friendly, but doesn't provide advanced grammar checking"
+    },
+    "offlineDicts": {
+      "title": "Offline dictionaries",
+      "description": "Comma-separated list of dictionaries used for offline spell checking. <a href='https://github.com/wooorm/dictionaries/tree/main/dictionaries'>Available options</a>. Example: <code>en,it</code>"
     }
   },
   "errors": {
@@ -55,9 +63,9 @@
     "cantRender": "This block contains elements, such as images or tables, that don't work well with the SySpell system.",
     "waitingForSuggestions": "Grammar suggestions for this block aren't ready yet, please close the menu and open it again after a few seconds.",
     "correctionNotEnabled": "The correction has been copied to your clipboard. <small>Suggestions can be auto-applied when selected. Visit the plugin's settings to enable.</small>",
-    "checkServer": "Failed to contact grammar checking server, make sure it's correctly set in the plugin settings.",
-    "fatal": "The grammar checking plugin will quit now. Please restart SiYuan.",
-    "notImplementedNotebookSettings": "Notebook-wide grammar checking settings aren't implemented yet, they will be added in a future version. Thanks for your patience!"
+    "checkServer": "Failed to contact grammar checking server, make sure it's correctly set or enable Offline Mode in the plugin settings.",
+    "notImplementedNotebookSettings": "Notebook-wide grammar checking settings aren't implemented yet, they will be added in a future version. Thanks for your patience!",
+    "hunspellLoadError": "Failed loading offline spell checker: "
   },
   "docMenu": {
     "documentStatus": "Document status",

src/espells.ts

@@ -0,0 +1,45 @@
+import {Language, SpellChecker, Suggestion} from "@/spellChecker";
+import { Espells } from "espells"
+
+export class ESpellChecker implements SpellChecker {
+
+    spellchecker: Espells
+    loadedLanguages: Language[]
+
+    constructor(languages: {aff: string, dic: string, language: Language}[]) {
+        this.spellchecker = new Espells({aff: languages[0].aff, dic: languages.map(l => l.dic)})
+        this.loadedLanguages = languages.map(l => l.language)
+    }
+
+    async check(text: string, _: string[]): Promise<Suggestion[]> {
+
+        let suggestions: Suggestion[] = []
+
+        const regex =   /[\w']+/g;
+        let match;
+
+        while ((match = regex.exec(text)) !== null) {
+            const word = match[0];
+            const counter = match.index;
+            const {correct} = this.spellchecker.lookup(word)
+            if(!correct) {
+                const hsSuggestions = this.spellchecker.suggest(word)
+                    suggestions.push({
+                        typeName: "UnknownWord",
+                        message: word,
+                        shortMessage: "Misspelled word",
+                        replacements: hsSuggestions,
+                        offset: counter,
+                        length: word.length
+                    })
+            }
+        }
+
+        return suggestions
+    }
+
+    async getLanguages(): Promise<Language[]> {
+        return this.loadedLanguages
+    }
+
+}

src/hunspellDictManager.ts

@@ -0,0 +1,51 @@
+import {getFile, putFile} from "@/api";
+
+export class HunspellDictManager {
+
+    private static pathBase = 'data/storage/petal/syspell'
+    private static urlBase = 'https://raw.githubusercontent.com/wooorm/dictionaries/refs/heads/main/dictionaries'
+
+    static async loadDictionary(language: string, downloadIfMissing: boolean): Promise<{ aff: string, dic: string }> {
+
+        const aff = await getFile(`${this.pathBase}/${language}.aff`)
+        const dic = await getFile(`${this.pathBase}/${language}.dic`)
+
+        if(aff.code == 404 || dic.code == 404) {
+            if(downloadIfMissing) {
+                await this.downloadDictionary(language)
+                return this.loadDictionary(language, false)
+            }else{
+                throw new Error(`Dictionary ${language} not found`)
+            }
+        }
+
+        return { aff, dic }
+
+    }
+
+    private static async downloadFile(url: string, filename: string) {
+
+        const res = await fetch(url);
+        const mimeType = res.headers.get('content-type')
+
+        if(res.status != 200) {
+            throw new Error(await res.text())
+        }
+
+        const blob = new Blob([await res.text()], { type: mimeType });
+        const file =  new File([blob], filename, { type: mimeType, lastModified: Date.now() });
+
+        await putFile(filename, false, file)
+
+    }
+
+    static async downloadDictionary(language: string) {
+        try {
+            await this.downloadFile(`${this.urlBase}/${language}/index.aff`, `${this.pathBase}/${language}.aff`);
+            await this.downloadFile(`${this.urlBase}/${language}/index.dic`, `${this.pathBase}/${language}.dic`);
+        }catch (e) {
+            throw new Error(`Download for dictionary '${language}' failed with ` + e)
+        }
+    }
+
+}

src/index.ts

@@ -6,6 +6,10 @@ import {SettingUtils} from "@/libs/setting-utils";
 import {Analytics} from "@/analytics";
 import {SuggestionEngine} from "@/suggestions";
 import {Menus} from "@/menus";
+import {ESpellChecker} from "@/espells";
+import {LanguageTool, LanguageToolSettings} from "@/languagetool";
+import {HunspellDictManager} from "@/hunspellDictManager";
+import {Language} from "@/spellChecker";
 
 
 export default class SpellCheckPlugin extends Plugin {
@@ -17,6 +21,9 @@ export default class SpellCheckPlugin extends Plugin {
     public analytics: Analytics
     public i18nx: any; // This object is just a copy of i18n, but with type "any" to not trigger type errors
 
+    public offlineSpellChecker: ESpellChecker
+    public onlineSpellChecker: LanguageTool
+
     public static ENABLED_ATTR = 'custom-spellcheck-enable'
     public static LANGUAGE_ATTR = 'custom-spellcheck-language'
 
@@ -24,11 +31,14 @@ export default class SpellCheckPlugin extends Plugin {
 
         this.i18nx = this.i18n
         new Icons(this);
+
         this.settingsUtil = await Settings.init(this)
         this.analytics = new Analytics(this.settingsUtil.get('analytics'));
         this.suggestions = new SuggestionEngine(this)
         this.menus = new Menus(this)
 
+        await this.prepareSpellCheckers()
+
         void this.analytics.sendEvent('load')
 
         const style = document.createElement('style');
@@ -83,7 +93,6 @@ export default class SpellCheckPlugin extends Plugin {
         })
 
         this.eventBus.on('open-menu-doctree', async (event) => {
-            console.log(event)
             const docID = ProtyleHelpers.getNodeId(event.detail.elements[0]) // @TODO this is ugly, why does the event not carry the docID?
             void this.menus.addSettingsToDocMenu(docID, event.detail.menu)
         })
@@ -119,4 +128,24 @@ export default class SpellCheckPlugin extends Plugin {
         void this.analytics.sendEvent('uninstall');
     }
 
+    private async prepareSpellCheckers() {
+
+        this.onlineSpellChecker = new LanguageTool(<LanguageToolSettings>this.settingsUtil.dump())
+        const offlineLanguages = this.settingsUtil.get('offlineDicts').split(',')
+
+        let langs: {aff: string, dic: string, language: Language}[] = []
+
+        try {
+            for(const lang of offlineLanguages) {
+                const { aff, dic } = await HunspellDictManager.loadDictionary(lang, true)
+                langs.push({aff: aff, dic: dic, language: {name: lang, code: lang, longCode: lang}})
+            }
+            this.offlineSpellChecker = new ESpellChecker(langs)
+        }catch (e){
+            console.error(e)
+            showMessage(this.i18nx.errors.hunspellLoadError + e, -1, 'error')
+        }
+
+    }
+
 }

src/languagetool.ts

@@ -1,6 +1,7 @@
-import {PluginSettings} from "@/settings";
+import {Language, Suggestion} from "@/spellChecker";
+import {SpellChecker} from "@/spellChecker";
 
-export type Suggestion = {
+type LanguageToolSuggestion = {
     message: string
     shortMessage: string
     replacements: Array<{
@@ -32,47 +33,73 @@ export type Suggestion = {
     ignoreForIncompleteSentence: boolean
     contextForSureMatch: number
 }
-
-export type Language = { name: string; code: string; longCode: string; }
 interface HTTPError extends Error {
     status?: number;
 }
+export type LanguageToolSettings = {
+    server: string
+    username: string
+    apiKey: string
+    picky: boolean
+    motherTongue: string
+    preferredVariants: string
+}
 
-export class LanguageTool {
+export class LanguageTool implements SpellChecker {
 
-    public static async check(text: string, language: string, settings: PluginSettings): Promise<Suggestion[]> {
+    private settings: LanguageToolSettings;
+
+    constructor(settings: LanguageToolSettings) {
+        this.settings = settings
+    }
+
+    public async check(text: string, languages: string[]): Promise<Suggestion[]> {
+
+        const language = languages.length > 0 ? languages[0] : 'auto';
 
         const body = new URLSearchParams({
             text: text,
             language: language,
-            level: settings.picky ? 'picky' : 'default',
-            motherTongue: settings.motherTongue == '' ? window.navigator.language : settings.motherTongue,
+            level: this.settings.picky ? 'picky' : 'default',
+            motherTongue: this.settings.motherTongue == '' ? window.navigator.language : this.settings.motherTongue,
         });
 
-        if(settings.username != '') {
-            body.append('username', settings.username);
+        if(this.settings.username != '') {
+            body.append('username', this.settings.username);
         }
-        if(settings.apiKey) {
-            body.append('apiKey', settings.apiKey);
+        if(this.settings.apiKey) {
+            body.append('apiKey', this.settings.apiKey);
         }
         if(language == 'auto') {
-            body.append('preferredVariants', settings.preferredVariants)
+            body.append('preferredVariants', this.settings.preferredVariants)
         }
 
-        const res = await fetch(settings.server + 'v2/check', {method: 'POST', body});
+        const res = await fetch(this.settings.server + 'v2/check', {method: 'POST', body});
         if(res.status != 200) {
             const err = new Error('Network error') as HTTPError
             err.status  = res.status;
             throw err
         }
 
-        const json = await res.json();
-        return json.matches;
+        const suggestions: LanguageToolSuggestion[] = (await res.json()).matches;
+        return suggestions.map((suggestion) => {
+            const ret: Suggestion  = {
+                message: suggestion.message,
+                shortMessage: suggestion.shortMessage,
+                replacements: suggestion.replacements.map((replacement) => {
+                    return replacement.value
+                }),
+                offset: suggestion.offset,
+                length: suggestion.length,
+                typeName: suggestion.type.typeName
+            }
+            return ret
+        });
 
     }
 
-    public static async getLanguages(settings: PluginSettings): Promise<Language[]> {
-        const res = await fetch(settings.server + 'v2/languages', {method: 'GET'});
+    public async getLanguages(): Promise<Language[]> {
+        const res = await fetch(this.settings.server + 'v2/languages', {method: 'GET'});
         return await res.json();
     }
 

src/menus.ts

@@ -1,8 +1,7 @@
 import {Menu, showMessage, subMenu} from 'siyuan';
 import SpellCheckPlugin from "@/index";
 import {getBlockAttrs, setBlockAttrs} from "@/api";
-import {LanguageTool} from "@/languagetool";
-import {PluginSettings, Settings} from "@/settings";
+import {Settings} from "@/settings";
 import {ProtyleHelpers} from "@/protyleHelpers";
 import {SuggestionEngine} from "@/suggestions";
 
@@ -31,14 +30,14 @@ export class Menus {
             }
         })
 
-        if(suggestion.type.typeName == 'UnknownWord') {
+        if(suggestion.typeName == 'UnknownWord') {
             // add to dictionary
             menu.addItem({
                 icon: 'add',
                 label: this.plugin.i18nx.textMenu.addToDictionary,
                 click: async () => {
                     void this.plugin.analytics.sendEvent('menu-click-add-to-dictionary');
-                    const word = SuggestionEngine.suggestionToWrongText(suggestion)
+                    const word = SuggestionEngine.suggestionToWrongText(suggestion, blockID)
                     await Settings.addToDictionary(word, this.plugin.settingsUtil)
                     showMessage(this.plugin.i18nx.textMenu.addedToDictionary + word, 5000, 'info')
                     await this.plugin.suggestions.renderSuggestions(blockID)
@@ -50,15 +49,15 @@ export class Menus {
         suggestion.replacements.forEach((replacement, correctionNumber) => {
             menu.addItem({
                 icon: 'spellcheck',
-                label: replacement.value,
+                label: replacement,
                 click: async () => {
                     void this.plugin.analytics.sendEvent('menu-click-correct', {
-                        'type': suggestion.rule.category.id
+                        'type': suggestion.typeName
                     });
                     if(this.plugin.settingsUtil.get('experimentalCorrect')) {
                         void this.plugin.suggestions.correctSuggestion(blockID, suggestionNumber, correctionNumber)
                     }else{
-                        void navigator.clipboard.writeText(replacement.value)
+                        void navigator.clipboard.writeText(replacement)
                         showMessage(this.plugin.i18nx.errors.correctionNotEnabled, 5000, 'info')
                     }
                 }
@@ -111,7 +110,7 @@ export class Menus {
             label: this.plugin.i18nx.docMenu.setDocumentLanguage,
             click: async (_, ev: MouseEvent) => {
                 void this.plugin.analytics.sendEvent('docmenu-click-setlang-1');
-                const languages = await LanguageTool.getLanguages(<PluginSettings>this.plugin.settingsUtil.dump())
+                const languages = await this.plugin.onlineSpellChecker.getLanguages()
                 const langMenu = new Menu('spellCheckLangMenu');
                 langMenu.addItem({
                     icon: 'autodetect',

src/settings.ts

@@ -1,21 +1,7 @@
 import {SettingUtils} from "@/libs/setting-utils";
 import {showMessage} from 'siyuan';
-import {LanguageTool} from "@/languagetool";
 import SpellCheckPlugin from "@/index";
-
-export type PluginSettings = {
-    server: string
-    username: string
-    apiKey: string
-    picky: boolean
-    motherTongue: string
-    preferredVariants: string
-    enabledByDefault: boolean
-    defaultLanguage: string
-    preferredLanguages: string
-    analytics: boolean
-}
-
+import {LanguageTool, LanguageToolSettings} from "@/languagetool";
 
 export class Settings {
 
@@ -61,13 +47,12 @@ export class Settings {
         await su.load() // needed to fetch languages from server
         let languagesKV = {}
         try {
-            let languages = await LanguageTool.getLanguages(<PluginSettings>su.dump())
+            let languages = await new LanguageTool(<LanguageToolSettings>{server: su.get('server')}).getLanguages()
             languages.forEach(language => {
                 languagesKV[language.longCode] = language.name + ' [' + language.longCode + ']'
             })
-        } catch {
+        } catch(e) {
             showMessage(plugin.i18nx.errors.checkServer, -1, 'error')
-            showMessage(plugin.i18nx.errors.fatal, -1, 'error')
         }
 
         su.addItem({
@@ -129,6 +114,22 @@ export class Settings {
             value: 'auto'
         })
 
+        su.addItem({
+            type: 'checkbox',
+            key: 'offline',
+            title: to.offline.title,
+            description: to.offline.description,
+            value: false
+        })
+
+        su.addItem({
+            type: 'textinput',
+            key: 'offlineDicts',
+            title: to.offlineDicts.title,
+            description: to.offlineDicts.description,
+            value: 'en'
+        })
+
         su.addItem({
             type: 'checkbox',
             key: 'analytics',
@@ -137,6 +138,12 @@ export class Settings {
             value: true
         })
 
+        su.save = async function (data?: any) {
+            data = data ?? this.dump();
+            await this.plugin.saveData(this.file, this.dump());
+            location.reload()
+        }.bind(su)
+
         await su.load()
         return su
 

src/spellChecker.ts

@@ -0,0 +1,15 @@
+export type Language = { name: string; code: string; longCode: string; }
+
+export type Suggestion = {
+    message: string
+    shortMessage: string
+    replacements: string[]
+    offset: number
+    length: number
+    typeName: string
+}
+
+export interface SpellChecker {
+    check(text: string, languages: string[]): Promise<Suggestion[]>
+    getLanguages(): Promise<Language[]>
+}

src/spellCheckerUI.ts

@@ -1,6 +1,6 @@
 import {ProtyleHelpers} from "@/protyleHelpers";
 
-export class SpellChecker {
+export class SpellCheckerUI {
 
     private readonly blockID: string;
     private readonly docID: string;
@@ -111,13 +111,13 @@ export class SpellChecker {
             const top = rect.bottom - editorRect.top - 2 + this.block.scrollTop;
             const width = rect.width;
 
-            const offset = SpellChecker.distance(this.overlay, this.block)
+            const offset = SpellCheckerUI.distance(this.overlay, this.block)
 
             underline.style.left = (left + offset.h) + 'px';
             underline.style.top = (top + 2 + offset.v) + 'px';
             underline.style.width = width + 'px';
 
-            if(!SpellChecker.checkDontUnderline(width, charsCount)) {
+            if(!SpellCheckerUI.checkDontUnderline(width, charsCount)) {
                 this.overlay.appendChild(underline);
             }
         }

src/suggestions.ts

@@ -1,13 +1,13 @@
 import {ProtyleHelpers} from "@/protyleHelpers";
-import {LanguageTool, Suggestion} from "@/languagetool";
-import {PluginSettings, Settings} from "@/settings";
+import {Settings} from "@/settings";
 import {getChildBlocks, updateBlock} from "@/api";
-import {SpellChecker} from "@/spellchecker";
+import {SpellCheckerUI} from "@/spellCheckerUI";
 import {showMessage} from "siyuan";
 import SpellCheckPlugin from "@/index";
+import {Suggestion} from "@/spellChecker";
 
 interface StoredBlock {
-    spellChecker: SpellChecker;
+    spellChecker: SpellCheckerUI;
     suggestions: Suggestion[];
 }
 
@@ -40,7 +40,7 @@ export class SuggestionEngine {
         const children = await getChildBlocks(blockID)
         if(children.length == 0) {
             if(!(blockID in this.blockStorage)) {
-                const spellChecker = new SpellChecker(blockID, this.documentID)
+                const spellChecker = new SpellCheckerUI(blockID, this.documentID)
                 this.blockStorage[blockID] = {
                     spellChecker: spellChecker,
                     suggestions: []
@@ -90,11 +90,16 @@ export class SuggestionEngine {
             return this.suggestForBlock(blockID)
         }
 
-        try {
-            suggestions = await LanguageTool.check(text, this.documentLanguage, <PluginSettings>this.plugin.settingsUtil.dump())
-        }catch (_) {
-            showMessage(this.plugin.i18nx.errors.checkServer, 5000, 'error')
+        if(this.plugin.settingsUtil.get('offline')) {
+            suggestions = await this.plugin.offlineSpellChecker.check(text, [this.documentLanguage])
+        }else{
+            try {
+                suggestions = await this.plugin.onlineSpellChecker.check(text, [this.documentLanguage])
+            }catch (_) {
+                showMessage(this.plugin.i18nx.errors.checkServer, 5000, 'error')
+            }
         }
+
         this.blockStorage[blockID].suggestions = suggestions
 
     }
@@ -109,14 +114,15 @@ export class SuggestionEngine {
         }
         this.blockStorage[blockID].spellChecker.clearUnderlines()
         this.blockStorage[blockID].suggestions.forEach(suggestion => {
-            if(!Settings.isInCustomDictionary(SuggestionEngine.suggestionToWrongText(suggestion), this.plugin.settingsUtil)) {
+            if(!Settings.isInCustomDictionary(SuggestionEngine.suggestionToWrongText(suggestion, blockID), this.plugin.settingsUtil)) {
                 this.blockStorage[blockID].spellChecker.highlightCharacterRange(suggestion.offset, suggestion.offset + suggestion.length)
             }
         })
     }
 
-    static suggestionToWrongText(suggestion: Suggestion): string {
-        return suggestion.context.text.slice(suggestion.context.offset, suggestion.context.offset + suggestion.context.length)
+    static suggestionToWrongText(suggestion: Suggestion, blockID: string): string {
+        const blockTxt = ProtyleHelpers.fastGetBlockText(blockID)
+        return blockTxt.slice(suggestion.offset, suggestion.offset + suggestion.length)
     }
 
     private getAbsoluteOffsetInBlock(range: Range, blockID: string): number {
@@ -162,7 +168,7 @@ export class SuggestionEngine {
         const suggestion = this.blockStorage[blockID].suggestions[suggestionNumber]
         const rich = ProtyleHelpers.fastGetBlockHTML(blockID)
         const fixedOffset = this.adjustIndexForTags(rich, suggestion.offset)
-        const newStr = rich.slice(0, fixedOffset) + suggestion.replacements[correctionNumber].value + rich.slice(fixedOffset + suggestion.length)
+        const newStr = rich.slice(0, fixedOffset) + suggestion.replacements[correctionNumber] + rich.slice(fixedOffset + suggestion.length)
 
         console.log("new str " + newStr);
         await updateBlock('markdown', window.Lute.New().BlockDOM2Md(newStr), blockID)