Add offline spell-checking

src/espells.ts

@@ -0,0 +1,45 @@
+import {Language, SpellChecker, Suggestion} from "@/spellChecker";
+import { Espells } from "espells"
+
+export class ESpellChecker implements SpellChecker {
+
+    spellchecker: Espells
+    loadedLanguages: Language[]
+
+    constructor(languages: {aff: string, dic: string, language: Language}[]) {
+        this.spellchecker = new Espells({aff: languages[0].aff, dic: languages.map(l => l.dic)})
+        this.loadedLanguages = languages.map(l => l.language)
+    }
+
+    async check(text: string, _: string[]): Promise<Suggestion[]> {
+
+        let suggestions: Suggestion[] = []
+
+        const regex =   /[\w']+/g;
+        let match;
+
+        while ((match = regex.exec(text)) !== null) {
+            const word = match[0];
+            const counter = match.index;
+            const {correct} = this.spellchecker.lookup(word)
+            if(!correct) {
+                const hsSuggestions = this.spellchecker.suggest(word)
+                    suggestions.push({
+                        typeName: "UnknownWord",
+                        message: word,
+                        shortMessage: "Misspelled word",
+                        replacements: hsSuggestions,
+                        offset: counter,
+                        length: word.length
+                    })
+            }
+        }
+
+        return suggestions
+    }
+
+    async getLanguages(): Promise<Language[]> {
+        return this.loadedLanguages
+    }
+
+}

src/hunspellDictManager.ts

@@ -0,0 +1,51 @@
+import {getFile, putFile} from "@/api";
+
+export class HunspellDictManager {
+
+    private static pathBase = 'data/storage/petal/syspell'
+    private static urlBase = 'https://raw.githubusercontent.com/wooorm/dictionaries/refs/heads/main/dictionaries'
+
+    static async loadDictionary(language: string, downloadIfMissing: boolean): Promise<{ aff: string, dic: string }> {
+
+        const aff = await getFile(`${this.pathBase}/${language}.aff`)
+        const dic = await getFile(`${this.pathBase}/${language}.dic`)
+
+        if(aff.code == 404 || dic.code == 404) {
+            if(downloadIfMissing) {
+                await this.downloadDictionary(language)
+                return this.loadDictionary(language, false)
+            }else{
+                throw new Error(`Dictionary ${language} not found`)
+            }
+        }
+
+        return { aff, dic }
+
+    }
+
+    private static async downloadFile(url: string, filename: string) {
+
+        const res = await fetch(url);
+        const mimeType = res.headers.get('content-type')
+
+        if(res.status != 200) {
+            throw new Error(await res.text())
+        }
+
+        const blob = new Blob([await res.text()], { type: mimeType });
+        const file =  new File([blob], filename, { type: mimeType, lastModified: Date.now() });
+
+        await putFile(filename, false, file)
+
+    }
+
+    static async downloadDictionary(language: string) {
+        try {
+            await this.downloadFile(`${this.urlBase}/${language}/index.aff`, `${this.pathBase}/${language}.aff`);
+            await this.downloadFile(`${this.urlBase}/${language}/index.dic`, `${this.pathBase}/${language}.dic`);
+        }catch (e) {
+            throw new Error(`Download for dictionary '${language}' failed with ` + e)
+        }
+    }
+
+}

src/index.ts

@@ -6,6 +6,10 @@ import {SettingUtils} from "@/libs/setting-utils";
 import {Analytics} from "@/analytics";
 import {SuggestionEngine} from "@/suggestions";
 import {Menus} from "@/menus";
+import {ESpellChecker} from "@/espells";
+import {LanguageTool, LanguageToolSettings} from "@/languagetool";
+import {HunspellDictManager} from "@/hunspellDictManager";
+import {Language} from "@/spellChecker";
 
 
 export default class SpellCheckPlugin extends Plugin {
@@ -17,6 +21,9 @@ export default class SpellCheckPlugin extends Plugin {
     public analytics: Analytics
     public i18nx: any; // This object is just a copy of i18n, but with type "any" to not trigger type errors
 
+    public offlineSpellChecker: ESpellChecker
+    public onlineSpellChecker: LanguageTool
+
     public static ENABLED_ATTR = 'custom-spellcheck-enable'
     public static LANGUAGE_ATTR = 'custom-spellcheck-language'
 
@@ -24,11 +31,14 @@ export default class SpellCheckPlugin extends Plugin {
 
         this.i18nx = this.i18n
         new Icons(this);
+
         this.settingsUtil = await Settings.init(this)
         this.analytics = new Analytics(this.settingsUtil.get('analytics'));
         this.suggestions = new SuggestionEngine(this)
         this.menus = new Menus(this)
 
+        await this.prepareSpellCheckers()
+
         void this.analytics.sendEvent('load')
 
         const style = document.createElement('style');
@@ -83,7 +93,6 @@ export default class SpellCheckPlugin extends Plugin {
         })
 
         this.eventBus.on('open-menu-doctree', async (event) => {
-            console.log(event)
             const docID = ProtyleHelpers.getNodeId(event.detail.elements[0]) // @TODO this is ugly, why does the event not carry the docID?
             void this.menus.addSettingsToDocMenu(docID, event.detail.menu)
         })
@@ -119,4 +128,24 @@ export default class SpellCheckPlugin extends Plugin {
         void this.analytics.sendEvent('uninstall');
     }
 
+    private async prepareSpellCheckers() {
+
+        this.onlineSpellChecker = new LanguageTool(<LanguageToolSettings>this.settingsUtil.dump())
+        const offlineLanguages = this.settingsUtil.get('offlineDicts').split(',')
+
+        let langs: {aff: string, dic: string, language: Language}[] = []
+
+        try {
+            for(const lang of offlineLanguages) {
+                const { aff, dic } = await HunspellDictManager.loadDictionary(lang, true)
+                langs.push({aff: aff, dic: dic, language: {name: lang, code: lang, longCode: lang}})
+            }
+            this.offlineSpellChecker = new ESpellChecker(langs)
+        }catch (e){
+            console.error(e)
+            showMessage(this.i18nx.errors.hunspellLoadError + e, -1, 'error')
+        }
+
+    }
+
 }

src/languagetool.ts

@@ -1,6 +1,7 @@
-import {PluginSettings} from "@/settings";
+import {Language, Suggestion} from "@/spellChecker";
+import {SpellChecker} from "@/spellChecker";
 
-export type Suggestion = {
+type LanguageToolSuggestion = {
     message: string
     shortMessage: string
     replacements: Array<{
@@ -32,47 +33,73 @@ export type Suggestion = {
     ignoreForIncompleteSentence: boolean
     contextForSureMatch: number
 }
-
-export type Language = { name: string; code: string; longCode: string; }
 interface HTTPError extends Error {
     status?: number;
 }
+export type LanguageToolSettings = {
+    server: string
+    username: string
+    apiKey: string
+    picky: boolean
+    motherTongue: string
+    preferredVariants: string
+}
 
-export class LanguageTool {
+export class LanguageTool implements SpellChecker {
 
-    public static async check(text: string, language: string, settings: PluginSettings): Promise<Suggestion[]> {
+    private settings: LanguageToolSettings;
+
+    constructor(settings: LanguageToolSettings) {
+        this.settings = settings
+    }
+
+    public async check(text: string, languages: string[]): Promise<Suggestion[]> {
+
+        const language = languages.length > 0 ? languages[0] : 'auto';
 
         const body = new URLSearchParams({
             text: text,
             language: language,
-            level: settings.picky ? 'picky' : 'default',
-            motherTongue: settings.motherTongue == '' ? window.navigator.language : settings.motherTongue,
+            level: this.settings.picky ? 'picky' : 'default',
+            motherTongue: this.settings.motherTongue == '' ? window.navigator.language : this.settings.motherTongue,
         });
 
-        if(settings.username != '') {
-            body.append('username', settings.username);
+        if(this.settings.username != '') {
+            body.append('username', this.settings.username);
         }
-        if(settings.apiKey) {
-            body.append('apiKey', settings.apiKey);
+        if(this.settings.apiKey) {
+            body.append('apiKey', this.settings.apiKey);
         }
         if(language == 'auto') {
-            body.append('preferredVariants', settings.preferredVariants)
+            body.append('preferredVariants', this.settings.preferredVariants)
         }
 
-        const res = await fetch(settings.server + 'v2/check', {method: 'POST', body});
+        const res = await fetch(this.settings.server + 'v2/check', {method: 'POST', body});
         if(res.status != 200) {
             const err = new Error('Network error') as HTTPError
             err.status  = res.status;
             throw err
         }
 
-        const json = await res.json();
-        return json.matches;
+        const suggestions: LanguageToolSuggestion[] = (await res.json()).matches;
+        return suggestions.map((suggestion) => {
+            const ret: Suggestion  = {
+                message: suggestion.message,
+                shortMessage: suggestion.shortMessage,
+                replacements: suggestion.replacements.map((replacement) => {
+                    return replacement.value
+                }),
+                offset: suggestion.offset,
+                length: suggestion.length,
+                typeName: suggestion.type.typeName
+            }
+            return ret
+        });
 
     }
 
-    public static async getLanguages(settings: PluginSettings): Promise<Language[]> {
-        const res = await fetch(settings.server + 'v2/languages', {method: 'GET'});
+    public async getLanguages(): Promise<Language[]> {
+        const res = await fetch(this.settings.server + 'v2/languages', {method: 'GET'});
         return await res.json();
     }
 

src/menus.ts

@@ -1,8 +1,7 @@
 import {Menu, showMessage, subMenu} from 'siyuan';
 import SpellCheckPlugin from "@/index";
 import {getBlockAttrs, setBlockAttrs} from "@/api";
-import {LanguageTool} from "@/languagetool";
-import {PluginSettings, Settings} from "@/settings";
+import {Settings} from "@/settings";
 import {ProtyleHelpers} from "@/protyleHelpers";
 import {SuggestionEngine} from "@/suggestions";
 
@@ -31,14 +30,14 @@ export class Menus {
             }
         })
 
-        if(suggestion.type.typeName == 'UnknownWord') {
+        if(suggestion.typeName == 'UnknownWord') {
             // add to dictionary
             menu.addItem({
                 icon: 'add',
                 label: this.plugin.i18nx.textMenu.addToDictionary,
                 click: async () => {
                     void this.plugin.analytics.sendEvent('menu-click-add-to-dictionary');
-                    const word = SuggestionEngine.suggestionToWrongText(suggestion)
+                    const word = SuggestionEngine.suggestionToWrongText(suggestion, blockID)
                     await Settings.addToDictionary(word, this.plugin.settingsUtil)
                     showMessage(this.plugin.i18nx.textMenu.addedToDictionary + word, 5000, 'info')
                     await this.plugin.suggestions.renderSuggestions(blockID)
@@ -50,15 +49,15 @@ export class Menus {
         suggestion.replacements.forEach((replacement, correctionNumber) => {
             menu.addItem({
                 icon: 'spellcheck',
-                label: replacement.value,
+                label: replacement,
                 click: async () => {
                     void this.plugin.analytics.sendEvent('menu-click-correct', {
-                        'type': suggestion.rule.category.id
+                        'type': suggestion.typeName
                     });
                     if(this.plugin.settingsUtil.get('experimentalCorrect')) {
                         void this.plugin.suggestions.correctSuggestion(blockID, suggestionNumber, correctionNumber)
                     }else{
-                        void navigator.clipboard.writeText(replacement.value)
+                        void navigator.clipboard.writeText(replacement)
                         showMessage(this.plugin.i18nx.errors.correctionNotEnabled, 5000, 'info')
                     }
                 }
@@ -111,7 +110,7 @@ export class Menus {
             label: this.plugin.i18nx.docMenu.setDocumentLanguage,
             click: async (_, ev: MouseEvent) => {
                 void this.plugin.analytics.sendEvent('docmenu-click-setlang-1');
-                const languages = await LanguageTool.getLanguages(<PluginSettings>this.plugin.settingsUtil.dump())
+                const languages = await this.plugin.onlineSpellChecker.getLanguages()
                 const langMenu = new Menu('spellCheckLangMenu');
                 langMenu.addItem({
                     icon: 'autodetect',

src/settings.ts

@@ -1,21 +1,7 @@
 import {SettingUtils} from "@/libs/setting-utils";
 import {showMessage} from 'siyuan';
-import {LanguageTool} from "@/languagetool";
 import SpellCheckPlugin from "@/index";
-
-export type PluginSettings = {
-    server: string
-    username: string
-    apiKey: string
-    picky: boolean
-    motherTongue: string
-    preferredVariants: string
-    enabledByDefault: boolean
-    defaultLanguage: string
-    preferredLanguages: string
-    analytics: boolean
-}
-
+import {LanguageTool, LanguageToolSettings} from "@/languagetool";
 
 export class Settings {
 
@@ -61,13 +47,12 @@ export class Settings {
         await su.load() // needed to fetch languages from server
         let languagesKV = {}
         try {
-            let languages = await LanguageTool.getLanguages(<PluginSettings>su.dump())
+            let languages = await new LanguageTool(<LanguageToolSettings>{server: su.get('server')}).getLanguages()
             languages.forEach(language => {
                 languagesKV[language.longCode] = language.name + ' [' + language.longCode + ']'
             })
-        } catch {
+        } catch(e) {
             showMessage(plugin.i18nx.errors.checkServer, -1, 'error')
-            showMessage(plugin.i18nx.errors.fatal, -1, 'error')
         }
 
         su.addItem({
@@ -129,6 +114,22 @@ export class Settings {
             value: 'auto'
         })
 
+        su.addItem({
+            type: 'checkbox',
+            key: 'offline',
+            title: to.offline.title,
+            description: to.offline.description,
+            value: false
+        })
+
+        su.addItem({
+            type: 'textinput',
+            key: 'offlineDicts',
+            title: to.offlineDicts.title,
+            description: to.offlineDicts.description,
+            value: 'en'
+        })
+
         su.addItem({
             type: 'checkbox',
             key: 'analytics',
@@ -137,6 +138,12 @@ export class Settings {
             value: true
         })
 
+        su.save = async function (data?: any) {
+            data = data ?? this.dump();
+            await this.plugin.saveData(this.file, this.dump());
+            location.reload()
+        }.bind(su)
+
         await su.load()
         return su
 

src/spellChecker.ts

@@ -0,0 +1,15 @@
+export type Language = { name: string; code: string; longCode: string; }
+
+export type Suggestion = {
+    message: string
+    shortMessage: string
+    replacements: string[]
+    offset: number
+    length: number
+    typeName: string
+}
+
+export interface SpellChecker {
+    check(text: string, languages: string[]): Promise<Suggestion[]>
+    getLanguages(): Promise<Language[]>
+}

src/spellCheckerUI.ts

@@ -1,6 +1,6 @@
 import {ProtyleHelpers} from "@/protyleHelpers";
 
-export class SpellChecker {
+export class SpellCheckerUI {
 
     private readonly blockID: string;
     private readonly docID: string;
@@ -111,13 +111,13 @@ export class SpellChecker {
             const top = rect.bottom - editorRect.top - 2 + this.block.scrollTop;
             const width = rect.width;
 
-            const offset = SpellChecker.distance(this.overlay, this.block)
+            const offset = SpellCheckerUI.distance(this.overlay, this.block)
 
             underline.style.left = (left + offset.h) + 'px';
             underline.style.top = (top + 2 + offset.v) + 'px';
             underline.style.width = width + 'px';
 
-            if(!SpellChecker.checkDontUnderline(width, charsCount)) {
+            if(!SpellCheckerUI.checkDontUnderline(width, charsCount)) {
                 this.overlay.appendChild(underline);
             }
         }

src/suggestions.ts

@@ -1,13 +1,13 @@
 import {ProtyleHelpers} from "@/protyleHelpers";
-import {LanguageTool, Suggestion} from "@/languagetool";
-import {PluginSettings, Settings} from "@/settings";
+import {Settings} from "@/settings";
 import {getChildBlocks, updateBlock} from "@/api";
-import {SpellChecker} from "@/spellchecker";
+import {SpellCheckerUI} from "@/spellCheckerUI";
 import {showMessage} from "siyuan";
 import SpellCheckPlugin from "@/index";
+import {Suggestion} from "@/spellChecker";
 
 interface StoredBlock {
-    spellChecker: SpellChecker;
+    spellChecker: SpellCheckerUI;
     suggestions: Suggestion[];
 }
 
@@ -40,7 +40,7 @@ export class SuggestionEngine {
         const children = await getChildBlocks(blockID)
         if(children.length == 0) {
             if(!(blockID in this.blockStorage)) {
-                const spellChecker = new SpellChecker(blockID, this.documentID)
+                const spellChecker = new SpellCheckerUI(blockID, this.documentID)
                 this.blockStorage[blockID] = {
                     spellChecker: spellChecker,
                     suggestions: []
@@ -90,11 +90,16 @@ export class SuggestionEngine {
             return this.suggestForBlock(blockID)
         }
 
-        try {
-            suggestions = await LanguageTool.check(text, this.documentLanguage, <PluginSettings>this.plugin.settingsUtil.dump())
-        }catch (_) {
-            showMessage(this.plugin.i18nx.errors.checkServer, 5000, 'error')
+        if(this.plugin.settingsUtil.get('offline')) {
+            suggestions = await this.plugin.offlineSpellChecker.check(text, [this.documentLanguage])
+        }else{
+            try {
+                suggestions = await this.plugin.onlineSpellChecker.check(text, [this.documentLanguage])
+            }catch (_) {
+                showMessage(this.plugin.i18nx.errors.checkServer, 5000, 'error')
+            }
         }
+
         this.blockStorage[blockID].suggestions = suggestions
 
     }
@@ -109,14 +114,15 @@ export class SuggestionEngine {
         }
         this.blockStorage[blockID].spellChecker.clearUnderlines()
         this.blockStorage[blockID].suggestions.forEach(suggestion => {
-            if(!Settings.isInCustomDictionary(SuggestionEngine.suggestionToWrongText(suggestion), this.plugin.settingsUtil)) {
+            if(!Settings.isInCustomDictionary(SuggestionEngine.suggestionToWrongText(suggestion, blockID), this.plugin.settingsUtil)) {
                 this.blockStorage[blockID].spellChecker.highlightCharacterRange(suggestion.offset, suggestion.offset + suggestion.length)
             }
         })
     }
 
-    static suggestionToWrongText(suggestion: Suggestion): string {
-        return suggestion.context.text.slice(suggestion.context.offset, suggestion.context.offset + suggestion.context.length)
+    static suggestionToWrongText(suggestion: Suggestion, blockID: string): string {
+        const blockTxt = ProtyleHelpers.fastGetBlockText(blockID)
+        return blockTxt.slice(suggestion.offset, suggestion.offset + suggestion.length)
     }
 
     private getAbsoluteOffsetInBlock(range: Range, blockID: string): number {
@@ -162,7 +168,7 @@ export class SuggestionEngine {
         const suggestion = this.blockStorage[blockID].suggestions[suggestionNumber]
         const rich = ProtyleHelpers.fastGetBlockHTML(blockID)
         const fixedOffset = this.adjustIndexForTags(rich, suggestion.offset)
-        const newStr = rich.slice(0, fixedOffset) + suggestion.replacements[correctionNumber].value + rich.slice(fixedOffset + suggestion.length)
+        const newStr = rich.slice(0, fixedOffset) + suggestion.replacements[correctionNumber] + rich.slice(fixedOffset + suggestion.length)
 
         console.log("new str " + newStr);
         await updateBlock('markdown', window.Lute.New().BlockDOM2Md(newStr), blockID)