Added a checker for sanitize

.gitignore

@@ -1,2 +1,3 @@
 disabled
 .env
+/sanitize

README.md

@@ -0,0 +1 @@
+In order to test the word lists for suspicious characters, you have to copy the sanitize folder from scribble.rs to sanitize/ and then run `go run test-words.go`

go.mod

@@ -0,0 +1,5 @@
+module test-words
+
+go 1.26.5
+
+require golang.org/x/text v0.41.0 // indirect

go.sum

@@ -0,0 +1,2 @@
+golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
+golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=

test-words.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"io/fs"
+	"path"
+	"bufio"
+	"log"
+	sanitize "test-words/sanitize"
+)
+
+var alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"
+
+func listWordLists(fsys fs.FS, basepath string, in []string) []string {
+	files, _ := fs.ReadDir(fsys, ".")
+
+	out := in
+	for _, file := range files {
+		fname := file.Name()
+		fpath := path.Join(basepath, fname)
+		if fname == "scribblers" {
+			continue
+		}
+		if file.IsDir() {
+			subfs, _ := fs.Sub(fsys, fname)
+			out = listWordLists(subfs, fpath, out)
+		} else {
+			out = append(out, fpath)
+		}
+	}
+	return out
+}
+
+func readFileWordList(path string) ([]string, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil,err
+	}
+	defer file.Close()
+
+	lines := []string{}
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	return lines,scanner.Err()
+}
+
+
+func InAlphabet(c rune) bool {
+	for _, x := range alphabet {
+		if x == c {
+			return true
+		}
+	}
+	return false
+}
+
+func main() {
+	lang := sanitize.AllLanguageData["french"]
+	basePath := "./lists/"
+	thisFolder := os.DirFS(basePath)
+	thisFolder = thisFolder.(fs.ReadDirFS)
+	mainList := listWordLists(thisFolder, basePath, []string{})
+	fmt.Println("Read wordlists:", len(mainList))
+
+	for _, wl := range mainList {
+		words, err := readFileWordList(wl)
+		if err != nil {
+			log.Println("Could not read wordlist %s", wl)
+			os.Exit(1)
+		}
+		log.Printf("%s -> %d words", wl, len(words))
+
+		for i, w := range words {
+			if w == "" {
+				log.Printf("Found empty word in list %s, line %d", wl, i)
+				os.Exit(2)
+			}
+
+			w = lang.Lowercaser().String(w)
+			for _, character := range w {
+				if lang.IsAlwaysVisibleCharacter(character) {
+					continue
+				}
+				if _, contains := lang.Transliterations[character]; contains {
+					continue
+				}
+				if InAlphabet(character) {
+					continue
+				}
+				log.Printf("Suspicious character '%s' in '%s' (%s, line %d)", character, w, wl, i)
+			}
+		}
+	}
+}