diff --git a/.gitignore b/.gitignore index 3cdaadf..aac6186 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ disabled .env +/sanitize diff --git a/README.md b/README.md new file mode 100644 index 0000000..b5c2d87 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +In order to test the word lists for suspicious characters, you have to copy the sanitize folder from scribble.rs to sanitize/ and then run `go run test-words.go` diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7ea07dc --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module test-words + +go 1.26.5 + +require golang.org/x/text v0.41.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..5a94f53 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= diff --git a/test-words.go b/test-words.go new file mode 100644 index 0000000..e682766 --- /dev/null +++ b/test-words.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + "os" + "io/fs" + "path" + "bufio" + "log" + sanitize "test-words/sanitize" +) + +var alphabet = "abcdefghijklmnopqrstuvwxyz0123456789" + +func listWordLists(fsys fs.FS, basepath string, in []string) []string { + files, _ := fs.ReadDir(fsys, ".") + + out := in + for _, file := range files { + fname := file.Name() + fpath := path.Join(basepath, fname) + if fname == "scribblers" { + continue + } + if file.IsDir() { + subfs, _ := fs.Sub(fsys, fname) + out = listWordLists(subfs, fpath, out) + } else { + out = append(out, fpath) + } + } + return out +} + +func readFileWordList(path string) ([]string, error) { + file, err := os.Open(path) + if err != nil { + return nil,err + } + defer file.Close() + + lines := []string{} + scanner := bufio.NewScanner(file) + for scanner.Scan() { + lines = append(lines, scanner.Text()) + } + return lines,scanner.Err() +} + + +func InAlphabet(c rune) bool { + for _, x := range alphabet { + if x == c { + return true + } + } + return false +} + +func main() { + lang := sanitize.AllLanguageData["french"] + basePath := "./lists/" + thisFolder := os.DirFS(basePath) + thisFolder = thisFolder.(fs.ReadDirFS) + mainList := listWordLists(thisFolder, basePath, []string{}) + fmt.Println("Read wordlists:", len(mainList)) + + for _, wl := range mainList { + words, err := readFileWordList(wl) + if err != nil { + log.Println("Could not read wordlist %s", wl) + os.Exit(1) + } + log.Printf("%s -> %d words", wl, len(words)) + + for i, w := range words { + if w == "" { + log.Printf("Found empty word in list %s, line %d", wl, i) + os.Exit(2) + } + + w = lang.Lowercaser().String(w) + for _, character := range w { + if lang.IsAlwaysVisibleCharacter(character) { + continue + } + if _, contains := lang.Transliterations[character]; contains { + continue + } + if InAlphabet(character) { + continue + } + log.Printf("Suspicious character '%s' in '%s' (%s, line %d)", character, w, wl, i) + } + } + } +}