98 lines
1.9 KiB
Go
98 lines
1.9 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"io/fs"
|
|
"path"
|
|
"bufio"
|
|
"log"
|
|
sanitize "test-words/sanitize"
|
|
)
|
|
|
|
var alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"
|
|
|
|
func listWordLists(fsys fs.FS, basepath string, in []string) []string {
|
|
files, _ := fs.ReadDir(fsys, ".")
|
|
|
|
out := in
|
|
for _, file := range files {
|
|
fname := file.Name()
|
|
fpath := path.Join(basepath, fname)
|
|
if fname == "scribblers" {
|
|
continue
|
|
}
|
|
if file.IsDir() {
|
|
subfs, _ := fs.Sub(fsys, fname)
|
|
out = listWordLists(subfs, fpath, out)
|
|
} else {
|
|
out = append(out, fpath)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func readFileWordList(path string) ([]string, error) {
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return nil,err
|
|
}
|
|
defer file.Close()
|
|
|
|
lines := []string{}
|
|
scanner := bufio.NewScanner(file)
|
|
for scanner.Scan() {
|
|
lines = append(lines, scanner.Text())
|
|
}
|
|
return lines,scanner.Err()
|
|
}
|
|
|
|
|
|
func InAlphabet(c rune) bool {
|
|
for _, x := range alphabet {
|
|
if x == c {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func main() {
|
|
lang := sanitize.AllLanguageData["french"]
|
|
basePath := "./lists/"
|
|
thisFolder := os.DirFS(basePath)
|
|
thisFolder = thisFolder.(fs.ReadDirFS)
|
|
mainList := listWordLists(thisFolder, basePath, []string{})
|
|
fmt.Println("Read wordlists:", len(mainList))
|
|
|
|
for _, wl := range mainList {
|
|
words, err := readFileWordList(wl)
|
|
if err != nil {
|
|
log.Println("Could not read wordlist %s", wl)
|
|
os.Exit(1)
|
|
}
|
|
log.Printf("%s -> %d words", wl, len(words))
|
|
|
|
for i, w := range words {
|
|
if w == "" {
|
|
log.Printf("Found empty word in list %s, line %d", wl, i)
|
|
os.Exit(2)
|
|
}
|
|
|
|
w = lang.Lowercaser().String(w)
|
|
for _, character := range w {
|
|
if lang.IsAlwaysVisibleCharacter(character) {
|
|
continue
|
|
}
|
|
if _, contains := lang.Transliterations[character]; contains {
|
|
continue
|
|
}
|
|
if InAlphabet(character) {
|
|
continue
|
|
}
|
|
log.Printf("Suspicious character '%s' in '%s' (%s, line %d)", character, w, wl, i)
|
|
}
|
|
}
|
|
}
|
|
}
|