package main

import (
	"fmt"
	"os"
	"io/fs"
	"path"
	"bufio"
	"log"
	sanitize "test-words/sanitize"
)

var alphabet = "abcdefghijklmnopqrstuvwxyz0123456789"

func listWordLists(fsys fs.FS, basepath string, in []string) []string {
	files, _ := fs.ReadDir(fsys, ".")

	out := in
	for _, file := range files {
		fname := file.Name()
		fpath := path.Join(basepath, fname)
		if fname == "scribblers" {
			continue
		}
		if file.IsDir() {
			subfs, _ := fs.Sub(fsys, fname)
			out = listWordLists(subfs, fpath, out)
		} else {
			out = append(out, fpath)
		}
	}
	return out
}

func readFileWordList(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil,err
	}
	defer file.Close()

	lines := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines,scanner.Err()
}


func InAlphabet(c rune) bool {
	for _, x := range alphabet {
		if x == c {
			return true
		}
	}
	return false
}

func main() {
	lang := sanitize.AllLanguageData["french"]
	basePath := "./lists/"
	thisFolder := os.DirFS(basePath)
	thisFolder = thisFolder.(fs.ReadDirFS)
	mainList := listWordLists(thisFolder, basePath, []string{})
	fmt.Println("Read wordlists:", len(mainList))

	for _, wl := range mainList {
		words, err := readFileWordList(wl)
		if err != nil {
			log.Println("Could not read wordlist %s", wl)
			os.Exit(1)
		}
		log.Printf("%s -> %d words", wl, len(words))

		for i, w := range words {
			if w == "" {
				log.Printf("Found empty word in list %s, line %d", wl, i)
				os.Exit(2)
			}

			w = lang.Lowercaser().String(w)
			for _, character := range w {
				if lang.IsAlwaysVisibleCharacter(character) {
					continue
				}
				if _, contains := lang.Transliterations[character]; contains {
					continue
				}
				if InAlphabet(character) {
					continue
				}
				log.Printf("Suspicious character '%s' in '%s' (%s, line %d)", character, w, wl, i)
			}
		}
	}
}