Ajout d'un format de poids dans nltk

This commit is contained in:
Mysaa 2023-11-27 20:28:05 +01:00
parent 84e9f74c3a
commit 7474b8cdb3
Signed by: Mysaa
GPG Key ID: 7054D5D6A90F084F
3 changed files with 854 additions and 1 deletions

View File

@ -21,7 +21,7 @@ Le chat la mange avec ses dents
Avec quoi le chat mange la souris ? Avec quoi le chat mange la souris ?
Le rat donne un fromage à la souris Le rat donne un fromage à la souris
Un fromage est donné par le rat à la souris Un fromage est donné par le rat à la souris
A quelle souris un fromage est donné par le rat ? À quelle souris un fromage est donné par le rat ?
Il le donne à la souris Il le donne à la souris
Il le lui donne Il le lui donne
Il souhaite que mon voisin lui donne le chat Il souhaite que mon voisin lui donne le chat

853
test.ipynb Normal file
View File

@ -0,0 +1,853 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from nltk.ccg import chart, lexicon\n",
"from nltk.ccg.lexicon import CCGLexicon, Token, augParseCategory\n",
"from nltk.ccg.chart import CCGChart,CCGLeafEdge\n",
"from nltk.tree import Tree\n",
"import pandas as pd\n",
"import numpy as np"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Weighed Lexicon"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from warnings import warn\n",
"\n",
"class WeighedToken(Token):\n",
" def __init__(self, token, categ, semantics=None, weight = 1.0):\n",
" super().__init__(token, categ, semantics= semantics)\n",
" self._weight = weight\n",
" def weight(self):\n",
" \"\"\"1.0 is considered the default weight for any token\"\"\"\n",
" try:\n",
" return self._weight\n",
" except AttributeError:\n",
" warn(f\"[{self.token} : {str(self)}] : this token has no weight attribute, defaulted to 1.0.\")\n",
" return 1.0\n",
"\n",
"class WeighedLexicon(CCGLexicon):\n",
" def __init__(self, start, primitives, families, entries):\n",
" super().__init__(start, primitives, families, entries)\n",
"\n",
" def weight(self, entry):\n",
" return entry.weight()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# CYK"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"valz = {\n",
" '>' : 0.8,\n",
" '<' : 0.7\n",
"}\n",
"def rweight(rule):\n",
" s = rule.__str__()\n",
" if s in valz:\n",
" return valz[s]\n",
" else:\n",
" return 1.0 # Base rules weight\n",
"\n",
"# Implements the CYK algorithm, code partly taken from nltk\n",
"def weightedParse(tokens, lex, rules):\n",
" \"\"\"made to take weighed tokens and lexicons\"\"\"\n",
" chart = CCGChart(list(tokens))\n",
" \n",
" # Initialize leaf edges.\n",
" for index in range(chart.num_leaves()):\n",
" for token in lex.categories(chart.leaf(index)):\n",
" new_edge = CCGLeafEdge(index, token, chart.leaf(index))\n",
" new_edge.weight = token.weight()\n",
" chart.insert(new_edge, ())\n",
"\n",
" # Select a span for the new edges\n",
" for span in range(2, chart.num_leaves() + 1):\n",
" for start in range(0, chart.num_leaves() - span + 1):\n",
" \n",
" bestedge = None\n",
" \n",
" # Try all possible pairs of edges that could generate\n",
" # an edge for that span\n",
" for part in range(1, span):\n",
" lstart = start\n",
" mid = start + part\n",
" rend = start + span\n",
"\n",
" for left in chart.select(span=(lstart, mid)):\n",
" for right in chart.select(span=(mid, rend)):\n",
" # Generate all possible combinations of the two edges\n",
" for rule in rules:\n",
" edgez = list(rule.apply(chart, lex, left, right))\n",
" if(len(edgez)==1):\n",
" edge = edgez[0]\n",
" edge.weight = rweight(rule) * left.weight * right.weight\n",
" edge.triple = (rule,left,right)\n",
" if (bestedge == None) or (bestedge.weight < edge.weight):\n",
" bestedge = edge\n",
" elif(len(edgez)!=0):\n",
" print(\"Too many new edges (unsupported rule used)\")\n",
" \n",
" # end for rule loop\n",
" # end for right loop\n",
" # end for left loop\n",
" # end for part loop\n",
" return chart\n",
"\n",
"def wpToTree(edge):\n",
" if isinstance(edge,CCGLeafEdge):\n",
" return Tree((edge.token(),\"Leaf\"),[Tree(edge.token(),[edge.leaf()])])\n",
" else:\n",
" return Tree(\n",
" (chart.Token(None,edge.categ()),edge.triple[0].__str__()),\n",
" [wpToTree(t) for t in (edge.triple[1:])])\n",
"\n",
"def bestTree(tokens, lex, rules):\n",
" # We build the weighgted parse tree using cky\n",
" wChart = weightedParse(tokens, lex, rules)\n",
" # We get the biggest edge\n",
" e = list(wChart.select(start=0,end=len(tokens)))[0]\n",
" # We get the tree that brought us to this edge\n",
" t = wChart._trees(e, True, dict(), Tree)[0]\n",
" # (wpToTree(e),e.weight)\n",
" return (t,e.weight)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Application"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"from numbers import Number\n",
"from nltk.sem.logic import Expression\n",
"from nltk.ccg.api import PrimitiveCategory\n",
"\n",
"def to_pseudo_entries(table, consider_semantics = False):\n",
" \"\"\"returns a list of lists in the format ['word', 'category', 'weight', None]\n",
" if consider_semantics == false else ['word', 'category', weight, 'semantic']\n",
" that is left to be converted into tokens by to_wlex_entries\"\"\"\n",
"\n",
" entries = list()\n",
" for line in range(len(table['MOT'])):\n",
" for wdi, word in enumerate(table['MOT'][line].replace(\" \", \"\").split('/')):\n",
" for j in range(3):\n",
" if isinstance(table['Cat'+str(j)][line],str):\n",
" category = table['Cat'+str(j)][line]\n",
" weight = float(table['Weights'+str(j)][line]) if isinstance(table['Weights'+str(j)][line], Number) else 1.0\n",
" if consider_semantics:\n",
" semantic = (table['Sem'+str(j)][line].replace('\\\\\\\\', '\\\\').split('/'))[wdi]\n",
" else:\n",
" semantic = None\n",
" entries.append([word, category, weight, semantic])\n",
" return entries\n",
"\n",
"def to_wlex_entries(pseudo_entries, primitives, families, var=None):\n",
" \"\"\"returns the entries to a weighed lexicon from pseudo_entries generated by to_pseudo_entries\"\"\"\n",
" entries = dict()\n",
" for entry in pseudo_entries:\n",
" if entry[0] not in entries:\n",
" entries[entry[0]] = list()\n",
" categ, _ = augParseCategory(entry[1], primitives, families, var)\n",
" token = WeighedToken(token= entry[0],\n",
" categ= categ,\n",
" semantics= Expression.fromstring(entry[-1]),\n",
" weight= entry[2])\n",
" entries[entry[0]].append(token)\n",
" return entries\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{}\n",
"\\n v.à(v,n)\n",
"\\n v.avec(v,n)\n",
"chat\n",
"\\n m.de(n,m)\n",
"dents\n",
"\\n.donne(n)\n",
"\\n m.donne(n,m)\n",
"\\n.mange(n)\n",
"\\n m.mange(n,m)\n",
"donner\n",
"\\n.donner(n)\n",
"\\n.dormir(n)\n",
"\\n.dormir(n)\n",
"elle\n",
"il\n",
"{}\n",
"{}\n",
"{}\n",
"\\n m.et(n,m)\n",
"\\x y.et(x,y)\n",
"\\v w n.et(v,w,n)\n",
"fromage\n",
"{}\n",
"{}\n",
"{}\n",
"\\P.exists x.P(x)\n",
"\\P.exists x.P(x)\n",
"\\P.exists x.P(x)\n",
"\\P.exists x.P(x)\n",
"\\P.exists x.P(x)\n",
"\\n.mangé(n)\n",
"\\n.mangé(n)\n",
"\\n.donné(n)\n",
"\\n.méchant(n)\n",
"\\n.méchant(n)\n",
"\\n.noir(n)\n",
"\\n.noir(n)\n",
"\\v n.paisiblement(v,n)\n",
"{}\n",
"{}\n",
"{}\n",
"{}\n",
"{}\n",
"{}\n",
"{}\n",
"{}\n",
"{}\n",
"{}\n",
"rat\n",
"soeur\n",
"\\n m.souhaite(m,n)\n",
"\\n m.pourchasse(m,n)\n",
"\\n m.attrappe(m,n)\n",
"souris\n",
"{}\n",
"voisin\n",
"1 le chat dort\n",
"Found derivation tree with weight 0.5599999999999999\n",
" le chat dort\n",
" (N/N) {\\P.exists x.P(x)} N {chat} (S\\N) {\\n.dormir(n)}\n",
"------------------------------------>\n",
" N {exists x.chat(x)}\n",
"----------------------------------------------------------<\n",
" S {dormir(exists x.chat(x))}\n",
"1 il dort\n",
"Found derivation tree with weight 0.7\n",
" il dort\n",
" N {il} (S\\N) {\\n.dormir(n)}\n",
"------------------------------<\n",
" S {dormir(il)}\n",
"1 le chat dort paisiblement\n",
"Found derivation tree with weight 0.39199999999999996\n",
" le chat dort paisiblement\n",
" (N/N) {\\P.exists x.P(x)} N {chat} (S\\N) {\\n.dormir(n)} ((S\\N)\\(S\\N)) {\\v n.paisiblement(v,n)}\n",
"------------------------------------>\n",
" N {exists x.chat(x)}\n",
" --------------------------------------------------------------<\n",
" (S\\N) {\\n.paisiblement(\\n.dormir(n),n)}\n",
"--------------------------------------------------------------------------------------------------<\n",
" S {paisiblement(\\n.dormir(n),exists x.chat(x))}\n",
"2 le chat noir dort\n",
"Found derivation tree with weight 0.39199999999999996\n",
" le chat noir dort\n",
" (N/N) {\\P.exists x.P(x)} N {chat} (N\\N) {\\n.noir(n)} (S\\N) {\\n.dormir(n)}\n",
" ------------------------------<\n",
" N {noir(chat)}\n",
"-------------------------------------------------------->\n",
" N {exists x.noir(chat,x)}\n",
"------------------------------------------------------------------------------<\n",
" S {dormir(exists x.noir(chat,x))}\n",
"1 le méchant chat dort\n",
"Found derivation tree with weight 0.44800000000000006\n",
" le méchant chat dort\n",
" (N/N) {\\P.exists x.P(x)} (N/N) {\\n.méchant(n)} N {chat} (S\\N) {\\n.dormir(n)}\n",
" --------------------------------->\n",
" N {méchant(chat)}\n",
"----------------------------------------------------------->\n",
" N {exists x.méchant(chat,x)}\n",
"---------------------------------------------------------------------------------<\n",
" S {dormir(exists x.méchant(chat,x))}\n",
"2 le très méchant chat dort\n",
"Found derivation tree with weight 0.35840000000000005\n",
" le très méchant chat dort\n",
" (N/N) {\\P.exists x.P(x)} ((N/N)/(N/N)) {{}} (N/N) {\\n.méchant(n)} N {chat} (S\\N) {\\n.dormir(n)}\n",
" ------------------------------------------->\n",
" (N/N) {{}(\\n.méchant(n))}\n",
" ----------------------------------------------------->\n",
" N {{}(\\n.méchant(n),chat)}\n",
"------------------------------------------------------------------------------->\n",
" N {exists x.{}(\\n.méchant(n),chat,x)}\n",
"-----------------------------------------------------------------------------------------------------<\n",
" S {dormir(exists x.{}(\\n.méchant(n),chat,x))}\n",
"7 le chat de la sœur de mon voisin dort\n",
"Found derivation tree with weight 0.11239423999999998\n",
" le chat de la sœur de mon voisin dort\n",
" (N/N) {\\P.exists x.P(x)} N {chat} ((N/N)\\N) {\\n m.de(n,m)} (N/N) {\\P.exists x.P(x)} N {soeur} ((N/N)\\N) {\\n m.de(n,m)} (N/N) {\\P.exists x.P(x)} N {voisin} (S\\N) {\\n.dormir(n)}\n",
" ------------------------------------<\n",
" (N/N) {\\m.de(chat,m)}\n",
" -------------------------------------<\n",
" (N/N) {\\m.de(soeur,m)}\n",
" -------------------------------------->\n",
" N {exists x.voisin(x)}\n",
" --------------------------------------------------------------------------->\n",
" N {de(soeur,exists x.voisin(x))}\n",
" ----------------------------------------------------------------------------------------------------->\n",
" N {exists x.de(soeur,exists x.voisin(x),x)}\n",
" ----------------------------------------------------------------------------------------------------------------------------------------->\n",
" N {de(chat,exists x.de(soeur,exists x.voisin(x),x))}\n",
"------------------------------------------------------------------------------------------------------------------------------------------------------------------->\n",
" N {exists x.de(chat,exists x.de(soeur,exists x.voisin(x),x),x)}\n",
"-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {dormir(exists x.de(chat,exists x.de(soeur,exists x.voisin(x),x),x))}\n",
"2 le chat que mon voisin lui donne mange\n",
"Found derivation tree with weight 0.14049279999999997\n",
" le chat que mon voisin lui donne mange\n",
" (N/N) {\\P.exists x.P(x)} N {chat} ((N\\N)/S) {{}} (N/N) {\\P.exists x.P(x)} N {voisin} ((S\\N)/(S\\N)) {{}} (S\\N) {\\n.donne(n)} (S\\N) {\\n.mange(n)}\n",
" -------------------------------------->\n",
" N {exists x.voisin(x)}\n",
" ----------------------------------------->\n",
" (S\\N) {{}(\\n.donne(n))}\n",
" -------------------------------------------------------------------------------<\n",
" S {{}(\\n.donne(n),exists x.voisin(x))}\n",
" ----------------------------------------------------------------------------------------------->\n",
" (N\\N) {{}({}(\\n.donne(n),exists x.voisin(x)))}\n",
" ---------------------------------------------------------------------------------------------------------<\n",
" N {{}({}(\\n.donne(n),exists x.voisin(x)),chat)}\n",
"----------------------------------------------------------------------------------------------------------------------------------->\n",
" N {exists x.{}({}(\\n.donne(n),exists x.voisin(x)),chat,x)}\n",
"--------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {mange(exists x.{}({}(\\n.donne(n),exists x.voisin(x)),chat,x))}\n",
"8 le chat qui dort est noir\n",
"Found derivation tree with weight 0.25087999999999994\n",
" le chat qui dort est noir\n",
" (N/N) {\\P.exists x.P(x)} N {chat} ((N\\N)/(S\\N)) {{}} (S\\N) {\\n.dormir(n)} ((S\\N)/(N/N)) {{}} (N\\N) {\\n.noir(n)}\n",
" ------------------------------------------>\n",
" (N\\N) {{}(\\n.dormir(n))}\n",
" ----------------------------------------------------<\n",
" N {{}(\\n.dormir(n),chat)}\n",
"------------------------------------------------------------------------------>\n",
" N {exists x.{}(\\n.dormir(n),chat,x)}\n",
" ---------------------------------------->\n",
" (S\\N) {{}(\\n.noir(n))}\n",
"----------------------------------------------------------------------------------------------------------------------<\n",
" S {{}(\\n.noir(n),exists x.{}(\\n.dormir(n),chat,x))}\n",
"1 le chat mange la souris\n",
"Found derivation tree with weight 0.35840000000000005\n",
" le chat mange la souris\n",
" (N/N) {\\P.exists x.P(x)} N {chat} ((S\\N)/N) {\\n m.mange(n,m)} (N/N) {\\P.exists x.P(x)} N {souris}\n",
"------------------------------------>\n",
" N {exists x.chat(x)}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ------------------------------------------------------------------->\n",
" (S\\N) {\\m.mange(exists x.souris(x),m)}\n",
"-------------------------------------------------------------------------------------------------------<\n",
" S {mange(exists x.souris(x),exists x.chat(x))}\n",
"1 le chat la mange\n",
"Found derivation tree with weight 0.44799999999999995\n",
" le chat la mange\n",
" (N/N) {\\P.exists x.P(x)} N {chat} ((S\\N)/((S\\N)/N)) {{}} ((S\\N)/N) {\\n m.mange(n,m)}\n",
"------------------------------------>\n",
" N {exists x.chat(x)}\n",
" ----------------------------------------------------->\n",
" (S\\N) {{}(\\n m.mange(n,m))}\n",
"-----------------------------------------------------------------------------------------<\n",
" S {{}(\\n m.mange(n,m),exists x.chat(x))}\n",
"1 il la mange\n",
"Found derivation tree with weight 0.5599999999999999\n",
" il la mange\n",
" N {il} ((S\\N)/((S\\N)/N)) {{}} ((S\\N)/N) {\\n m.mange(n,m)}\n",
" ----------------------------------------------------->\n",
" (S\\N) {{}(\\n m.mange(n,m))}\n",
"-------------------------------------------------------------<\n",
" S {{}(\\n m.mange(n,m),il)}\n",
"1 quel chat mange la souris ?\n",
"Found derivation tree with weight 0.2867200000000001\n",
" quel chat mange la souris ?\n",
" ((S/(S\\N))/N) {{}} N {chat} ((S\\N)/N) {\\n m.mange(n,m)} (N/N) {\\P.exists x.P(x)} N {souris} (S\\S) {{}}\n",
"------------------------------>\n",
" (S/(S\\N)) {{}(chat)}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ------------------------------------------------------------------->\n",
" (S\\N) {\\m.mange(exists x.souris(x),m)}\n",
"------------------------------------------------------------------------------------------------->\n",
" S {{}(chat,\\m.mange(exists x.souris(x),m))}\n",
"-------------------------------------------------------------------------------------------------------------<\n",
" S {{}({}(chat,\\m.mange(exists x.souris(x),m)))}\n",
"1 qui mange la souris ?\n",
"Found derivation tree with weight 0.35840000000000005\n",
" qui mange la souris ?\n",
" (S/(S\\N)) {{}} ((S\\N)/N) {\\n m.mange(n,m)} (N/N) {\\P.exists x.P(x)} N {souris} (S\\S) {{}}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ------------------------------------------------------------------->\n",
" (S\\N) {\\m.mange(exists x.souris(x),m)}\n",
"----------------------------------------------------------------------------------->\n",
" S {{}(\\m.mange(exists x.souris(x),m))}\n",
"-----------------------------------------------------------------------------------------------<\n",
" S {{}({}(\\m.mange(exists x.souris(x),m)))}\n",
"1 quelle souris mange le chat ?\n",
"Found derivation tree with weight 0.2867200000000001\n",
" quelle souris mange le chat ?\n",
" ((S/(S\\N))/N) {{}} N {souris} ((S\\N)/N) {\\n m.mange(n,m)} (N/N) {\\P.exists x.P(x)} N {chat} (S\\S) {{}}\n",
"-------------------------------->\n",
" (S/(S\\N)) {{}(souris)}\n",
" ------------------------------------>\n",
" N {exists x.chat(x)}\n",
" ----------------------------------------------------------------->\n",
" (S\\N) {\\m.mange(exists x.chat(x),m)}\n",
"------------------------------------------------------------------------------------------------->\n",
" S {{}(souris,\\m.mange(exists x.chat(x),m))}\n",
"-------------------------------------------------------------------------------------------------------------<\n",
" S {{}({}(souris,\\m.mange(exists x.chat(x),m)))}\n",
"1 la souris est mangée par le chat\n",
"Found derivation tree with weight 0.20070400000000002\n",
" la souris est mangée par le chat\n",
" (N/N) {\\P.exists x.P(x)} N {souris} ((S\\N)/Pp) {{}} Pp {\\n.mangé(n)} (((S\\N)\\(S\\N))/N) {{}} (N/N) {\\P.exists x.P(x)} N {chat}\n",
"-------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ----------------------------------->\n",
" (S\\N) {{}(\\n.mangé(n))}\n",
" ------------------------------------>\n",
" N {exists x.chat(x)}\n",
" ------------------------------------------------------------>\n",
" ((S\\N)\\(S\\N)) {{}(exists x.chat(x))}\n",
" -----------------------------------------------------------------------------------------------<\n",
" (S\\N) {{}(exists x.chat(x),{}(\\n.mangé(n)))}\n",
"-------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {{}(exists x.chat(x),{}(\\n.mangé(n)),exists x.souris(x))}\n",
"1 elle est mangée par le chat\n",
"Found derivation tree with weight 0.25088000000000005\n",
" elle est mangée par le chat\n",
" N {elle} ((S\\N)/Pp) {{}} Pp {\\n.mangé(n)} (((S\\N)\\(S\\N))/N) {{}} (N/N) {\\P.exists x.P(x)} N {chat}\n",
" ----------------------------------->\n",
" (S\\N) {{}(\\n.mangé(n))}\n",
" ------------------------------------>\n",
" N {exists x.chat(x)}\n",
" ------------------------------------------------------------>\n",
" ((S\\N)\\(S\\N)) {{}(exists x.chat(x))}\n",
" -----------------------------------------------------------------------------------------------<\n",
" (S\\N) {{}(exists x.chat(x),{}(\\n.mangé(n)))}\n",
"---------------------------------------------------------------------------------------------------------<\n",
" S {{}(exists x.chat(x),{}(\\n.mangé(n)),elle)}\n",
"1 quelle souris est mangée par le chat ?\n",
"Found derivation tree with weight 0.16056320000000004\n",
" quelle souris est mangée par le chat ?\n",
" ((S/(S\\N))/N) {{}} N {souris} ((S\\N)/Pp) {{}} Pp {\\n.mangé(n)} (((S\\N)\\(S\\N))/N) {{}} (N/N) {\\P.exists x.P(x)} N {chat} (S\\S) {{}}\n",
"-------------------------------->\n",
" (S/(S\\N)) {{}(souris)}\n",
" ----------------------------------->\n",
" (S\\N) {{}(\\n.mangé(n))}\n",
" ------------------------------------>\n",
" N {exists x.chat(x)}\n",
" ------------------------------------------------------------>\n",
" ((S\\N)\\(S\\N)) {{}(exists x.chat(x))}\n",
" -----------------------------------------------------------------------------------------------<\n",
" (S\\N) {{}(exists x.chat(x),{}(\\n.mangé(n)))}\n",
"------------------------------------------------------------------------------------------------------------------------------->\n",
" S {{}(souris,{}(exists x.chat(x),{}(\\n.mangé(n))))}\n",
"-------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {{}({}(souris,{}(exists x.chat(x),{}(\\n.mangé(n)))))}\n",
"1 le chat mange la souris avec ses dents\n",
"Found derivation tree with weight 0.16056320000000004\n",
" le chat mange la souris avec ses dents\n",
" (N/N) {\\P.exists x.P(x)} N {chat} ((S\\N)/N) {\\n m.mange(n,m)} (N/N) {\\P.exists x.P(x)} N {souris} (((S\\N)\\(S\\N))/N) {\\n v.avec(v,n)} (N/N) {\\P.exists x.P(x)} N {dents}\n",
"------------------------------------>\n",
" N {exists x.chat(x)}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ------------------------------------------------------------------->\n",
" (S\\N) {\\m.mange(exists x.souris(x),m)}\n",
" ------------------------------------->\n",
" N {exists x.dents(x)}\n",
" ------------------------------------------------------------------------->\n",
" ((S\\N)\\(S\\N)) {\\v.avec(v,exists x.dents(x))}\n",
" --------------------------------------------------------------------------------------------------------------------------------------------<\n",
" (S\\N) {avec(\\m.mange(exists x.souris(x),m),exists x.dents(x))}\n",
"--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {avec(\\m.mange(exists x.souris(x),m),exists x.dents(x),exists x.chat(x))}\n",
"1 le chat la mange avec ses dents\n",
"Found derivation tree with weight 0.20070400000000002\n",
" le chat la mange avec ses dents\n",
" (N/N) {\\P.exists x.P(x)} N {chat} ((S\\N)/((S\\N)/N)) {{}} ((S\\N)/N) {\\n m.mange(n,m)} (((S\\N)\\(S\\N))/N) {\\n v.avec(v,n)} (N/N) {\\P.exists x.P(x)} N {dents}\n",
"------------------------------------>\n",
" N {exists x.chat(x)}\n",
" ----------------------------------------------------->\n",
" (S\\N) {{}(\\n m.mange(n,m))}\n",
" ------------------------------------->\n",
" N {exists x.dents(x)}\n",
" ------------------------------------------------------------------------->\n",
" ((S\\N)\\(S\\N)) {\\v.avec(v,exists x.dents(x))}\n",
" ------------------------------------------------------------------------------------------------------------------------------<\n",
" (S\\N) {avec({}(\\n m.mange(n,m)),exists x.dents(x))}\n",
"------------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {avec({}(\\n m.mange(n,m)),exists x.dents(x),exists x.chat(x))}\n",
"0 avec quoi le chat mange la souris ?\n",
"Pas de dérivation tout court :/\n",
"1 le rat donne un fromage à la souris\n",
"Found derivation tree with weight 0.16056320000000004\n",
" le rat donne un fromage à la souris\n",
" (N/N) {\\P.exists x.P(x)} N {rat} ((S\\N)/N) {\\n m.donne(n,m)} (N/N) {\\P.exists x.P(x)} N {fromage} (((S\\N)\\(S\\N))/N) {\\n v.à(v,n)} (N/N) {\\P.exists x.P(x)} N {souris}\n",
"----------------------------------->\n",
" N {exists x.rat(x)}\n",
" --------------------------------------->\n",
" N {exists x.fromage(x)}\n",
" -------------------------------------------------------------------->\n",
" (S\\N) {\\m.donne(exists x.fromage(x),m)}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ----------------------------------------------------------------------->\n",
" ((S\\N)\\(S\\N)) {\\v.à(v,exists x.souris(x))}\n",
" -------------------------------------------------------------------------------------------------------------------------------------------<\n",
" (S\\N) {à(\\m.donne(exists x.fromage(x),m),exists x.souris(x))}\n",
"------------------------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {à(\\m.donne(exists x.fromage(x),m),exists x.souris(x),exists x.rat(x))}\n",
"1 un fromage est donné par le rat à la souris\n",
"Found derivation tree with weight 0.08991539200000002\n",
" un fromage est donné par le rat à la souris\n",
" (N/N) {\\P.exists x.P(x)} N {fromage} ((S\\N)/Pp) {{}} Pp {\\n.donné(n)} (((S\\N)\\(S\\N))/N) {{}} (N/N) {\\P.exists x.P(x)} N {rat} (((S\\N)\\(S\\N))/N) {\\n v.à(v,n)} (N/N) {\\P.exists x.P(x)} N {souris}\n",
"--------------------------------------->\n",
" N {exists x.fromage(x)}\n",
" ----------------------------------->\n",
" (S\\N) {{}(\\n.donné(n))}\n",
" ----------------------------------->\n",
" N {exists x.rat(x)}\n",
" ----------------------------------------------------------->\n",
" ((S\\N)\\(S\\N)) {{}(exists x.rat(x))}\n",
" ----------------------------------------------------------------------------------------------<\n",
" (S\\N) {{}(exists x.rat(x),{}(\\n.donné(n)))}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ----------------------------------------------------------------------->\n",
" ((S\\N)\\(S\\N)) {\\v.à(v,exists x.souris(x))}\n",
" ---------------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" (S\\N) {à({}(exists x.rat(x),{}(\\n.donné(n))),exists x.souris(x))}\n",
"------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {à({}(exists x.rat(x),{}(\\n.donné(n))),exists x.souris(x),exists x.fromage(x))}\n",
"0 à quelle souris un fromage est donné par le rat ?\n",
"Pas de dérivation tout court :/\n",
"1 il le donne à la souris\n",
"Found derivation tree with weight 0.25088000000000005\n",
" il le donne à la souris\n",
" N {il} ((S\\N)/((S\\N)/N)) {{}} ((S\\N)/N) {\\n m.donne(n,m)} (((S\\N)\\(S\\N))/N) {\\n v.à(v,n)} (N/N) {\\P.exists x.P(x)} N {souris}\n",
" ----------------------------------------------------->\n",
" (S\\N) {{}(\\n m.donne(n,m))}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ----------------------------------------------------------------------->\n",
" ((S\\N)\\(S\\N)) {\\v.à(v,exists x.souris(x))}\n",
" ----------------------------------------------------------------------------------------------------------------------------<\n",
" (S\\N) {à({}(\\n m.donne(n,m)),exists x.souris(x))}\n",
"------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {à({}(\\n m.donne(n,m)),exists x.souris(x),il)}\n",
"0 il le lui donne\n",
"Pas de dérivation tout court :/\n",
"1 il souhaite que mon voisin lui donne le chat\n",
"Found derivation tree with weight 0.12845056000000002\n",
" il souhaite que mon voisin lui donne le chat\n",
" N {il} ((S\\N)/N) {\\n m.souhaite(m,n)} (N/S) {{}} (N/N) {\\P.exists x.P(x)} N {voisin} ((S\\N)/(S\\N)) {{}} ((S\\N)/N) {\\n m.donne(n,m)} (N/N) {\\P.exists x.P(x)} N {chat}\n",
" -------------------------------------->\n",
" N {exists x.voisin(x)}\n",
" ------------------------------------>\n",
" N {exists x.chat(x)}\n",
" ----------------------------------------------------------------->\n",
" (S\\N) {\\m.donne(exists x.chat(x),m)}\n",
" ------------------------------------------------------------------------------------->\n",
" (S\\N) {{}(\\m.donne(exists x.chat(x),m))}\n",
" ---------------------------------------------------------------------------------------------------------------------------<\n",
" S {{}(\\m.donne(exists x.chat(x),m),exists x.voisin(x))}\n",
" --------------------------------------------------------------------------------------------------------------------------------------->\n",
" N {{}({}(\\m.donne(exists x.chat(x),m),exists x.voisin(x)))}\n",
" ----------------------------------------------------------------------------------------------------------------------------------------------------------------------->\n",
" (S\\N) {\\m.souhaite(m,{}({}(\\m.donne(exists x.chat(x),m),exists x.voisin(x))))}\n",
"-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {souhaite(il,{}({}(\\m.donne(exists x.chat(x),m),exists x.voisin(x))))}\n",
"1 il souhaite donner le chat à mon voisin\n",
"Found derivation tree with weight 0.16056320000000004\n",
" il souhaite donner le chat à mon voisin\n",
" N {il} ((S\\N)/N) {\\n m.souhaite(m,n)} (N/N) {\\n.donner(n)} (N/N) {\\P.exists x.P(x)} N {chat} (((S\\N)\\(S\\N))/N) {\\n v.à(v,n)} (N/N) {\\P.exists x.P(x)} N {voisin}\n",
" ------------------------------------>\n",
" N {exists x.chat(x)}\n",
" ---------------------------------------------------------->\n",
" N {donner(exists x.chat(x))}\n",
" ------------------------------------------------------------------------------------------>\n",
" (S\\N) {\\m.souhaite(m,donner(exists x.chat(x)))}\n",
" -------------------------------------->\n",
" N {exists x.voisin(x)}\n",
" ----------------------------------------------------------------------->\n",
" ((S\\N)\\(S\\N)) {\\v.à(v,exists x.voisin(x))}\n",
" -----------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" (S\\N) {à(\\m.souhaite(m,donner(exists x.chat(x))),exists x.voisin(x))}\n",
"-------------------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {à(\\m.souhaite(m,donner(exists x.chat(x))),exists x.voisin(x),il)}\n",
"0 le chat de mon voisin pourchasse et attrape la souris\n",
"Pas de dérivation tout court :/\n",
"2 la souris dort et le chat de mon voisin attrape la souris\n",
"Found derivation tree with weight 0.05035261952\n",
" la souris dort et le chat de mon voisin attrape la souris\n",
" (N/N) {\\P.exists x.P(x)} N {souris} (S\\N) {\\n.dormir(n)} ((S/S)\\S) {\\x y.et(x,y)} (N/N) {\\P.exists x.P(x)} N {chat} ((N/N)\\N) {\\n m.de(n,m)} (N/N) {\\P.exists x.P(x)} N {voisin} ((S\\N)/N) {\\n m.attrappe(m,n)} (N/N) {\\P.exists x.P(x)} N {souris}\n",
"-------------------------------------->\n",
" N {exists x.souris(x)}\n",
"------------------------------------------------------------<\n",
" S {dormir(exists x.souris(x))}\n",
"--------------------------------------------------------------------------------------<\n",
" (S/S) {\\y.et(dormir(exists x.souris(x)),y)}\n",
" ------------------------------------<\n",
" (N/N) {\\m.de(chat,m)}\n",
" -------------------------------------->\n",
" N {exists x.voisin(x)}\n",
" -------------------------------------------------------------------------->\n",
" N {de(chat,exists x.voisin(x))}\n",
" ---------------------------------------------------------------------------------------------------->\n",
" N {exists x.de(chat,exists x.voisin(x),x)}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ---------------------------------------------------------------------->\n",
" (S\\N) {\\m.attrappe(m,exists x.souris(x))}\n",
" --------------------------------------------------------------------------------------------------------------------------------------------------------------------------<\n",
" S {attrappe(exists x.de(chat,exists x.voisin(x),x),exists x.souris(x))}\n",
"---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------->\n",
" S {et(dormir(exists x.souris(x)),attrappe(exists x.de(chat,exists x.voisin(x),x),exists x.souris(x)))}\n",
"1 le chat dort et la souris dort\n",
"Found derivation tree with weight 0.17561599999999997\n",
" le chat dort et la souris dort\n",
" (N/N) {\\P.exists x.P(x)} N {chat} (S\\N) {\\n.dormir(n)} ((S/S)\\S) {\\x y.et(x,y)} (N/N) {\\P.exists x.P(x)} N {souris} (S\\N) {\\n.dormir(n)}\n",
"------------------------------------>\n",
" N {exists x.chat(x)}\n",
"----------------------------------------------------------<\n",
" S {dormir(exists x.chat(x))}\n",
"------------------------------------------------------------------------------------<\n",
" (S/S) {\\y.et(dormir(exists x.chat(x)),y)}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" ------------------------------------------------------------<\n",
" S {dormir(exists x.souris(x))}\n",
"------------------------------------------------------------------------------------------------------------------------------------------------>\n",
" S {et(dormir(exists x.chat(x)),dormir(exists x.souris(x)))}\n",
"2 le chat et la souris dorment\n",
"Found derivation tree with weight 0.25088\n",
" le chat et la souris dorment\n",
" (N/N) {\\P.exists x.P(x)} N {chat} ((N/N)\\N) {\\n m.et(n,m)} (N/N) {\\P.exists x.P(x)} N {souris} (S\\N) {\\n.dormir(n)}\n",
" ------------------------------------<\n",
" (N/N) {\\m.et(chat,m)}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" -------------------------------------------------------------------------->\n",
" N {et(chat,exists x.souris(x))}\n",
"---------------------------------------------------------------------------------------------------->\n",
" N {exists x.et(chat,exists x.souris(x),x)}\n",
"--------------------------------------------------------------------------------------------------------------------------<\n",
" S {dormir(exists x.et(chat,exists x.souris(x),x))}\n",
"2 le chat et la souris dorment\n",
"Found derivation tree with weight 0.25088\n",
" le chat et la souris dorment\n",
" (N/N) {\\P.exists x.P(x)} N {chat} ((N/N)\\N) {\\n m.et(n,m)} (N/N) {\\P.exists x.P(x)} N {souris} (S\\N) {\\n.dormir(n)}\n",
" ------------------------------------<\n",
" (N/N) {\\m.et(chat,m)}\n",
" -------------------------------------->\n",
" N {exists x.souris(x)}\n",
" -------------------------------------------------------------------------->\n",
" N {et(chat,exists x.souris(x))}\n",
"---------------------------------------------------------------------------------------------------->\n",
" N {exists x.et(chat,exists x.souris(x),x)}\n",
"--------------------------------------------------------------------------------------------------------------------------<\n",
" S {dormir(exists x.et(chat,exists x.souris(x),x))}\n"
]
}
],
"source": [
"# Catégories primitives et familles\n",
"primitives = ['S', 'N', 'Pp']\n",
"V = augParseCategory(\"S\\\\N\", primitives = primitives, families={})\n",
"families = {'V': V}\n",
"\n",
"# On importe notre lexique sous forme de tableur\n",
"table = pd.read_excel(\"CategoriesGramaticalesCombinatoire.ods\", engine=\"odf\")\n",
"#print(table.keys())\n",
"\n",
"# On le convertit en Lexique pondéré\n",
"pe = to_pseudo_entries(table, consider_semantics = True)\n",
"#print(pe)\n",
"wEntries = to_wlex_entries(pseudo_entries= pe, primitives= primitives, families= families)\n",
"#print([list(map(lambda x: f\"{k} : \"+ str(x) + str(x._semantics), L)) for k, L in wEntries.items()])\n",
"lex = WeighedLexicon(start= 'S', primitives= primitives, families= families, entries= wEntries)\n",
"\n",
"\n",
"# On récupère le nombre de mots qui ont été définis\n",
"# n = len(table['MOT'])\n",
"\n",
"# On donne la liste des catégories primitives\n",
"# lexstring = ':- S,N,Pp\\n'\n",
"# On ajoute la notation V pour N\\S\n",
"# lexstring += 'V :: S\\\\N\\n'\n",
"\n",
"# On lis les données depuis le tableur en une chaine de caractère parsable\n",
"#for i in range(n):\n",
"# for j in range(3):\n",
"# if isinstance(table['Cat'+str(j)][i],str):\n",
"# for mot in table['MOT'][i].split('/'):\n",
"# lexstring+=mot+' => ' + table['Cat'+str(j)][i] + '\\n'\n",
"\n",
"# Pour inverser les slash dans le lexicon\n",
"#lexstring = lexstring.replace('\\\\','#').replace('/','\\\\').replace('#','/')\n",
"\n",
"# On crée notre lexique\n",
"# lex = lexicon.fromstring(lexstring)\n",
"\n",
"# On crée le parser, on donne l'ensemble des règles qu'il est cencé connaître\n",
"parser = chart.CCGChartParser(lex, chart.DefaultRuleSet)\n",
"#parser = chart.CCGChartParser(lex, chart.ApplicationRuleSet)\n",
"\n",
"printTotal=True\n",
"printDerivations=not printTotal\n",
"\n",
"# On lit les phrases dans le fichier\n",
"with open('phrases.txt') as f:\n",
" lines = f.readlines()\n",
"\n",
" lines.append(\"le chat et la souris dorment\")\n",
" \n",
" for phrase in lines:\n",
" # On met tout en minuscule\n",
" phrase = phrase.lower().strip()\n",
" if printDerivations:\n",
" print(\"=\"*77)\n",
" print('#',phrase)\n",
" # lex = lexicon.fromstring(lexstring)\n",
" parser = chart.CCGChartParser(lex, chart.ApplicationRuleSet)\n",
"\n",
" # Et on affiche tous les arbres de dérivation trouvés\n",
" i=0\n",
" for parse in parser.parse(phrase.split()):\n",
" i+=1\n",
" if printDerivations:\n",
" chart.printCCGDerivation(parse)\n",
" \n",
" if printTotal:\n",
" print(i,phrase)\n",
" \n",
" \n",
" # On affiche la dérivation la meilleure pour l'arbre\n",
" if (i==0):\n",
" print(\"Pas de dérivation tout court :/\")\n",
" else:\n",
"\n",
" t,d = bestTree(phrase.split(), lex, chart.ApplicationRuleSet)\n",
" print(\"Found derivation tree with weight\",d)\n",
" chart.printCCGDerivation(t)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"? => (S\\S) {{}}\n",
"attrape => ((S\\N)/N) {\\n m.attrappe(m,n)}\n",
"avec => (((S\\N)\\(S\\N))/N) {\\n v.avec(v,n)}\n",
"chat => N {chat}\n",
"de => ((N/N)\\N) {\\n m.de(n,m)}\n",
"dents => N {dents}\n",
"donne => (S\\N) {\\n.donne(n)} | ((S\\N)/N) {\\n m.donne(n,m)}\n",
"donner => N {donner} | (N/N) {\\n.donner(n)}\n",
"donné => Pp {\\n.donné(n)}\n",
"dorment => (S\\N) {\\n.dormir(n)}\n",
"dort => (S\\N) {\\n.dormir(n)}\n",
"elle => N {elle}\n",
"est => ((S\\N)/Pp) {{}} | ((S\\N)/(N/N)) {{}} | ((S\\N)/(N\\N)) {{}}\n",
"et => ((N/N)\\N) {\\n m.et(n,m)} | ((S/S)\\S) {\\x y.et(x,y)} | (((S\\N)/(S\\N))\\(S\\N)) {\\v w n.et(v,w,n)}\n",
"fromage => N {fromage}\n",
"il => N {il}\n",
"la => ((S\\N)/((S\\N)/N)) {{}} | (N/N) {\\P.exists x.P(x)}\n",
"le => ((S\\N)/((S\\N)/N)) {{}} | (N/N) {\\P.exists x.P(x)}\n",
"lui => ((S\\N)/(S\\N)) {{}}\n",
"mange => (S\\N) {\\n.mange(n)} | ((S\\N)/N) {\\n m.mange(n,m)}\n",
"mangé => Pp {\\n.mangé(n)}\n",
"mangée => Pp {\\n.mangé(n)}\n",
"mon => (N/N) {\\P.exists x.P(x)}\n",
"méchant => (N/N) {\\n.méchant(n)} | (N\\N) {\\n.méchant(n)}\n",
"noir => (N\\N) {\\n.noir(n)} | (N/N) {\\n.noir(n)}\n",
"paisiblement => ((S\\N)\\(S\\N)) {\\v n.paisiblement(v,n)}\n",
"par => (((S\\N)\\(S\\N))/N) {{}}\n",
"pourchasse => ((S\\N)/N) {\\n m.pourchasse(m,n)}\n",
"que => ((N\\N)/S) {{}} | (N/S) {{}}\n",
"quel => (((((S\\N)\\(S\\N))/N)\\(S/S))/N) {{}} | ((S/(S\\N))/N) {{}}\n",
"quelle => (((((S\\N)\\(S\\N))/N)\\(S/S))/N) {{}} | ((S/(S\\N))/N) {{}}\n",
"qui => ((N\\N)/(S\\N)) {{}} | (S/(S\\N)) {{}}\n",
"quoi => (((S/S)\\(((S\\N)\\(S\\N))/N))/N) {{}}\n",
"rat => N {rat}\n",
"ses => (N/N) {\\P.exists x.P(x)}\n",
"souhaite => ((S\\N)/N) {\\n m.souhaite(m,n)}\n",
"souris => N {souris}\n",
"sœur => N {soeur}\n",
"très => ((N/N)/(N/N)) {{}}\n",
"un => (N/N) {\\P.exists x.P(x)}\n",
"voisin => N {voisin}\n",
"à => (((S\\N)\\(S\\N))/N) {\\n v.à(v,n)}\n"
]
}
],
"source": [
"print(lex)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}