From c8a17ee264a269a59651936b34a960f4d40a9074 Mon Sep 17 00:00:00 2001
From: Santhosh Thottingal <santhosh.thottingal@gmail.com>
Date: Sun, 19 Jul 2009 18:32:04 +0530
Subject: JSON RPC Based new architecture and corresponding changes

---
 silpa/modules/stemmer/stemmer.py | 128 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 128 insertions(+)
 create mode 100755 silpa/modules/stemmer/stemmer.py

(limited to 'silpa/modules/stemmer/stemmer.py')

diff --git a/silpa/modules/stemmer/stemmer.py b/silpa/modules/stemmer/stemmer.py
new file mode 100755
index 0000000..64bfaea
--- /dev/null
+++ b/silpa/modules/stemmer/stemmer.py
@@ -0,0 +1,128 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys  
+import codecs  
+import os  
+import string
+import curses.ascii 
+from common import SilpaModule
+class Stemmer(SilpaModule):
+
+	def __init__(self):
+		self.rules_file = "./modules/stemmer/stemmer_ml.rules"
+		self.rulesDict = dict()
+		
+	def lemmatize(self, text):
+		result = ""
+		self.rulesDict = self.LoadRules()
+		words=text.split(" ")
+		word_count=len(words)
+		result_dict = dict()
+		word_iter=0
+		word=""
+		while word_iter < word_count:
+			word = words[word_iter]
+			word = self.trim(word)
+			word_length = len(word)
+			suffix_pos_itr = 2
+			word_stemmed=""
+			while suffix_pos_itr < word_length :
+				suffix = word[suffix_pos_itr:word_length]
+				if suffix in self.rulesDict:
+					word_stemmed= word[0:suffix_pos_itr] +  self.rulesDict[suffix]
+					break;
+				suffix_pos_itr = suffix_pos_itr+1	
+			word_iter = word_iter+1
+			if(word_stemmed==""):
+				word_stemmed=word
+			result_dict[ word ] = word_stemmed
+		return result_dict
+					
+	def LoadRules(self):	
+		print "Loading the rules..."
+		rules_dict = dict()
+		line = []
+		line_number = 0
+		rule_number = 0
+		rules_file = codecs. open(self.rules_file,encoding='utf-8', errors='ignore')
+		while 1:
+			line_number = line_number +1 
+   			text = unicode( rules_file.readline())
+			if text == "":
+			      break
+			if text[0] == '#': 
+			      continue  #this is a comment - ignore
+			text = text.split("#")[0]   #remove the comment part of the line     
+			line_number = line_number +1       
+			line = text.strip()  # remove unwanted space
+			if(line == ""):
+				  continue 
+			if(len(line.split("=")) != 2):
+					print "[Error] Syntax Error in the Rules. Line number: ",  line_number
+				  	print "Line: "+ text
+				  	continue 
+	 		lhs = line.split("=") [ 0 ]  .strip()
+	 		rhs = line.split("=") [ 1 ]  .strip()
+	 		if(len(rhs)>0):
+	 			if(lhs[0]=='"'):
+	 				lhs=lhs[1:len(lhs)] # if the string is "quoted"
+	 			if(lhs[len(lhs)-1]=='"'):
+	 				lhs=lhs[0:len(lhs)-1] # if the string is "quoted"
+	 		if(len(rhs)>0):
+	 			if(rhs[0]=='"'):
+	 				rhs=rhs[1:len(rhs)]  # if the string is "quoted"
+	 			if(rhs[len(rhs)-1]=='"'):
+	 				rhs=rhs[0:len(rhs)-1]	 # if the string is "quoted"			
+	 		rule_number=rule_number+1
+			rules_dict[lhs]=rhs
+			#print "[", rule_number ,"] " +lhs + " : " +rhs
+		print "Found ",rule_number, " rules."
+		return rules_dict
+	
+	def trim(self,word):
+		punctuations=['~','!','@','#','$','%','^','&','*','(',')','-','+','_','=','{','}','|' ,':',';','<','>','\,','.','?']
+		word=word.strip()
+		index=len(word)-1
+		while index>0:
+			if word[index] in punctuations:
+				word=word[0:index]
+			else:
+				break 
+			index=index-1	
+		return word
+	def process(self, form):
+		response = """
+		<h2>Stemmer</h2></hr>
+		<p>Enter the text for stemming in the below text area.
+		 Language of each  word will be detected. 
+		 You can give the text in any language and even with mixed language
+		</p>
+		<form action="" method="post">
+		<textarea cols='100' rows='25' name='input_text' id='id1'>%s</textarea>
+		<input  type="submit" id="Stem" value="Stem"  name="action" style="width:12em;"/>
+		<input type="reset" value="Clear" style="width:12em;"/>
+		</br>
+		</form>
+		"""
+		if(form.has_key('input_text')):
+			text = form['input_text'].value	.decode('utf-8')
+			response=response % text
+			result_dict = self.lemmatize(text)
+			response = response+"<h2>Stemming Results</h2></hr>"
+			response = response+"<table class=\"table1\"><tr><th>Word</th><th>Stem</th></tr>"
+			for key in result_dict:
+				response = response+"<tr><td>"+key+"</td><td>"+result_dict[key]+"</td></tr>"
+			response = response+"</table>	"
+		else:
+			response=response % ""	
+		return response
+	def get_module_name(self):
+		return "Stemmer"
+	def get_info(self):
+		return 	"Malayalam Stemmer(Experimental)"
+		
+def getInstance():
+	return Stemmer()	
+
+	
-- 
cgit