diff options
Diffstat (limited to 'silpa/modules/ngram/sample_ngram.py')
-rw-r--r-- | silpa/modules/ngram/sample_ngram.py | 57 |
1 files changed, 42 insertions, 15 deletions
diff --git a/silpa/modules/ngram/sample_ngram.py b/silpa/modules/ngram/sample_ngram.py index d373aa8..ec4b184 100644 --- a/silpa/modules/ngram/sample_ngram.py +++ b/silpa/modules/ngram/sample_ngram.py @@ -26,6 +26,27 @@ import codecs import pickle import sys from visualizer import NGramVisualizer +from optparse import OptionParser + +def getData(new_file_name): + line = [] + line_number = 0 + rule_number = 0 + corpus="" + data_file = codecs. open(new_file_name,encoding='utf-8', errors='ignore') + while 1: + line_number = line_number +1 + text = unicode( data_file.readline()) + if text == "": + break + if text[0] == '#': + continue + line_number = line_number +1 + line = text.strip() + if(line == ""): + continue + corpus=corpus+" "+line + return corpus def printGraph(corpus,start_word): ngv=NGramVisualizer () @@ -38,18 +59,24 @@ def printGraph(corpus,start_word): print graph.to_string().encode("utf-8") if __name__ == "__main__": - """ - python sample_ngram.py <input file> <corpus path> 1 - this will generate the corpus for the given input file, if corpus specified at - corpus path is empty. Else it will recreate the corpus with the additional data. - - python sample_ngram.py <corpus path> <start word> 2 - this will generate the graph for the given start word in the given corpus at corpus path. - - This is just a crude attempt, a lot more improvement is to be done. - """ - if sys.argv[3] == "1": - ngv=NGramVisualizer () - ngv.loadCorpus(sys.argv[1],sys.argv[2]) - elif sys.argv[3] == "2": - printGraph(sys.argv[1],sys.argv[2]) + usage = "usage: %prog [options] INPUTDATA CORPUSFILE" + parser = OptionParser(version="%prog 1.0",description="Sample program to add data INPUTDATA to the corpus in CORPUSFILE") + parser.set_usage(usage) + parser.add_option("-s", "--start-word", dest="start_word",action="store_true",default=False,help="Creates a graph beginning from INPUTDATA") + parser.add_option("-f", "--file", action="store_true",default=False,dest="infile",help="Gets Data from file INPUTDATA") + (options, args) = parser.parse_args() + if len(args) != 2 : + parser.error("incorrect number of arguments") + if options.infile and options.start_word: + parser.error("options -f and -s are mutually exclusive") + ngv=NGramVisualizer () + if options.infile: + data = getData(args[0]) + # print data + ngv.loadCorpus(data,args[1]) + elif options.start_word: + printGraph(args[1],args[0]) + else: + # print args[0] + data = args[0].decode("utf-8") + ngv.loadCorpus(data,args[1]) |