diff options
author | Peng Wu <alexepico@gmail.com> | 2011-07-24 20:57:40 +0800 |
---|---|---|
committer | Peng Wu <alexepico@gmail.com> | 2011-07-24 20:57:40 +0800 |
commit | b3b768cc6806d51b9693348c759b40fc46deabc3 (patch) | |
tree | 33694a6006725e5e2edd1006afa40cacbfd73965 | |
parent | 36b055a68d3dbb9c16a85dec9aa5449b042d2090 (diff) | |
download | trainer-b3b768cc6806d51b9693348c759b40fc46deabc3.tar.gz trainer-b3b768cc6806d51b9693348c759b40fc46deabc3.tar.xz trainer-b3b768cc6806d51b9693348c759b40fc46deabc3.zip |
write tryprune.py in progress
-rwxr-xr-x | estimate.py | 2 | ||||
-rw-r--r-- | tryprune.py | 89 |
2 files changed, 85 insertions, 6 deletions
diff --git a/estimate.py b/estimate.py index 3ca48db..5a05ccf 100755 --- a/estimate.py +++ b/estimate.py @@ -39,7 +39,7 @@ def handleOneModel(modelfile): modelfile] subprocess = Popen(cmdline, shell=False, stdout=PIPE, \ - close_fds= True) + close_fds=True) for line in subprocess.stdout.readlines(): #remove trailing '\n' diff --git a/tryprune.py b/tryprune.py index 6c03e7a..7d0a32d 100644 --- a/tryprune.py +++ b/tryprune.py @@ -1,21 +1,100 @@ #!/usr/bin/python3 +import os +import sys from subprocess import Popen, PIPE from argparse import ArgumentParser +from myconfig import MyConfig + + +config = MyConfig() + +#change cwd to the libpinyin utils/training directory +libpinyin_dir = config.getToolsDir() +libpinyin_sub_dir = os.path.join(libpinyin_dir, 'utils', 'training') +os.chdir(libpinyin_sub_dir) +#chdir done def validateModel(modelfile): - pass + #begin processing + cmdline = ['./validate_k_mixture_model', \ + modelfile] + + subprocess = Popen(cmdline, shell=False, close_fds=True) + #check os.waitpid doc + (pid, status) = os.waitpid(subprocess.pid, 0) + if status != 0: + sys.exit('Corrupted model found when validating:' + modelfile) + #end processing def exportModel(modelfile, textmodel): - pass + #begin processing + cmdline = ['./export_k_mixture_model', \ + '--k-mixture-model-file', \ + modelfile] + + subprocess = Popen(cmdline, shell=False, stdout=PIPE, \ + close_fds=True) + + with open(textmodel, 'wb') as f: + f.writelines(subprocess.stdout.readlines()) + f.close() + + #check os.waitpid doc + (pid, status) = os.waitpid(subprocess.pid, 0) + if status != 0: + sys.exit('Corrupted model found when exporting:' + modelfile) + #end processing def mergeOneModel(mergedmodel, onemodel): - pass + #begin processing + cmdline = ['./merge_k_mixture_model', \ + '--result-file', \ + mergedmodel, \ + onemodel] + + subprocess = Popen(cmdline, shell=False, close_fds=True) + #check os.waitpid doc + (pid, status) = os.waitpid(subprocess.pid, 0) + if status != 0: + sys.exit('Corrupted model found when merging:' + onemodel) + #end processing def mergeSomeModels(indexfile, mergenum): pass def pruneModel(modelfile, k, CDF): - pass + #begin processing + cmdline = ['./prune_k_mixture_model', \ + '-k', k, '--CDF', CDF, + modelfile] + + subprocess = Popen(cmdline, shell=False, close_fds=True) + #check os.waitpid doc + (pid, status) = os.waitpid(subprocess.pid, 0) + if (status != 0): + sys.exit('Corrupted model found when pruning:' + modelfile) + #end processing if __name__ == '__main__': - pass + parser = ArgumentParser(description='Try prune models.') + parser.add_argument('--modeldir', action='store', \ + help='model directory', \ + default=config.getModelDir()) + + parser.add_argument('--mergenumber', action='store', \ + help='number of documents to be merged', \ + default='10') + + parser.add_argument('-k', action='store', \ + help='k parameter of k mixture model prune', \ + default='3') + + parser.add_argument('--CDF', action='store', \ + help='CDF parameter of k mixture model prune', \ + default='0.99') + + parser.add_argument('tryname', action='store', \ + help='the storage directory') + + args = parser.parse_args() + print(args) |