diff options
| -rw-r--r-- | docs/fileformat | 4 | ||||
| -rw-r--r-- | lib/myconfig.py | 3 | ||||
| -rwxr-xr-x[-rw-r--r--] | tryprune.py | 42 |
3 files changed, 37 insertions, 12 deletions
diff --git a/docs/fileformat b/docs/fileformat index 6a30aba..1bcac86 100644 --- a/docs/fileformat +++ b/docs/fileformat @@ -46,9 +46,9 @@ Status File Format The lines are sorted by <score>. Prune Status Files 1. 'merged.db', 'kmm_merged.text' , 'pruned.db', 'kmm_pruned.text', 'interpolation.text' are generated when running prune tools in 'finals/try<name>' sub-directory. - 2. 'prune.status' file are generated also, like: + 2. 'cwd.status' file are generated also, like: {'PruneEpoch': 4, 'PruneMergeNumber': 1000, 'PruneK':2, 'PruneCDF': 0.6} Evaluate Status Files - 1. 'evaluate.status' file are generated, like: + 1. 'cwd.status' file are generated, like: {'EvaluateEpoch': 5, 'EvaluateAverageLambda': 0.66, 'EvaluateCorrectionRate': 0.77}
\ No newline at end of file diff --git a/lib/myconfig.py b/lib/myconfig.py index 0032348..b39d012 100644 --- a/lib/myconfig.py +++ b/lib/myconfig.py @@ -96,3 +96,6 @@ class MyConfig: def getFinalModelFileName(self): return 'interpolation.text' + + def getFinalStatusFileName(self): + return 'cwd.status' diff --git a/tryprune.py b/tryprune.py index d0f3c8e..4a85b79 100644..100755 --- a/tryprune.py +++ b/tryprune.py @@ -6,6 +6,7 @@ import sys from subprocess import Popen, PIPE from argparse import ArgumentParser from myconfig import MyConfig +import utils config = MyConfig() @@ -90,7 +91,7 @@ def mergeOneModel(mergedmodel, onemodel, score): sys.exit('Corrupted model found when merging:' + onemodel) #end processing -def mergeSomeModels(tryname, mergedmodel, sortedindexname, mergenum): +def mergeSomeModels(mergedmodel, sortedindexname, mergenum): last_score = 1. #begin processing indexfile = open(sortedindexname, 'r') @@ -113,11 +114,11 @@ def mergeSomeModels(tryname, mergedmodel, sortedindexname, mergenum): #validate merged model validateModel(mergedmodel) -def pruneModel(modelfile, k, CDF): +def pruneModel(prunedmodel, k, CDF): #begin processing cmdline = ['./prune_k_mixture_model', \ '-k', k, '--CDF', CDF, - modelfile] + prunedmodel] subprocess = Popen(cmdline, shell=False, close_fds=True) #check os.waitpid doc @@ -126,6 +127,9 @@ def pruneModel(modelfile, k, CDF): sys.exit('Corrupted model found when pruning:' + modelfile) #end processing + #validate pruned model + validateModel(prunedmodel) + if __name__ == '__main__': parser = ArgumentParser(description='Try prune models.') parser.add_argument('--modeldir', action='store', \ @@ -150,33 +154,51 @@ if __name__ == '__main__': args = parser.parse_args() print(args) tryname = 'try' + args.tryname + + trydir = os.path.join(config.getFinalDir(), tryname) + + #check try<name> directory + if os.access(trydir, os.F_OK): + sys.exit('try' + tryname + ' exists.') + + os.makedirs(trydir) + cwdstatuspath = os.path.join(trydir, config.getFinalStatusFileName()) + cwdstatus = {} + cwdstatus['PruneMergeNumber'] = args.mergenumber + cwdstatus['PruneK'] = args.k + cwdstatus['PruneCDF'] = args.CDF + utils.store_status(cwdstatuspath, cwdstatus) + #merge model candidates print('merging') - mergedmodel = os.path.join(config.getFinalDir(), tryname, 'merged.db') + mergedmodel = os.path.join(trydir, 'merged.db') sortedindexname = os.path.join(args.modeldir, \ config.getSortedEstimateIndex()) - mergeSomeModels(tryname, mergedmodel, sortedindexname, args.mergenumber) + mergeSomeModels(mergedmodel, sortedindexname, args.mergenumber) #export textual format print('exporting') - exportfile = os.path.join(config.getFinalDir(), tryname, 'kmm_merged.text') + exportfile = os.path.join(trydir, 'kmm_merged.text') exportModel(mergedmodel, exportfile) #prune merged model print('pruning') - prunedmodel = os.path.join(config.getFinalDir(), tryname, 'pruned.db') + prunedmodel = os.path.join(trydir, 'pruned.db') #backup merged model shutil.copyfile(mergedmodel, prunedmodel) pruneModel(prunedmodel, args.k, args.CDF) #export textual format print('exporting') - exportfile = os.path.join(config.getFinalDir(), tryname, 'kmm_pruned.text') + exportfile = os.path.join(trydir, 'kmm_pruned.text') exportModel(prunedmodel, exportfile) #convert to interpolation print('converting') kmm_model = exportfile - inter_model = os.path.join(config.getFinalDir(), tryname, \ - config.getFinalModelFileName()) + inter_model = os.path.join(trydir, config.getFinalModelFileName()) convertModel(kmm_model, inter_model) + + #sign status epoch + utils.sign_epoch(cwdstatus, 'Prune') + utils.store_status(cwdstatuspath, cwdstatus) |