fixes estimate.py

3 files changed, 20 insertions, 5 deletions

diff --git a/estimate.py b/estimate.py
index eaec334..ef5c40a 100755
--- a/estimate.py
+++ b/estimate.py
@@ -22,7 +22,7 @@ def handleError(error):
     sys.exit(error)
 
 
-def handleOneModel(modelfile):
+def handleOneModel(modelfile, reportfile):
     modelfilestatuspath = modelfile + config.getStatusPostfix()
     modelfilestatus = utils.load_status(modelfilestatuspath)
     if not utils.check_epoch(modelfilestatus, 'Generate'):
@@ -30,6 +30,8 @@ def handleOneModel(modelfile):
     if utils.check_epoch(modelfilestatus, 'Estimate'):
         return
 
+    reporthandle = open(reportfile, 'wb')
+
     result_line_prefix = "average lambda:"
     avg_lambda = 0.
 
@@ -44,14 +46,21 @@ def handleOneModel(modelfile):
                            close_fds=True)
 
     for line in subprocess.stdout.readlines():
+        reporthandle.writelines([line])
         #remove trailing '\n'
+        line = line.decode('utf-8')
         line = line.rstrip(os.linesep)
         if line.startswith(result_line_prefix):
             avg_lambda = float(line[len(result_line_prefix):])
 
-    os.waitpid(subprocess.pid, 0)
+    reporthandle.close()
+
+    (pid, status) = os.waitpid(subprocess.pid, 0)
+    if status != 0:
+        sys.exit('estimate k mixture model returns error.')
     #end processing
 
+    print('average lambda:', avg_lambda)
     modelfilestatus['EstimateScore'] = avg_lambda
     utils.sign_epoch(modelfilestatus, 'Estimate')
     utils.store_status(modelfilestatuspath, modelfilestatus)
@@ -63,13 +72,16 @@ def walkThroughModels(path):
             filepath = os.path.join(root, onefile)
             if onefile.endswith(config.getModelPostfix()):
                 subpath = os.path.relpath(filepath, path)
+                reportfile = filepath + config.getReportPostfix()
                 print("Processing " + subpath)
-                handleOneModel(filepath)
+                handleOneModel(filepath, reportfile)
                 print("Processed " + subpath)
             elif onefile.endswith(config.getStatusPostfix()):
                 pass
             elif onefile.endswith(config.getIndexPostfix()):
                 pass
+            elif onefile.endswith(config.getReportPostfix()):
+                pass
             else:
                 print('Unexpected file:' + filepath)
 
@@ -102,6 +114,8 @@ def gatherModels(path, indexname):
                 pass
             elif onefile.endswith(config.getIndexPostfix()):
                 pass
+            elif onefile.endswith(config.getReportPostfix()):
+                pass
             else:
                 print('Unexpected file:' + filepath)
     indexfile.close()
diff --git a/generate.py b/generate.py
index 6f98814..f3cc09c 100755
--- a/generate.py
+++ b/generate.py
@@ -148,6 +148,7 @@ def handleOneIndex(indexpath, subdir, indexname):
     #end processing
 
     #save current progress in status file
+    modelnum += 1
     indexstatus['GenerateTextEnd'] = nexttextnum
     indexstatus['GenerateModelEnd'] = modelnum
 
diff --git a/lib/myconfig.py b/lib/myconfig.py
index f40d2a4..a5f4eec 100644
--- a/lib/myconfig.py
+++ b/lib/myconfig.py
@@ -39,7 +39,7 @@ class MyConfig:
         return self.m_evals_dir
 
     def getEstimatesModel(self):
-        estimates_model = m_tools_dir + '/data/estimates.db'
+        estimates_model = self.m_tools_dir + '/data/estimates.db'
         return estimates_model
 
     def getEstimateIndex(self):
@@ -62,7 +62,7 @@ class MyConfig:
 
     #the trained corpus size of model candidates
     def getCandidateModelSize(self):
-        candidate_model_size = 11.9 * 1024 * 1024
+        candidate_model_size = 11.9 * 1024 * 1024 * 3
         return candidate_model_size
 
     def getModelPostfix(self):