switch to walkIndex

3 files changed, 6 insertions, 61 deletions

diff --git a/partialwordthreshold.py b/partialwordthreshold.py
index 77b6a96..da1daa0 100644
--- a/partialwordthreshold.py
+++ b/partialwordthreshold.py
@@ -5,7 +5,7 @@ from argparse import ArgumentParser
 from operator import itemgetter
 import utils
 from myconfig import MyConfig
-
+from dirwalk import walkIndex
 
 SELECT_WORD_DML = '''
 SELECT freq from ngram where words = ?;
@@ -19,10 +19,6 @@ os.chdir(words_dir)
 #chdir done
 
 
-def handleError(error):
-    sys.exit(error)
-
-
 def getWordFrequency(conn, word):
     sep = config.getWordSep()
     word_str = sep + word + sep
@@ -99,21 +95,6 @@ def handleOneIndex(indexpath, subdir, indexname):
     utils.store_status(indexstatuspath, indexstatus)
 
 
-def walkThroughIndex(path):
-    for root, dirs, files in os.walk(path, topdown=True, onerror=handleError):
-        for onefile in files:
-            filepath = os.path.join(root, onefile)
-            indexpostfix = config.getIndexPostfix()
-            if onefile.endswith(indexpostfix):
-                subdir = os.path.relpath(root, path)
-                indexname = onefile[:-len(indexpostfix)]
-                handleOneIndex(filepath, subdir, indexname)
-            elif onefile.endswith(config.getStatusPostfix()):
-                pass
-            else:
-                print('Unexpected file:' + filepath)
-
-
 if __name__ == '__main__':
     parser = ArgumentParser(description='Partial word threshold.')
     parser.add_argument('--indexdir', action = 'store', \
@@ -122,5 +103,5 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
     print(args)
-    walkThroughIndex(args.indexdir)
+    walkIndex(handleOneIndex, args.indexdir)
     print('done')
diff --git a/populatebigram.py b/populatebigram.py
index cad54ca..a71fe72 100644
--- a/populatebigram.py
+++ b/populatebigram.py
@@ -5,7 +5,7 @@ import sqlite3
 from argparse import ArgumentParser
 import utils
 from myconfig import MyConfig
-
+from dirwalk import walkIndex
 
 CREATE_BIGRAM_DDL = '''
 CREATE TABLE bigram (
@@ -32,10 +32,6 @@ os.chdir(words_dir)
 #chdir done
 
 
-def handleError(error):
-    sys.exit(error)
-
-
 def createBigramSqlite(indexpath, workdir):
     print(indexpath, workdir, 'create bigram')
 
@@ -116,21 +112,6 @@ def handleOneIndex(indexpath, subdir, indexname):
     utils.store_status(indexstatuspath, indexstatus)
 
 
-def walkThroughIndex(path):
-    for root, dirs, files in os.walk(path, topdown=True, onerror=handleError):
-        for onefile in files:
-            filepath = os.path.join(root, onefile)
-            indexpostfix = config.getIndexPostfix()
-            if onefile.endswith(indexpostfix):
-                subdir = os.path.relpath(root, path)
-                indexname = onefile[:-len(indexpostfix)]
-                handleOneIndex(filepath, subdir, indexname)
-            elif onefile.endswith(config.getStatusPostfix()):
-                pass
-            else:
-                print('Unexpected file:' + filepath)
-
-
 if __name__ == '__main__':
     parser = ArgumentParser(description='Populate bi-gram.')
     parser.add_argument('--indexdir', action='store', \
@@ -139,5 +120,5 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
     print(args)
-    walkThroughIndex(args.indexdir)
+    walkIndex(handleOneIndex, args.indexdir)
     print('done')
diff --git a/prepare.py b/prepare.py
index c7325ea..5322bf4 100644
--- a/prepare.py
+++ b/prepare.py
@@ -5,6 +5,7 @@ import sqlite3
 from argparse import ArgumentParser
 import utils
 from myconfig import MyConfig
+from dirwalk import walkIndex
 
 
 CREATE_NGRAM_DDL = '''
@@ -31,10 +32,6 @@ os.chdir(words_dir)
 #chdir done
 
 
-def handleError(error):
-    sys.exit(error)
-
-
 def createSqliteDatabases(onedir):
     print(onedir)
 
@@ -83,20 +80,6 @@ def handleOneIndex(indexpath, subdir, indexname):
     utils.sign_epoch(indexstatus, 'Prepare')
     utils.store_status(indexstatuspath, indexstatus)
 
-def walkThroughIndex(path):
-    for root, dirs, files in os.walk(path, topdown=True, onerror=handleError):
-        for onefile in files:
-            filepath = os.path.join(root, onefile)
-            indexpostfix = config.getIndexPostfix()
-            if onefile.endswith(indexpostfix):
-                subdir = os.path.relpath(root, path)
-                indexname = onefile[:-len(indexpostfix)]
-                handleOneIndex(filepath, subdir, indexname)
-            elif onefile.endswith(config.getStatusPostfix()):
-                pass
-            else:
-                print('Unexpected file:' + filepath)
-
 
 if __name__ == '__main__':
     parser = ArgumentParser(description='Prepare word recognizer.')
@@ -106,5 +89,5 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
     print(args)
-    walkThroughIndex(args.indexdir)
+    walkIndex(handleOneIndex, args.indexdir)
     print('done')