diff options
-rw-r--r-- | partialwordthreshold.py | 23 | ||||
-rw-r--r-- | populatebigram.py | 23 | ||||
-rw-r--r-- | prepare.py | 21 |
3 files changed, 6 insertions, 61 deletions
diff --git a/partialwordthreshold.py b/partialwordthreshold.py index 77b6a96..da1daa0 100644 --- a/partialwordthreshold.py +++ b/partialwordthreshold.py @@ -5,7 +5,7 @@ from argparse import ArgumentParser from operator import itemgetter import utils from myconfig import MyConfig - +from dirwalk import walkIndex SELECT_WORD_DML = ''' SELECT freq from ngram where words = ?; @@ -19,10 +19,6 @@ os.chdir(words_dir) #chdir done -def handleError(error): - sys.exit(error) - - def getWordFrequency(conn, word): sep = config.getWordSep() word_str = sep + word + sep @@ -99,21 +95,6 @@ def handleOneIndex(indexpath, subdir, indexname): utils.store_status(indexstatuspath, indexstatus) -def walkThroughIndex(path): - for root, dirs, files in os.walk(path, topdown=True, onerror=handleError): - for onefile in files: - filepath = os.path.join(root, onefile) - indexpostfix = config.getIndexPostfix() - if onefile.endswith(indexpostfix): - subdir = os.path.relpath(root, path) - indexname = onefile[:-len(indexpostfix)] - handleOneIndex(filepath, subdir, indexname) - elif onefile.endswith(config.getStatusPostfix()): - pass - else: - print('Unexpected file:' + filepath) - - if __name__ == '__main__': parser = ArgumentParser(description='Partial word threshold.') parser.add_argument('--indexdir', action = 'store', \ @@ -122,5 +103,5 @@ if __name__ == '__main__': args = parser.parse_args() print(args) - walkThroughIndex(args.indexdir) + walkIndex(handleOneIndex, args.indexdir) print('done') diff --git a/populatebigram.py b/populatebigram.py index cad54ca..a71fe72 100644 --- a/populatebigram.py +++ b/populatebigram.py @@ -5,7 +5,7 @@ import sqlite3 from argparse import ArgumentParser import utils from myconfig import MyConfig - +from dirwalk import walkIndex CREATE_BIGRAM_DDL = ''' CREATE TABLE bigram ( @@ -32,10 +32,6 @@ os.chdir(words_dir) #chdir done -def handleError(error): - sys.exit(error) - - def createBigramSqlite(indexpath, workdir): print(indexpath, workdir, 'create bigram') @@ -116,21 +112,6 @@ def handleOneIndex(indexpath, subdir, indexname): utils.store_status(indexstatuspath, indexstatus) -def walkThroughIndex(path): - for root, dirs, files in os.walk(path, topdown=True, onerror=handleError): - for onefile in files: - filepath = os.path.join(root, onefile) - indexpostfix = config.getIndexPostfix() - if onefile.endswith(indexpostfix): - subdir = os.path.relpath(root, path) - indexname = onefile[:-len(indexpostfix)] - handleOneIndex(filepath, subdir, indexname) - elif onefile.endswith(config.getStatusPostfix()): - pass - else: - print('Unexpected file:' + filepath) - - if __name__ == '__main__': parser = ArgumentParser(description='Populate bi-gram.') parser.add_argument('--indexdir', action='store', \ @@ -139,5 +120,5 @@ if __name__ == '__main__': args = parser.parse_args() print(args) - walkThroughIndex(args.indexdir) + walkIndex(handleOneIndex, args.indexdir) print('done') @@ -5,6 +5,7 @@ import sqlite3 from argparse import ArgumentParser import utils from myconfig import MyConfig +from dirwalk import walkIndex CREATE_NGRAM_DDL = ''' @@ -31,10 +32,6 @@ os.chdir(words_dir) #chdir done -def handleError(error): - sys.exit(error) - - def createSqliteDatabases(onedir): print(onedir) @@ -83,20 +80,6 @@ def handleOneIndex(indexpath, subdir, indexname): utils.sign_epoch(indexstatus, 'Prepare') utils.store_status(indexstatuspath, indexstatus) -def walkThroughIndex(path): - for root, dirs, files in os.walk(path, topdown=True, onerror=handleError): - for onefile in files: - filepath = os.path.join(root, onefile) - indexpostfix = config.getIndexPostfix() - if onefile.endswith(indexpostfix): - subdir = os.path.relpath(root, path) - indexname = onefile[:-len(indexpostfix)] - handleOneIndex(filepath, subdir, indexname) - elif onefile.endswith(config.getStatusPostfix()): - pass - else: - print('Unexpected file:' + filepath) - if __name__ == '__main__': parser = ArgumentParser(description='Prepare word recognizer.') @@ -106,5 +89,5 @@ if __name__ == '__main__': args = parser.parse_args() print(args) - walkThroughIndex(args.indexdir) + walkIndex(handleOneIndex, args.indexdir) print('done') |