1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
# -*- coding: utf-8 -*-
"""
Copyright (C) 2012 Aurélien Bompard <abompard@fedoraproject.org>
Author: Aurélien Bompard <abompard@fedoraproject.org>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or (at
your option) any later version.
See http://www.gnu.org/copyleft/gpl.html for the full text of the
license.
"""
from __future__ import absolute_import
import os
import shutil
import sys
from zope.interface import providedBy
from whoosh.index import create_in, exists_in, open_dir
from whoosh.fields import Schema, ID, TEXT, DATETIME, KEYWORD, BOOLEAN
from whoosh.analysis import StemmingAnalyzer
from whoosh.qparser import MultifieldParser
from whoosh.query import Term
from mailman.interfaces.archiver import ArchivePolicy
from mailman.interfaces.messages import IMessage
def email_to_search_doc(email):
if not IMessage.providedBy(email):
raise ValueError("not an instance of the Email class")
private_list = (email.mlist.archive_policy == ArchivePolicy.private)
search_doc = {
"list_name": email.list_name,
"message_id": email.message_id,
"sender": u"%s %s" % (email.sender_name, email.sender_email),
"user_id": email.user_id,
"subject": email.subject,
"content": email.content,
"date": email.date, # UTC
"private_list": private_list,
}
attachments = [a.name for a in email.attachments]
if attachments:
search_doc["attachments"] = " ".join(attachments)
return search_doc
class SearchEngine(object):
def __init__(self, location):
self.location = location
self._index = None
def _get_schema(self):
stem_ana = StemmingAnalyzer()
return Schema(
list_name=ID(stored=True),
message_id=ID(stored=True, unique=True),
sender=TEXT(field_boost=1.5),
user_id=TEXT,
subject=TEXT(field_boost=2.0, analyzer=stem_ana),
content=TEXT(analyzer=stem_ana),
date=DATETIME(),
attachments=TEXT,
tags=KEYWORD(commas=True, scorable=True),
private_list=BOOLEAN(),
)
@property
def index(self):
if self._index is None:
self._index = open_dir(self.location)
return self._index
def add(self, doc):
writer = self.index.writer()
if IMessage.providedBy(doc):
doc = email_to_search_doc(doc)
try:
writer.add_document(**doc)
except Exception:
writer.cancel()
raise
else:
writer.commit()
def search(self, query, list_name=None, page=None, limit=10,
sortedby=None, reverse=False):
"""
TODO: Should the searcher be shared?
http://pythonhosted.org/Whoosh/threads.html#concurrency
"""
query = MultifieldParser(
["sender", "subject", "content", "attachments"],
self.index.schema).parse(query)
if list_name:
results_filter = Term("list_name", list_name)
else:
# When searching all lists, only the public lists are searched
results_filter = Term("private_list", False)
return_value = {"total": 0, "results": []}
with self.index.searcher() as searcher:
if page:
results = searcher.search_page(
query, page, pagelen=limit, sortedby=sortedby,
reverse=reverse, filter=results_filter)
return_value["total"] = results.total
else:
results = searcher.search(
query, limit=limit, sortedby=sortedby,
reverse=reverse, filter=results_filter)
# http://pythonhosted.org/Whoosh/searching.html#results-object
if results.has_exact_length():
return_value["total"] = len(results)
else:
return_value["total"] = results.estimated_length()
return_value["results"] = [ dict(r) for r in results ]
return return_value
def optimize(self):
return self.index.optimize()
def add_batch(self, documents, debug=False):
"""
See http://pythonhosted.org/Whoosh/batch.html
"""
if debug:
sys.stdout.write("Indexing all messages")
sys.stdout.flush()
# Don't use optimizations below, it will eat up lots of memory and can
# go as far as preventing forking (OSError), tested on a 3GB VM with
# the Fedora archives
#writer = self.index.writer(limitmb=256, procs=4, multisegment=True)
writer = self.index.writer(multisegment=True)
# remove the LRU cache limit from the stemanalyzer
for component in writer.schema["content"].analyzer:
try:
component.cachesize = -1
component.clear()
except AttributeError:
continue
try:
for num, doc in enumerate(documents):
if IMessage.providedBy(doc):
doc = email_to_search_doc(doc)
writer.add_document(**doc)
if debug and num % 100 == 0:
sys.stdout.write(".")
sys.stdout.flush()
except Exception:
writer.cancel()
raise
else:
writer.commit()
if debug:
sys.stdout.write("\n")
sys.stdout.flush()
def initialize_with(self, store):
"""Create and populate the index with the contents of a Store"""
if not os.path.isdir(self.location):
os.makedirs(self.location)
self._index = create_in(self.location, self._get_schema())
self.add_batch(store.get_all_messages(), store.debug)
def needs_upgrade(self):
if not exists_in(self.location):
return True
if "user_id" not in self.index.schema:
return True
new_schema = self._get_schema()
for field_name, field_type in new_schema.items():
if field_name not in self.index.schema:
return True
return False
def upgrade(self, store):
"""Upgrade the schema"""
if not exists_in(self.location):
self.initialize_with(store)
if "user_id" not in self.index.schema:
print "Rebuilding the search index to include the new user_id field..."
shutil.rmtree(self.location)
self.initialize_with(store)
new_schema = self._get_schema()
writer = self.index.writer()
for field_name, field_type in new_schema.items():
if field_name not in self.index.schema:
print "Adding field %s to the search index" % field_name
writer.add_field(field_name, field_type)
writer.commit(optimize=True)
|
