1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
# -*- coding: utf-8 -*-
# Copyright (C) 2011-2012 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
# USA.
"""
Various utility scripts.
Author: Aurelien Bompard <abompard@fedoraproject.org>
"""
from optparse import OptionParser
from kittystore import get_store
#
# Manual database update
#
def updatedb():
parser = OptionParser(usage="%prog -s store_url")
parser.add_option("-s", "--store", help="the URL to the store database")
parser.add_option("-d", "--debug", action="store_true",
help="show SQL queries")
opts, args = parser.parse_args()
if opts.store is None:
parser.error("the store URL is missing (eg: "
"sqlite:///kittystore.sqlite).")
if args:
parser.error("no arguments allowed.")
print 'Upgrading the database schema if necessary...'
store = get_store(opts.store, debug=opts.debug)
version = list(store.db.execute(
"SELECT patch.version FROM patch "
"ORDER BY version DESC LIMIT 1"
))[0][0]
print "Done, the current schema version is %d." % version
#
# Mailman 2 archives downloader
#
import os
import urllib2
import gzip
import itertools
from multiprocessing import Pool
from datetime import date
MONTHS = ['January', 'February', 'March', 'April', 'May', 'June', 'July',
'August', 'September', 'October', 'November', 'December']
def dl_archives():
parser = OptionParser(usage="%prog -u URL -l LIST_NAME [-d destdir]")
parser.add_option("-u", "--url", help="URL to the mailman installation")
parser.add_option("-l", "--list-name", help="mailing-list name")
parser.add_option("-d", "--destination", default=os.getcwd(),
help="directory to download the archives to. Defaults "
"to the current directory (%default)")
parser.add_option("-s", "--start", default="2002",
help="first year to start looking for archives")
parser.add_option("-v", "--verbose", action="store_true",
help="show more information")
opts, args = parser.parse_args()
if not opts.url:
parser.error("an URL must be provided")
if not opts.list_name:
parser.error("a list name must be provided")
if "@" in opts.list_name:
opts.list_name = opts.list_name[:opts.list_name.index("@")]
years = range(int(opts.start), date.today().year + 1)
p = Pool(5)
p.map(_archive_downloader, itertools.product([opts], years, MONTHS))
def _archive_downloader(args):
opts, year, month = args
if not year or not month:
return
basename = "{0}-{1}.txt.gz".format(year, month)
filepath = os.path.join(opts.destination, basename)
if os.path.exists(filepath):
if opts.verbose:
print "{0} already downloaded, skipping".format(basename)
return
url = "{0}/pipermail/{1}/{2}".format(
opts.url, opts.list_name, basename)
if opts.verbose:
print "Downloading from {0}".format(url)
try:
request = urllib2.urlopen(url)
with open(filepath, "w") as f:
f.write(request.read())
except urllib2.URLError, e:
if isinstance(e, urllib2.HTTPError) and e.code == 404:
print ("This archive hasn't been created on the server yet: "
+ basename)
else:
print "Error: %s" % e.reason
return
pos = str(MONTHS.index(month) + 1).rjust(2, "0")
newname = '{0}-{1}-{2}-{3}.txt'.format(opts.list_name, year, pos, month)
with open(os.path.join(opts.destination, newname), "w") as f:
f.write(gzip.open(filepath).read())
print "Downloaded archive for {0} {1} from {2}".format(month, year, url)
|
