From 5234bf2126af899a910dfc29c4e72ca9af04aa3b Mon Sep 17 00:00:00 2001 From: Todd Zullinger Date: Thu, 27 Mar 2008 20:16:27 -0400 Subject: replace PackageOwners() with BaseClient from python-fedora --- PackageOwners.py | 277 ------------------------------------------------------ check-licenses.py | 82 ++++++++-------- 2 files changed, 41 insertions(+), 318 deletions(-) delete mode 100644 PackageOwners.py diff --git a/PackageOwners.py b/PackageOwners.py deleted file mode 100644 index 8a8723e..0000000 --- a/PackageOwners.py +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/python -# -*- mode: Python; indent-tabs-mode: nil; -*- -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - -import commands -import errno -import os, sys, time -import shutil -import tempfile -from urllib import FancyURLopener - - -class AccountsURLopener(FancyURLopener): - """Subclass of urllib.FancyURLopener to allow passing http basic auth info""" - def __init__(self, username, password): - FancyURLopener.__init__(self) - self.username = username - self.password = password - - def prompt_user_passwd(self, host, realm): - return (self.username, self.password) - - -class PackageOwners: - """interface to Fedora package owners list (and Fedora Extras owners/owners.list file)""" - - def __init__(self): - self.dict = {} - self.how = 'unknown' - - - def FromURL(self, retries=3, retrysecs=300, url='https://admin.fedoraproject.org/pkgdb/acls/bugzilla?tg_format=plain', - pkgdb=True, repoid='Fedora', username=None, password=None): - # old url='http://cvs.fedora.redhat.com/viewcvs/*checkout*/owners/owners.list?root=extras' - if pkgdb: - self.how = 'pkgdb' - else: - self.how = 'url' - self.url = url - self.repoid = repoid - self.retries = retries - self.retrysecs = retrysecs - self.username = username - self.password = password - return self._refresh() - - - def FromCVS(self, retries=3, retrysecs=300, command='LC_ALL=C CVS_RSH=ssh cvs -f -d :pserver:anonymous@cvs.fedora.redhat.com:/cvs/extras co owners', workdir='',repoid='Fedora'): - self.how = 'cvs' - self.command = command - self.repoid = repoid - self.retries = retries - self.retrysecs = retrysecs - self.workdir = workdir - self.ownersfile = os.path.join('owners', 'owners.list') - self.cwdstack = [] - return self._refresh() - - - def __getitem__(self,rpmname): - """return e-mail address from initialowner field""" - return self.GetOwner(rpmname) - - - def GetOwner(self,rpmname): - """return e-mail address from initialowner field""" - try: - r = self.dict[rpmname]['mailto'] - except KeyError: - r = '' - return r - - - def GetOwners(self,rpmname): - """return list of e-mail addresses from initialowner+initialcclist fields""" - r = self.GetCoOwnerList(rpmname) - r2 = self.GetOwner(rpmname) - if len(r2): - r.append(r2) - return r - - - def GetCoOwnerList(self,rpmname): - """return list of e-mail addresses from initialcclist field""" - try: - r = self.dict[rpmname]['cc'] - except KeyError: - r = [] - return r - - - def _enterworkdir(self): - self.cwdstack.append( os.getcwd() ) - if self.workdir != '': - os.chdir(self.workdir) - - - def _leaveworkdir(self): - if len(self.cwdstack): - os.chdir( self.cwdstack.pop() ) - - - def _refresh(self): - self.dict = {} # map package name to email address, dict[name] - return self._download() - - - def _parse(self,ownerslist): - for line in ownerslist: - if line.startswith('#') or line.isspace(): - continue - try: - (repo,pkgname,summary,emails,qacontact,cc) = line.rstrip().split('|') - # This is commented, because we don't need the summary. - #summary.replace(r'\u007c','|').replace('\u005c','\\') - - # The PkgDb includes repo's other than Fedora (Fedora EPEL, - # Fedora OLPC, and Red Hat Linux, for example). Skip them. - if repo != self.repoid: - continue - def fixaddr(a): - # Old Fedora CVS owners.list contains e-mail addresses. - # PkgDb plain output contains usernames only. - if not self.how == 'pkgdb': - return a - if not self.usermap.has_key(a): - return a - return self.usermap[a] - - addrs = [] - mailto = '' # primary pkg owner - if len(emails): - if emails.find(',')>=0: - (addrs) = emails.split(',') - mailto = addrs[0] - addrs = addrs[1:] - else: - mailto = emails - mailto = fixaddr(mailto) - - ccaddrs = [] - if len(cc): - (ccaddrs) = cc.split(',') - addrs += ccaddrs - addrs = map(lambda a: fixaddr(a), addrs) - - self.dict[pkgname] = { - 'mailto' : mailto, - 'cc' : addrs - } - except: - print 'ERROR: owners.list is broken' - print line - - - def _downloadfromcvs(self): - self._enterworkdir() - # Dumb caching. Check that file exists and is "quite recent". - cached = False - try: - fstats = os.stat(self.ownersfile) - if ( fstats.st_size and - ((time.time() - fstats.st_ctime) < 3600*2) ): - cached = True - except OSError: - pass - - if not cached: - # Remove 'owners' directory contents, if it exists. - for root, dirs, files in os.walk( 'owners', topdown=False ): - for fname in files: - os.remove(os.path.join( root, fname )) - for dname in dirs: - os.rmdir(os.path.join( root, dname )) - # Retry CVS checkout a few times. - for count in range(self.retries): - (rc, rv) = commands.getstatusoutput(self.command) - if not rc: - break - print rv - time.sleep(self.retrysecs) - if rc: - # TODO: customise behaviour on error conditions - self._leaveworkdir() - return False - - try: - f = file( self.ownersfile ) - except IOError, (err, strerr): - print 'ERROR: %s' % strerr - # TODO: customise behaviour on error conditions - self._leaveworkdir() - return err - ownerslist = f.readlines() - f.close() - self._parse(ownerslist) - self._leaveworkdir() - return True - - - def _getlinesfromurl(self,url): - err = 0 - strerr = '' - # Retry URL download a few times. - for count in range(self.retries): - if count != 0: - time.sleep(self.retrysecs) - try: - opener = AccountsURLopener(self.username, self.password) - f = opener.open(url) - rc = 0 - if 'www-authenticate' in f.headers: - rc = 1 - strerr = 'Authentication is required to access %s' % url - break - except IOError, (_err, _strerr): - rc = 1 - print url - print _strerr - (err,strerr) = (_err,_strerr) - if rc: - raise IOError, (err, strerr) - else: - l = f.readlines() - f.close() - return l - - - def _downloadfromurl(self): - self._parse(self._getlinesfromurl(self.url)) - return True - - - def _downloadfrompkgdb(self): - fasdump = self._getlinesfromurl('https://admin.fedoraproject.org/accounts/dump-group.cgi') - self.usermap = {} - for line in fasdump: - fields = line.split(',') - try: - user = fields[0] - addr = fields[1] - except IndexError: - print line - raise - if (addr.find('@') < 0): # unexpected, no addr - print 'No email in:', line - raise Exception - self.usermap[user] = addr - self._parse(self._getlinesfromurl(self.url)) - return True - - - def _download(self): - if self.how == 'url': - return self._downloadfromurl() - elif self.how == 'pkgdb': - return self._downloadfrompkgdb() - elif self.how == 'cvs': - return self._downloadfromcvs() - else: - self.__init__() - return False - - diff --git a/check-licenses.py b/check-licenses.py index 8894cdd..f18ca3e 100755 --- a/check-licenses.py +++ b/check-licenses.py @@ -3,25 +3,10 @@ """Report problems with License: tags in Fedora spec files. This script checks the License: tag in spec files. The default is to check spec -files in a checkout of the devel branch. Download a daily checkout seed from: +files in a checkout of the devel branch. You can use the get-cvs-tree script to +get a current checkout. -http://cvs.fedoraproject.org/webfiles/ - -You will need to fix up the CVS/Root entries: - -find devel -wholename '*/CVS/Root' | \ - xargs sed -i 's/^/:pserver:anonymous@cvs.fedora.redhat.com:/' - -To generate a report sorted by owner, you will need to grab PackageOwners.py -from the extras-repoclosure module in fedora CVS: - -http://cvs.fedoraproject.org/viewcvs/extras-repoclosure/?root=fedora - -You also need to create an FASauth.py file that contains your Fedora Account -System username and password. Something like: - -username = 'username' -password = 'password' +Finally, enter your Fedora Account System login info in FASauth.py. """ # TODO @@ -38,7 +23,7 @@ import string import optparse from time import gmtime, strftime from FASauth import username, password -from PackageOwners import PackageOwners +from fedora.tg.client import BaseClient, AuthError, ServerError blacklist = ['glibc32', 'glibc64', 'olpc-logos'] no_owner = '_No_Owner_in_PackageDB' @@ -95,43 +80,57 @@ def split_license(license): return map(string.strip, [l for l in regex.split(license) if l]) +def get_packages(): + try: + pkgdb_url = 'https://admin.fedoraproject.org/pkgdb' + pkgdb_client = BaseClient(pkgdb_url, username, password) + pkgdb_data = pkgdb_client.send_request('acls/bugzilla', auth=True) + packages = pkgdb_data['bugzillaAcls']['Fedora'] + except AuthError, e: + raise SystemExit('Authentication error: %s' % e) + except ServerError, e: + raise SystemExit('Server error: %s' % e) + + return packages + + def get_owners(): - owners = PackageOwners() - if not owners.FromURL(username=username, password=password): - print 'Unable to retrieve package owner data' - raise SystemExit(1) - addressmap = dict((v, k) for k, v in owners.usermap.iteritems()) - return owners, addressmap + try: + acct_url = 'https://admin.fedoraproject.org/accounts/' + acct_client = BaseClient(acct_url, username, password) + acct_data = acct_client.send_request('group/dump', auth=True) + owners = {} + for p in acct_data['people']: + user, addr, name = p + owners[user] = {'addr': addr, 'name': name} + except AuthError, e: + raise SystemExit('Authentication error: %s' % e) + except ServerError, e: + raise SystemExit('Server error: %s' % e) + + return owners -def status_by_owner(bogus, owners, addressmap, show_addrs): +def status_by_owner(bogus, owners, packages, show_addrs): bad = {} for srpm, licenses in bogus.items(): - owner = owners.GetOwner(srpm) - if owner: - owner = addressmap[owner] - else: - owner = no_owner + owner = packages[srpm]['owner'] bad.setdefault(owner, {}).setdefault(srpm, licenses) for owner, data in sorted(bad.items()): print owner, if show_addrs and owner != no_owner: - print '(%s)' % owners.usermap[owner].replace('@', ' at '), + print '(%s)' % owners[owner]['addr'].replace('@', ' at '), print '[%d]:' % len(data) for srpm, licenses in sorted(data.items()): print '\t%s: %s' % (srpm, ' '.join(['"%s"' % l for l in licenses])) print -def status_by_package(bogus, owners, addressmap): +def status_by_package(bogus, packages): for srpm, licenses in sorted(bogus.items()): - owner = owners.GetOwner(srpm) - if owner: - owner = addressmap[owner] - else: - owner = no_owner + owner = packages[srpm]['owner'] print '%s (%s): %s' % (srpm, owner, ' '.join(['"%s"' % l for l in licenses])) print @@ -205,11 +204,12 @@ def main(): bad = len(bogus) all = len(specs) percent = (bad / float(all)) * 100 - owners, addressmap = get_owners() + owners = get_owners() + packages = get_packages() if opts.owners: - status_by_owner(bogus, owners, addressmap, opts.addrs) + status_by_owner(bogus, owners, packages, opts.addrs) elif opts.packages: - status_by_package(bogus, owners, addressmap) + status_by_package(bogus, packages) if not opts.license_regex: datestr = strftime('%Y-%m-%d %H:%M UTC', gmtime()) -- cgit