From a549f749323b56679f3bba657a045078207e18c3 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 12 Apr 2011 18:15:58 -0400 Subject: basic model and scraping for dependencies. --- leafymiracle/models.py | 18 +++++++++++++++++- leafymiracle/populate.py | 19 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/leafymiracle/models.py b/leafymiracle/models.py index 0e86aa1..0edbdd4 100644 --- a/leafymiracle/models.py +++ b/leafymiracle/models.py @@ -13,7 +13,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from sqlalchemy import Integer, Column, Unicode, UnicodeText, ForeignKey +from sqlalchemy import Integer, Column, Unicode, UnicodeText, ForeignKey, Table from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship, scoped_session, sessionmaker @@ -115,6 +115,22 @@ class Package(Base): """.format(**self.__dict__) } +dependencies_mapping = Table( + 'packages_dependencies_mapping', Base.metadata, + Column('depender_id', Integer, + ForeignKey('packages.id'), primary_key=True), + Column('dependee_id', Integer, + ForeignKey('packages.id'), primary_key=True)) + +Package.__mapper__.add_property('dependencies', relationship( + Package, + primaryjoin=Package.id==dependencies_mapping.c.dependee_id, + secondaryjoin=dependencies_mapping.c.depender_id==Package.id, + secondary=dependencies_mapping, + doc="List of this packages' dependencies!", +)) + + def initialize_sql(engine): DBSession.configure(bind=engine) diff --git a/leafymiracle/populate.py b/leafymiracle/populate.py index 9f78fc5..cb3c7d4 100644 --- a/leafymiracle/populate.py +++ b/leafymiracle/populate.py @@ -19,7 +19,8 @@ from kitchen.text.converters import to_unicode from models import Root, Category, Group, Package, DBSession, initialize_sql -def populate(comps='comps-f16'): + +def populate(comps='comps-f16', do_dependencies=True): from yum.comps import Comps session = DBSession() @@ -58,6 +59,22 @@ def populate(comps='comps-f16'): session.flush() + if do_dependencies: + yumobj = yum.YumBase() + yumobj.setCacheDir() + for package in session.query(Package).all(): + deps = yumobj.pkgSack.searchNevra(name=package.name)[0] + deps_d = pkg.findDeps([pkg]) + deps = [tup[0] for tup in deps_d[deps_d.keys()[0]].keys()] + + for dep in deps: + dep_as_package = session.query(Package)\ + .filter_by(name=dep).one() + if dep_as_package not in package.dependencies: + package.dependencies.append(dep_as_package) + + + session.commit() def build_comps(): -- cgit From 2de3cb1c4de6e17184e7b70be4800ebd8b54c58e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 12 Apr 2011 19:20:25 -0400 Subject: Script to link in system modules --- README.txt | 1 + link_in_system_modules.sh | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100755 link_in_system_modules.sh diff --git a/README.txt b/README.txt index 0a308c4..bbd5f8c 100644 --- a/README.txt +++ b/README.txt @@ -33,6 +33,7 @@ [ Running ] $ virtualenv --no-site-packages env && source env/bin/activate +$ ./link_in_system_modules.sh $ python setup.py develop $ python leafymiracle/populate.py $ paster serve development.ini diff --git a/link_in_system_modules.sh b/link_in_system_modules.sh new file mode 100755 index 0000000..36bd81c --- /dev/null +++ b/link_in_system_modules.sh @@ -0,0 +1,10 @@ +#!/bin/bash + + +ln -s /usr/lib/python2.*/site-packages/yum* $WORKON_HOME/leafymiracle/lib/python*/site-packages +ln -s /usr/lib/python2.*/site-packages/rpm* $WORKON_HOME/leafymiracle/lib/python*/site-packages +ln -s /usr/lib/python2.*/site-packages/urlgrabber* $WORKON_HOME/leafymiracle/lib/python*/site-packages +ln -s /usr/lib/python2.*/site-packages/libxml2* $WORKON_HOME/leafymiracle/lib/python*/site-packages +ln -s /usr/lib/python2.*/site-packages/pycurl* $WORKON_HOME/leafymiracle/lib/python*/site-packages +ln -s /usr/lib/python2.*/site-packages/sqlitecache* $WORKON_HOME/leafymiracle/lib/python*/site-packages +ln -s /usr/lib/python2.*/site-packages/_sqlitecache* $WORKON_HOME/leafymiracle/lib/python*/site-packages -- cgit From cfa45253b6495622fbbe49e41a1cd86bc070098f Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 12 Apr 2011 19:21:16 -0400 Subject: Ignoring more stuff --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dc09afb..8027a99 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ *.pyc *.pyo -*.db +*.db* build dist env *.egg-info *.swp +comps -- cgit From 2aabeb571240760f015e306ef74dac13709b1b9e Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 12 Apr 2011 19:21:26 -0400 Subject: Package deps crawl/scrape --- leafymiracle/populate.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/leafymiracle/populate.py b/leafymiracle/populate.py index cb3c7d4..9859d58 100644 --- a/leafymiracle/populate.py +++ b/leafymiracle/populate.py @@ -19,6 +19,9 @@ from kitchen.text.converters import to_unicode from models import Root, Category, Group, Package, DBSession, initialize_sql +from yum import YumBase +yumobj = YumBase() +yumobj.setCacheDir() def populate(comps='comps-f16', do_dependencies=True): from yum.comps import Comps @@ -60,22 +63,36 @@ def populate(comps='comps-f16', do_dependencies=True): session.flush() if do_dependencies: - yumobj = yum.YumBase() - yumobj.setCacheDir() for package in session.query(Package).all(): - deps = yumobj.pkgSack.searchNevra(name=package.name)[0] - deps_d = pkg.findDeps([pkg]) - deps = [tup[0] for tup in deps_d[deps_d.keys()[0]].keys()] + add_dependencies(package, session) - for dep in deps: - dep_as_package = session.query(Package)\ - .filter_by(name=dep).one() - if dep_as_package not in package.dependencies: - package.dependencies.append(dep_as_package) + session.commit() +def add_dependencies(package, session): + try: + pkg = yumobj.pkgSack.searchNevra(name=package.name)[0] + deps_d = yumobj.findDeps([pkg]) + deps = [tup[0] for tup in deps_d[deps_d.keys()[0]].keys()] + for dep in deps: + base_query = session.query(Package).filter_by(name=dep) + if base_query.count() == 0: + _new_package = Package(name=dep) + session.add(_new_package) + session.flush() + add_dependencies(_new_package, session) - session.commit() + dep_as_package = base_query.one() + + if dep_as_package not in package.dependencies: + package.dependencies.append(dep_as_package) + + print "package: %s has (%i/%i) deps" % ( + package.name, len(package.dependencies), len(deps)) + session.flush() + except Exception as e: + # TODO -- figure out why some stuff breaks here. + pass def build_comps(): import subprocess -- cgit From 8ecba28a63d36e656b03ef9d7b753608bb6655ee Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Tue, 12 Apr 2011 19:25:04 -0400 Subject: No longer need to exclude since package-nodes look so "busy" now --- leafymiracle/widgets.py | 1 - 1 file changed, 1 deletion(-) diff --git a/leafymiracle/widgets.py b/leafymiracle/widgets.py index d2b38c0..7816da4 100644 --- a/leafymiracle/widgets.py +++ b/leafymiracle/widgets.py @@ -37,7 +37,6 @@ class LeafyGraph(SQLARadialGraph): show_attributes = False imply_relations = True auto_labels = False - excluded_columns = ['group'] deep_linking = True #transition = JSSymbol(src='$jit.Trans.Back.easeInOut') duration = 400 -- cgit From 13d9a8971b2a0113ee4535e5a6f3e644032cb8fe Mon Sep 17 00:00:00 2001 From: Luke Macken Date: Tue, 12 Apr 2011 21:33:31 -0400 Subject: Improve populate.add_dependencies --- leafymiracle/populate.py | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/leafymiracle/populate.py b/leafymiracle/populate.py index 9859d58..87d578a 100644 --- a/leafymiracle/populate.py +++ b/leafymiracle/populate.py @@ -69,30 +69,34 @@ def populate(comps='comps-f16', do_dependencies=True): session.commit() def add_dependencies(package, session): - try: - pkg = yumobj.pkgSack.searchNevra(name=package.name)[0] - deps_d = yumobj.findDeps([pkg]) - deps = [tup[0] for tup in deps_d[deps_d.keys()[0]].keys()] + deps = set() + pkg = yumobj.pkgSack.searchNevra(name=package.name) + if not pkg: + print "Cannot find package: %s" % package + return - for dep in deps: - base_query = session.query(Package).filter_by(name=dep) - if base_query.count() == 0: - _new_package = Package(name=dep) - session.add(_new_package) - session.flush() - add_dependencies(_new_package, session) + deps_d = yumobj.findDeps([pkg[0]]) + for dep in deps_d.itervalues(): + for req in dep.itervalues(): + deps.add(req[0].name) - dep_as_package = base_query.one() + for dep in deps: + base_query = session.query(Package).filter_by(name=dep) + if base_query.count() == 0: + _new_package = Package(name=dep) + session.add(_new_package) + session.flush() + add_dependencies(_new_package, session) - if dep_as_package not in package.dependencies: - package.dependencies.append(dep_as_package) + dep_as_package = base_query.one() - print "package: %s has (%i/%i) deps" % ( - package.name, len(package.dependencies), len(deps)) - session.flush() - except Exception as e: - # TODO -- figure out why some stuff breaks here. - pass + if dep_as_package not in package.dependencies: + package.dependencies.append(dep_as_package) + + print "package: %s has (%i/%i) deps" % ( + package.name, len(package.dependencies), len(deps)) + + session.flush() def build_comps(): import subprocess -- cgit