Some hacking on srpm builder

4 files changed, 204 insertions, 15 deletions

diff --git a/rpmci/artifact.py b/rpmci/artifact.py
index 6a4a5cb..4be1e76 100644
--- a/rpmci/artifact.py
+++ b/rpmci/artifact.py
@@ -45,8 +45,9 @@ class ArtifactSet(object):
         return targets
 
     @classmethod
-    def from_config(cls, config, section):
-        artifacts_str = config.get('releases', 'artifacts')
+    def from_config(cls, config):
+        section = 'releases'
+        artifacts_str = config.get(section, 'artifacts')
         artifact_list = map(str.strip, artifacts_str.split(' '))
         artifacts = []
         
@@ -68,7 +69,7 @@ class ArtifactSet(object):
 
             artifacts.append(Artifact(artifact_build_targets))
         return cls(artifacts)
-            
+
 def fedora_git_url_for_build_target(config, buildtarget):
     fedora_os_master = config.get('fedora', 'master')
     base = '%s/%s.git' % (FEDORA_ANONGIT_URL, buildtarget.module)
@@ -76,3 +77,19 @@ def fedora_git_url_for_build_target(config, buildtarget):
         return base
     else:
         return '%s#%s/master' % (base, buildtarget.os)
+
+def upstream_git_url_for_build_target(config, buildtarget):
+    fedora_url = fedora_git_url_for_build_target(config, buildtarget)
+    escaped_url = urllib.quote(url, '')
+    vcsdir = os.path.join(mirror_dir, escaped_url)
+    if not os.path.isdir(vcsdir):
+        raise SystemExit("Not a directory: %r" % (vcsdir, ))
+    specpath = None
+    for filename in os.listdir(vcsdir):
+        if filename.endswith('.spec'):
+            specpath = os.path.join(vcsdir, filename)
+            break
+    assert specpath is not None
+    spec_obj = spec.Spec(specpath)
+    return spec_obj.get_vcs()
+
diff --git a/rpmci/dynrepo.py b/rpmci/dynrepo.py
new file mode 100644
index 0000000..e6cdc90
--- /dev/null
+++ b/rpmci/dynrepo.py
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+
+# dynrepo.py:
+# Monitor set of RPMs in a directory.
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+import os
+import sys
+import logging
+
+import glib
+import rpm
+import rpmUtils
+import rpmUtils.miscutils
+
+from . import async_subprocess
+
+class Repo(object):
+    def __init__(self, dirpath):
+        self._dir = dirpath
+        self._dir_gfile = gio.File(path=dirpath)
+        self._monitor = self._dir_gfile.monitor(gio.FILE_MONITOR_NONE)
+        self._monitor.connect('changed', self._on_dir_changed)
+
+   def get_rpms(self):
+       return self._rpms
+
+   def get_rpms_for_name(self, name, arch):
+       result = []
+       for filename,(n,a,e,v,r) in self._rpms.iteritems():
+           if n == name and (arch is None or a == arch):
+               result.append((filename, n, a, e, v, r))
+       return result
+
+   def get_latest_srpm_for_name(self, name):
+       rpms = self.get_rpms_for_name(name, 'src')
+       cmpevr=rpmUtils.miscutils.compareEVR
+       if rpms:
+           rpms.sort(lambda a,b: cmpevr(a[3:6], b[3:6]))
+           return rpms[-1]
+       return None
+
+   def add_srpm(self, srpm_path):
+       basename = os.path.basename(srpm_path)
+       os.link(srpm_path, os.path.join(self._dir, basename))
+
+   def _delete_old_rpms_in_dir(self, dirpath):
+       proc = popen_verbose(['repomanage', '-o', '.'], stdout=subprocess.PIPE,
+                            stderr=sys.stderr,
+                            cwd=dirpath)
+       output = proc.communicate()[0]
+       for line in output.split('\n'):
+           if line.endswith('.rpm') and os.path.exists(line):
+               os.unlink(line)
+
+   def update_repo_sync(self):
+       self._delete_old_rpms_in_dir(dirpath)
+       subprocess.check_call(['createrepo', '.'], cwd=self._dir)
+
+   def _headers_from_packages(self, rpmlist):
+       result = {}
+       ts = rpm.TransactionSet()
+       ts.setVSFlags(~(rpm._RPMVSF_NOPAYLOAD))
+       for pkg in rpmlist:
+           pkg_path = os.path.join(self._dir, pkg)
+           try:
+               header = rpmUtils.miscutils.hdrFromPackage(ts, pkg_path)
+           except rpmUtils.RpmUtilsError, e:
+               logging.exception(e)
+               continue
+           (n,a,e,v,r) = rpmUtils.miscutils.pkgTupleFromHeader(header)
+           del header
+           result[pkg] = (n,a,e,v,r)
+       return result
+
+   def _on_dir_changed(self, mon, gfile, other, event):
+       self._reload()
+
+   def _reload(self):
+       dir_contents = os.listdir(self._dirpath)
+       messages = set()
+       rpmlist = set()
+       for filename in dir_contents:
+           if filename.endswith('.tmp'):
+               continue
+           if not filename.endswith('.rpm'):
+               continue
+           rpmlist.add(filename)
+        
+       deleted = []
+       for rpm in self._rpms:
+           if rpm not in rpmlist:
+               deleted.add(rpm)
+       for rpm in deleted:
+           del self._rpms[rpm]
+       new = []
+       for rpm in self._rpms:
+           if rpm not in rpmlist:
+               new.append(rpm)
+       for rpm,data in self._headers_from_packages(new).iteritems():
+           self._rpms[rpm] = data
diff --git a/rpmci/rpmci_srpm_builder_main.py b/rpmci/rpmci_srpm_builder_main.py
new file mode 100644
index 0000000..2bfbef4
--- /dev/null
+++ b/rpmci/rpmci_srpm_builder_main.py
@@ -0,0 +1,80 @@
+#!/usr/bin/python
+
+# rpmci_srpm_builder_main.py:
+# Implementation of rpm-vcs-mirror
+#
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+# Copyright (C) 2010 Red Hat, Inc.
+# Written by Colin Walters <walters@verbum.org>
+
+import os
+import sys
+import time
+import shutil
+import optparse
+from ConfigParser import SafeConfigParser
+import logging
+import urllib
+
+import glib
+import gobject
+import gio
+
+from . import msgqueue
+from . import dynrepo
+from . import lame_vcs_abstraction
+
+class SRPMBuilder(object):
+    def __init__(self, options, config):
+        self.config = config
+        self.urls = urls
+
+        self._options = options
+
+        vcs_msgqueue_dir = config.get('VCS', 'msgqueue')
+        self._vcs_msgqueue = msgqueue.MessageQueue(vcs_msgqueue_dir)
+
+        self._artifactset = artifact.ArtifactSet.from_config(config)
+        self._vcs_by_url = {}
+        for target in self._artifactset.get_build_targets():
+            fedora_url = artifact.fedora_git_url_for_build_target(target)
+            upstream_url = artifact.upstream_git_url_for_build_target(target)
+            self._vcs_by_url[fedora_url] = lame_vcs_abstraction.VCS.new_from_spec(fedora_url)
+            self._vcs_by_url[upstream_url] = lame_vcs_abstraction.VCS.new_from_spec(upstream_url)
+
+        self._srcrepo = dynrepo.Repo(config.get('SRPM', 'srpm_dir'))
+
+    def start(self):
+        self._vcs_msgqueue.connect(self._on_vcs_message)
+        
+    def _on_vcs_message(self, q, messages):
+        for msg in messages:
+            url = msg.payload.url
+            q.consume(msg)
+            vcs = self._vcs_by_url[url]
+            
+            
+
+def main():
+    glib.threads_init()
+
+    opts = optparse.OptionParser("usage: %prog [options]")
+    opts.add_option('-c', '--config', dest='config', help="Path to configuration file")
+    opts.add_option('', '--debug', action='store_true', help="Print verbose debugging")
+
+    (options, args) = opts.parse_args()
+
+    if options.config is None:
+        print "Must specify --config"
+        sys.exit(1)
+
+    config = SafeConfigParser({'home': os.environ['HOME']})
+    config.read(options.config)
+    level = logging.DEBUG if options.debug else logging.INFO
+    logging.basicConfig(stream=sys.stderr, level=level)
+
+    builder = SRPMBuilder(options, config, urls)
+    builder.start()
+    
+    loop = glib.MainLoop()
+    loop.run()
diff --git a/rpmci/rpmci_update_config_main.py b/rpmci/rpmci_update_config_main.py
index 98355c1..8e36d36 100644
--- a/rpmci/rpmci_update_config_main.py
+++ b/rpmci/rpmci_update_config_main.py
@@ -61,18 +61,6 @@ def update_config(options, config):
                                         
     all_urls = set(fedora_git_urls)
     for url in fedora_git_urls:
-        escaped_url = urllib.quote(url, '')
-        vcsdir = os.path.join(mirror_dir, escaped_url)
-        if not os.path.isdir(vcsdir):
-            raise SystemExit("Not a directory: %r" % (vcsdir, ))
-        specpath = None
-        for filename in os.listdir(vcsdir):
-            if filename.endswith('.spec'):
-                specpath = os.path.join(vcsdir, filename)
-                break
-        assert specpath is not None
-        spec_obj = spec.Spec(specpath)
-        upstream_vcs_url = spec_obj.get_vcs()
         all_urls.add(upstream_vcs_url)
             
     _write_vcs_urls(options, config, all_urls)