all 2-week Atomic composes have been done with Pungi 4 for a
while now, we don't need these icky workarounds any more. Yay!
Check that job scheduling still works.
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| Path | Packages | |||
|---|---|---|---|---|
| M | scheduler/fedora_openqa_schedule/consumer.py (15 lines) | |||
| M | scheduler/fedora_openqa_schedule/schedule.py (39 lines) |
| Commit | Tree | Parents | Author | Summary | Date |
|---|---|---|---|---|---|
| 6087626a1124 | 37a0cd2a8275 | 181af84d9af7 | Adam Williamson | scheduler: drop old 2-week Atomic compose workarounds (Show More…) | Oct 6 2016, 4:15 PM |
| Show All 26 Lines | |||||
| 27 | import fedora_openqa_schedule.schedule as schedule | 27 | import fedora_openqa_schedule.schedule as schedule | ||
| 28 | import fedora_openqa_schedule.report as report | 28 | import fedora_openqa_schedule.report as report | ||
| 29 | 29 | | |||
| 30 | 30 | | |||
| 31 | class OpenQAConsumer(fedmsg.consumers.FedmsgConsumer): | 31 | class OpenQAConsumer(fedmsg.consumers.FedmsgConsumer): | ||
| 32 | """A fedmsg consumer that schedules openQA jobs when a new compose | 32 | """A fedmsg consumer that schedules openQA jobs when a new compose | ||
| 33 | appears. | 33 | appears. | ||
| 34 | """ | 34 | """ | ||
| 35 | topic = ["org.fedoraproject.prod.pungi.compose.status.change", | 35 | topic = ["org.fedoraproject.prod.pungi.compose.status.change"] | ||
| 36 | "org.fedoraproject.prod.compose.*"] | | |||
| 37 | config_key = "fedora_openqa_schedule.consumer.enabled" | 36 | config_key = "fedora_openqa_schedule.consumer.enabled" | ||
| 38 | 37 | | |||
| 39 | def _log(self, level, message): | 38 | def _log(self, level, message): | ||
| 40 | """Convenience function for sticking the class name on the | 39 | """Convenience function for sticking the class name on the | ||
| 41 | front of the log message as an identifier. | 40 | front of the log message as an identifier. | ||
| 42 | """ | 41 | """ | ||
| 43 | logfnc = getattr(self.log, level) | 42 | logfnc = getattr(self.log, level) | ||
| 44 | logfnc("%s: %s", self.__class__.__name__, message) | 43 | logfnc("%s: %s", self.__class__.__name__, message) | ||
| 45 | 44 | | |||
| 46 | def consume(self, message): | 45 | def consume(self, message): | ||
| 47 | """Consume incoming message.""" | 46 | """Consume incoming message.""" | ||
| 48 | # as two-week atomic message topics are a bit awkward, we have | 47 | status = message['body']['msg'].get('status') | ||
| 49 | # to do a bit more filtering here | | |||
| 50 | if not message['topic'] == "org.fedoraproject.prod.pungi.compose.status.change": | | |||
| 51 | if not message['topic'].endswith(".cloudimg-staging.done"): | | |||
| 52 | return | | |||
| 53 | | ||||
| 54 | # two-week atomic messages don't indicate status, so we'll just | | |||
| 55 | # always run for those, by setting default value 'FINISHED' | | |||
| 56 | status = message['body']['msg'].get('status', 'FINISHED') | | |||
| 57 | location = message['body']['msg'].get('location') | 48 | location = message['body']['msg'].get('location') | ||
| 58 | compstr = message['body']['msg'].get('compose_id', location) | 49 | compstr = message['body']['msg'].get('compose_id', location) | ||
| 59 | 50 | | |||
| 60 | if 'FINISHED' in status and location: | 51 | if 'FINISHED' in status and location: | ||
| 61 | # We have a complete pungi4 compose or a 2-week atomic compose | 52 | # We have a complete pungi4 compose | ||
| 62 | self._log('info', "Scheduling openQA jobs for {0}".format(compstr)) | 53 | self._log('info', "Scheduling openQA jobs for {0}".format(compstr)) | ||
| 63 | try: | 54 | try: | ||
| 64 | (compose, jobs) = schedule.jobs_from_compose(location) | 55 | (compose, jobs) = schedule.jobs_from_compose(location) | ||
| 65 | except schedule.TriggerException as err: | 56 | except schedule.TriggerException as err: | ||
| 66 | self._log('warning', "No openQA jobs run! {0}".format(err)) | 57 | self._log('warning', "No openQA jobs run! {0}".format(err)) | ||
| 67 | return | 58 | return | ||
| 68 | if jobs: | 59 | if jobs: | ||
| 69 | self._log('info', "openQA jobs run on {0}: " | 60 | self._log('info', "openQA jobs run on {0}: " | ||
| ▲ Show 20 Lines • Show All 71 Lines • Show Last 20 Lines | |||||
| Show First 20 Lines • Show All 46 Lines • ▼ Show 20 Line(s) | 44 | FORMAT_TO_PARAM = { | |||
|---|---|---|---|---|---|
| 47 | "raw.xz": "HDD_2_DECOMPRESS_URL" | 47 | "raw.xz": "HDD_2_DECOMPRESS_URL" | ||
| 48 | } | 48 | } | ||
| 49 | 49 | | |||
| 50 | 50 | | |||
| 51 | class TriggerException(Exception): | 51 | class TriggerException(Exception): | ||
| 52 | pass | 52 | pass | ||
| 53 | 53 | | |||
| 54 | 54 | | |||
| 55 | def _get_atomic_id(location): | | |||
| 56 | """This is an extremely ugly, temporary way to get a compose ID | | |||
| 57 | for Two Week Atomic nightly composes. They are not done with Pungi | | |||
| 58 | 4 at present so do not have an official compose ID. This should | | |||
| 59 | die as soon as all composes we want to test are done in Pungi 4 | | |||
| 60 | style. | | |||
| 61 | """ | | |||
| 62 | return location.split('/')[-1] | | |||
| 63 | | ||||
| 64 | | ||||
| 65 | def _get_atomic_installer(location, cid): | | |||
| 66 | """This is an extremely ugly, temporary way to get the sole image | | |||
| 67 | we know we need from the Two Week Atomic nightly composes. They | | |||
| 68 | are not done with Pungi 4 at present so do not have the metadata | | |||
| 69 | we want to use. This should die as soon as all composes we want to | | |||
| 70 | test are done in Pungi 4 style. | | |||
| 71 | """ | | |||
| 72 | url = '{0}/Cloud_Atomic/x86_64/iso/Fedora-Cloud_Atomic-x86_64-{1}.iso'.format( | | |||
| 73 | location, cid) | | |||
| 74 | try: | | |||
| 75 | resp = urlopen(url) | | |||
| 76 | return [('Atomic-boot-iso', 'x86_64', 0, {'ISO_URL': url}, 'Atomic', 'boot')] | | |||
| 77 | except (ValueError, URLError, HTTPError): | | |||
| 78 | raise TriggerException("Compose not found, or failed!") | | |||
| 79 | | ||||
| 80 | | ||||
| 81 | def _get_compose_id(location): | 55 | def _get_compose_id(location): | ||
| 82 | """Given a compose location, find the compose ID. Really we'd like | 56 | """Given a compose location, find the compose ID. Really we'd like | ||
| 83 | taskotron to give us this, as fedmsg provides it, but taskotron is | 57 | taskotron to give us this, as fedmsg provides it, but taskotron is | ||
| 84 | kinda tied to giving us just *one* variable property of the fedmsg | 58 | kinda tied to giving us just *one* variable property of the fedmsg | ||
| 85 | message. So we read it from the compose metadata's known location. | 59 | message. So we read it from the compose metadata's known location. | ||
| 86 | """ | 60 | """ | ||
| 87 | try: | 61 | try: | ||
| 88 | resp = urlopen('{0}/metadata/composeinfo.json'.format(location)) | 62 | resp = urlopen('{0}/metadata/composeinfo.json'.format(location)) | ||
| ▲ Show 20 Lines • Show All 194 Lines • ▼ Show 20 Line(s) | 236 | def jobs_from_compose(location, wanted=WANTED, force=False, extraparams=None, create_resultsdb_job=None): | |||
| 283 | a dict (or anything else that can be passed to `dict.update()`) | 257 | a dict (or anything else that can be passed to `dict.update()`) | ||
| 284 | containing arbitrary extra parameters to be included in the ISO | 258 | containing arbitrary extra parameters to be included in the ISO | ||
| 285 | post request (usually this will be to specify extra openQA vars). | 259 | post request (usually this will be to specify extra openQA vars). | ||
| 286 | 260 | | |||
| 287 | create_resultsdb_job controls whether to create job in ResultsDB for | 261 | create_resultsdb_job controls whether to create job in ResultsDB for | ||
| 288 | this compose. | 262 | this compose. | ||
| 289 | """ | 263 | """ | ||
| 290 | location = location.strip('/') | 264 | location = location.strip('/') | ||
| 291 | # trigger ugly special-casing for non-Pungi4-ified Two Week Atomic | 265 | compose = _get_compose_id(location) | ||
| 292 | # nightlies | 266 | logger.debug("Finding images for compose %s in location %s", compose, location) | ||
| 293 | if 'alt/atomic/testing' in location: | 267 | images = _get_images(location, wanted=wanted) | ||
| 294 | compose = _get_atomic_id(location) | | |||
| 295 | logger.debug("Finding images for Atomic compose %s in location %s", compose, location) | | |||
| 296 | images = _get_atomic_installer(location, compose) | | |||
| 297 | else: | | |||
| 298 | compose = _get_compose_id(location) | | |||
| 299 | logger.debug("Finding images for compose %s in location %s", compose, location) | | |||
| 300 | images = _get_images(location, wanted=wanted) | | |||
| 301 | if len(images) == 0: | 268 | if len(images) == 0: | ||
| 302 | raise TriggerException("Compose found, but no available images") | 269 | raise TriggerException("Compose found, but no available images") | ||
| 303 | jobs = [] | 270 | jobs = [] | ||
| 304 | univs = {} | 271 | univs = {} | ||
| 305 | 272 | | |||
| 306 | # create jobs instance in resultsdb if necessary | 273 | # create jobs instance in resultsdb if necessary | ||
| 307 | rdb_job_id = None | 274 | rdb_job_id = None | ||
| 308 | 275 | | |||
| Show All 35 Lines | |||||