| tflink | |
| jskladan |
| libtaskotron |
You can now exclude arches in koji_directive or koji_utils. This is
useful for task-abicheck, which doesn't need to download noarch RPMs.
This also modified arch_utils to include a mapping from base arches to
binary arches, and improved behavior in koji_utils when the user
requests only base arches to be downloaded - we now correctly convert it
to concrete binary arches, because that's what Koji requires as input.
As a minor point, typo in armv7hl has been fixed.
unit tests adjusted, tested manually with modified task-abicheck
| Automatic diff as part of commit; lint not applicable. |
| Automatic diff as part of commit; unit tests not applicable. |
| libtaskotron/directives/koji_directive.py | ||
|---|---|---|
| 166–172 | I don't see it being used in testing. I think originally this was split into several methods and that's why all the variables were saved as class attributes. Is that a problem? I think it doesn't really matter at all. | |
| libtaskotron/directives/koji_directive.py | ||
|---|---|---|
| 166–172 | If it does not matter, then please remove the self. from those variables. Creating instance attributes at "random" places (without a serious reason) is one of the no-nos in Python. Thx | |
| libtaskotron/directives/koji_directive.py | ||
|---|---|---|
| 166–172 | I'll make it a follow-up patch. Do you have any article explaining this in more detail? | |
| Path | Packages | |||
|---|---|---|---|---|
| M | libtaskotron/arch_utils.py (38 lines) | |||
| M | libtaskotron/directives/koji_directive.py (91 lines) | |||
| M | libtaskotron/ext/fedora/koji_utils.py (56 lines) | |||
| M | testing/test_koji_directive.py (136 lines) | |||
| M | testing/test_koji_utils.py (137 lines) |
| Commit | Tree | Parents | Author | Summary | Date |
|---|---|---|---|---|---|
| 4bda9eff4cc7 | 86720e902ff9 | 525b4f12b915 | Kamil Páral | koji: allow to exclude arches (Show More…) | May 20 2016, 2:58 PM |
| 1 | # -*- coding: utf-8 -*- | 1 | # -*- coding: utf-8 -*- | ||
|---|---|---|---|---|---|
| 2 | # Copyright 2009-2014, Red Hat, Inc. | 2 | # Copyright 2009-2016, Red Hat, Inc. | ||
| 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | ||
| 4 | # See the LICENSE file for more details on Licensing | 4 | # See the LICENSE file for more details on Licensing | ||
| 5 | 5 | | |||
| 6 | from __future__ import absolute_import | 6 | from __future__ import absolute_import | ||
| 7 | import os | 7 | import os | ||
| 8 | import itertools | ||||
| 8 | 9 | | |||
| 9 | class Arches(): | 10 | | ||
| 11 | class Arches(object): | ||||
| 10 | ''' | 12 | ''' | ||
| 11 | Helper class for working with supported arches inside taskotron | 13 | Helper class for working with supported architectures inside taskotron | ||
| 12 | ''' | 14 | ''' | ||
| 13 | 15 | | |||
| 14 | #: all supported architectures | 16 | #: a mapping from base arches to concrete binary arches | ||
| 15 | all = ['i386', 'i486', 'i586', 'i686', | 17 | binary = { | ||
| 16 | 'x86_64', | 18 | 'i386': ['i386', 'i486', 'i586', 'i686'], | ||
| 17 | 'armhfp', 'arm7hl', | 19 | 'x86_64': ['x86_64'], | ||
| 18 | 'noarch', 'src'] | 20 | 'armhfp': ['armhfp', 'armv7hl'], | ||
| 21 | } | ||||
| 19 | 22 | | |||
| 20 | #: base architectures | 23 | #: base architectures | ||
| 21 | base = ['i386', 'x86_64', 'armhfp'] | 24 | base = binary.keys() | ||
| 22 | 25 | | |||
| 23 | #: meta architectures | 26 | #: meta arches | ||
| 24 | meta = ['noarch', 'src'] | 27 | meta = ['noarch', 'src'] | ||
| 25 | 28 | | |||
| 29 | #: all known architectures | ||||
| 30 | known = sorted(set(binary.keys() + list(itertools.chain(*binary.values())) + meta)) | ||||
| 31 | | ||||
| 32 | | ||||
| 26 | def basearch(arch=None): | 33 | def basearch(arch=None): | ||
| 27 | ''' | 34 | ''' | ||
| 28 | This returns the 'base' architecture identifier for a specified architecture | 35 | This returns the 'base' architecture identifier for a specified architecture | ||
| 29 | (e.g. ``i386`` for ``i[3-6]86``), to be used by YUM etc. | 36 | (e.g. ``i386`` for ``i[3-6]86``), to be used by YUM etc. | ||
| 30 | 37 | | |||
| 31 | :param str arch: an architecture to be converted to a basearch. If ``None``, | 38 | :param str arch: an architecture to be converted to a basearch. If ``None``, | ||
| 32 | then the arch of the current machine is used. | 39 | then the arch of the current machine is used. | ||
| 33 | :return: basearch, or ``arch`` if no basearch was found for it | 40 | :return: basearch, or ``arch`` if no basearch was found for it | ||
| 34 | :rtype: str | 41 | :rtype: str | ||
| 35 | ''' | 42 | ''' | ||
| 36 | if arch is None: | 43 | if arch is None: | ||
| 37 | arch = os.uname()[4] | 44 | arch = os.uname()[4] | ||
| 38 | if arch in ['i386', 'i486', 'i586', 'i686']: | 45 | | ||
| 39 | arch = 'i386' | 46 | for base, binary in Arches.binary.items(): | ||
| 40 | if arch in ['armhfp', 'arm7hl']: | 47 | if arch == base or arch in binary: | ||
| 41 | arch = 'armhfp' | 48 | return base | ||
| 42 | return arch | 49 | | ||
| 43 | No newline at end of file | 50 | return arch | ||
| 1 | # -*- coding: utf-8 -*- | 1 | # -*- coding: utf-8 -*- | ||
|---|---|---|---|---|---|
| 2 | # Copyright 2009-2014, Red Hat, Inc. | 2 | # Copyright 2009-2016, Red Hat, Inc. | ||
| 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | ||
| 4 | # See the LICENSE file for more details on Licensing | 4 | # See the LICENSE file for more details on Licensing | ||
| 5 | 5 | | |||
| 6 | from __future__ import absolute_import | 6 | from __future__ import absolute_import | ||
| 7 | 7 | | |||
| 8 | DOCUMENTATION = """ | 8 | DOCUMENTATION = """ | ||
| 9 | module: koji_directive | 9 | module: koji_directive | ||
| 10 | short_description: download builds and tags from Koji | 10 | short_description: download builds and tags from Koji | ||
| 11 | description: | | 11 | description: | | ||
| 12 | The koji directive interfaces with `Koji <http://koji.fedoraproject.org/>`_ | 12 | The koji directive interfaces with `Koji <http://koji.fedoraproject.org/>`_ | ||
| 13 | to facilitate various Koji actions. You can either download all RPMs from | 13 | to facilitate various Koji actions. You can either download all RPMs from | ||
| 14 | a specific build, or you can download all RPMs from all builds belonging | 14 | a specific build, or you can download all RPMs from all builds belonging | ||
| 15 | to a specific Koji tag. | 15 | to a specific Koji tag. | ||
| 16 | parameters: | 16 | parameters: | ||
| 17 | action: | 17 | action: | ||
| 18 | required: true | 18 | required: true | ||
| 19 | description: choose whether to download a single build (``download`` value) | 19 | description: choose whether to download a single build (``download`` value) | ||
| 20 | or all builds belonging to a Koji tag (``download_tag`` value) | 20 | or all builds belonging to a Koji tag (``download_tag`` value) | ||
| 21 | type: str | 21 | type: str | ||
| 22 | choices: [download, download_tag, download_latest_stable] | 22 | choices: [download, download_tag, download_latest_stable] | ||
| 23 | arch: | 23 | arch: | ||
| 24 | required: true | 24 | required: true | ||
| 25 | description: | | 25 | description: | | ||
| 26 | a list of architectures for which to download RPMs for the requested | 26 | a list of architectures for which to download RPMs for the requested | ||
| 27 | build/tag. If you want to download RPMs for all arches, use ``['all']``. | 27 | build/tag. If you want to download RPMs for all arches, use ``['all']``. If | ||
| 28 | 28 | you don't want to download any binary arch (i.e. you're only interested in | |||
| 29 | Note: ``noarch`` RPMs are always automatically downloaded even when not | 29 | SRPMs, which is controlled by `src` option), use an empty list ``[]``. | ||
| 30 | requested, unless ``arch=[]`` and ``src=True``. | 30 | | ||
| 31 | Note: ``noarch`` RPMs are always automatically downloaded unless ``arch=[]``, | ||||
| 32 | or you specifically exclude them using ``arch_exclude``. Also, if you specify base arches | ||||
| 33 | (like ``i386``), all concrete binary arches for that base arch will be automatically added | ||||
| 34 | (e.g. ``i[3-6]86``). | ||||
| 35 | type: list of str | ||||
| 36 | choices: [supported architectures, all] | ||||
| 37 | arch_exclude: | ||||
| 38 | required: false | ||||
| 39 | description: | | ||||
| 40 | a list of architectures to exclude from downloading (overrides ``arch`` | ||||
| 41 | value). You can use it to exclude some specific archicture while using | ||||
| 42 | ``arch=['all']``. Example: ``['armhfp', 'noarch']`` | ||||
| 31 | type: list of str | 43 | type: list of str | ||
| 32 | choices: [supported architectures, ['all']] | 44 | choices: [supported architectures] | ||
| 33 | koji_build: | 45 | koji_build: | ||
| 34 | required: true | 46 | required: true | ||
| 35 | description: | | 47 | description: | | ||
| 36 | N(E)VR of a Koji build to download (for ``action="download"``) or to search | 48 | N(E)VR of a Koji build to download (for ``action="download"``) or to search | ||
| 37 | the latest stable build for (for ``action="download_latest_stable"``). Not | 49 | the latest stable build for (for ``action="download_latest_stable"``). Not | ||
| 38 | required for ``action="download_tag"``. Example: ``xchat-2.8.8-21.fc20`` | 50 | required for ``action="download_tag"``. Example: ``xchat-2.8.8-21.fc20`` | ||
| 39 | type: str | 51 | type: str | ||
| 40 | koji_tag: | 52 | koji_tag: | ||
| Show All 19 Lines | 68 | target_dir: | |||
| 60 | default: ${workdir} | 72 | default: ${workdir} | ||
| 61 | returns: | | 73 | returns: | | ||
| 62 | A dictionary containing following items: | 74 | A dictionary containing following items: | ||
| 63 | 75 | | |||
| 64 | * `downloaded_rpms`: (list of str) a list of local filenames of the downloaded | 76 | * `downloaded_rpms`: (list of str) a list of local filenames of the downloaded | ||
| 65 | RPMs | 77 | RPMs | ||
| 66 | raises: | | 78 | raises: | | ||
| 67 | * :class:`.TaskotronRemoteError`: if downloading failed | 79 | * :class:`.TaskotronRemoteError`: if downloading failed | ||
| 68 | * :class:`.TaskotronValueError`: if ``arch=[]`` and ``src=False``, therefore | 80 | * :class:`.TaskotronValueError`: if the combination of ``arch``, ``arch_exclude`` and ``src`` | ||
| 69 | there is nothing to download | 81 | results in nothing to be downloaded | ||
| 70 | version_added: 0.4 | 82 | version_added: 0.4 | ||
| 71 | """ | 83 | """ | ||
| 72 | 84 | | |||
| 73 | EXAMPLES = """ | 85 | EXAMPLES = """ | ||
| 74 | Rpmlint needs to download a specific build from Koji, all architectures | 86 | Rpmlint needs to download a specific build from Koji, all architectures | ||
| 75 | including src.rpm:: | 87 | including src.rpm:: | ||
| 76 | 88 | | |||
| 77 | - name: download rpms from koji | 89 | - name: download rpms from koji | ||
| 78 | koji: | 90 | koji: | ||
| 79 | action: download | 91 | action: download | ||
| 80 | koji_build: ${koji_build} | 92 | koji_build: ${koji_build} | ||
| 81 | arch: ['all'] | 93 | arch: ['all'] | ||
| 82 | src: True | 94 | src: True | ||
| 83 | 95 | | |||
| 84 | Depcheck needs to download all builds in a specific Koji tag for the current | 96 | Depcheck needs to download all builds in a specific Koji tag for the current | ||
| 85 | architecture:: | 97 | architecture:: | ||
| 86 | 98 | | |||
| 87 | - name: download koji tag | 99 | - name: download koji tag | ||
| 88 | koji: | 100 | koji: | ||
| 89 | action: download_tag | 101 | action: download_tag | ||
| 90 | koji_tag: ${koji_tag} | 102 | koji_tag: ${koji_tag} | ||
| 91 | arch: ${arch} | 103 | arch: [${arch}] | ||
| 92 | target_dir: ${workdir}/downloaded_tag/ | 104 | target_dir: ${workdir}/downloaded_tag/ | ||
| 93 | 105 | | |||
| 94 | Abidiff needs to download the latest stable build of a package:: | 106 | Abicheck downloads both current build and latest stable build. It checks all binary arches, but | ||
| 107 | doesn't need ``noarch``:: | ||||
| 95 | 108 | | |||
| 96 | - name: download rpms of the latest stable build of a given build from koji | 109 | - name: download latest stable rpms for the package | ||
| 97 | koji: | 110 | koji: | ||
| 98 | action: download_latest_stable | 111 | action: download_latest_stable | ||
| 99 | koji_build: ${koji_build} | 112 | koji_build: ${koji_build} | ||
| 100 | arch: ${arch} | 113 | arch: ['all'] | ||
| 101 | target_dir: ${workdir}/stable/ | 114 | arch_exclude: ['noarch'] | ||
| 115 | debuginfo: True | ||||
| 116 | target_dir: ${workdir}/stable | ||||
| 117 | | ||||
| 118 | - name: download rpms for the tested build | ||||
| 119 | koji: | ||||
| 120 | action: download | ||||
| 121 | koji_build: ${koji_build} | ||||
| 122 | arch: ['all'] | ||||
| 123 | arch_exclude: ['noarch'] | ||||
| 124 | debuginfo: True | ||||
| 125 | target_dir: ${workdir}/update | ||||
| 126 | | ||||
| 102 | """ | 127 | """ | ||
| 103 | 128 | | |||
| 104 | from libtaskotron.directives import BaseDirective | 129 | from libtaskotron.directives import BaseDirective | ||
| 105 | import libtaskotron.exceptions as exc | 130 | import libtaskotron.exceptions as exc | ||
| 106 | from libtaskotron.ext.fedora.koji_utils import KojiClient | 131 | from libtaskotron.ext.fedora.koji_utils import KojiClient | ||
| 107 | from libtaskotron.ext.fedora import rpm_utils | 132 | from libtaskotron.ext.fedora import rpm_utils | ||
| 108 | 133 | | |||
| 109 | 134 | | |||
| Show All 23 Lines | 157 | else: | |||
| 133 | self.target_dir = params['target_dir'] | 158 | self.target_dir = params['target_dir'] | ||
| 134 | 159 | | |||
| 135 | if 'arch' not in params: | 160 | if 'arch' not in params: | ||
| 136 | detected_args = ', '.join(params.keys()) | 161 | detected_args = ', '.join(params.keys()) | ||
| 137 | raise exc.TaskotronDirectiveError( | 162 | raise exc.TaskotronDirectiveError( | ||
| 138 | "The koji directive requires 'arch' as an argument. Detected " | 163 | "The koji directive requires 'arch' as an argument. Detected " | ||
| 139 | "arguments: %s" % detected_args) | 164 | "arguments: %s" % detected_args) | ||
| 140 | 165 | | |||
| 141 | self.arches = params['arch'] | 166 | self.arches = list(params['arch']) | ||
| 142 | if self.arches and ('noarch' not in self.arches): | 167 | if self.arches and ('all' not in self.arches) and ('noarch' not in self.arches): | ||
| 143 | self.arches.append('noarch') | 168 | self.arches.append('noarch') | ||
| 144 | 169 | | |||
| 170 | self.arch_exclude = params.get('arch_exclude', []) | ||||
| 145 | self.debuginfo = params.get('debuginfo', False) | 171 | self.debuginfo = params.get('debuginfo', False) | ||
| 146 | | ||||
| 147 | self.src = params.get('src', False) | 172 | self.src = params.get('src', False) | ||
| 148 | 173 | | |||
| 149 | # download files | 174 | # download files | ||
| 150 | output_data = {} | 175 | output_data = {} | ||
| 151 | 176 | | |||
| 152 | if action == 'download': | 177 | if action == 'download': | ||
| 153 | if 'koji_build' not in params: | 178 | if 'koji_build' not in params: | ||
| 154 | detected_args = ', '.join(params.keys()) | 179 | detected_args = ', '.join(params.keys()) | ||
| 155 | raise exc.TaskotronDirectiveError( | 180 | raise exc.TaskotronDirectiveError( | ||
| 156 | "The koji directive requires 'koji_build' for the 'download' " | 181 | "The koji directive requires 'koji_build' for the 'download' " | ||
| 157 | "action. Detected arguments: %s" % detected_args) | 182 | "action. Detected arguments: %s" % detected_args) | ||
| 158 | 183 | | |||
| 159 | nvr = rpm_utils.rpmformat(params['koji_build'], 'nvr') | 184 | nvr = rpm_utils.rpmformat(params['koji_build'], 'nvr') | ||
| 160 | 185 | | |||
| 161 | output_data['downloaded_rpms'] = self.koji.get_nvr_rpms(nvr, | 186 | output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( | ||
| 162 | self.target_dir, | 187 | nvr, self.target_dir, arches=self.arches, arch_exclude=self.arch_exclude, | ||
| 163 | arches=self.arches, | 188 | debuginfo=self.debuginfo, src=self.src) | ||
| 164 | debuginfo=self.debuginfo, | 189 | | ||
| 165 | src=self.src) | | |||
| 166 | elif action == 'download_tag': | 190 | elif action == 'download_tag': | ||
| 167 | if 'koji_tag' not in params: | 191 | if 'koji_tag' not in params: | ||
| 168 | detected_args = ', '.join(params.keys()) | 192 | detected_args = ', '.join(params.keys()) | ||
| 169 | raise exc.TaskotronDirectiveError( | 193 | raise exc.TaskotronDirectiveError( | ||
| 170 | "The koji directive requires 'koji_tag' for the 'download_tag' " | 194 | "The koji directive requires 'koji_tag' for the 'download_tag' " | ||
| 171 | "action. Detected arguments: %s" % detected_args) | 195 | "action. Detected arguments: %s" % detected_args) | ||
| 172 | 196 | | |||
| 173 | koji_tag = params['koji_tag'] | 197 | koji_tag = params['koji_tag'] | ||
| 174 | 198 | | |||
| 175 | output_data['downloaded_rpms'] = self.koji.get_tagged_rpms(koji_tag, | 199 | output_data['downloaded_rpms'] = self.koji.get_tagged_rpms( | ||
| 176 | self.target_dir, | 200 | koji_tag, self.target_dir, arches=self.arches, arch_exclude=self.arch_exclude, | ||
| 177 | arches=self.arches, | 201 | debuginfo=self.debuginfo, src=self.src) | ||
| 178 | debuginfo=self.debuginfo, | 202 | | ||
| 179 | src=self.src) | | |||
| 180 | elif action == 'download_latest_stable': | 203 | elif action == 'download_latest_stable': | ||
| 181 | if 'koji_build' not in params: | 204 | if 'koji_build' not in params: | ||
| 182 | detected_args = ', '.join(params.keys()) | 205 | detected_args = ', '.join(params.keys()) | ||
| 183 | raise exc.TaskotronDirectiveError( | 206 | raise exc.TaskotronDirectiveError( | ||
| 184 | "The koji directive requires 'koji_build' for the 'download_latest_stable' " | 207 | "The koji directive requires 'koji_build' for the 'download_latest_stable' " | ||
| 185 | "action. Detected arguments: %s" % detected_args) | 208 | "action. Detected arguments: %s" % detected_args) | ||
| 186 | 209 | | |||
| 187 | name = rpm_utils.rpmformat(params['koji_build'], 'n') | 210 | name = rpm_utils.rpmformat(params['koji_build'], 'n') | ||
| 188 | disttag = rpm_utils.get_dist_tag(params['koji_build']) | 211 | disttag = rpm_utils.get_dist_tag(params['koji_build']) | ||
| 189 | # we need to do 'fc22' -> 'f22' conversion | 212 | # we need to do 'fc22' -> 'f22' conversion | ||
| 190 | tag = disttag.replace('c', '') | 213 | tag = disttag.replace('c', '') | ||
| 191 | 214 | | |||
| 192 | # first we need to check updates tag and if that fails, the latest | 215 | # first we need to check updates tag and if that fails, the latest | ||
| 193 | # stable nvr is in the base repo | 216 | # stable nvr is in the base repo | ||
| 194 | tags = ['%s-updates' % tag, tag] | 217 | tags = ['%s-updates' % tag, tag] | ||
| 195 | latest_stable_nvr = self.koji.latest_by_tag(tags, name) | 218 | latest_stable_nvr = self.koji.latest_by_tag(tags, name) | ||
| 196 | 219 | | |||
| 197 | output_data['downloaded_rpms'] = self.koji.get_nvr_rpms(latest_stable_nvr, | 220 | output_data['downloaded_rpms'] = self.koji.get_nvr_rpms( | ||
| 198 | self.target_dir, | 221 | latest_stable_nvr, self.target_dir, arch_exclude=self.arch_exclude, | ||
| 199 | arches=self.arches, | 222 | arches=self.arches, debuginfo=self.debuginfo, src=self.src) | ||
| 200 | debuginfo=self.debuginfo, | | |||
| 201 | src=self.src) | | |||
| 202 | 223 | | |||
| 203 | return output_data | 224 | return output_data | ||
| 1 | # -*- coding: utf-8 -*- | 1 | # -*- coding: utf-8 -*- | ||
|---|---|---|---|---|---|
| 2 | # Copyright 2009-2014, Red Hat, Inc. | 2 | # Copyright 2009-2016, Red Hat, Inc. | ||
| 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | ||
| 4 | # See the LICENSE file for more details on Licensing | 4 | # See the LICENSE file for more details on Licensing | ||
| 5 | 5 | | |||
| 6 | ''' Utility methods related to Koji ''' | 6 | ''' Utility methods related to Koji ''' | ||
| 7 | 7 | | |||
| 8 | from __future__ import absolute_import | 8 | from __future__ import absolute_import | ||
| 9 | import os | 9 | import os | ||
| 10 | 10 | | |||
| 11 | import collections | 11 | import collections | ||
| 12 | import koji | 12 | import koji | ||
| 13 | import hawkey | 13 | import hawkey | ||
| 14 | 14 | | |||
| 15 | from libtaskotron import file_utils | 15 | from libtaskotron import file_utils | ||
| 16 | from libtaskotron.logger import log | 16 | from libtaskotron.logger import log | ||
| 17 | from libtaskotron import exceptions as exc | 17 | from libtaskotron import exceptions as exc | ||
| 18 | from libtaskotron import config | 18 | from libtaskotron import config | ||
| 19 | from libtaskotron import arch_utils | ||||
| 19 | 20 | | |||
| 20 | from . import rpm_utils | 21 | from . import rpm_utils | ||
| 21 | 22 | | |||
| 22 | class KojiClient(object): | 23 | class KojiClient(object): | ||
| 23 | '''Helper Koji methods. | 24 | '''Helper Koji methods. | ||
| 24 | 25 | | |||
| 25 | :ivar koji.ClientSession session: Koji client session | 26 | :ivar koji.ClientSession session: Koji client session | ||
| 26 | ''' | 27 | ''' | ||
| ▲ Show 20 Lines • Show All 106 Lines • ▼ Show 20 Line(s) | 74 | def rpms_to_build(self, rpms): | |||
| 133 | build_to_buildinfo = dict(zip(builds, buildinfos)) | 134 | build_to_buildinfo = dict(zip(builds, buildinfos)) | ||
| 134 | result = [] | 135 | result = [] | ||
| 135 | for rpminfo in rpminfos: | 136 | for rpminfo in rpminfos: | ||
| 136 | build_id = rpminfo[0]['build_id'] | 137 | build_id = rpminfo[0]['build_id'] | ||
| 137 | result.append(build_to_buildinfo[build_id][0]) | 138 | result.append(build_to_buildinfo[build_id][0]) | ||
| 138 | 139 | | |||
| 139 | return result | 140 | return result | ||
| 140 | 141 | | |||
| 141 | def nvr_to_urls(self, nvr, arches=['all'], debuginfo=False, src=True): | 142 | def nvr_to_urls(self, nvr, arches=['all'], arch_exclude=[], debuginfo=False, src=True): | ||
| 142 | '''Get list of URLs for RPMs corresponding to a build. | 143 | '''Get list of URLs for RPMs corresponding to a build. | ||
| 143 | 144 | | |||
| 144 | :param str nvr: build NVR | 145 | :param str nvr: build NVR | ||
| 145 | :param arches: restrict the arches of builds to provide URLs for. By | 146 | :param arches: restrict the arches of builds to provide URLs for. By default, all | ||
| 146 | default, all architectures are considered. If you want to consider | 147 | architectures are considered. If you want to consider just some selected arches, | ||
| 147 | just some selected arches, provide their names in a list. | 148 | provide their names in a list. | ||
| 148 | 149 | | |||
| 149 | .. note:: If basearch ``i386`` is in the list, ``i686`` arch is | 150 | .. note:: If you specify base arches (like ``i386``), all concrete binary arches for | ||
| 150 | automatically added (since that's the arch Koji API uses). | 151 | that base arch will be automatically added (e.g. ``i[3-6]86``), because Koji query | ||
| 152 | requires concrete binary arches. | ||||
| 151 | :type arches: list of str | 153 | :type arches: list of str | ||
| 154 | :param arch_exclude: exclude some specific arches, even if they are specified in | ||||
| 155 | ``arches`` or they are implicit in the default ``'all'`` value | ||||
| 156 | :type arch_exclude: list of str | ||||
| 152 | :param bool debuginfo: whether to provide URLs for debuginfo RPM files | 157 | :param bool debuginfo: whether to provide URLs for debuginfo RPM files | ||
| 153 | or ignore them | 158 | or ignore them | ||
| 154 | :param bool src: whether to include a URL for the source RPM | 159 | :param bool src: whether to include a URL for the source RPM | ||
| 155 | :rtype: list of str | 160 | :rtype: list of str | ||
| 156 | :raise TaskotronRemoteError: when the requested build doesn't exist | 161 | :raise TaskotronRemoteError: when the requested build doesn't exist | ||
| 157 | :raise TaskotronValueError: if ``arches=[]`` and ``src=False``, | 162 | :raise TaskotronValueError: if ``arches=[]`` and ``src=False``, | ||
| 158 | therefore there is nothing to query for | 163 | therefore there is nothing to query for | ||
| 159 | ''' | 164 | ''' | ||
| 160 | log.info('Querying Koji for a list of RPMS for: %s', nvr) | 165 | log.info('Querying Koji for a list of RPMS for: %s', nvr) | ||
| 161 | 166 | | |||
| 162 | # add i686 arch if i386 is present in arches | 167 | # add binary arches if base arches are specified | ||
| 163 | if 'i386' in arches and 'i686' not in arches: | 168 | archlist = list(arches) | ||
| 164 | arches.append('i686') | 169 | for arch in arches: | ||
| 170 | if arch in arch_utils.Arches.base: | ||||
| 171 | archlist.extend(arch_utils.Arches.binary[arch]) | ||||
| 172 | arches = list(set(archlist)) | ||||
| 165 | 173 | | |||
| 166 | # if src is enabled, we need to add it so that it is included in the | 174 | # if src is enabled, we need to add it so that it is included in the | ||
| 167 | # Koji query | 175 | # Koji query | ||
| 168 | if src and ('src' not in arches): | 176 | if src and ('src' not in arches): | ||
| 169 | arches.append('src') | 177 | arches.append('src') | ||
| 170 | 178 | | |||
| 179 | # exclude unwanted arches | ||||
| 180 | arches = [arch for arch in arches if arch not in arch_exclude] | ||||
| 181 | | ||||
| 171 | # if "nothing" is requested, it's probably an error | 182 | # if "nothing" is requested, it's probably an error | ||
| 172 | if not arches: | 183 | if not arches: | ||
| 173 | raise exc.TaskotronValueError('Nothing to query for, `arches` was set to an empty ' | 184 | raise exc.TaskotronValueError('Nothing to query for, the combination of `arches`, ' | ||
| 174 | 'list and `src` was disabled. At least one of them ' | 185 | '`arch_exclude` and `src` ended up an empty list') | ||
| 175 | 'need to be non-empty.') | | |||
| 176 | 186 | | |||
| 177 | # find the koji build | 187 | # find the koji build | ||
| 178 | info = self.session.getBuild(nvr) | 188 | info = self.session.getBuild(nvr) | ||
| 179 | if info is None: | 189 | if info is None: | ||
| 180 | raise exc.TaskotronRemoteError("No such build found in Koji: %s" % nvr) | 190 | raise exc.TaskotronRemoteError("No such build found in Koji: %s" % nvr) | ||
| 181 | 191 | | |||
| 182 | # list its RPM files | 192 | # list its RPM files | ||
| 183 | req_arches = None if 'all' in arches else arches | 193 | req_arches = None if 'all' in arches else arches | ||
| 184 | rpms = self.session.listRPMs(buildID=info['id'], arches=req_arches) | 194 | rpms = self.session.listRPMs(buildID=info['id'], arches=req_arches) | ||
| 185 | if not debuginfo: | 195 | if not debuginfo: | ||
| 186 | rpms = [r for r in rpms if (not r['name'].endswith('-debuginfo')) | 196 | rpms = [r for r in rpms if (not r['name'].endswith('-debuginfo')) and | ||
| 187 | and ('-debuginfo-' not in r['name'])] | 197 | ('-debuginfo-' not in r['name'])] | ||
| 188 | if not src: | 198 | if not src: | ||
| 189 | rpms = [r for r in rpms if not r['arch'] == 'src'] | 199 | rpms = [r for r in rpms if not r['arch'] == 'src'] | ||
| 200 | # exclude arches once again, in case `arch=['all']` was used | ||||
| 201 | rpms = [r for r in rpms if not r['arch'] in arch_exclude] | ||||
| 190 | 202 | | |||
| 191 | # create URLs | 203 | # create URLs | ||
| 192 | baseurl = '/'.join((config.get_config().pkg_url, info['package_name'], | 204 | baseurl = '/'.join((config.get_config().pkg_url, info['package_name'], | ||
| 193 | info['version'], info['release'])) | 205 | info['version'], info['release'])) | ||
| 194 | urls = ['%s/%s' % (baseurl, koji.pathinfo.rpm(r)) for r in rpms] | 206 | urls = ['%s/%s' % (baseurl, koji.pathinfo.rpm(r)) for r in rpms] | ||
| 195 | 207 | | |||
| 196 | return sorted(urls) | 208 | return sorted(urls) | ||
| 197 | 209 | | |||
| 198 | def get_nvr_rpms(self, nvr, dest, arches=['all'], debuginfo=False, src=False): | 210 | def get_nvr_rpms(self, nvr, dest, arches=['all'], arch_exclude=[], debuginfo=False, src=False): | ||
| 199 | '''Retrieve the RPMs associated with a build NVR into the specified | 211 | '''Retrieve the RPMs associated with a build NVR into the specified | ||
| 200 | directory. | 212 | directory. | ||
| 201 | 213 | | |||
| 214 | For parameters undocumented here, see :meth:`nvr_to_urls`. | ||||
| 215 | | ||||
| 202 | :param str nvr: build NVR | 216 | :param str nvr: build NVR | ||
| 203 | :param str dest: location where to store the RPMs | 217 | :param str dest: location where to store the RPMs | ||
| 204 | :param arches: see :meth:`nvr_to_urls` | | |||
| 205 | :param debuginfo: see :meth:`nvr_to_urls` | | |||
| 206 | :param src: see :meth:`nvr_to_urls` | | |||
| 207 | :return: list of local filenames of the grabbed RPMs (might be empty, | 218 | :return: list of local filenames of the grabbed RPMs (might be empty, | ||
| 208 | according to your option choices and the particular NVR) | 219 | according to your option choices and the particular NVR) | ||
| 209 | :rtype: list of str | 220 | :rtype: list of str | ||
| 210 | :raise TaskotronRemoteError: if the files can't be downloaded | 221 | :raise TaskotronRemoteError: if the files can't be downloaded | ||
| 211 | :raise TaskotronValueError: if ``arches=[]`` and ``src=False``, | 222 | :raise TaskotronValueError: if ``arches=[]`` and ``src=False``, | ||
| 212 | therefore there is nothing to download | 223 | therefore there is nothing to download | ||
| 213 | ''' | 224 | ''' | ||
| 214 | # always create dest dir, even if nothing gets downloaded | 225 | # always create dest dir, even if nothing gets downloaded | ||
| 215 | file_utils.makedirs(dest) | 226 | file_utils.makedirs(dest) | ||
| 216 | 227 | | |||
| 217 | rpm_urls = self.nvr_to_urls(nvr, arches, debuginfo, src) | 228 | rpm_urls = self.nvr_to_urls(nvr, arches, arch_exclude, debuginfo, src) | ||
| 218 | 229 | | |||
| 219 | rpm_files = [] | 230 | rpm_files = [] | ||
| 220 | log.info('Fetching %s RPMs for: %s (into %s)', len(rpm_urls), nvr, dest) | 231 | log.info('Fetching %s RPMs for: %s (into %s)', len(rpm_urls), nvr, dest) | ||
| 221 | 232 | | |||
| 222 | # RPMs are safe to be cached, Koji guarantees they are immutable | 233 | # RPMs are safe to be cached, Koji guarantees they are immutable | ||
| 223 | cachedir = None | 234 | cachedir = None | ||
| 224 | if config.get_config().download_cache_enabled: | 235 | if config.get_config().download_cache_enabled: | ||
| 225 | cachedir = config.get_config().cachedir | 236 | cachedir = config.get_config().cachedir | ||
| 226 | 237 | | |||
| 227 | for url in rpm_urls: | 238 | for url in rpm_urls: | ||
| 228 | rpm_file = file_utils.download(url, dest, cachedir=cachedir) | 239 | rpm_file = file_utils.download(url, dest, cachedir=cachedir) | ||
| 229 | rpm_files.append(rpm_file) | 240 | rpm_files.append(rpm_file) | ||
| 230 | 241 | | |||
| 231 | return rpm_files | 242 | return rpm_files | ||
| 232 | 243 | | |||
| 233 | def get_tagged_rpms(self, tag, dest, arches=['all'], debuginfo=False, src=False): | 244 | def get_tagged_rpms(self, tag, dest, arches=['all'], arch_exclude=[], debuginfo=False, | ||
| 245 | src=False): | ||||
| 234 | '''Downloads all RPMs of all NVRs tagged by a specific Koji tag. | 246 | '''Downloads all RPMs of all NVRs tagged by a specific Koji tag. | ||
| 235 | 247 | | |||
| 236 | Note: This works basically the same as :meth:`get_nvr_rpms`, it just | 248 | Note: This works basically the same as :meth:`get_nvr_rpms`, it just | ||
| 237 | downloads a lot of builds instead of a single one. For description of | 249 | downloads a lot of builds instead of a single one. For description of | ||
| 238 | all shared parameters and return values, please see that method. | 250 | all shared parameters and return values, please see that method. | ||
| 239 | 251 | | |||
| 240 | :param str tag: Koji tag to be queried for available builds, e.g. | 252 | :param str tag: Koji tag to be queried for available builds, e.g. | ||
| 241 | ``f20-updates-pending`` | 253 | ``f20-updates-pending`` | ||
| 242 | ''' | 254 | ''' | ||
| 243 | # always create dest dir, even if nothing gets downloaded | 255 | # always create dest dir, even if nothing gets downloaded | ||
| 244 | file_utils.makedirs(dest) | 256 | file_utils.makedirs(dest) | ||
| 245 | 257 | | |||
| 246 | log.info('Querying Koji for tag: %s', tag) | 258 | log.info('Querying Koji for tag: %s', tag) | ||
| 247 | tag_data = self.session.listTagged(tag) | 259 | tag_data = self.session.listTagged(tag) | ||
| 248 | 260 | | |||
| 249 | nvrs = sorted([x['nvr'] for x in tag_data]) | 261 | nvrs = sorted([x['nvr'] for x in tag_data]) | ||
| 250 | rpms = [] | 262 | rpms = [] | ||
| 251 | log.info("Fetching %s builds for tag: %s", len(nvrs), tag) | 263 | log.info("Fetching %s builds for tag: %s", len(nvrs), tag) | ||
| 252 | log.debug('Builds to be downloaded:\n %s', '\n '.join(nvrs)) | 264 | log.debug('Builds to be downloaded:\n %s', '\n '.join(nvrs)) | ||
| 253 | 265 | | |||
| 254 | for nvr in nvrs: | 266 | for nvr in nvrs: | ||
| 255 | rpms.extend(self.get_nvr_rpms(nvr, dest, arches, debuginfo, src)) | 267 | rpms.extend(self.get_nvr_rpms(nvr, dest, arches, arch_exclude, debuginfo, src)) | ||
| 256 | 268 | | |||
| 257 | return rpms | 269 | return rpms | ||
| 258 | 270 | | |||
| 259 | 271 | | |||
| 260 | def getNEVR(build): | 272 | def getNEVR(build): | ||
| 261 | '''Extract RPM version identifier in NEVR format from Koji build object | 273 | '''Extract RPM version identifier in NEVR format from Koji build object | ||
| 262 | 274 | | |||
| 263 | :param dict build: Koji buildinfo dictionary (as returned e.g. from | 275 | :param dict build: Koji buildinfo dictionary (as returned e.g. from | ||
| Show All 17 Lines | |||||
| 1 | # -*- coding: utf-8 -*- | 1 | # -*- coding: utf-8 -*- | ||
|---|---|---|---|---|---|
| 2 | # Copyright 2009-2014, Red Hat, Inc. | 2 | # Copyright 2009-2016, Red Hat, Inc. | ||
| 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | ||
| 4 | # See the LICENSE file for more details on Licensing | 4 | # See the LICENSE file for more details on Licensing | ||
| 5 | 5 | | |||
| 6 | from dingus import Dingus | 6 | from dingus import Dingus | ||
| 7 | import pytest | ||||
| 7 | 8 | | |||
| 8 | from libtaskotron.directives import koji_directive | 9 | from libtaskotron.directives import koji_directive | ||
| 9 | 10 | | |||
| 11 | ACTIONS = ('download', 'download_tag', 'download_latest_stable') | ||||
| 12 | | ||||
| 10 | class TestKojiDirective(): | 13 | class TestKojiDirective(): | ||
| 11 | def setup_method(self, method): | 14 | def setup_method(self, method): | ||
| 15 | self.koji_calls = ('get_nvr_rpms', 'get_tagged_rpms') | ||||
| 16 | | ||||
| 12 | self.ref_nvr = 'foo-1.2-3.fc99' | 17 | self.ref_nvr = 'foo-1.2-3.fc99' | ||
| 13 | self.ref_previous_nvr = 'foo-1.2-2.fc99' | 18 | self.ref_previous_nvr = 'foo-1.2-2.fc99' | ||
| 14 | self.ref_arch = ['noarch'] | 19 | self.ref_arch = ['noarch'] | ||
| 15 | self.ref_tag = 'tagfoo' | 20 | self.ref_tag = 'tagfoo' | ||
| 16 | self.ref_name = 'foo' | 21 | self.ref_name = 'foo' | ||
| 17 | self.ref_version = '1.2' | 22 | self.ref_version = '1.2' | ||
| 18 | self.ref_release = '3.fc99' | 23 | self.ref_release = '3.fc99' | ||
| 19 | self.ref_buildid = 123456 | 24 | self.ref_buildid = 123456 | ||
| ▲ Show 20 Lines • Show All 81 Lines • ▼ Show 20 Line(s) | 89 | def test_parse_download_latest_stable_command(self): | |||
| 101 | requested_src = getrpm_calls[2]['src'] | 106 | requested_src = getrpm_calls[2]['src'] | ||
| 102 | requested_debuginfo = getrpm_calls[2]['debuginfo'] | 107 | requested_debuginfo = getrpm_calls[2]['debuginfo'] | ||
| 103 | 108 | | |||
| 104 | assert len(koji_calls) == 2 | 109 | assert len(koji_calls) == 2 | ||
| 105 | assert requested_nvr == self.ref_previous_nvr | 110 | assert requested_nvr == self.ref_previous_nvr | ||
| 106 | assert not requested_src | 111 | assert not requested_src | ||
| 107 | assert not requested_debuginfo | 112 | assert not requested_debuginfo | ||
| 108 | 113 | | |||
| 109 | def test_parse_multiple_arches(self): | 114 | @pytest.mark.parametrize('action', ACTIONS) | ||
| 110 | self.ref_arch = ['noarch', 'i386'] | 115 | def test_parse_multiple_arches(self, action): | ||
| 111 | 116 | ref_arch = ['noarch', 'i386'] | |||
| 112 | self.ref_input = {'action': 'download_tag', | 117 | ref_input = {'action': action, 'arch': ref_arch, | ||
| 113 | 'arch': self.ref_arch, | 118 | 'koji_build': self.ref_nvr, 'koji_tag': self.ref_tag} | ||
| 114 | 'koji_tag': self.ref_tag} | | |||
| 115 | 119 | | |||
| 116 | stub_koji = Dingus('koji_utils') | 120 | stub_koji = Dingus('koji_utils') | ||
| 117 | 121 | | |||
| 118 | test_helper = koji_directive.KojiDirective(stub_koji) | 122 | test_helper = koji_directive.KojiDirective(stub_koji) | ||
| 119 | test_helper.process(self.ref_input, self.ref_arg_data) | 123 | test_helper.process(ref_input, self.ref_arg_data) | ||
| 120 | 124 | | |||
| 121 | assert test_helper.arches == self.ref_arch | 125 | assert test_helper.arches == ref_arch | ||
| 122 | 126 | | |||
| 123 | def test_parse_single_arch(self): | 127 | @pytest.mark.parametrize('action', ACTIONS) | ||
| 124 | self.ref_input = {'action': 'download_tag', | 128 | def test_parse_single_arch(self, action): | ||
| 125 | 'arch': self.ref_arch, | 129 | ref_input = {'action': action, 'arch': self.ref_arch, | ||
| 126 | 'koji_tag': self.ref_tag} | 130 | 'koji_build': self.ref_nvr, 'koji_tag': self.ref_tag} | ||
| 127 | 131 | | |||
| 128 | stub_koji = Dingus('koji_utils') | 132 | stub_koji = Dingus('koji_utils') | ||
| 129 | 133 | | |||
| 130 | test_helper = koji_directive.KojiDirective(stub_koji) | 134 | test_helper = koji_directive.KojiDirective(stub_koji) | ||
| 131 | test_helper.process(self.ref_input, self.ref_arg_data) | 135 | test_helper.process(ref_input, self.ref_arg_data) | ||
| 132 | 136 | | |||
| 133 | assert test_helper.arches == self.ref_arch | 137 | assert test_helper.arches == self.ref_arch | ||
| 134 | 138 | | |||
| 135 | def test_add_noarch_download_command(self): | 139 | @pytest.mark.parametrize('action', ACTIONS) | ||
| 136 | ref_input = {'action': 'download', 'arch': ['x86_64'], | 140 | def test_add_noarch(self, action): | ||
| 137 | 'koji_build': self.ref_nvr} | 141 | ref_input = {'action': action, 'arch': ['x86_64'], | ||
| 138 | ref_arg_data = {'workdir': '/var/tmp/foo'} | 142 | 'koji_build': self.ref_nvr, 'koji_tag': self.ref_tag} | ||
| 139 | 143 | | |||
| 140 | stub_koji = Dingus() | 144 | stub_koji = Dingus() | ||
| 141 | test_helper = koji_directive.KojiDirective(stub_koji) | 145 | test_helper = koji_directive.KojiDirective(stub_koji) | ||
| 142 | test_helper.process(ref_input, ref_arg_data) | 146 | test_helper.process(ref_input, self.ref_arg_data) | ||
| 143 | 147 | | |||
| 144 | getrpm_calls = stub_koji.calls() | 148 | getrpm_calls = [call for call in stub_koji.calls() if call[0] in self.koji_calls] | ||
| 145 | requested_arches = getrpm_calls[0][2]['arches'] | 149 | assert len(getrpm_calls) >= 1 | ||
| 146 | 150 | for call in getrpm_calls: | |||
| 147 | assert len(getrpm_calls) == 1 | 151 | requested_arches = call[2]['arches'] | ||
| 148 | assert 'x86_64' in requested_arches | 152 | for arch in ref_input['arch']: | ||
| 149 | assert 'noarch' in requested_arches | 153 | assert arch in requested_arches | ||
| 154 | assert 'noarch' in requested_arches | ||||
| 150 | 155 | | |||
| 151 | def test_add_noarch_download_tag_command(self): | 156 | @pytest.mark.parametrize('action', ACTIONS) | ||
| 152 | ref_input = {'action': 'download_tag', 'arch': ['x86_64'], | 157 | def test_exclude_arch(self, action): | ||
| 153 | 'koji_tag': self.ref_tag} | 158 | ref_input = {'action': action, 'arch': ['all'], 'arch_exclude': ['x86_64', 'noarch'], | ||
| 154 | ref_arg_data = {'workdir': '/var/tmp/foo'} | 159 | 'koji_build': self.ref_nvr, 'koji_tag': self.ref_tag} | ||
| 155 | 160 | | |||
| 156 | stub_koji = Dingus(get_tagged_rpms__returns=self.ref_rpms) | 161 | stub_koji = Dingus() | ||
| 157 | test_helper = koji_directive.KojiDirective(stub_koji) | 162 | test_helper = koji_directive.KojiDirective(stub_koji) | ||
| 158 | test_helper.process(ref_input, ref_arg_data) | 163 | test_helper.process(ref_input, self.ref_arg_data) | ||
| 159 | 164 | | |||
| 160 | getrpm_calls = stub_koji.calls() | 165 | getrpm_calls = [call for call in stub_koji.calls() if call[0] in self.koji_calls] | ||
| 161 | requested_arches = getrpm_calls[0][2]['arches'] | 166 | assert len(getrpm_calls) >= 1 | ||
| 162 | 167 | for call in getrpm_calls: | |||
| 163 | assert len(getrpm_calls) == 1 | 168 | requested_arches = call[2]['arches'] | ||
| 164 | assert 'x86_64' in requested_arches | 169 | excluded_arches = call[2]['arch_exclude'] | ||
| 165 | assert 'noarch' in requested_arches | 170 | assert requested_arches == ref_input['arch'] | ||
| 166 | 171 | for arch in ref_input['arch_exclude']: | |||
| 167 | def test_download_command_src(self): | 172 | assert arch in excluded_arches | ||
| 168 | ref_input = {'action': 'download', 'arch': [], 'src': True, | 173 | assert len(excluded_arches) == len(ref_input['arch_exclude']) | ||
| 169 | 'koji_build': self.ref_nvr} | 174 | | ||
| 170 | ref_arg_data = {'workdir': '/var/tmp/foo'} | 175 | @pytest.mark.parametrize('action', ACTIONS) | ||
| 176 | def test_src(self, action): | ||||
| 177 | ref_input = {'action': action, 'arch': [], 'src': True, | ||||
| 178 | 'koji_build': self.ref_nvr, 'koji_tag': self.ref_tag} | ||||
| 171 | 179 | | |||
| 172 | stub_koji = Dingus() | 180 | stub_koji = Dingus() | ||
| 173 | test_helper = koji_directive.KojiDirective(stub_koji) | 181 | test_helper = koji_directive.KojiDirective(stub_koji) | ||
| 174 | test_helper.process(ref_input, ref_arg_data) | 182 | test_helper.process(ref_input, self.ref_arg_data) | ||
| 175 | 183 | | |||
| 176 | getrpm_calls = stub_koji.calls() | 184 | getrpm_calls = [call for call in stub_koji.calls() if call[0] in self.koji_calls] | ||
| 177 | requested_arches = getrpm_calls[0][2]['arches'] | 185 | assert len(getrpm_calls) >= 1 | ||
| 178 | requested_src = getrpm_calls[0][2]['src'] | 186 | for call in getrpm_calls: | ||
| 187 | requested_arches = call[2]['arches'] | ||||
| 188 | requested_src = call[2]['src'] | ||||
| 189 | assert requested_arches == ref_input['arch'] | ||||
| 190 | assert requested_src | ||||
| 179 | 191 | | |||
| 180 | assert len(getrpm_calls) == 1 | 192 | @pytest.mark.parametrize('action', ACTIONS) | ||
| 181 | assert [] == requested_arches | 193 | def test_debuginfo(self, action): | ||
| 182 | assert requested_src | 194 | ref_input = {'action': action, 'arch': [], 'debuginfo': True, | ||
| 183 | 195 | 'koji_build': self.ref_nvr, 'koji_tag': self.ref_tag} | |||
| 184 | def test_download_command_debuginfo(self): | | |||
| 185 | ref_input = {'action': 'download', 'arch': [], 'debuginfo': True, | | |||
| 186 | 'koji_build': self.ref_nvr} | | |||
| 187 | ref_arg_data = {'workdir': '/var/tmp/foo'} | | |||
| 188 | 196 | | |||
| 189 | stub_koji = Dingus() | 197 | stub_koji = Dingus() | ||
| 190 | test_helper = koji_directive.KojiDirective(stub_koji) | 198 | test_helper = koji_directive.KojiDirective(stub_koji) | ||
| 191 | test_helper.process(ref_input, ref_arg_data) | 199 | test_helper.process(ref_input, self.ref_arg_data) | ||
| 192 | 200 | | |||
| 193 | getrpm_calls = stub_koji.calls() | 201 | getrpm_calls = [call for call in stub_koji.calls() if call[0] in self.koji_calls] | ||
| 194 | requested_arches = getrpm_calls[0][2]['arches'] | 202 | assert len(getrpm_calls) >= 1 | ||
| 195 | requested_debuginfo = getrpm_calls[0][2]['debuginfo'] | 203 | for call in getrpm_calls: | ||
| 196 | 204 | requested_arches = call[2]['arches'] | |||
| 197 | assert len(getrpm_calls) == 1 | 205 | requested_debuginfo = call[2]['debuginfo'] | ||
| 198 | assert [] == requested_arches | 206 | assert requested_arches == ref_input['arch'] | ||
| 199 | assert requested_debuginfo | 207 | assert requested_debuginfo | ||
| 1 | # -*- coding: utf-8 -*- | 1 | # -*- coding: utf-8 -*- | ||
|---|---|---|---|---|---|
| 2 | # Copyright 2009-2014, Red Hat, Inc. | 2 | # Copyright 2009-2016, Red Hat, Inc. | ||
| 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | 3 | # License: GPL-2.0+ <http://spdx.org/licenses/GPL-2.0+> | ||
| 4 | # See the LICENSE file for more details on Licensing | 4 | # See the LICENSE file for more details on Licensing | ||
| 5 | 5 | | |||
| 6 | '''Unit tests for libtaskotron/koji_utils.py''' | 6 | '''Unit tests for libtaskotron/koji_utils.py''' | ||
| 7 | 7 | | |||
| 8 | import pytest | 8 | import pytest | ||
| 9 | from dingus import Dingus | 9 | from dingus import Dingus | ||
| 10 | import os | 10 | import os | ||
| Show All 19 Lines | 21 | def create_multicall(first, second): | |||
| 30 | 30 | | |||
| 31 | return multicall | 31 | return multicall | ||
| 32 | 32 | | |||
| 33 | 33 | | |||
| 34 | class TestKojiClient(): | 34 | class TestKojiClient(): | ||
| 35 | def setup_method(self, method): | 35 | def setup_method(self, method): | ||
| 36 | self.ref_nvr = 'foo-1.2-3.fc99' | 36 | self.ref_nvr = 'foo-1.2-3.fc99' | ||
| 37 | self.ref_latest_stable = 'foo-1.2-2.fc99' | 37 | self.ref_latest_stable = 'foo-1.2-2.fc99' | ||
| 38 | self.ref_arch = 'noarch' | 38 | self.ref_arch = 'x86_64' | ||
| 39 | self.ref_name = 'foo' | 39 | self.ref_name = 'foo' | ||
| 40 | self.ref_version = '1.2' | 40 | self.ref_version = '1.2' | ||
| 41 | self.ref_release = '3.fc99' | 41 | self.ref_release = '3.fc99' | ||
| 42 | self.ref_buildid = 123456 | 42 | self.ref_buildid = 123456 | ||
| 43 | self.ref_filename = "%s.%s.rpm" % (self.ref_nvr, self.ref_arch) | 43 | self.ref_filename = "%s.%s.rpm" % (self.ref_nvr, self.ref_arch) | ||
| 44 | self.ref_tags = ['f99-updates', 'f99'] | 44 | self.ref_tags = ['f99-updates', 'f99'] | ||
| 45 | 45 | | |||
| 46 | self.ref_build = {'package_name': self.ref_name, | 46 | self.ref_build = {'package_name': self.ref_name, | ||
| 47 | 'version': self.ref_version, | 47 | 'version': self.ref_version, | ||
| 48 | 'release': self.ref_release, | 48 | 'release': self.ref_release, | ||
| 49 | 'id': self.ref_buildid, | 49 | 'id': self.ref_buildid, | ||
| 50 | 'nvr': '%s-%s-%s' % (self.ref_name, self.ref_version, | 50 | 'nvr': '%s-%s-%s' % (self.ref_name, self.ref_version, | ||
| 51 | self.ref_release)} | 51 | self.ref_release)} | ||
| 52 | 52 | | |||
| 53 | # rpms: standard, src, 2x debuginfo, noarch | ||||
| 53 | self.ref_rpms = [{'name': self.ref_name, 'version': self.ref_version, | 54 | self.ref_rpms = [{'name': self.ref_name, 'version': self.ref_version, | ||
| 54 | 'release': self.ref_release, 'nvr': self.ref_nvr, | 55 | 'release': self.ref_release, 'nvr': self.ref_nvr, | ||
| 55 | 'arch': self.ref_arch, 'build_id': self.ref_buildid,}, | 56 | 'arch': self.ref_arch, 'build_id': self.ref_buildid}, | ||
| 56 | {'name': self.ref_name, 'version': self.ref_version, | 57 | {'name': self.ref_name, 'version': self.ref_version, | ||
| 57 | 'release': self.ref_release, 'nvr': self.ref_nvr, | 58 | 'release': self.ref_release, 'nvr': self.ref_nvr, | ||
| 58 | 'arch': 'src', 'build_id': self.ref_buildid, }, | 59 | 'arch': 'src', 'build_id': self.ref_buildid}, | ||
| 59 | {'name': self.ref_name + '-debuginfo', | 60 | {'name': self.ref_name + '-debuginfo', | ||
| 60 | 'version': self.ref_version, | 61 | 'version': self.ref_version, | ||
| 61 | 'release': self.ref_release, 'nvr': self.ref_nvr, | 62 | 'release': self.ref_release, 'nvr': self.ref_nvr, | ||
| 62 | 'arch': 'src', 'build_id': self.ref_buildid, }, | 63 | 'arch': self.ref_arch, 'build_id': self.ref_buildid}, | ||
| 63 | {'name': self.ref_name + '-debuginfo-common', | 64 | {'name': self.ref_name + '-debuginfo-common', | ||
| 64 | 'version': self.ref_version, | 65 | 'version': self.ref_version, | ||
| 65 | 'release': self.ref_release, 'nvr': self.ref_nvr, | 66 | 'release': self.ref_release, 'nvr': self.ref_nvr, | ||
| 66 | 'arch': 'src', 'build_id': self.ref_buildid, }] | 67 | 'arch': self.ref_arch, 'build_id': self.ref_buildid}, | ||
| 68 | {'name': self.ref_name + '-data', | ||||
| 69 | 'version': self.ref_version, | ||||
| 70 | 'release': self.ref_release, 'nvr': self.ref_nvr, | ||||
| 71 | 'arch': 'noarch', 'build_id': self.ref_buildid}] | ||||
| 67 | 72 | | |||
| 68 | def test_latest_by_tag_first_tag_miss(self): | 73 | def test_latest_by_tag_first_tag_miss(self): | ||
| 69 | stub_koji = Dingus(listTagged__returns=None, | 74 | stub_koji = Dingus(listTagged__returns=None, | ||
| 70 | multiCall__returns=[[[]], [[{'nvr': self.ref_latest_stable}]]]) | 75 | multiCall__returns=[[[]], [[{'nvr': self.ref_latest_stable}]]]) | ||
| 71 | 76 | | |||
| 72 | test_koji = koji_utils.KojiClient(stub_koji) | 77 | test_koji = koji_utils.KojiClient(stub_koji) | ||
| 73 | outcome = test_koji.latest_by_tag(self.ref_tags, self.ref_name) | 78 | outcome = test_koji.latest_by_tag(self.ref_tags, self.ref_name) | ||
| 74 | 79 | | |||
| ▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Line(s) | 140 | stub_koji = Dingus(getBuild__returns=None, | |||
| 136 | getRPM__returns=None, | 141 | getRPM__returns=None, | ||
| 137 | multiCall=create_multicall([[self.ref_rpms[0]]], | 142 | multiCall=create_multicall([[self.ref_rpms[0]]], | ||
| 138 | [[None]])) | 143 | [[None]])) | ||
| 139 | 144 | | |||
| 140 | test_koji = koji_utils.KojiClient(stub_koji) | 145 | test_koji = koji_utils.KojiClient(stub_koji) | ||
| 141 | with pytest.raises(exc.TaskotronRemoteError): | 146 | with pytest.raises(exc.TaskotronRemoteError): | ||
| 142 | test_koji.rpms_to_build([self.ref_filename]) | 147 | test_koji.rpms_to_build([self.ref_filename]) | ||
| 143 | 148 | | |||
| 144 | 149 | def test_get_urls(self): | |||
| 145 | def test_get_noarch_rpmurls_from_nvr(self): | 150 | stub_koji = Dingus(getBuild__returns=self.ref_build, | ||
| 146 | stub_koji = Dingus(getBuild__returns = self.ref_build, | 151 | listRPMs__returns=self.ref_rpms) | ||
| 147 | listRPMs__returns = self.ref_rpms) | | |||
| 148 | | ||||
| 149 | test_koji = koji_utils.KojiClient(stub_koji) | 152 | test_koji = koji_utils.KojiClient(stub_koji) | ||
| 150 | | ||||
| 151 | test_urls = test_koji.nvr_to_urls(self.ref_nvr) | | |||
| 152 | | ||||
| 153 | koji_baseurl = config.get_config().pkg_url | 153 | koji_baseurl = config.get_config().pkg_url | ||
| 154 | ref_urls = ['%s/%s/%s/%s/%s/%s.%s.rpm' % (koji_baseurl, self.ref_name, | | |||
| 155 | self.ref_version, self.ref_release, self.ref_arch, | | |||
| 156 | self.ref_nvr, self.ref_arch), | | |||
| 157 | '%s/%s/%s/%s/src/%s.src.rpm' % (koji_baseurl, self.ref_name, | | |||
| 158 | self.ref_version, self.ref_release, self.ref_nvr), | | |||
| 159 | ] | | |||
| 160 | 154 | | |||
| 161 | assert test_urls == ref_urls | 155 | test_urls = test_koji.nvr_to_urls(self.ref_nvr) | ||
| 156 | for url in test_urls: | ||||
| 157 | assert url.startswith(koji_baseurl) | ||||
| 158 | assert url.endswith('.rpm') | ||||
| 162 | 159 | | |||
| 163 | def should_i386_rpmurls_query_i686(self): | 160 | def should_i386_rpmurls_query_i686(self): | ||
| 164 | self.ref_arch = 'i386' | 161 | self.ref_arch = 'i386' | ||
| 165 | 162 | stub_koji = Dingus(getBuild__returns=self.ref_build, | |||
| 166 | stub_koji = Dingus(getBuild__returns = self.ref_build) | 163 | listRPMs__returns=self.ref_rpms) | ||
| 167 | | ||||
| 168 | test_koji = koji_utils.KojiClient(stub_koji) | 164 | test_koji = koji_utils.KojiClient(stub_koji) | ||
| 169 | 165 | | |||
| 170 | # this isn't setup to actually return urls since we don't care about | 166 | test_koji.nvr_to_urls(self.ref_nvr, arches=[self.ref_arch]) | ||
| 171 | # that for this test | | |||
| 172 | # it should throw an exception in this case. | | |||
| 173 | try: | | |||
| 174 | test_koji.nvr_to_urls(self.ref_nvr, arches = [self.ref_arch]) | | |||
| 175 | except exc.TaskotronRemoteError: | | |||
| 176 | pass | | |||
| 177 | 167 | | |||
| 178 | listrpm_calls = stub_koji.calls('listRPMs') | 168 | listrpm_calls = stub_koji.calls('listRPMs') | ||
| 179 | requested_arches = listrpm_calls[0][2]['arches'] | 169 | requested_arches = listrpm_calls[0][2]['arches'] | ||
| 180 | | ||||
| 181 | assert 'i686' in requested_arches | 170 | assert 'i686' in requested_arches | ||
| 182 | assert 'i386' in requested_arches | 171 | assert 'i386' in requested_arches | ||
| 183 | 172 | | |||
| 184 | def should_not_throw_exception_norpms(self): | 173 | def should_not_throw_exception_norpms(self): | ||
| 185 | '''It's possible to have no RPMs (for the given arch) in a build''' | 174 | '''It's possible to have no RPMs (for the given arch) in a build''' | ||
| 186 | stub_koji = Dingus(getBuild__returns = self.ref_build) | 175 | stub_koji = Dingus(getBuild__returns = self.ref_build) | ||
| 187 | test_koji = koji_utils.KojiClient(stub_koji) | 176 | test_koji = koji_utils.KojiClient(stub_koji) | ||
| 188 | 177 | | |||
| 189 | test_koji.nvr_to_urls(self.ref_nvr, arches = [self.ref_arch]) | 178 | test_koji.nvr_to_urls(self.ref_nvr, arches = [self.ref_arch]) | ||
| 190 | 179 | | |||
| 191 | def test_nvr_to_urls_src_filtering(self): | 180 | def test_nvr_to_urls_src_filtering(self): | ||
| 192 | stub_koji = Dingus(getBuild__returns=self.ref_build, | 181 | stub_koji = Dingus(getBuild__returns=self.ref_build, | ||
| 193 | listRPMs__returns=self.ref_rpms) | 182 | listRPMs__returns=self.ref_rpms) | ||
| 194 | test_koji = koji_utils.KojiClient(stub_koji) | 183 | test_koji = koji_utils.KojiClient(stub_koji) | ||
| 195 | 184 | | |||
| 196 | # arch and src enabled | 185 | # src enabled | ||
| 197 | urls = test_koji.nvr_to_urls(self.ref_nvr, arches=[self.ref_arch]) | 186 | urls = test_koji.nvr_to_urls(self.ref_nvr, src=True, debuginfo=True) | ||
| 198 | assert len(urls) == 2 | 187 | assert len(urls) == len(self.ref_rpms) | ||
| 199 | assert 'src' in urls[0] or 'src' in urls[1] | 188 | assert any(['src' in url for url in urls]) | ||
| 200 | 189 | | |||
| 201 | # arch enabled, src disabled | 190 | # src disabled | ||
| 202 | urls = test_koji.nvr_to_urls(self.ref_nvr, arches=[self.ref_arch], | 191 | urls = test_koji.nvr_to_urls(self.ref_nvr, src=False, debuginfo=True) | ||
| 203 | src=False) | 192 | assert len(urls) == len([rpm for rpm in self.ref_rpms | ||
| 204 | assert len(urls) == 1 | 193 | if rpm['arch'] != 'src']) | ||
| 205 | assert 'src' not in urls[0] | 194 | assert all(['src' not in url for url in urls]) | ||
| 206 | 195 | | |||
| 207 | def test_nvr_to_urls_debuginfo(self): | 196 | def test_nvr_to_urls_debuginfo(self): | ||
| 208 | stub_koji = Dingus(getBuild__returns=self.ref_build, | 197 | stub_koji = Dingus(getBuild__returns=self.ref_build, | ||
| 209 | listRPMs__returns=self.ref_rpms) | 198 | listRPMs__returns=self.ref_rpms) | ||
| 210 | test_koji = koji_utils.KojiClient(stub_koji) | 199 | test_koji = koji_utils.KojiClient(stub_koji) | ||
| 211 | 200 | | |||
| 212 | # arch enabled, src enabled, debuginfo enabled | 201 | # debuginfo enabled | ||
| 213 | urls = test_koji.nvr_to_urls(self.ref_nvr, arches=[self.ref_arch], | 202 | urls = test_koji.nvr_to_urls(self.ref_nvr, debuginfo=True) | ||
| 214 | debuginfo=True) | 203 | assert len(urls) == len(self.ref_rpms) | ||
| 215 | assert len(urls) == 4 | 204 | assert any(['debuginfo' in url for url in urls]) | ||
| 216 | assert len([url for url in urls if '-debuginfo' in url]) == 2 | 205 | | ||
| 206 | # debuginfo disabled | ||||
| 207 | urls = test_koji.nvr_to_urls(self.ref_nvr, debuginfo=False) | ||||
| 208 | assert len(urls) == len([rpm for rpm in self.ref_rpms | ||||
| 209 | if 'debuginfo' not in rpm['name']]) | ||||
| 210 | assert all(['debuginfo' not in url for url in urls]) | ||||
| 217 | 211 | | |||
| 218 | def test_nvr_to_urls_only_src(self): | 212 | def test_nvr_to_urls_only_src(self): | ||
| 219 | stub_koji = Dingus(getBuild__returns=self.ref_build) | 213 | stub_koji = Dingus(getBuild__returns=self.ref_build) | ||
| 220 | test_koji = koji_utils.KojiClient(stub_koji) | 214 | test_koji = koji_utils.KojiClient(stub_koji) | ||
| 221 | 215 | | |||
| 222 | # arch disabled, src enabled | 216 | # arch disabled, src enabled | ||
| 223 | test_koji.nvr_to_urls(self.ref_nvr, arches=[], src=True) | 217 | test_koji.nvr_to_urls(self.ref_nvr, arches=[], src=True) | ||
| 224 | listrpm_calls = stub_koji.calls('listRPMs') | 218 | listrpm_calls = stub_koji.calls('listRPMs') | ||
| 225 | requested_arches = listrpm_calls[0][2]['arches'] | 219 | requested_arches = listrpm_calls[0][2]['arches'] | ||
| 226 | assert len(requested_arches) == 1 | 220 | assert len(requested_arches) == 1 | ||
| 227 | assert requested_arches[0] == 'src' | 221 | assert requested_arches[0] == 'src' | ||
| 228 | 222 | | |||
| 229 | def should_raise_no_arch_no_src(self): | 223 | def should_raise_no_arch_no_src(self): | ||
| 230 | test_koji = koji_utils.KojiClient(Dingus()) | 224 | test_koji = koji_utils.KojiClient(Dingus()) | ||
| 231 | 225 | | |||
| 232 | with pytest.raises(exc.TaskotronValueError): | 226 | with pytest.raises(exc.TaskotronValueError): | ||
| 233 | test_koji.nvr_to_urls(self.ref_nvr, arches=[], src=False) | 227 | test_koji.nvr_to_urls(self.ref_nvr, arches=[], src=False) | ||
| 234 | 228 | | |||
| 229 | def test_arch_exclude(self): | ||||
| 230 | stub_koji = Dingus(getBuild__returns=self.ref_build, | ||||
| 231 | listRPMs__returns=self.ref_rpms) | ||||
| 232 | test_koji = koji_utils.KojiClient(stub_koji) | ||||
| 233 | ref_exclude = ['x86_64'] | ||||
| 234 | | ||||
| 235 | # arch_exclude disabled | ||||
| 236 | urls = test_koji.nvr_to_urls(self.ref_nvr, src=True, debuginfo=True) | ||||
| 237 | assert len(urls) == len(self.ref_rpms) | ||||
| 238 | assert any(['src' in url for url in urls]) | ||||
| 239 | | ||||
| 240 | # arch_exclude enabled | ||||
| 241 | urls = test_koji.nvr_to_urls(self.ref_nvr, arch_exclude=ref_exclude, src=True, | ||||
| 242 | debuginfo=True) | ||||
| 243 | assert len(urls) == len([rpm for rpm in self.ref_rpms | ||||
| 244 | if rpm['arch'] not in ref_exclude]) | ||||
| 245 | for arch in ref_exclude: | ||||
| 246 | assert all([arch not in url for url in urls]) | ||||
| 247 | | ||||
| 248 | def test_arch_exclude_override_explicit(self): | ||||
| 249 | '''arch_exclude should work even when the same arch is explicitly stated in arches''' | ||||
| 250 | stub_koji = Dingus(getBuild__returns=self.ref_build, | ||||
| 251 | listRPMs__returns=self.ref_rpms) | ||||
| 252 | test_koji = koji_utils.KojiClient(stub_koji) | ||||
| 253 | ref_exclude = ['x86_64'] | ||||
| 254 | ref_arch = set([rpm['arch'] for rpm in self.ref_rpms]) | ||||
| 255 | | ||||
| 256 | # arch_exclude enabled | ||||
| 257 | urls = test_koji.nvr_to_urls(self.ref_nvr, arches=ref_arch, arch_exclude=ref_exclude, | ||||
| 258 | src=True, debuginfo=True) | ||||
| 259 | assert len(urls) == len([rpm for rpm in self.ref_rpms | ||||
| 260 | if rpm['arch'] not in ref_exclude]) | ||||
| 261 | for arch in ref_exclude: | ||||
| 262 | assert all([arch not in url for url in urls]) | ||||
| 263 | | ||||
| 235 | 264 | | |||
| 236 | # === get_nvr_rpms === | 265 | # === get_nvr_rpms === | ||
| 237 | 266 | | |||
| 238 | def test_get_nvr_rpms_simple(self, monkeypatch): | 267 | def test_get_nvr_rpms_simple(self, monkeypatch): | ||
| 239 | '''NVR contains a few RPMs''' | 268 | '''NVR contains a few RPMs''' | ||
| 240 | test_koji = koji_utils.KojiClient(Dingus()) | 269 | test_koji = koji_utils.KojiClient(Dingus()) | ||
| 241 | 270 | | |||
| 242 | stub_urls = [ | 271 | stub_urls = [ | ||
| ▲ Show 20 Lines • Show All 134 Lines • Show Last 20 Lines | |||||
I guess the self. for these variables (plus self.target_dir) is for testing purposes, as it really does not make much sense otherwise. Or is there any other reason?