Differential D712

Enhance dodo.py to work better with CI. Fixes T683
ClosedPublic
Actions

Authored by tflink on Jan 7 2016, 10:51 PM.

Tags

None

Subscribers

None

Details

Reviewers

kparal
mkrizek
jskladan

Maniphest Tasks

T683: enhance libtaskotron's dodo.py to work better in CI environments

Commits

rLTRNbfb49eaf136c: Enhance dodo.py to work better with CI. Fixes T683

Summary

This patch does a couple different things - it adds the ability to include the current branch name in the output dir if buildtype=release is added to args. It also supports a basedir=/place/to/output arg which affects where output is copied. Lastly, it adds a updatelatest task which makes sure a relative latest symlink points to the build just done

Test Plan

test locally with builds, use on stg. buildmaster for CI

Diff Detail

Repository

rLTRN libtaskotron

Lint

Automatic diff as part of commit; lint not applicable.

Unit

Automatic diff as part of commit; unit tests not applicable.

tflink retitled this revision from to Enhance dodo.py to work better with CI. Fixes T683.Jan 7 2016, 10:51 PM

tflink updated this object.

tflink edited the test plan for this revision. (Show Details)

tflink added reviewers: kparal, jskladan, mkrizek.

Bah, I forgot to squash my commits before submitting the review. Will take care of that if/when the review is accepted

LGTM

This revision is now accepted and ready to land.Jan 8 2016, 9:42 AM

Looks good in local usage

Closed by commit rLTRNbfb49eaf136c: Enhance dodo.py to work better with CI. Fixes T683 (authored by tflink). · Explain WhyJan 8 2016, 9:54 PM

This revision was automatically updated to reflect the committed changes.

Revision Contents

			Path	Packages
M			dodo.py (73 lines)

Diff 1837

dodo.py

Show First 20 Lines • Show All 49 Lines • ▼ Show 20 Line(s)		49	if os.system('which rpmspec &>/dev/null') != 0:
50	raise RuntimeError('rpmspec was not found in the system, '	50		raise RuntimeError('rpmspec was not found in the system, '
51	'run "dnf install rpm-build" to install it')	51		'run "dnf install rpm-build" to install it')
52		52
53	rpmspec_command = ['rpmspec', '-q', '--queryformat="%{RELEASE}\\n"', specfile]	53		rpmspec_command = ['rpmspec', '-q', '--queryformat="%{RELEASE}\\n"', specfile]
54	raw_output = subprocess.check_output(rpmspec_command).split()[0].lstrip('"')	54		raw_output = subprocess.check_output(rpmspec_command).split()[0].lstrip('"')
55	return raw_output.split('.')[0]	55		return raw_output.split('.')[0]
56		56
57		57
		58		def listdirs(targetdir):
		59		"""List all dirs in a targetdir, separated out as dirs and symlinks"""
		60		output_contents = [os.path.join(targetdir, d) for d in os.listdir(targetdir)]
		61
		62		dirs = [d for d in output_contents if os.path.isdir(d) and not os.path.islink(d)]
		63
		64		links = [d for d in output_contents if os.path.islink(d)]
		65
		66		return {'dirs': dirs, 'links': links}
		67
		68
		69		def get_gitbranch():
		70		"""retrieve the current git branch"""
		71
		72		gitbranch_command = ['git', 'rev-parse', '--abbrev-ref', 'HEAD']
		73		raw_output = subprocess.check_output(gitbranch_command)
		74		return raw_output.strip()
		75
58	################################################################################	76		################################################################################
59	# GENERAL VARIABLES	77		# GENERAL VARIABLES
60	# These variables are computed based on project definitions and current state	78		# These variables are computed based on project definitions and current state
61	# as a convenience for later functions	79		# as a convenience for later functions
62	################################################################################	80		################################################################################
63		81
64	VERSION = get_version()	82		VERSION = get_version()
65	TARGETNAME = "{}-{}".format(NAME, VERSION)	83		TARGETNAME = "{}-{}".format(NAME, VERSION)
66	SPECFILE = '{}.spec'.format(NAME)	84		SPECFILE = '{}.spec'.format(NAME)
67	RPMRELEASE = get_rpmrelease(os.path.abspath(SPECFILE))	85		RPMRELEASE = get_rpmrelease(os.path.abspath(SPECFILE))
68	NVR = '{}-{}-{}.{}'.format(NAME, VERSION, RPMRELEASE, TARGETDIST)	86		NVR = '{}-{}-{}.{}'.format(NAME, VERSION, RPMRELEASE, TARGETDIST)
69	SHATYPE = 'sha256'	87		SHATYPE = 'sha256'
70		88
71	HERE = os.path.abspath(os.path.dirname(__file__))	89		HERE = os.path.abspath(os.path.dirname(__file__))
72	OUTDIR = os.path.join(HERE, 'builds/{}'.format(VERSION))
73		90
74	# these are temporary, should be options with defaults
75	BRANCH = get_var('branch', 'HEAD')	91		BRANCH = get_var('branch', 'HEAD')
		92		BUILDTYPE = get_var('buildtype', 'NORMAL')
		93
		94		CURBRANCH = get_gitbranch()
		95
		96		# many branches have slashes in them, make sure they'll be one dir
		97		SAFEBRANCH = CURBRANCH.replace('/', '_')
		98
		99		BASEDIR = get_var('basedir', os.path.join(HERE, 'builds'))
		100
		101
		102		# if we're releasing, want to have the branch name in the basedir
		103		if BUILDTYPE.upper() == 'RELEASE':
		104		BASEDIR = os.path.join(BASEDIR, SAFEBRANCH)
		105
		106		OUTDIR = os.path.join(BASEDIR, VERSION)
		107
76		108
77		109
78	################################################################################	110		################################################################################
79	# PROJECT TASKS	111		# PROJECT TASKS
80	# These tasks should be pretty much consistent for all qadevel projects, driven	112		# These tasks should be pretty much consistent for all qadevel projects, driven
81	# by the variables listed above	113		# by the variables listed above
82	################################################################################	114		################################################################################
83		115
▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Line(s)		185	def task_releasedocs():
154	"""Put generated docs in the release output"""	186		"""Put generated docs in the release output"""
155	return {'actions': ['mkdir -p {}'.format(OUTDIR),	187		return {'actions': ['mkdir -p {}'.format(OUTDIR),
156	'cp -r docs/build/html {}/docs'.format(OUTDIR)],	188		'cp -r docs/build/html {}/docs'.format(OUTDIR)],
157	'task_dep': ['builddocs'],	189		'task_dep': ['builddocs'],
158	'targets': ['{}/docs'.format(OUTDIR)]	190		'targets': ['{}/docs'.format(OUTDIR)]
159	}	191		}
160		192
161		193
		194		def set_symlink(basedir, source, linkdest):
		195		"""make relative symlink between source and linkdest in basedir"""
		196
		197		fullsource = os.path.join(basedir, source)
		198		fulldest = os.path.join(basedir, linkdest)
		199
		200		# check if the source exists
		201		if not os.path.exists(fullsource):
		202		raise Exception("Attempted to create symlink to non existant \
		203		directory: {}".format(fullsource))
		204
		205		# check if the symlink already exists
		206		if os.path.exists(fulldest):
		207		# remove the existing link so we can re-create it
		208		print("Removing old latest symlink")
		209		os.remove(fulldest)
		210
		211		cwd = os.getcwd()
		212		os.chdir(basedir)
		213		os.symlink(source, linkdest)
		214		os.chdir(cwd)
		215
		216
		217		def task_updatelatest():
		218		"""Make sure that the 'latest' symlink in output directory points to the correct version"""
		219
		220		def updatelatest():
		221		if not os.path.exists(OUTDIR):
		222		raise Exception("BASEDIR ({}) does not exist, cannot fix symlinks".format(BASEDIR))
		223
		224		set_symlink(BASEDIR, VERSION, 'latest')
		225
		226		return {'actions': [updatelatest],
		227		'targets': [],
		228		'task_dep': ['releasedocs'],
		229		}
		230
162	def task_test():	231		def task_test():
163	"""Run the unit and functional tests"""	232		"""Run the unit and functional tests"""
164	return {'actions': ['py.test -F testing/']	233		return {'actions': ['py.test -F testing/']
165	}	234		}
166		235
167		236
168	def task__details():	237		def task__details():
169	""" Display some calculated variables used in the script for debug purposes"""	238		""" Display some calculated variables used in the script for debug purposes"""
170		239
171	return {'actions': ['echo "taskotron version: {}"'.format(VERSION),	240		return {'actions': ['echo "taskotron version: {}"'.format(VERSION),
172	'echo "target name: {}"'.format(TARGETNAME),	241		'echo "target name: {}"'.format(TARGETNAME),
173	'echo "output dir: {}"'.format(OUTDIR)],	242		'echo "output dir: {}"'.format(OUTDIR)],
174	'verbosity': 2	243		'verbosity': 2
175	}	244		}