From 5325b04af1cb194463aee03f528780f75d0f33d2 Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Mon, 28 Oct 2019 22:51:12 +0100 Subject: registry: cache component loading Each component (builder, publishers...) is loaded everytime dispatch() is called. Its main purpose is to lookup the type of the component. However pkg_resources.EntryPoint.load() is a costly operation and it was done every single time dispatch() is called. Against Wikimedia configurations files [0] it is extremely noticeable. The entrypoint is loaded solely to figure out its type. I do not see any good reason for it to change, thus introduce a new cache _component_type_cache. That then let us prevent invoking load() every single time. Run time to generate Wikimedia configuration files went from 17 seconds to a mere 4 seconds! [0] https://gerrit.wikimedia.org/g/integration/config/+/master/jjb/ Change-Id: I49bc5dc91e3ba69b7f15c5f5ded8a42fb61a60c2 --- jenkins_jobs/registry.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py index 38ee4a55..7ac93d77 100644 --- a/jenkins_jobs/registry.py +++ b/jenkins_jobs/registry.py @@ -32,6 +32,7 @@ logger = logging.getLogger(__name__) class ModuleRegistry(object): _entry_points_cache = {} + _component_type_cache = {} def __init__(self, jjb_config, plugins_list=None): self.modules = [] @@ -128,6 +129,17 @@ class ModuleRegistry(object): def set_parser_data(self, parser_data): self.__parser_data = parser_data + def get_component_list_type(self, entry_point): + if entry_point in self._component_type_cache: + return self._component_type_cache[entry_point] + + # pkg_resources.EntryPoint.load() is costly, cache it. + component_list_type = entry_point.load().component_list_type + logging.info("Caching type %s of %s", component_list_type, entry_point) + self._component_type_cache[entry_point] = component_list_type + + return component_list_type + def dispatch(self, component_type, xml_parent, component, template_data={}): """This is a method that you can call from your implementation of Base.gen_xml or component. It allows modules to define a type @@ -154,7 +166,7 @@ class ModuleRegistry(object): ) entry_point = self.modules_by_component_type[component_type] - component_list_type = entry_point.load().component_list_type + component_list_type = self.get_component_list_type(entry_point) if isinstance(component, dict): # The component is a singleton dictionary of name: dict(args) -- cgit