Small optimizations to pagure-sync-bugzilla.py

1 files changed, 45 insertions, 24 deletions

diff --git a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2 b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2
index 863bcd654..70f5152a7 100644
--- a/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2
+++ b/roles/distgit/pagure/templates/pagure-sync-bugzilla.py.j2
@@ -45,6 +45,7 @@ except ImportError:
     from email.message import EmailMessage as Message
 
 import bugzilla
+import dogpile.cache
 import requests
 import yaml
 from six import string_types
@@ -54,6 +55,12 @@ from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 
 
+cache = dogpile.cache.make_region().configure(
+	'dogpile.cache.memory',
+	expiration_time=3600,
+)
+
+
 def retry_session():
     session = requests.Session()
     retry = Retry(
@@ -447,16 +454,8 @@ def notify_users(errors):
         json.dump(new_data, stream)
 
 
-def pagure_project_to_acl_schema(pagure_project, product):
-    """
-    This function translates the JSON of a Pagure project to what PkgDB used to
-    output in the Bugzilla API.
-    :param pagure_project: a dictionary of the JSON of a Pagure project
-    :return: a dictionary of the content that the Bugzilla API would output
-    """
-    session = retry_session()
-    base_error_msg = ('The connection to "{0}" failed with the status code '
-                      '{1} and output "{2}"')
+@cache.cache_on_arguments()
+def _get_watchers_rv_json(pagure_project):
     watchers_api_url = '{0}/api/0/{1}/{2}/watchers'.format(
         PAGURE_DIST_GIT_URL.rstrip('/'), pagure_project['namespace'],
         pagure_project['name'])
@@ -467,7 +466,36 @@ def pagure_project_to_acl_schema(pagure_project, product):
         error_msg = base_error_msg.format(
             watchers_api_url, watchers_rv.status_code, watchers_rv.text)
         raise RuntimeError(error_msg)
-    watchers_rv_json = watchers_rv.json()
+    return watchers_rv.json()
+
+
+@cache.cache_on_arguments()
+def _get_override_yaml(project):
+    pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format(
+        PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'],
+        project['name'])
+
+    if DRY_RUN:
+        print('Querying {0}'.format(pagure_override_url))
+    override_rv = session.get(pagure_override_url, timeout=30)
+    if override_rv.status_code == 200:
+        override_yaml = yaml.load(override_rv.text)
+        return override_yaml.get('bugzilla_contact', {})
+    return {}
+
+
+def pagure_project_to_acl_schema(pagure_project, product):
+    """
+    This function translates the JSON of a Pagure project to what PkgDB used to
+    output in the Bugzilla API.
+    :param pagure_project: a dictionary of the JSON of a Pagure project
+    :return: a dictionary of the content that the Bugzilla API would output
+    """
+    session = retry_session()
+    base_error_msg = ('The connection to "{0}" failed with the status code '
+                      '{1} and output "{2}"')
+
+    watchers_rv_json = _get_watchers_rv_json(pagure_project)
 
     user_cc_list = []
     for user, watch_levels in watchers_rv_json['watchers'].items():
@@ -492,17 +520,10 @@ def pagure_project_to_acl_schema(pagure_project, product):
 
     # Check if the Bugzilla ticket assignee has been overridden
     owner = pagure_project['access_users']['owner'][0]
-    pagure_override_url = '{0}/{1}/raw/master/f/{2}/{3}'.format(
-        PAGUREURL.rstrip('/'), BUGZILLA_OVERRIDE_REPO, project['namespace'],
-        project['name'])
-
-    override_rv = session.get(pagure_override_url, timeout=30)
-    if override_rv.status_code == 200:
-        override_yaml = yaml.load(override_rv.text)
-        override_yaml = override_yaml.get('bugzilla_contact', {})
-        if override_yaml.get(product) \
-                and isinstance(override_yaml[product], string_types):
-            owner = override_yaml[product]
+    override_yaml = _get_override_yaml(project)
+    if override_yaml.get(product) \
+            and isinstance(override_yaml[product], string_types):
+        owner = override_yaml[product]
 
     return {
         'cclist': {
@@ -542,7 +563,7 @@ if __name__ == '__main__':
         'Fedora Container': {},
         'Fedora EPEL': {},
     }
-    pagure_rpms_api_url = ('{0}/api/0/projects?&namespace=rpms&page=1&'
+    pagure_rpms_api_url = ('{0}/api/0/projects?fork=false&namespace=rpms&page=1&'
                            'per_page=100'.format(
                                PAGURE_DIST_GIT_URL.rstrip('/')))
     session = retry_session()
@@ -578,7 +599,7 @@ if __name__ == '__main__':
             break
 
     pagure_container_api_url = (
-        '{0}/api/0/projects?&namespace=container&page=1&per_page=100'
+        '{0}/api/0/projects?fork=false&namespace=container&page=1&per_page=100'
         .format(PAGURE_DIST_GIT_URL))
     while True:
         if DRY_RUN: