Apply cloud stats work from ticket 3893. Thanks doteast and many others that worked on this!

4 files changed, 108 insertions, 0 deletions

diff --git a/playbooks/hosts/logserver.yml b/playbooks/hosts/logserver.yml
index 8decfd1da..15a112dbe 100644
--- a/playbooks/hosts/logserver.yml
+++ b/playbooks/hosts/logserver.yml
@@ -45,3 +45,11 @@
   handlers:
   - include: "{{ handlers }}/restart_services.yml"
   - include: "{{ handlers }}/semanage.yml"
+
+- name: Cloud Image stats
+  hosts: log01.phx2.fedoraproject.org
+  user: root
+  gather_facts: False
+
+  roles:
+  - cloudstats
diff --git a/roles/cloudstats/files/cloud-image-stat.cron b/roles/cloudstats/files/cloud-image-stat.cron
new file mode 100644
index 000000000..08b8cb314
--- /dev/null
+++ b/roles/cloudstats/files/cloud-image-stat.cron
@@ -0,0 +1 @@
+0 0 1 * * root /usr/bin/python /usr/local/bin/cloud-image-stat.py >& /dev/null
diff --git a/roles/cloudstats/files/cloud-image-stat.py b/roles/cloudstats/files/cloud-image-stat.py
new file mode 100644
index 000000000..ab212db9d
--- /dev/null
+++ b/roles/cloudstats/files/cloud-image-stat.py
@@ -0,0 +1,84 @@
+from glob import glob
+from datetime import date
+from os import walk,listdir
+from os.path import exists
+from bz2 import BZ2File
+import smtplib
+from email.mime.text import MIMEText
+
+stat_qcow2_x86_64 = 0
+stat_rawxz_x86_64 = 0
+stat_qcow2_i386 = 0
+stat_rawxz_i386 = 0
+stat_x86_64 = 0
+stat_i386 = 0
+stat_qcow2 = 0
+stat_rawxz = 0
+email_sender = 'cloud-image-stat-cron@fedoraproject.org'
+email_receiver = 'cloud@lists.fedoraproject.org'
+log_file = 'download.fedoraproject.org-access.log.bz2'
+
+proxy_logs = glob('/var/log/hosts/proxy*')
+date = date.today()
+year,month,day = date.isoformat().split('-')
+
+if month == '01':             # in case it is january
+    qyear = str(date.year-1)
+    qmonth = '12'
+else:                         # for all other months
+    qyear = year
+    qmonth = str(date.month-1)
+
+if int(qmonth) < 10:
+    qmonth = '0'+qmonth
+
+for proxy_log in (proxy_log for proxy_log in proxy_logs if '.stg.' not in proxy_log):
+    proxy_month_log = '{0}/{1}/{2}'.format(proxy_log,qyear,qmonth)
+    if exists(proxy_month_log):
+        for day in listdir(proxy_month_log):
+            proxy_download_log = '{0}/{1}/{2}/{3}'.format(proxy_month_log,day,'http',log_file)
+            if exists(proxy_download_log):
+                fd = BZ2File(proxy_download_log,'r')
+                try:
+                    for line in fd:
+                        if 'sda.qcow2' in line:
+                            if 'x86_64' in line:
+                                stat_qcow2_x86_64 += 1
+                            elif 'i386' in line:
+                                stat_qcow2_i386 += 1
+                        elif 'sda.raw.xz' in line:
+                            if 'x86_64' in line:
+                                stat_rawxz_x86_64 += 1
+                            elif 'i386' in line:
+                                stat_rawxz_i386 += 1
+                finally:
+                    fd.close()
+
+stat_x86_64 = stat_qcow2_x86_64 + stat_rawxz_x86_64
+stat_i386 = stat_qcow2_i386 + stat_rawxz_i386
+stat_qcow2 = stat_qcow2_x86_64 + stat_qcow2_i386
+stat_rawxz = stat_rawxz_x86_64 + stat_rawxz_i386
+
+report = '''Download statistics for cloud images in month {0}-{1} :\n
+- 32-bit arch :\n
+    total = {2},  qcow2 = {3},  raw.xz = {4}\n
+- 64-bit arch :\n 
+    total = {5}, qcow2 = {6}, raw.xz = {7}\n
+- qcow2 images :\n
+    total = {8}, 32-bit = {9}, 64-bit = {10}\n
+- raw.xz images :\n
+    total = {11}, 32-bit = {12}, 64-bit = {13}\n'''
+
+report = report.format(qmonth, qyear,
+                       stat_i386, stat_qcow2_i386, stat_rawxz_i386,
+                       stat_x86_64, stat_qcow2_x86_64, stat_rawxz_x86_64,
+                       stat_qcow2, stat_qcow2_i386, stat_qcow2_x86_64,
+                       stat_rawxz, stat_rawxz_i386, stat_rawxz_x86_64)
+
+email = MIMEText(report)
+email['Subject'] = 'Monthly download statistics for cloud images'
+email['From'] = email_sender
+email['To'] = email_receiver
+server = smtplib.SMTP('localhost')
+server.sendmail(email_sender,email_receiver,email.as_string())
+server.quit()
diff --git a/roles/cloudstats/tasks/main.yml b/roles/cloudstats/tasks/main.yml
new file mode 100644
index 000000000..d7c0c38e6
--- /dev/null
+++ b/roles/cloudstats/tasks/main.yml
@@ -0,0 +1,15 @@
+- name: Install the cloud-image-stat script
+  copy: >
+    src=cloud-image-stat.py dest=/usr/local/bin/cloud-image-stat.py
+    owner=root group=root mode=0755
+  tags:
+  - files
+  - cloudstats
+
+- name: Install the cloud-image-stats cronjob to collect stats from log01
+  copy: >
+    src=cloud-image-stat.cron dest=/etc/cron.d/cloud-image-stat.cron
+    owner=root group=root mode=0644
+  tags:
+  - cron
+  - cloudstats