Update the mailman role for Mailman 3

16 files changed, 193 insertions, 96 deletions

diff --git a/roles/mailman/templates/mailman.logrotate.j2 b/files/lists-dev/mailman.logrotate.j2
index 597581426..048e3a305 100644
--- a/roles/mailman/templates/mailman.logrotate.j2
+++ b/files/lists-dev/mailman.logrotate.j2
@@ -3,7 +3,7 @@
     sharedscripts
     su mailman mailman
     postrotate
-        /bin/kill -HUP `cat /run/mailman3/master.pid 2>/dev/null` 2>/dev/null || true
+        /bin/kill -HUP `cat {{ mailman_webui_basedir }}/var/master.pid 2>/dev/null` 2>/dev/null || true
         # Don't run "mailman3 reopen" with SELinux on here in the logrotate
         # context, it will be blocked
         #/usr/bin/mailman3 reopen >/dev/null 2>&1 || true
diff --git a/roles/mailman/templates/mailman3.service.j2 b/files/lists-dev/mailman3.service.j2
index 402cbb7cd..402cbb7cd 100644
--- a/roles/mailman/templates/mailman3.service.j2
+++ b/files/lists-dev/mailman3.service.j2
diff --git a/files/lists-dev/pgpass.j2 b/files/lists-dev/pgpass.j2
index b0b229729..a7bd44af6 100644
--- a/files/lists-dev/pgpass.j2
+++ b/files/lists-dev/pgpass.j2
@@ -1,3 +1,2 @@
 *:*:mailman:mailmanadmin:{{ lists_dev_mm_db_pass }}
 *:*:hyperkitty:hyperkittyadmin:{{ lists_dev_hk_db_pass }}
-*:*:kittystore:kittystoreadmin:{{ lists_dev_ks_db_pass }}
diff --git a/files/mailman/mailman2-import.sh b/files/mailman/mailman2-import.sh
deleted file mode 100644
index 71f0821d1..000000000
--- a/files/mailman/mailman2-import.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-HKCONFDIR="/etc/hyperkitty/sites/default"
-MMDIR=$1
-DOMAIN=$2
-
-if [ -z "$MMDIR" ]; then
-    echo  "Usage: $0 <mailman-lib-directory>"
-    exit 2
-fi
-
-[ -z "$DOMAIN" ] && DOMAIN=lists.fedoraproject.org
-
-existinglists=`mktemp`
-trap "rm -f $existinglists" EXIT
-sudo -u mailman mailman3 lists -q > $existinglists
-
-for listname in `ls $MMDIR/lists`; do
-    listaddr="$listname@$DOMAIN"
-    if ! grep -qs $listaddr $existinglists; then
-        echo "sudo -u mailman mailman3 create -d $listaddr"
-        echo "sudo -u mailman PYTHONPATH=/usr/lib/mailman mailman3 import21 $listaddr $MMDIR/lists/$listname/config.pck"
-    fi
-    echo "sudo kittystore-import -p $HKCONFDIR -s settings_admin -l $listaddr --continue $MMDIR/archives/private/${listname}.mbox/${listname}.mbox"
-done
diff --git a/files/mailman/pgpass.j2 b/files/mailman/pgpass.j2
index bfb3161ad..dae76e65d 100644
--- a/files/mailman/pgpass.j2
+++ b/files/mailman/pgpass.j2
@@ -1,7 +1,5 @@
 *:*:mailman:mailman:{{ mailman_mm_db_pass }}
 *:*:hyperkitty:hyperkittyapp:{{ mailman_hk_db_pass }}
 *:*:hyperkitty:hyperkittyadmin:{{ mailman_hk_admin_db_pass }}
-*:*:kittystore:kittystoreapp:{{ mailman_ks_db_pass }}
-*:*:kittystore:kittystoreadmin:{{ mailman_ks_admin_db_pass }}
 *:*:postorius:postoriusapp:{{ mailman_ps_db_pass }}
 *:*:postorius:postoriusadmin:{{ mailman_ps_admin_db_pass }}
diff --git a/playbooks/groups/mailman.yml b/playbooks/groups/mailman.yml
index f50df37ad..10afa2878 100644
--- a/playbooks/groups/mailman.yml
+++ b/playbooks/groups/mailman.yml
@@ -86,16 +86,11 @@
     postgresql_user: name=hyperkittyadmin password={{ mailman_hk_admin_db_pass }}
   - name: hyperkitty DB user
     postgresql_user: name=hyperkittyapp password={{ mailman_hk_db_pass }}
-  - name: kittystore DB admin user
-    postgresql_user: name=kittystoreadmin password={{ mailman_ks_admin_db_pass }}
-  - name: kittystore DB user
-    postgresql_user: name=kittystoreapp password={{ mailman_ks_db_pass }}
   - name: databases creation
     postgresql_db: name={{ item }} owner="{{ item }}admin" encoding=UTF-8
     with_items:
     - mailman
     - hyperkitty
-    - kittystore
   - name: test database creation
     postgresql_db: name=test_hyperkitty owner=hyperkittyadmin encoding=UTF-8
 
@@ -116,8 +111,6 @@
     mailman_mailman_db_pass: "{{ mailman_mm_db_pass }}"
     mailman_hyperkitty_admin_db_pass: "{{ mailman_hk_admin_db_pass }}"
     mailman_hyperkitty_db_pass: "{{ mailman_hk_db_pass }}"
-    mailman_kittystore_admin_db_pass: "{{ mailman_ks_admin_db_pass }}"
-    mailman_kittystore_db_pass: "{{ mailman_ks_db_pass }}"
     mailman_hyperkitty_cookie_key: "{{ mailman_hk_cookie_key }}"
   - fedmsg/base
 
@@ -126,7 +119,6 @@
     yum: pkg={{ item }} state=present
     with_items:
     - tar
-    - mailman # transition from mailman2.1
     tags:
     - packages
 
diff --git a/playbooks/hosts/lists-dev.cloud.fedoraproject.org.yml b/playbooks/hosts/lists-dev.cloud.fedoraproject.org.yml
index 2780daa57..1e9627c31 100644
--- a/playbooks/hosts/lists-dev.cloud.fedoraproject.org.yml
+++ b/playbooks/hosts/lists-dev.cloud.fedoraproject.org.yml
@@ -151,6 +151,15 @@
     notify:
     - reload aliases
 
+  # Mailman is installed from source, not from the RPM
+  - name: systemd service file for mailman3
+    template: src="{{ files }}mailman3.service.j2" dest=/etc/systemd/system/multi-user.target.wants/mailman3.service
+    tags:
+    - config
+
+  - name: mailman logging -- rotation
+    template: src=mailman.logrotate.j2 dest=/etc/logrotate.d/mailman3
+
 
   handlers:
   - include: "{{ handlers }}/restart_services.yml"
diff --git a/roles/base/files/postfix/main.cf/main.cf.lists-dev.cloud.fedoraproject.org b/roles/base/files/postfix/main.cf/main.cf.lists-dev.cloud.fedoraproject.org
index f1e306de9..dd08f22be 100644
--- a/roles/base/files/postfix/main.cf/main.cf.lists-dev.cloud.fedoraproject.org
+++ b/roles/base/files/postfix/main.cf/main.cf.lists-dev.cloud.fedoraproject.org
@@ -688,10 +688,7 @@ message_size_limit = 20971520
 
 # Mailman, see MTA.rst
 owner_request_special = no
-#transport_maps = hash:/var/lib/mailman3/data/postfix_lmtp
-#local_recipient_maps = hash:/var/lib/mailman3/data/postfix_lmtp
-#relay_domains = hash:/var/lib/mailman3/data/postfix_domains
-# Until the mailman3 package is fixed:
+# Mailman is installed from source
 transport_maps = hash:/srv/webui/var/data/postfix_lmtp
 local_recipient_maps = hash:/srv/webui/var/data/postfix_lmtp
 relay_domains = hash:/srv/webui/var/data/postfix_domains
diff --git a/roles/mailman/files/clean-sessions.sh b/roles/mailman/files/clean-sessions.sh
deleted file mode 100755
index cda65189d..000000000
--- a/roles/mailman/files/clean-sessions.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-CONFFILE=/etc/mailman-migration.conf
-
-set -e 
-export PATH=$PATH:$(dirname $(realpath $0)) # make yamlget available
-
-CONFDIR=`yamlget confdir $CONFFILE`
-django-admin clearsessions --pythonpath $CONFDIR --settings settings
diff --git a/roles/mailman/tasks/main.yml b/roles/mailman/tasks/main.yml
index 8cd7974e6..34935fda5 100644
--- a/roles/mailman/tasks/main.yml
+++ b/roles/mailman/tasks/main.yml
@@ -65,6 +65,8 @@
   - python-beautifulsoup4
   - python-mock
   - python-whoosh
+  - python-tox
+  - python-vcrpy
   tags:
   - packages
 
@@ -72,13 +74,14 @@
 #
 # Initialize mailman (must be done after settings up the DBs)
 #
-- name: add mailman to the apache group
-  user: name=mailman groups=apache append=yes
-  tags:
-  - config
-  #notify:
-  #  - restart mailman3
-
+#- name: add mailman to the apache group
+#  user: name=mailman groups=apache append=yes
+#  tags:
+#  - config
+#  #notify:
+#  #  - restart mailman3
+
+# access to the aliases files generated by mailman
 - name: add postfix to the mailman group
   user: name=postfix groups=mailman append=yes
   tags:
@@ -86,26 +89,23 @@
   notify:
     - restart postfix
 
-# for access to the full-text index
-- name: add apache to the mailman group
-  user: name=apache groups=mailman append=yes
-  tags:
-  - config
-  notify:
-    - restart httpd
+## for access to the full-text index
+#- name: add apache to the mailman group
+#  user: name=apache groups=mailman append=yes
+#  tags:
+#  - config
+#  notify:
+#    - restart httpd
 
 - name: set the mailman conffile
-  template: src=mailman.cfg.j2 dest=/etc/mailman.cfg
-  tags:
-  - config
-  #notify:
-  #  - restart mailman3
-
-# XXX: to remove when the RPM can be used
-- name: systemd service file for mailman3
-  template: src=mailman3.service.j2 dest=/etc/systemd/system/multi-user.target.wants/mailman3.service
+  template: src={{ item }} dest=/etc/mailman.cfg
+  with_first_found:
+  - mailman.cfg.{{ ansible_hostname }}.j2
+  - mailman.cfg.j2
   tags:
   - config
+  notify:
+    - restart mailman3
 
 
 #
@@ -124,11 +124,6 @@
   copy: src=hyperkitty.logrotate.conf
         dest=/etc/logrotate.d/hyperkitty
 
-# XXX: to remove when the RPM can be used
-- name: mailman logging -- rotation
-  template: src=mailman.logrotate.j2
-        dest=/etc/logrotate.d/mailman3
-
 
 #
 # HyperKitty + Postorius setup
@@ -207,16 +202,16 @@
         dest="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg"
   tags:
   - config
-  #notify:
-  #  - restart mailman3
+  notify:
+    - restart mailman3
 
 - name: symlink the hyperkitty conffile in mailman directory
   file: src="{{ mailman_webui_confdir }}/mailman-hyperkitty.cfg"
         dest=/etc/mailman3.d/hyperkitty.cfg state=link
   tags:
   - config
-  #notify:
-  #  - restart mailman3
+  notify:
+    - restart mailman3
 
 
 #
@@ -251,12 +246,14 @@
         owner=root group=apache mode=0640
 
 
-# Sync databases
+# Sync databases and collect static files on RPM install/upgrade
 - name: install the post-transaction trigger
   template: src=post-transaction.action.j2
             dest=/etc/yum/post-actions/hyperkitty.action
+
 #
 # Only run this on mailman01 for now. 
+# TODO: run it on lists-dev too
 # 
 - name: run the post-update script
   command: "{{ mailman_webui_basedir }}/bin/post-update.sh"
@@ -273,9 +270,6 @@
   notify:
   - restart memcached
 
-- name: start memcached
-  service: state=started enabled=yes name=memcached
-
 # Start services
 - name: start services
   service: state=started enabled=yes name={{ item }}
diff --git a/roles/mailman/templates/apache.conf.j2 b/roles/mailman/templates/apache.conf.j2
index 58f8a0c5c..7e5c7df5d 100644
--- a/roles/mailman/templates/apache.conf.j2
+++ b/roles/mailman/templates/apache.conf.j2
@@ -1,4 +1,3 @@
-Alias /robots.txt  {{ mailman_webui_basedir }}/static/hyperkitty/robots.txt
 Alias /favicon.ico {{ mailman_webui_basedir }}/static/hyperkitty/favicon.ico
 Alias /static      {{ mailman_webui_basedir }}/static
 
diff --git a/roles/mailman/templates/crontab.j2 b/roles/mailman/templates/crontab.j2
index 79a0847d1..380ffb5bf 100644
--- a/roles/mailman/templates/crontab.j2
+++ b/roles/mailman/templates/crontab.j2
@@ -8,3 +8,4 @@
 @monthly apache  django-admin runjobs monthly --pythonpath {{ mailman_webui_confdir }} --settings settings
 @yearly  apache  django-admin runjobs yearly  --pythonpath {{ mailman_webui_confdir }} --settings settings
 2,17,32,47 * * * * apache  django-admin runjobs quarter_hourly --pythonpath {{ mailman_webui_confdir }} --settings settings
+* * * * *  apache  django-admin runjobs minutely  --pythonpath {{ mailman_webui_confdir }} --settings settings
diff --git a/roles/mailman/templates/mailman-migration.conf.j2 b/roles/mailman/templates/mailman-migration.conf.j2
index 5220339e3..8e4381408 100644
--- a/roles/mailman/templates/mailman-migration.conf.j2
+++ b/roles/mailman/templates/mailman-migration.conf.j2
@@ -1,4 +1,3 @@
 basedir: {{ mailman_webui_basedir }}
 confdir: {{ mailman_webui_confdir }}
-mm21codedir: /usr/lib/mailman
 domain: lists.fedoraproject.org
diff --git a/roles/mailman/templates/mailman.cfg.j2 b/roles/mailman/templates/mailman.cfg.j2
index 4240a4a26..3299a03ea 100644
--- a/roles/mailman/templates/mailman.cfg.j2
+++ b/roles/mailman/templates/mailman.cfg.j2
@@ -9,9 +9,7 @@
 site_owner: root@localhost
 
 # Set the paths to be Fedora-compliant
-#layout: fhs
-# Until the mailman3 rpm is back:
-layout: dev
+layout: fhs
 
 [paths.dev]
 var_dir = {{ mailman_webui_basedir }}/var
@@ -25,9 +23,9 @@ lock_dir: /run/lock/mailman3
 ext_dir: /etc/mailman3.d
 pid_file: /run/mailman3/master.pid
 
-#[database]
-#class: mailman.database.postgresql.PostgreSQLDatabase
-#url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
+[database]
+class: mailman.database.postgresql.PostgreSQLDatabase
+url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
 
 [archiver.hyperkitty]
 class: mailman_hyperkitty.Archiver
diff --git a/roles/mailman/templates/mailman.cfg.lists-dev.cloud.fedoraproject.org.j2 b/roles/mailman/templates/mailman.cfg.lists-dev.cloud.fedoraproject.org.j2
new file mode 100644
index 000000000..91b846e1c
--- /dev/null
+++ b/roles/mailman/templates/mailman.cfg.lists-dev.cloud.fedoraproject.org.j2
@@ -0,0 +1,145 @@
+# This is the absolute bare minimum base configuration file.  User supplied
+# configurations are pushed onto this.
+
+[mailman]
+# This address is the "site owner" address.  Certain messages which must be
+# delivered to a human, but which can't be delivered to a list owner (e.g. a
+# bounce from a list owner), will be sent to this address.  It should point to
+# a human.
+site_owner: root@localhost
+
+# Set the paths to be Fedora-compliant
+#layout: fhs
+# Mailman 3 is installed from source
+layout: dev
+
+[paths.dev]
+var_dir = {{ mailman_webui_basedir }}/var
+
+[paths.fhs]
+bin_dir: /usr/libexec/mailman3
+var_dir: /var/lib/mailman3
+queue_dir: /var/spool/mailman3
+log_dir: /var/log/mailman3
+lock_dir: /run/lock/mailman3
+ext_dir: /etc/mailman3.d
+pid_file: /run/mailman3/master.pid
+
+#[database]
+#class: mailman.database.postgresql.PostgreSQLDatabase
+#url: postgresql://mailmanadmin:{{ mailman_mailman_db_pass }}@{{ mailman_db_server }}/mailman
+
+[archiver.hyperkitty]
+class: mailman_hyperkitty.Archiver
+enable: yes
+configuration: {{ mailman_webui_confdir }}/mailman-hyperkitty.cfg
+
+#[archiver.fedmsg]
+#class: mailman3_fedmsg_plugin.Archiver
+#enable: yes
+
+[archiver.prototype]
+enable: yes
+
+
+# http://www.lingoes.net/en/translator/langcode.htm
+
+[language.pt]
+description: Protuguese
+charset: iso-8859-15
+enabled: yes
+
+[language.cs]
+description: Czech
+charset: utf-8
+enabled: yes
+
+[language.ca]
+description: Catalan
+charset: utf-8
+enabled: yes
+
+[language.ja]
+description: Japanese
+charset: utf-8
+enabled: yes
+
+[language.ar]
+description: Arabic
+charset: utf-8
+enabled: yes
+
+[language.nl]
+description: Dutch
+charset: utf-8
+enabled: yes
+
+[language.pl]
+description: Polish
+charset: utf-8
+enabled: yes
+
+[language.es]
+description: Spanish
+charset: utf-8
+enabled: yes
+
+[language.pt_BR]
+description: Protuguese (Brazil)
+charset: iso-8859-15
+enabled: yes
+
+[language.zh_CN]
+description: Chinese (S)
+charset: utf-8
+enabled: yes
+
+[language.zh_TW]
+description: Chinese (T)
+charset: utf-8
+enabled: yes
+
+[language.ru]
+description: Russian
+charset: utf-8
+enabled: yes
+
+[language.vi]
+description: Vietnamese
+charset: utf-8
+enabled: yes
+
+[language.it]
+description: Italian
+charset: utf-8
+enabled: yes
+
+[language.fr]
+description: French
+charset: utf-8
+enabled: yes
+
+[language.ro]
+description: Romanian
+charset: utf-8
+enabled: yes
+
+[language.de]
+description: German
+charset: utf-8
+enabled: yes
+
+[language.hu]
+description: Hungarian
+charset: utf-8
+enabled: yes
+
+[language.ko]
+description: Korean
+charset: utf-8
+enabled: yes
+
+[language.uk]
+description: Ukrainian
+charset: utf-8
+enabled: yes
diff --git a/roles/mailman/templates/post-transaction.action.j2 b/roles/mailman/templates/post-transaction.action.j2
index c2e124bb7..0da392efa 100644
--- a/roles/mailman/templates/post-transaction.action.j2
+++ b/roles/mailman/templates/post-transaction.action.j2
@@ -2,5 +2,5 @@
 
 hyperkitty:install:{{ mailman_webui_basedir }}/bin/post-update.sh
 hyperkitty:update:{{ mailman_webui_basedir }}/bin/post-update.sh
-kittystore:install:{{ mailman_webui_basedir }}/bin/post-update.sh
-kittystore:update:{{ mailman_webui_basedir }}/bin/post-update.sh
+postorius:install:{{ mailman_webui_basedir }}/bin/post-update.sh
+postorius:update:{{ mailman_webui_basedir }}/bin/post-update.sh