Package ipapython, ipalib, ipaplatform, ipatests for Python 3

Running make with PYTHON=/usr/bin/python3 will build/install the
bits for Python 3.

Executable scripts in ipatests have symlinks Python version suffixes
as per Fedora guidelines. Suffix-less names point to the Python 2 versions.

Reviewed-By: Jan Cholasta <jcholast@redhat.com>

10 files changed, 289 insertions, 32 deletions

diff --git a/.gitignore b/.gitignore
index 13f9eb34a..06b017df2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -67,6 +67,9 @@ freeipa2-dev-doc
 
 /ipatests/setup.py
 
+/ipalib/setup.py
+!/ipalib/Makefile
+
 /ipapython/setup.py
 /ipapython/version.py
 !/ipapython/Makefile
diff --git a/Makefile b/Makefile
index ab2524db1..7b9f95a1d 100644
--- a/Makefile
+++ b/Makefile
@@ -3,7 +3,7 @@
 
 include VERSION
 
-SUBDIRS=asn1 daemons install ipapython ipa-client
+SUBDIRS=asn1 daemons install ipapython ipalib ipa-client
 CLIENTDIRS=ipapython ipa-client asn1
 
 PRJ_PREFIX=freeipa
@@ -143,6 +143,8 @@ version-update: release-update
 		> ipapython/setup.py
 	sed -e s/__VERSION__/$(IPA_VERSION)/ ipaplatform/setup.py.in \
 		> ipaplatform/setup.py
+	sed -e s/__VERSION__/$(IPA_VERSION)/ ipalib/setup.py.in \
+		> ipalib/setup.py
 	sed -e s/__VERSION__/$(IPA_VERSION)/ ipapython/version.py.in \
 		> ipapython/version.py
 	sed -e s/__VERSION__/$(IPA_VERSION)/ ipatests/setup.py.in \
@@ -240,7 +242,7 @@ rpms: rpmroot rpmdistdir version-update lint tarballs
 	cp dist/sources/$(TARBALL) $(RPMBUILD)/SOURCES/.
 	rpmbuild --define "_topdir $(RPMBUILD)" -ba freeipa.spec
 	cp $(RPMBUILD)/RPMS/*/$(PRJ_PREFIX)-*-$(IPA_VERSION)-*.rpm dist/rpms/
-	cp $(RPMBUILD)/RPMS/*/python2-ipa*-$(IPA_VERSION)-*.rpm dist/rpms/
+	cp $(RPMBUILD)/RPMS/*/python?-ipa*-$(IPA_VERSION)-*.rpm dist/rpms/
 	cp $(RPMBUILD)/SRPMS/$(PRJ_PREFIX)-$(IPA_VERSION)-*.src.rpm dist/srpms/
 	rm -rf $(RPMBUILD)
 
@@ -248,7 +250,7 @@ client-rpms: rpmroot rpmdistdir version-update lint tarballs
 	cp dist/sources/$(TARBALL) $(RPMBUILD)/SOURCES/.
 	rpmbuild --define "_topdir $(RPMBUILD)" --define "ONLY_CLIENT 1" -ba freeipa.spec
 	cp $(RPMBUILD)/RPMS/*/$(PRJ_PREFIX)-*-$(IPA_VERSION)-*.rpm dist/rpms/
-	cp $(RPMBUILD)/RPMS/*/python2-ipa*-$(IPA_VERSION)-*.rpm dist/rpms/
+	cp $(RPMBUILD)/RPMS/*/python?-ipa*-$(IPA_VERSION)-*.rpm dist/rpms/
 	cp $(RPMBUILD)/SRPMS/$(PRJ_PREFIX)-$(IPA_VERSION)-*.src.rpm dist/srpms/
 	rm -rf $(RPMBUILD)
 
diff --git a/freeipa.spec.in b/freeipa.spec.in
index d8d9f113c..0cc37aa19 100644
--- a/freeipa.spec.in
+++ b/freeipa.spec.in
@@ -2,6 +2,12 @@
 # subpackages
 %{!?ONLY_CLIENT:%global ONLY_CLIENT 0}
 
+%if 0%{?rhel}
+%global with_python3 0
+%else
+%global with_python3 1
+%endif
+
 %global alt_name ipa
 %if 0%{?rhel}
 %global samba_version 4.0.5-1
@@ -104,6 +110,10 @@ BuildRequires:  dbus-python
 BuildRequires:  libcmocka-devel
 BuildRequires:  nss_wrapper
 
+%if 0%{?with_python3}
+BuildRequires:  python3-devel
+%endif  # with_python3
+
 %description
 IPA is an integrated solution to provide centrally managed Identity (users,
 hosts, services), Authentication (SSO, 2FA), and Authorization
@@ -472,6 +482,50 @@ and integration with Active Directory based infrastructures (Trusts).
 If you are using IPA, you need to install this package.
 
 
+%if 0%{?with_python3}
+
+%package -n python3-ipalib
+Summary: Python3 libraries used by IPA
+Group: System Environment/Libraries
+%{?python_provide:%python_provide python3-ipalib}
+%{?python_provide:%{?_isa:%python_provide python3-ipalib%{_isa}}}
+Provides: python3-ipapython = %{version}-%{release}
+%{?python_provide:%python_provide python3-ipapython}
+Provides: python3-ipaplatform = %{version}-%{release}
+%{?python_provide:%python_provide python3-ipaplatform}
+Requires: %{name}-common = %{version}-%{release}
+Requires: python3-gssapi >= 1.1.2
+Requires: gnupg
+Requires: iproute
+Requires: keyutils
+Requires: python3-pyOpenSSL
+Requires: python3-nss >= 0.16
+Requires: python3-cryptography
+Requires: python3-lxml
+Requires: python3-netaddr
+Requires: python3-libipa_hbac
+Requires: python3-qrcode-core >= 5.0.0
+Requires: python3-pyasn1
+Requires: python3-dateutil
+Requires: python3-yubico >= 1.2.3
+Requires: python3-sss-murmur
+Requires: curl
+Requires: python3-dbus
+Requires: python3-setuptools
+Requires: python3-six
+Requires: python3-jwcrypto
+
+%description -n python3-ipalib
+IPA is an integrated solution to provide centrally managed Identity (users,
+hosts, services), Authentication (SSO, 2FA), and Authorization
+(host access control, SELinux user roles, services). The solution provides
+features for further integration with Linux based clients (SUDO, automount)
+and integration with Active Directory based infrastructures (Trusts).
+If you are using IPA with Python 3, you need to install this package.
+
+%endif # with_python3
+
+
 %package common
 Summary: Common files used by IPA
 Group: System Environment/Libraries
@@ -525,6 +579,34 @@ features for further integration with Linux based clients (SUDO, automount)
 and integration with Active Directory based infrastructures (Trusts).
 This package contains tests that verify IPA functionality.
 
+
+%if 0%{?with_python3}
+
+%package -n python3-ipatests
+Summary: IPA tests and test tools
+BuildArch: noarch
+%{?python_provide:%python_provide python3-ipatests}
+Requires: %{name}-client-common = %{version}-%{release}
+Requires: python3-ipalib%{?_isa} = %{version}-%{release}
+Requires: tar
+Requires: xz
+Requires: python3-nose
+Requires: python3-pytest >= 2.6
+Requires: python3-coverage
+Requires: python3-polib
+Requires: python3-pytest-multihost >= 0.5
+Requires: python3-pytest-sourceorder
+
+%description -n python3-ipatests
+IPA is an integrated solution to provide centrally managed Identity (users,
+hosts, services), Authentication (SSO, 2FA), and Authorization
+(host access control, SELinux user roles, services). The solution provides
+features for further integration with Linux based clients (SUDO, automount)
+and integration with Active Directory based infrastructures (Trusts).
+This package contains tests that verify IPA functionality under Python 3.
+
+%endif # with_python3
+
 %endif # ONLY_CLIENT
 
 
@@ -554,6 +636,10 @@ cd daemons; ../autogen.sh --prefix=%{_usr} --sysconfdir=%{_sysconfdir} --localst
 cd install; ../autogen.sh --prefix=%{_usr} --sysconfdir=%{_sysconfdir} --localstatedir=%{_localstatedir} --libdir=%{_libdir} --mandir=%{_mandir}; cd ..
 %endif # ONLY_CLIENT
 
+%if 0%{?with_python3}
+(cd ipapython/ipap11helper && make PYTHON=%{__python3} IPA_VERSION_IS_GIT_SNAPSHOT=no %{?_smp_mflags} all)
+%endif
+
 %if ! %{ONLY_CLIENT}
 make IPA_VERSION_IS_GIT_SNAPSHOT=no %{?_smp_mflags} all
 %else
@@ -578,9 +664,38 @@ rm -f ipaplatform/constants.py
 make version-update
 %if ! %{ONLY_CLIENT}
 make install DESTDIR=%{buildroot}
+
+mv %{buildroot}%{_bindir}/ipa-run-tests %{buildroot}%{_bindir}/ipa-run-tests-%{python2_version}
+mv %{buildroot}%{_bindir}/ipa-test-config %{buildroot}%{_bindir}/ipa-test-config-%{python2_version}
+mv %{buildroot}%{_bindir}/ipa-test-task %{buildroot}%{_bindir}/ipa-test-task-%{python2_version}
+
+%if 0%{?with_python3}
+(cd ipatests && %{__python3} setup.py install --root %{buildroot})
+mv %{buildroot}%{_bindir}/ipa-run-tests %{buildroot}%{_bindir}/ipa-run-tests-%{python3_version}
+mv %{buildroot}%{_bindir}/ipa-test-config %{buildroot}%{_bindir}/ipa-test-config-%{python3_version}
+mv %{buildroot}%{_bindir}/ipa-test-task %{buildroot}%{_bindir}/ipa-test-task-%{python3_version}
+ln -s %{_bindir}/ipa-run-tests-%{python3_version} %{buildroot}%{_bindir}/ipa-run-tests-3
+ln -s %{_bindir}/ipa-test-config-%{python3_version} %{buildroot}%{_bindir}/ipa-test-config-3
+ln -s %{_bindir}/ipa-test-task-%{python3_version} %{buildroot}%{_bindir}/ipa-test-task-3
+%endif # with_python3
+
+ln -s %{_bindir}/ipa-run-tests-%{python2_version} %{buildroot}%{_bindir}/ipa-run-tests-2
+ln -s %{_bindir}/ipa-test-config-%{python2_version} %{buildroot}%{_bindir}/ipa-test-config-2
+ln -s %{_bindir}/ipa-test-task-%{python2_version} %{buildroot}%{_bindir}/ipa-test-task-2
+ln -s %{_bindir}/ipa-run-tests-%{python2_version} %{buildroot}%{_bindir}/ipa-run-tests
+ln -s %{_bindir}/ipa-test-config-%{python2_version} %{buildroot}%{_bindir}/ipa-test-config
+ln -s %{_bindir}/ipa-test-task-%{python2_version} %{buildroot}%{_bindir}/ipa-test-task
+
 %else
 make client-install DESTDIR=%{buildroot}
 %endif # ONLY_CLIENT
+
+%if 0%{?with_python3}
+(cd ipalib && make PYTHON=%{__python3} IPA_VERSION_IS_GIT_SNAPSHOT=no %{?_smp_mflags} DESTDIR=%{buildroot} install)
+(cd ipapython && make PYTHON=%{__python3} IPA_VERSION_IS_GIT_SNAPSHOT=no %{?_smp_mflags} DESTDIR=%{buildroot} install)
+(cd ipaplatform && %{__python3} setup.py install --root %{buildroot})
+%endif # with_python3
+
 %find_lang %{gettext_domain}
 
 mkdir -p %{buildroot}%{_usr}/share/ipa
@@ -1196,6 +1311,7 @@ fi
 %attr(0644,root,root) %{python_sitearch}/default_encoding_utf8.so
 %attr(0644,root,root) %{python_sitearch}/_ipap11helper.so
 %{python_sitelib}/ipapython-*.egg-info
+%{python_sitelib}/ipalib-*.egg-info
 %{python_sitelib}/freeipa-*.egg-info
 %{python_sitelib}/ipaplatform-*.egg-info
 %{python_sitearch}/python_default_encoding-*.egg-info
@@ -1208,6 +1324,25 @@ fi
 %license COPYING
 
 
+%if 0%{?with_python3}
+
+%files -n python3-ipalib
+%defattr(-,root,root,-)
+%doc README Contributors.txt
+%license COPYING
+
+%{python3_sitelib}/ipapython/
+%{python3_sitelib}/ipalib/
+%{python3_sitelib}/ipaplatform/
+%{python3_sitelib}/ipapython-*.egg-info
+%{python3_sitelib}/ipalib-*.egg-info
+%{python3_sitelib}/ipaplatform-*.egg-info
+%attr(0644,root,root) %{python3_sitearch}/_ipap11helper.cpython-*.so
+%{python3_sitearch}/_ipap11helper-*.egg-info
+
+%endif # with_python3
+
+
 %if ! %{ONLY_CLIENT}
 
 %files -n python2-ipatests -f tests-python.list
@@ -1228,11 +1363,35 @@ fi
 %{_bindir}/ipa-run-tests
 %{_bindir}/ipa-test-config
 %{_bindir}/ipa-test-task
+%{_bindir}/ipa-run-tests-2
+%{_bindir}/ipa-test-config-2
+%{_bindir}/ipa-test-task-2
+%{_bindir}/ipa-run-tests-%{python2_version}
+%{_bindir}/ipa-test-config-%{python2_version}
+%{_bindir}/ipa-test-task-%{python2_version}
 %{python_sitelib}/ipatests-*.egg-info
 %{_mandir}/man1/ipa-run-tests.1.gz
 %{_mandir}/man1/ipa-test-config.1.gz
 %{_mandir}/man1/ipa-test-task.1.gz
 
+%if 0%{?with_python3}
+
+%files -n python3-ipatests
+%defattr(-,root,root,-)
+%doc README Contributors.txt
+%license COPYING
+
+%{python3_sitelib}/ipatests/
+%{_bindir}/ipa-run-tests-3
+%{_bindir}/ipa-test-config-3
+%{_bindir}/ipa-test-task-3
+%{_bindir}/ipa-run-tests-%{python3_version}
+%{_bindir}/ipa-test-config-%{python3_version}
+%{_bindir}/ipa-test-task-%{python3_version}
+%{python3_sitelib}/ipatests-*.egg-info
+
+%endif # with_python3
+
 %endif # ONLY_CLIENT
 
 
diff --git a/ipalib/Makefile b/ipalib/Makefile
new file mode 100644
index 000000000..5bd84fa4a
--- /dev/null
+++ b/ipalib/Makefile
@@ -0,0 +1,25 @@
+PYTHON ?= /usr/bin/python2
+PYTHONLIBDIR ?= $(shell $(PYTHON) -c "from distutils.sysconfig import *; print(get_python_lib())")
+
+all:
+	# Pure Python; no need to build
+	true
+
+check:
+
+.PHONY: install
+install:
+	if [ "$(DESTDIR)" = "" ]; then \
+		$(PYTHON) setup.py install; \
+	else \
+		$(PYTHON) setup.py install --root $(DESTDIR); \
+	fi
+
+clean:
+	rm -f *~ *.pyc __pycache__/
+
+distclean: clean
+	rm -f setup.py
+
+maintainer-clean: distclean
+	rm -rf build
diff --git a/ipalib/cli.py b/ipalib/cli.py
index 44ef61d30..567b59946 100644
--- a/ipalib/cli.py
+++ b/ipalib/cli.py
@@ -47,6 +47,7 @@ except ImportError:
     # this is already installed and since it is installed with IPA therein
     # lies the problem. Skip it for now so ipalib can be imported in-tree
     # even in cases that IPA isn't installed on the dev machine.
+    # Also, under Python 3, default_encoding_utf8 is not built at all.
     pass
 
 from ipalib import frontend
diff --git a/ipalib/setup.py.in b/ipalib/setup.py.in
new file mode 100644
index 000000000..47481dbe0
--- /dev/null
+++ b/ipalib/setup.py.in
@@ -0,0 +1,73 @@
+#!/usr/bin/python2
+# Copyright (C) 2007  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""FreeIPA common python library
+
+FreeIPA is a server for identity, policy, and audit.
+"""
+
+DOCLINES = __doc__.split("\n")
+
+import os
+import sys
+import distutils.sysconfig
+
+CLASSIFIERS = """\
+Intended Audience :: System Environment/Base
+License :: GPL
+Programming Language :: Python
+Operating System :: POSIX
+Operating System :: Unix
+"""
+
+# BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
+# update it when the contents of directories change.
+if os.path.exists('MANIFEST'): os.remove('MANIFEST')
+
+def setup_package():
+
+    from distutils.core import setup
+
+    old_path = os.getcwd()
+    local_path = os.path.dirname(os.path.abspath(sys.argv[0]))
+    os.chdir(local_path)
+    sys.path.insert(0,local_path)
+
+    try:
+        setup(
+            name = "ipalib",
+            version = "__VERSION__",
+            license = "GPL",
+            url = "http://www.freeipa.org/",
+            description = DOCLINES[0],
+            long_description = "\n".join(DOCLINES[2:]),
+            download_url = "http://www.freeipa.org/page/Downloads",
+            classifiers=[line for line in CLASSIFIERS.split('\n') if line],
+            platforms = ["Linux", "Solaris", "Unix"],
+            package_dir = {'ipalib': ''},
+            packages = ["ipalib",
+                        "ipalib.plugins",
+                       ],
+        )
+    finally:
+        del sys.path[0]
+        os.chdir(old_path)
+    return
+
+if __name__ == '__main__':
+    setup_package()
diff --git a/ipapython/Makefile b/ipapython/Makefile
index a865ca758..833f3cdc5 100644
--- a/ipapython/Makefile
+++ b/ipapython/Makefile
@@ -1,7 +1,5 @@
-PYTHONLIBDIR ?= $(shell  python2 -c "from distutils.sysconfig import *; print get_python_lib()")
-PACKAGEDIR ?= $(DESTDIR)/$(PYTHONLIBDIR)/ipa
-CONFIGDIR ?= $(DESTDIR)/etc/ipa
-TESTS = $(wildcard test/*.py)
+PYTHON ?= /usr/bin/python2
+PYTHONLIBDIR ?= $(shell $(PYTHON) -c "from distutils.sysconfig import *; print(get_python_lib())")
 
 SUBDIRS = py_default_encoding ipap11helper
 
@@ -15,16 +13,16 @@ check:
 .PHONY: install
 install:
 	if [ "$(DESTDIR)" = "" ]; then \
-		python2 setup.py install; \
+		$(PYTHON) setup.py install; \
 	else \
-		python2 setup.py install --root $(DESTDIR); \
+		$(PYTHON) setup.py install --root $(DESTDIR); \
 	fi
 	@for subdir in $(SUBDIRS); do \
 		(cd $$subdir && $(MAKE) $@) || exit 1; \
 	done
 
 clean:
-	rm -f *~ *.pyc
+	rm -f *~ *.pyc __pycache__/
 	@for subdir in $(SUBDIRS); do \
 		(cd $$subdir && $(MAKE) $@) || exit 1; \
 	done
@@ -40,9 +38,3 @@ maintainer-clean: distclean
 	@for subdir in $(SUBDIRS); do \
 		(cd $$subdir && $(MAKE) $@) || exit 1; \
 	done
-
-.PHONY: test
-test: $(subst .py,.tst,$(TESTS))
-
-%.tst: %.py
-	python2 $<
diff --git a/ipapython/ipap11helper/Makefile b/ipapython/ipap11helper/Makefile
index 88f17f705..f66edb82e 100644
--- a/ipapython/ipap11helper/Makefile
+++ b/ipapython/ipap11helper/Makefile
@@ -1,15 +1,14 @@
-PYTHONLIBDIR ?= $(shell  python2 -c "from distutils.sysconfig import *; print get_python_lib()")
-PACKAGEDIR ?= $(DESTDIR)/$(PYTHONLIBDIR)/ipa
-CONFIGDIR ?= $(DESTDIR)/etc/ipa
+PYTHON ?= /usr/bin/python2
+PYTHONLIBDIR ?= $(shell $(PYTHON) -c "from distutils.sysconfig import *; print(get_python_lib())")
 
 all:
-	python2 setup.py build
+	$(PYTHON) setup.py build
 
 install:
 	if [ "$(DESTDIR)" = "" ]; then \
-		python2 setup.py install; \
+		$(PYTHON) setup.py install; \
 	else \
-		python2 setup.py install --root $(DESTDIR); \
+		$(PYTHON) setup.py install --root $(DESTDIR); \
 	fi
 
 clean:
diff --git a/ipapython/py_default_encoding/Makefile b/ipapython/py_default_encoding/Makefile
index 88f17f705..a73f429db 100644
--- a/ipapython/py_default_encoding/Makefile
+++ b/ipapython/py_default_encoding/Makefile
@@ -1,15 +1,20 @@
-PYTHONLIBDIR ?= $(shell  python2 -c "from distutils.sysconfig import *; print get_python_lib()")
-PACKAGEDIR ?= $(DESTDIR)/$(PYTHONLIBDIR)/ipa
-CONFIGDIR ?= $(DESTDIR)/etc/ipa
+PYTHON ?= /usr/bin/python2
+PYTHONLIBDIR ?= $(shell $(PYTHON) -c "from distutils.sysconfig import *; print(get_python_lib())")
+PYTHONVERSION ?= $(shell $(PYTHON) -c "import sys; print(sys.version_info[0])")
 
 all:
-	python2 setup.py build
+	if [ "$(PYTHONVERSION)" = "2" ]; then \
+		python2 setup.py build; \
+	fi
 
 install:
-	if [ "$(DESTDIR)" = "" ]; then \
-		python2 setup.py install; \
-	else \
-		python2 setup.py install --root $(DESTDIR); \
+	# Skip this module under Python 3
+	if [ "$(PYTHONVERSION)" = "2" ]; then \
+		if [ "$(DESTDIR)" = "" ]; then \
+			python2 setup.py install; \
+		else \
+			python2 setup.py install --root $(DESTDIR); \
+		fi; \
 	fi
 
 clean:
diff --git a/setup.py b/setup.py
index 330503d61..006f34e70 100755
--- a/setup.py
+++ b/setup.py
@@ -75,8 +75,6 @@ setup(
     license='GPLv3+',
     url='http://freeipa.org/',
     packages=[
-        'ipalib',
-        'ipalib.plugins',
         'ipaserver',
         'ipaserver.advise',
         'ipaserver.advise.plugins',