Arrange stripping .po files

The .po files we use for translations have two shortcomings when used in Git:
- They include file locations, which change each time the source is updated.
  This results in large, unreadable diffs that don't merge well.
- They include source strings for untranslated messages, wasting space
  unnecessarily.

Update the Makefile so that the extraneous information is stripped when the
files are updated or pulled form Transifex, and empty translation files are
removed entirely.
Also, translations are normalized to a common style. This should help diffs
and merges.

The validator requires file location comments to identify the programming
language, and to produce good error reports.
To make this work, merge the comments in before validation.

First patch for: https://fedorahosted.org/freeipa/ticket/2435

4 files changed, 49 insertions, 6 deletions

diff --git a/install/configure.ac b/install/configure.ac
index 827ddbab..9e781a68 100644
--- a/install/configure.ac
+++ b/install/configure.ac
@@ -48,6 +48,11 @@ if test "x$MSGCMP" = "xno"; then
     AC_MSG_ERROR([msgcmp not found, install gettext])
 fi
 
+AC_PATH_PROG(MSGATTRIB, msgattrib, [no])
+if test "x$MSGATTRIB" = "xno"; then
+    AC_MSG_ERROR([msgattrib not found, install gettext])
+fi
+
 AC_PATH_PROG(TX, tx, [/usr/bin/tx])
 
 AC_ARG_WITH([gettext_domain],
diff --git a/install/po/Makefile.in b/install/po/Makefile.in
index 9a3dde78..bc91a933 100644
--- a/install/po/Makefile.in
+++ b/install/po/Makefile.in
@@ -14,6 +14,7 @@ MSGFMT = @MSGFMT@
 MSGINIT = @MSGINIT@
 MSGMERGE = @MSGMERGE@
 MSGCMP = @MSGCMP@
+MSGATTRIB = @MSGATTRIB@
 TX = @TX@
 IPA_TEST_I18N = ../../tests/i18n.py
 
@@ -67,7 +68,7 @@ C_POTFILES = $(C_FILES) $(H_FILES)
 
 .SUFFIXES:
 .SUFFIXES: .po .mo
-.PHONY: all create-po update-po update-pot install mostlyclean clean distclean test mo-files debug
+.PHONY: all create-po update-po update-pot install mostlyclean clean distclean test mo-files debug strip-po merge-po $(po_files)
 
 all:
 
@@ -86,6 +87,19 @@ $(po_files): $(DOMAIN).pot
 	echo Merging $(DOMAIN).pot into $@; \
 	$(MSGMERGE) --no-fuzzy-matching -o $@ $@ $(DOMAIN).pot
 
+strip-po:
+	@for po_file in $(po_files); do \
+		echo Stripping $$po_file; \
+		$(MSGATTRIB) --translated --no-fuzzy --no-location $$po_file > $$po_file.tmp; \
+		mv $$po_file.tmp $$po_file; \
+	done
+	@export FILES_TO_REMOVE=`find . -name '*.po' -empty`; \
+	if [ "$$FILES_TO_REMOVE" != "" ]; then \
+		echo Removing empty translation files; \
+		rm -v $$FILES_TO_REMOVE; \
+		echo; echo Please remove the deleted files from LINGUAS!; echo; \
+	fi
+
 create-po: $(DOMAIN).pot
 	@for po_file in $(po_files); do \
 	    if [ ! -e $$po_file ]; then \
@@ -98,10 +112,14 @@ create-po: $(DOMAIN).pot
 
 pull-po:
 	cd ../..; $(TX) pull -f
+	$(MAKE) strip-po
 
-update-po: update-pot
+merge-po: update-pot
 	$(MAKE) $(po_files)
 
+update-po: merge-po
+	$(MAKE) strip-po
+
 update-pot:
 	@rm -f $(DOMAIN).pot.update
 	@pushd ../.. ; \
diff --git a/install/po/README b/install/po/README
index ada7df40..6894a063 100644
--- a/install/po/README
+++ b/install/po/README
@@ -6,10 +6,17 @@ A: Edit Makefile.in and add the source file to the appropriate *_POTFILES list.
    NOTE: Now this i only necessary for python files that lack the .py
          extension. All .py, .c and .h files are automatically sourced.
 
+Q: Untranslated strings and file locations are missing from my .po file.
+   How do I add them?
+
+A: make merge-po
+   Untranslated strings are left out of the files in SCM. The merge-po command
+   runs msgmerge to add them again.
+
 Q: How do I pick up new strings to translate from the source files after the
    source have been modified?
 
-A: make update-po
+A: make merge-po
    This regenerates the pot template file by scanning all the source files.
    Then the new strings are merged into each .po file from the new pot file.
 
@@ -18,6 +25,11 @@ Q: How do I just regenerate the pot template file without regenerating all the
 
 A: make update-pot
 
+Q: I am done translating. How do I commit my changes?
+
+A: Run `make strip-po` to remove unneeded information from the po files, then
+   add your changes to SCM.
+
 Q: How do I add a new language for translation?
 
 A: Edit the LINGUAS file and add the new language. Then run "make create-po".
@@ -27,7 +39,7 @@ A: Edit the LINGUAS file and add the new language. Then run "make create-po".
    http://www.gnu.org/software/hello/manual/gettext/Plural-forms.html
    However, if this line is wrong, it is often an indicator that the locale
    value is incorrect.  For example, using 'jp' for Japanese in stead of 'ja'
-   will result in an invailid Plural's line.
+   will result in an invalid Plurals line.
 
 Q: What files must be under source code control?
 
diff --git a/tests/i18n.py b/tests/i18n.py
index 703dc8bb..9c8479bb 100755
--- a/tests/i18n.py
+++ b/tests/i18n.py
@@ -367,7 +367,7 @@ def validate_positional_substitutions(s, prog_langs, s_name='string'):
 
     return errors
 
-def validate_file(file_path, validation_mode):
+def validate_file(file_path, validation_mode, reference_pot=None):
     '''
     Given a pot or po file scan all it's entries looking for problems
     with variable substitutions. See the following functions for
@@ -378,6 +378,9 @@ def validate_file(file_path, validation_mode):
     * validate_positional_substitutions()
 
     Returns the number of entries with errors.
+
+    For po files, ``reference_pot`` gives a pot file to merge with (to recover
+    comments and file locations)
     '''
 
     def emit_messages():
@@ -419,6 +422,9 @@ def validate_file(file_path, validation_mode):
         emit_messages()
         return Result(n_entries=n_entries, n_msgids=n_msgids, n_msgstrs=n_msgstrs, n_warnings=n_warnings, n_errors=n_errors)
 
+    if validation_mode == 'po' and reference_pot:
+        # Merge the .pot file for comments and file locations
+        po.merge(reference_pot)
 
     if validation_mode == 'po':
         plural_forms = po.metadata.get('Plural-Forms')
@@ -754,12 +760,14 @@ def main():
             if not files:
                 files = [options.pot_file]
             validation_mode = 'pot'
+            reference_pot = None
         elif options.mode == 'validate_po':
             files = args
             if not files:
                 print >> sys.stderr, 'ERROR: no po files specified'
                 return 1
             validation_mode = 'po'
+            reference_pot = polib.pofile(options.pot_file)
         else:
             print >> sys.stderr, 'ERROR: unknown validation mode "%s"' % (options.mode)
             return 1
@@ -771,7 +779,7 @@ def main():
         total_errors = 0
 
         for f in files:
-            result = validate_file(f, validation_mode)
+            result = validate_file(f, validation_mode, reference_pot)
             total_entries += result.n_entries
             total_msgids += result.n_msgids
             total_msgstrs += result.n_msgstrs