From 50cae762b1b7a70e22b61f50b6d873a8909fe9be Mon Sep 17 00:00:00 2001 From: Matt Odden Date: Mon, 15 Apr 2013 13:17:11 +0000 Subject: Add basic lazy gettext implementation This is the first part of an implementation towards a deferred localization functionality in OpenStack. This change adds a Message class for encapsulating a translatable message and its relevant data, as well as an example LogHandler of how such a class can be used in the logging context. bp delayed-message-translation Change-Id: I8485a346d32925327ea9185e0da3822e4e19c2f5 --- openstack/common/gettextutils.py | 176 ++++++++++++++++++ tests/unit/test_gettext.py | 372 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 547 insertions(+), 1 deletion(-) diff --git a/openstack/common/gettextutils.py b/openstack/common/gettextutils.py index e816f14..d6b5f10 100644 --- a/openstack/common/gettextutils.py +++ b/openstack/common/gettextutils.py @@ -2,6 +2,7 @@ # Copyright 2012 Red Hat, Inc. # All Rights Reserved. +# Copyright 2013 IBM Corp. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -23,8 +24,11 @@ Usual usage in an openstack.common module: from openstack.common.gettextutils import _ """ +import copy import gettext +import logging.handlers import os +import UserString _localedir = os.environ.get('oslo'.upper() + '_LOCALEDIR') _t = gettext.translation('oslo', localedir=_localedir, fallback=True) @@ -48,3 +52,175 @@ def install(domain): gettext.install(domain, localedir=os.environ.get(domain.upper() + '_LOCALEDIR'), unicode=True) + + +""" +Lazy gettext functionality. + +The following is an attempt to introduce a deferred way +to do translations on messages in OpenStack. We attempt to +override the standard _() function and % (format string) operation +to build Message objects that can later be translated when we have +more information. Also included is an example LogHandler that +translates Messages to an associated locale, effectively allowing +many logs, each with their own locale. +""" + + +def get_lazy_gettext(domain): + """Assemble and return a lazy gettext function for a given domain. + + Factory method for a project/module to get a lazy gettext function + for its own translation domain (i.e. nova, glance, cinder, etc.) + """ + + def _lazy_gettext(msg): + """ + Create and return a Message object encapsulating a string + so that we can translate it later when needed. + """ + return Message(msg, domain) + + return _lazy_gettext + + +class Message(UserString.UserString, object): + """Class used to encapsulate translatable messages.""" + def __init__(self, msg, domain): + # _msg is the gettext msgid and should never change + self._msg = msg + self._left_extra_msg = '' + self._right_extra_msg = '' + self.params = None + self.locale = None + self.domain = domain + + @property + def data(self): + # NOTE(mrodden): this should always resolve to a unicode string + # that best represents the state of the message currently + + localedir = os.environ.get(self.domain.upper() + '_LOCALEDIR') + if self.locale: + lang = gettext.translation(self.domain, + localedir=localedir, + languages=[self.locale], + fallback=True) + else: + # use system locale for translations + lang = gettext.translation(self.domain, + localedir=localedir, + fallback=True) + + full_msg = (self._left_extra_msg + + lang.ugettext(self._msg) + + self._right_extra_msg) + + if self.params is not None: + full_msg = full_msg % self.params + + return unicode(full_msg) + + def _save_parameters(self, other): + # we check for None later to see if + # we actually have parameters to inject, + # so encapsulate if our parameter is actually None + if other is None: + self.params = (other, ) + else: + self.params = copy.deepcopy(other) + + return self + + # overrides to be more string-like + def __unicode__(self): + return self.data + + def __str__(self): + return self.data.encode('utf-8') + + def __getstate__(self): + to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg', + 'domain', 'params', 'locale'] + new_dict = self.__dict__.fromkeys(to_copy) + for attr in to_copy: + new_dict[attr] = copy.deepcopy(self.__dict__[attr]) + + return new_dict + + def __setstate__(self, state): + for (k, v) in state.items(): + setattr(self, k, v) + + # operator overloads + def __add__(self, other): + copied = copy.deepcopy(self) + copied._right_extra_msg += other.__str__() + return copied + + def __radd__(self, other): + copied = copy.deepcopy(self) + copied._left_extra_msg += other.__str__() + return copied + + def __mod__(self, other): + # do a format string to catch and raise + # any possible KeyErrors from missing parameters + self.data % other + copied = copy.deepcopy(self) + return copied._save_parameters(other) + + def __mul__(self, other): + return self.data * other + + def __rmul__(self, other): + return other * self.data + + def __getitem__(self, key): + return self.data[key] + + def __getslice__(self, start, end): + return self.data.__getslice__(start, end) + + def __getattribute__(self, name): + # NOTE(mrodden): handle lossy operations that we can't deal with yet + # These override the UserString implementation, since UserString + # uses our __class__ attribute to try and build a new message + # after running the inner data string through the operation. + # At that point, we have lost the gettext message id and can just + # safely resolve to a string instead. + ops = ['capitalize', 'center', 'decode', 'encode', + 'expandtabs', 'ljust', 'lstrip', 'replace', 'rjust', 'rstrip', + 'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill'] + if name in ops: + return getattr(self.data, name) + else: + return UserString.UserString.__getattribute__(self, name) + + +class LocaleHandler(logging.Handler): + """Handler that can have a locale associated to translate Messages. + + A quick example of how to utilize the Message class above. + LocaleHandler takes a locale and a target logging.Handler object + to forward LogRecord objects to after translating the internal Message. + """ + + def __init__(self, locale, target): + """ + Initialize a LocaleHandler + + :param locale: locale to use for translating messages + :param target: logging.Handler object to forward + LogRecord objects to after translation + """ + logging.Handler.__init__(self) + self.locale = locale + self.target = target + + def emit(self, record): + if isinstance(record.msg, Message): + # set the locale and resolve to a string + record.msg.locale = self.locale + + self.target.emit(record) diff --git a/tests/unit/test_gettext.py b/tests/unit/test_gettext.py index 3a86782..cd139ee 100644 --- a/tests/unit/test_gettext.py +++ b/tests/unit/test_gettext.py @@ -2,6 +2,7 @@ # Copyright 2012 Red Hat, Inc. # All Rights Reserved. +# Copyright 2013 IBM Corp. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -15,7 +16,10 @@ # License for the specific language governing permissions and limitations # under the License. -import logging +import copy +import gettext +import logging.handlers +import os import mock @@ -42,3 +46,369 @@ class GettextTest(utils.BaseTestCase): gettext_install.assert_called_once_with('blaa', localedir='/foo/bar', unicode=True) + + +class MessageTestCase(utils.BaseTestCase): + """Unit tests for locale Message class.""" + + def setUp(self): + super(MessageTestCase, self).setUp() + self._lazy_gettext = gettextutils.get_lazy_gettext('oslo') + + def tearDown(self): + # need to clean up stubs early since they interfere + # with super class clean up operations + self.mox.UnsetStubs() + super(MessageTestCase, self).tearDown() + + def test_message_equal_to_string(self): + msgid = "Some msgid string" + result = self._lazy_gettext(msgid) + + self.assertEqual(result, msgid) + + def test_message_not_equal(self): + msgid = "Some msgid string" + result = self._lazy_gettext(msgid) + + self.assertNotEqual(result, "Other string %s" % msgid) + + def test_message_equal_with_param(self): + msgid = "Some string with params: %s" + params = (0, ) + + message = msgid % params + + result = self._lazy_gettext(msgid) % params + + self.assertEqual(result, message) + + result_str = '%s' % result + self.assertEqual(result_str, message) + + def test_message_injects_nonetype(self): + msgid = "Some string with param: %s" + params = None + + message = msgid % params + + result = self._lazy_gettext(msgid) % params + + self.assertEqual(result, message) + + result_str = '%s' % result + self.assertIn('None', result_str) + self.assertEqual(result_str, message) + + def test_message_iterate(self): + msgid = "Some string with params: %s" + params = 'blah' + + message = msgid % params + + result = self._lazy_gettext(msgid) % params + + # compare using iterators + for (c1, c2) in zip(result, message): + self.assertEqual(c1, c2) + + def test_message_equal_with_dec_param(self): + """Verify we can inject numbers into Messages.""" + msgid = "Some string with params: %d" + params = [0, 1, 10, 24124] + + messages = [] + results = [] + for param in params: + messages.append(msgid % param) + results.append(self._lazy_gettext(msgid) % param) + + for message, result in zip(messages, results): + self.assertEqual(type(result), gettextutils.Message) + self.assertEqual(result, message) + + # simulate writing out as string + result_str = '%s' % result + self.assertEqual(result_str, message) + + def test_message_equal_with_extra_params(self): + msgid = "Some string with params: %(param1)s %(param2)s" + params = {'param1': 'test', + 'param2': 'test2', + 'param3': 'notinstring'} + + result = self._lazy_gettext(msgid) % params + + self.assertEqual(result, msgid % params) + + def test_message_object_param_copied(self): + """Verify that injected parameters get copied.""" + some_obj = SomeObject() + some_obj.tag = 'stub_object' + msgid = "Found object: %(some_obj)s" + + result = self._lazy_gettext(msgid) % {'some_obj': some_obj} + + old_some_obj = copy.copy(some_obj) + some_obj.tag = 'switched_tag' + + self.assertEqual(result, msgid % {'some_obj': old_some_obj}) + + def test_interpolation_with_missing_param(self): + msgid = ("Some string with params: %(param1)s %(param2)s" + " and a missing one %(missing)s") + params = {'param1': 'test', + 'param2': 'test2'} + + test_me = lambda: self._lazy_gettext(msgid) % params + + self.assertRaises(KeyError, test_me) + + def test_operator_add(self): + msgid = "Some msgid string" + result = self._lazy_gettext(msgid) + + additional = " with more added" + expected = msgid + additional + result = result + additional + + self.assertEqual(type(result), gettextutils.Message) + self.assertEqual(result, expected) + + def test_operator_radd(self): + msgid = "Some msgid string" + result = self._lazy_gettext(msgid) + + additional = " with more added" + expected = additional + msgid + result = additional + result + + self.assertEqual(type(result), gettextutils.Message) + self.assertEqual(result, expected) + + def test_get_index(self): + msgid = "Some msgid string" + result = self._lazy_gettext(msgid) + + expected = 'm' + result = result[2] + + self.assertEqual(result, expected) + + def test_getitem_string(self): + """Verify using string indexes on Message does not work.""" + msgid = "Some msgid string" + result = self._lazy_gettext(msgid) + + test_me = lambda: result['blah'] + + self.assertRaises(TypeError, test_me) + + def test_contains(self): + msgid = "Some msgid string" + result = self._lazy_gettext(msgid) + + self.assertIn('msgid', result) + self.assertNotIn('blah', result) + + def test_locale_set_does_translation(self): + msgid = "Some msgid string" + result = self._lazy_gettext(msgid) + result.domain = 'test_domain' + result.locale = 'test_locale' + os.environ['TEST_DOMAIN_LOCALEDIR'] = '/tmp/blah' + + self.mox.StubOutWithMock(gettext, 'translation') + fake_lang = self.mox.CreateMock(gettext.GNUTranslations) + + gettext.translation('test_domain', + languages=['test_locale'], + fallback=True, + localedir='/tmp/blah').AndReturn(fake_lang) + fake_lang.ugettext(msgid).AndReturn(msgid) + + self.mox.ReplayAll() + result = result.data + os.environ.pop('TEST_DOMAIN_LOCALEDIR') + self.assertEqual(msgid, result) + + def _get_testmsg_inner_params(self): + return {'params': {'test1': 'blah1', + 'test2': 'blah2', + 'test3': SomeObject()}, + 'domain': 'test_domain', + 'locale': 'en_US', + '_left_extra_msg': 'Extra. ', + '_right_extra_msg': '. More Extra.'} + + def _get_full_test_message(self): + msgid = "Some msgid string: %(test1)s %(test2)s %(test3)s" + message = self._lazy_gettext(msgid) + attrs = self._get_testmsg_inner_params() + for (k, v) in attrs.items(): + setattr(message, k, v) + + return copy.deepcopy(message) + + def test_message_copyable(self): + message = self._get_full_test_message() + copied_msg = copy.copy(message) + + self.assertIsNot(message, copied_msg) + + for k in self._get_testmsg_inner_params(): + self.assertEqual(getattr(message, k), + getattr(copied_msg, k)) + + self.assertEqual(message, copied_msg) + + message._msg = 'Some other msgid string' + + self.assertNotEqual(message, copied_msg) + + def test_message_copy_deepcopied(self): + message = self._get_full_test_message() + inner_obj = SomeObject() + message.params['test3'] = inner_obj + + copied_msg = copy.copy(message) + + self.assertIsNot(message, copied_msg) + + inner_obj.tag = 'different' + self.assertNotEqual(message, copied_msg) + + def test_add_returns_copy(self): + msgid = "Some msgid string: %(test1)s %(test2)s" + message = self._lazy_gettext(msgid) + m1 = '10 ' + message + ' 10' + m2 = '20 ' + message + ' 20' + + self.assertIsNot(message, m1) + self.assertIsNot(message, m2) + self.assertIsNot(m1, m2) + self.assertEqual(m1, '10 %s 10' % msgid) + self.assertEqual(m2, '20 %s 20' % msgid) + + def test_mod_returns_copy(self): + msgid = "Some msgid string: %(test1)s %(test2)s" + message = self._lazy_gettext(msgid) + m1 = message % {'test1': 'foo', 'test2': 'bar'} + m2 = message % {'test1': 'foo2', 'test2': 'bar2'} + + self.assertIsNot(message, m1) + self.assertIsNot(message, m2) + self.assertIsNot(m1, m2) + self.assertEqual(m1, msgid % {'test1': 'foo', 'test2': 'bar'}) + self.assertEqual(m2, msgid % {'test1': 'foo2', 'test2': 'bar2'}) + + def test_comparator_operators(self): + """Verify Message comparison is equivalent to string comparision.""" + m1 = self._get_full_test_message() + m2 = copy.deepcopy(m1) + m3 = "1" + m1 + + # m1 and m2 are equal + self.assertEqual(m1 >= m2, str(m1) >= str(m2)) + self.assertEqual(m1 <= m2, str(m1) <= str(m2)) + self.assertEqual(m2 >= m1, str(m2) >= str(m1)) + self.assertEqual(m2 <= m1, str(m2) <= str(m1)) + + # m1 is greater than m3 + self.assertEqual(m1 >= m3, str(m1) >= str(m3)) + self.assertEqual(m1 > m3, str(m1) > str(m3)) + + # m3 is not greater than m1 + self.assertEqual(m3 >= m1, str(m3) >= str(m1)) + self.assertEqual(m3 > m1, str(m3) > str(m1)) + + # m3 is less than m1 + self.assertEqual(m3 <= m1, str(m3) <= str(m1)) + self.assertEqual(m3 < m1, str(m3) < str(m1)) + + # m3 is not less than m1 + self.assertEqual(m1 <= m3, str(m1) <= str(m3)) + self.assertEqual(m1 < m3, str(m1) < str(m3)) + + def test_mul_operator(self): + message = self._get_full_test_message() + message_str = str(message) + + self.assertEqual(message * 10, message_str * 10) + self.assertEqual(message * 20, message_str * 20) + self.assertEqual(10 * message, 10 * message_str) + self.assertEqual(20 * message, 20 * message_str) + + def test_to_unicode(self): + message = self._get_full_test_message() + message_str = unicode(message) + + self.assertEqual(message, message_str) + self.assertTrue(isinstance(message_str, unicode)) + + +class LocaleHandlerTestCase(utils.BaseTestCase): + + def setUp(self): + super(LocaleHandlerTestCase, self).setUp() + self._lazy_gettext = gettextutils.get_lazy_gettext('oslo') + self.buffer_handler = logging.handlers.BufferingHandler(40) + self.locale_handler = gettextutils.LocaleHandler( + 'zh_CN', self.buffer_handler) + self.logger = logging.getLogger('localehander_logger') + self.logger.propogate = False + self.logger.setLevel(logging.DEBUG) + self.logger.addHandler(self.locale_handler) + + def test_emit_message(self): + msgid = 'Some logrecord message.' + message = self._lazy_gettext(msgid) + self.emit_called = False + + def emit(record): + self.assertEqual(record.msg.locale, 'zh_CN') + self.assertEqual(record.msg, msgid) + self.assertTrue(isinstance(record.msg, + gettextutils.Message)) + self.emit_called = True + self.stubs.Set(self.buffer_handler, 'emit', emit) + + self.logger.info(message) + + self.assertTrue(self.emit_called) + + def test_emit_nonmessage(self): + msgid = 'Some logrecord message.' + self.emit_called = False + + def emit(record): + self.assertEqual(record.msg, msgid) + self.assertFalse(isinstance(record.msg, + gettextutils.Message)) + self.emit_called = True + self.stubs.Set(self.buffer_handler, 'emit', emit) + + self.logger.info(msgid) + + self.assertTrue(self.emit_called) + + +class SomeObject(object): + + def __init__(self, tag='default'): + self.tag = tag + + def __str__(self): + return self.tag + + def __getstate__(self): + return self.__dict__ + + def __setstate__(self, state): + for (k, v) in state.items(): + setattr(self, k, v) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return self.tag == other.tag + return False -- cgit