prewikka/master: String encoding fixes, do not mix unicode and bytestring
[email protected] Fri, 3 Jul 2009 10:24:34 +0200 (CEST)
| Newsgroups | gmane.comp.security.ids.prelude.cvs |
|---|---|
| Message-ID | <[email protected]> |
commit af548128b8626f18686ed4abaf6361b183116e5b Author: Yoann Vandoorselaere <[email protected]> Date: Thu Jul 2 13:47:46 2009 +0200 String encoding fixes, do not mix unicode and bytestring Do not call str() on the template, but rather directly use template.respond(). Encode template output to UTF8, or any other user specified encoding, and convert unknown character to XML references. Fix our HTML escaping function so that it doesn't fail on unicode object, and more generally, make sure that we don't call str() on unicode object, or store unicode in Python bytestring. Translate all input string to unicode using the new utils.toUnicode() function which should be able to handle most of the input cases. Make sure all string that leave prewikka to call lower layer C functions are converted back to UTF-8 bytestring. This fixes a lot of possible exception with specific user input, or with localized Prewikka. ======================================== conf/prewikka.conf | 4 +++- prewikka/Core.py | 6 ++++-- prewikka/IDMEFDatabase.py | 10 +++++----- prewikka/localization.py | 6 +++--- prewikka/utils.py | 34 +++++++++++++++++++++++++++++++++- prewikka/view.py | 5 ++++- prewikka/views/alertlisting.py | 5 ++++- prewikka/views/commands.py | 1 + 8 files changed, 57 insertions(+), 14 deletions(-) ======================================== diff --git a/conf/prewikka.conf b/conf/prewikka.conf index 7bb07f5..cb36c01 100644 --- a/conf/prewikka.conf +++ b/conf/prewikka.conf @@ -22,7 +22,6 @@ external_link_new_window #max_aggregated_target: 10 #max_aggregated_classification: 10 - # Asynchronous DNS resolution (require twisted.names and twisted.internet) # # While rendering view containing address scheduled for asynchronous @@ -40,6 +39,9 @@ external_link_new_window # Default locale to use (default is English): # default_locale: fr +# Default encoding to use (default is UTF8): +# encoding: utf8 + [interface] software: Prewikka diff --git a/prewikka/Core.py b/prewikka/Core.py index ed2f42c..9afc5b7 100644 --- a/prewikka/Core.py +++ b/prewikka/Core.py @@ -326,6 +326,7 @@ class Core: login = None view = None user = None + encoding = self._env.config.general.getOptionValue("encoding", "utf8") try: if self._prelude_version_error: @@ -373,10 +374,11 @@ class Core: resolve.process(self._env.dns_max_delay) try: - request.content = str(template) + request.content = template.respond() except Exception, e: error = self.prepareError(e, request, user, login, view) - request.content = str(load_template(error.template, error.dataset)) + request.content = load_template(error.template, error.dataset).respond() + request.content = request.content.encode(encoding, "xmlcharrefreplace") request.sendResponse() diff --git a/prewikka/IDMEFDatabase.py b/prewikka/IDMEFDatabase.py index 8d1ff57..10a061e 100644 --- a/prewikka/IDMEFDatabase.py +++ b/prewikka/IDMEFDatabase.py @@ -159,7 +159,7 @@ class Message: return value def _get_raw_value(self, key): - path = idmef_path_new_fast(key) + path = idmef_path_new_fast(key.encode("utf8")) idmef_value = idmef_path_get(path, self._res) if idmef_value: @@ -186,7 +186,7 @@ class Message: if type(criteria) is list: criteria = " && ".join(criteria) - criteria = idmef_criteria_new_from_string(criteria) + criteria = idmef_criteria_new_from_string(criteria.encode("utf8")) ret = idmef_criteria_match(criteria, self._res) idmef_criteria_destroy(criteria) @@ -375,7 +375,7 @@ class IDMEFDatabase: criteria = " && ".join(criteria) if criteria: - criteria = idmef_criteria_new_from_string(criteria) + criteria = idmef_criteria_new_from_string(criteria.encode("utf8")) idents = [ ] @@ -457,11 +457,11 @@ class IDMEFDatabase: criteria = " && ".join([ "(" + c + ")" for c in criteria ]) if criteria: - criteria = idmef_criteria_new_from_string(criteria) + criteria = idmef_criteria_new_from_string(criteria.encode("utf8")) my_selection = preludedb_path_selection_new() for selected in selection: - my_selected = preludedb_selected_path_new_string(selected) + my_selected = preludedb_selected_path_new_string(selected.encode("utf8")) preludedb_path_selection_add(my_selection, my_selected) try: diff --git a/prewikka/localization.py b/prewikka/localization.py index a4fed1d..6e1356e 100644 --- a/prewikka/localization.py +++ b/prewikka/localization.py @@ -60,12 +60,12 @@ __builtin__.ngettext = _safeNgettext _LANGUAGES = { "Deutsch": "de", - "Español": "es", + u"Español": "es", "English": "en", - "Français": "fr", + u"Français": "fr", "Polski": "pl", "Portuguese (Brazilian)": "pt_BR", - "Русский": "ru" + u"Русский": "ru" } diff --git a/prewikka/utils.py b/prewikka/utils.py index 4a5e36f..aba7c15 100644 --- a/prewikka/utils.py +++ b/prewikka/utils.py @@ -118,7 +118,9 @@ def boolean_property(name, parameter, value=False): def escape_html_string(s): - s = str(s) + if type(s) is not str and type(s) is not unicode: + s = str(s) + s = s.replace("&", "&") s = s.replace("<", "<") s = s.replace(">", ">") @@ -150,3 +152,33 @@ def hexdump(content): i += 16 return content + + +def isUTF8(text): + try: + text = unicode(text, 'UTF-8', 'strict') + return True + except UnicodeDecodeError: + return False + +def toUnicode(text): + r""" + >>> toUnicode('ascii') + u'ascii' + >>> toUnicode(u'utf\xe9'.encode('UTF-8')) + u'utf\xe9' + >>> toUnicode(u'unicode') + u'unicode' + """ + if isinstance(text, unicode): + return text + + if not isinstance(text, str): + text = str(text) + + try: + return unicode(text, "utf8") + except UnicodeError: + pass + + return unicode(text, "ISO-8859-1") diff --git a/prewikka/view.py b/prewikka/view.py index 430ce94..d9683b8 100644 --- a/prewikka/view.py +++ b/prewikka/view.py @@ -19,7 +19,7 @@ from copy import copy -import Error, Log +import Error, Log, utils class ParameterError(Exception): pass @@ -87,6 +87,9 @@ class Parameters(dict): do_load = True for name, value in self.items(): + if isinstance(value, str): + value = self[name] = utils.toUnicode(value) + try: value = self._parseValue(name, value) except KeyError: diff --git a/prewikka/views/alertlisting.py b/prewikka/views/alertlisting.py index 2ebc534..1b2c6ee 100644 --- a/prewikka/views/alertlisting.py +++ b/prewikka/views/alertlisting.py @@ -911,6 +911,9 @@ class AlertListing(MessageListing, view.View): return "%s %s '%s'" % (object, operator, utils.escape_criteria(self._adjustFilterValue(operator, value))) def _getOperatorForPath(self, path, value): + path = path.encode("utf8") + value = value.encode("utf8") + # Check whether the path can handle substring comparison # this need to be done first, since enum check with * won't work with "=" operator. try: @@ -1067,7 +1070,7 @@ class AlertListing(MessageListing, view.View): def _getPathValueType(self, path): - p = prelude.idmef_path_new(path) + p = prelude.idmef_path_new(path.encode("utf8")) t = prelude.idmef_path_get_value_type(p, -1) prelude.idmef_path_destroy(p) return t diff --git a/prewikka/views/commands.py b/prewikka/views/commands.py index c8e2b09..0fd0210 100644 --- a/prewikka/views/commands.py +++ b/prewikka/views/commands.py @@ -49,6 +49,7 @@ class Command(view.View): command = command.replace("$host", self.parameters["host"]).split(" ") output = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True).communicate()[0] + output = utils.toUnicode(output) output = utils.escape_html_string(output).replace(" ", " ").replace("\n", "<br/>") self.dataset["command_output"] = output _______________________________________________ Prelude-cvslog site list [email protected] http://lists.prelude-ids.org/mailman/listinfo/prelude-cvslog