prewikka/master: Continued bytestring -> unicode conversion work
[email protected] Mon, 6 Jul 2009 17:06:32 +0200 (CEST)
| Newsgroups | gmane.comp.security.ids.prelude.cvs |
|---|---|
| Message-ID | <[email protected]> |
commit 07cb31c7e77fbcb93c194030df07fac1d18044a3 Author: Yoann Vandoorselaere <[email protected]> Date: Fri Jul 3 14:24:19 2009 +0200 Continued bytestring -> unicode conversion work This make it possible to use unicode within login/password and configuration options. Correct invalid str() conversion within the code, and make sure data returned from the database is converted to Unicode. Data sent to libpreludedb is converted to UTF8. ======================================== prewikka/Auth.py | 16 +++++++--------- prewikka/Core.py | 11 ++++++----- prewikka/Database.py | 20 +++++++++++--------- prewikka/Filter.py | 2 +- prewikka/IDMEFDatabase.py | 11 ++++++----- prewikka/MyConfigParser.py | 5 +++-- prewikka/User.py | 2 +- prewikka/localization.py | 10 +++++----- prewikka/modules/auth/cgi/cgi.py | 2 ++ .../modules/auth/loginpassword/loginpassword.py | 2 +- prewikka/resolve.py | 9 +++++---- prewikka/templates/AlertListing.tmpl | 12 ++++++------ prewikka/templates/ClassicLayout.tmpl | 2 +- prewikka/templates/SensorListing.tmpl | 4 ++-- prewikka/view.py | 6 ++++++ prewikka/views/alertlisting.py | 6 +++--- prewikka/views/sensor.py | 2 +- 17 files changed, 67 insertions(+), 55 deletions(-) ======================================== diff --git a/prewikka/Auth.py b/prewikka/Auth.py index 820d613..b538050 100644 --- a/prewikka/Auth.py +++ b/prewikka/Auth.py @@ -22,10 +22,7 @@ import md5 import random from prewikka.Error import PrewikkaError, PrewikkaUserError -from prewikka import DataSet -from prewikka import Database -from prewikka import Log -from prewikka import User +from prewikka import DataSet, Database, Log, User, utils class AuthError(PrewikkaUserError): @@ -116,20 +113,21 @@ class LoginPasswordAuth(Auth, Session): Session.__init__(self, session_expiration) def getUser(self, request): - if request.arguments.has_key("_login"): - login = request.arguments["_login"] + login = request.arguments.get("_login", None) + if not login: + login = utils.toUnicode(self.checkSession(request)) + else: del request.arguments["_login"] - password = request.arguments.get("_password", "") + password = utils.toUnicode(request.arguments.get("_password", "")) try: del request.arguments["_password"] except KeyError: pass + login = utils.toUnicode(login) self.checkPassword(login, password) self.createSession(request, login) self.log.info("User login", request, login) - else: - login = self.checkSession(request) return self.db.getUser(login) diff --git a/prewikka/Core.py b/prewikka/Core.py index 9afc5b7..0951c0e 100644 --- a/prewikka/Core.py +++ b/prewikka/Core.py @@ -84,7 +84,7 @@ def init_dataset(dataset, config, request): if name == "view" and value == "logout": continue - dataset["arguments"].append((name, value)) + dataset["arguments"].append((name, utils.toUnicode(value))) def load_template(name, dataset): @@ -287,7 +287,7 @@ class Core: print key + ":" self._printDataSet(value, level + 1) else: - print "%s: %s" % (key, value) + print "%s: %s" % (key, repr(value)) def _checkPermissions(self, request, view, user): if user and view.has_key("permissions"): @@ -316,8 +316,9 @@ class Core: return user def prepareError(self, e, request, user, login, view): - self._env.log.error("%s" % str(e), request, login) - error = Error.PrewikkaUserError("Prewikka internal error", str(e), + e = unicode(e) + self._env.log.error(e, request, login) + error = Error.PrewikkaUserError("Prewikka internal error", e, display_traceback=not self._env.config.general.has_key("disable_error_traceback")) self._setupDataSet(error.dataset, request, user, view=view) return error @@ -356,7 +357,7 @@ class Core: except Error.PrewikkaUserError, e: if e._log_priority: - self._env.log.log(e._log_priority, "%s" % str(e), request=request, user=login or e._log_user) + self._env.log.log(e._log_priority, unicode(e), request=request, user=login or e._log_user) self._setupDataSet(e.dataset, request, user, view=view) dataset, template_name = e.dataset, e.template diff --git a/prewikka/Database.py b/prewikka/Database.py index ba84759..7e71404 100644 --- a/prewikka/Database.py +++ b/prewikka/Database.py @@ -79,20 +79,19 @@ class Database: ("pass", None)): value = config.get(name, default) if value: - preludedb_sql_settings_set(settings, name, value) + preludedb_sql_settings_set(settings, name.encode("utf8"), value.encode("utf8")) db_type = config.get("type", "mysql") - - self._sql = preludedb_sql_new(db_type, settings) + self._sql = preludedb_sql_new(db_type.encode("utf8"), settings) if config.has_key("log"): - preludedb_sql_enable_query_logging(self._sql, config["log"]) + preludedb_sql_enable_query_logging(self._sql, config["log"].encode("utf8")) # check if the database has been created try: version = self.query("SELECT version FROM Prewikka_Version")[0][0] except PreludeDBError, e: - raise DatabaseSchemaError(str(e)) + raise DatabaseSchemaError(unicode(utils.toUnicode(e))) if version != self.required_version: d = { "version": version, "reqversion": self.required_version } @@ -111,7 +110,7 @@ class Database: def query(self, query): try: - _table = preludedb_sql_query(self._sql, query) + _table = preludedb_sql_query(self._sql, query.encode("utf8")) if not _table: return [ ] @@ -127,7 +126,7 @@ class Database: for col in range(columns): _field = preludedb_sql_row_fetch_field(_row, col) if _field: - row.append(preludedb_sql_field_to_string(_field)) + row.append(utils.toUnicode(preludedb_sql_field_to_string(_field))) else: row.append(None) @@ -151,7 +150,10 @@ class Database: return def escape(self, data): - return preludedb_sql_escape(self._sql, data) + if data: + data = data.encode("utf8") + + return utils.toUnicode(preludedb_sql_escape(self._sql, data)) def datetime(self, t): if t is None: @@ -200,7 +202,7 @@ class Database: if not config[view].has_key(name): config[view][name] = value else: - if type(config[view][name]) is str: + if isinstance(config[view][name], (str, unicode)): config[view][name] = [ config[view][name] ] config[view][name] = config[view][name] + [ value ] diff --git a/prewikka/Filter.py b/prewikka/Filter.py index f60e2f6..9dc71b1 100644 --- a/prewikka/Filter.py +++ b/prewikka/Filter.py @@ -63,7 +63,7 @@ class Filter: self.elements = elements self.formula = formula - crit = prelude.idmef_criteria_new_from_string(str(self)) + crit = prelude.idmef_criteria_new_from_string(unicode(self).encode("utf8")) prelude.idmef_criteria_destroy(crit) def _replace(self, element): diff --git a/prewikka/IDMEFDatabase.py b/prewikka/IDMEFDatabase.py index eeed776..130682c 100644 --- a/prewikka/IDMEFDatabase.py +++ b/prewikka/IDMEFDatabase.py @@ -341,12 +341,13 @@ class IDMEFDatabase: def __init__(self, config): settings = preludedb_sql_settings_new() for param in "file", "host", "port", "name", "user", "pass": - if config.getOptionValue(param): - preludedb_sql_settings_set(settings, param, config.getOptionValue(param)) + value = config.getOptionValue(param) + if value: + preludedb_sql_settings_set(settings, param, value.encode("utf8")) - sql = preludedb_sql_new(config.getOptionValue("type", "mysql"), settings) + sql = preludedb_sql_new(config.getOptionValue("type", "mysql").encode("utf8"), settings) if config.getOptionValue("log"): - preludedb_sql_enable_query_logging(sql, config.getOptionValue("log")) + preludedb_sql_enable_query_logging(sql, config.getOptionValue("log").encode("utf8")) cur = ver = None wanted_version = "0.9.12" @@ -410,7 +411,7 @@ class IDMEFDatabase: if analyzerid is None: criteria = "! %s.analyzer(-1).analyzerid" % (type) else: - criteria = "%s.analyzer(-1).analyzerid == '%s'" % (type, str(analyzerid)) + criteria = "%s.analyzer(-1).analyzerid == '%s'" % (type, unicode(analyzerid)) idents = get_message_idents(criteria, limit=1) diff --git a/prewikka/MyConfigParser.py b/prewikka/MyConfigParser.py index d3159c7..bde4427 100644 --- a/prewikka/MyConfigParser.py +++ b/prewikka/MyConfigParser.py @@ -19,6 +19,7 @@ import re +from prewikka import utils class Error(Exception): pass @@ -91,8 +92,8 @@ class ConfigParserSection(OrderedDict): class ConfigParserOption: def __init__(self, name, value, lineno, line): - self.name = name - self.value = value + self.name = utils.toUnicode(name) + self.value = utils.toUnicode(value) self.lineno = lineno self.line = line diff --git a/prewikka/User.py b/prewikka/User.py index 3513c81..4494a58 100644 --- a/prewikka/User.py +++ b/prewikka/User.py @@ -86,7 +86,7 @@ class User: self._db.query("DELETE FROM Prewikka_User_Configuration WHERE view = %s AND login = %s AND name = %s" % (v, login, k)) if not type(value) is list: self._db.query("INSERT INTO Prewikka_User_Configuration (view, login, name, value) VALUES (%s,%s,%s,%s)" % - (v, login, k, self._db.escape(str(value)))) + (v, login, k, self._db.escape(unicode(value)))) else: for val in value: self._db.query("INSERT INTO Prewikka_User_Configuration (view, login, name, value) VALUES (%s, %s,%s,%s)" % diff --git a/prewikka/localization.py b/prewikka/localization.py index 6e1356e..c04796c 100644 --- a/prewikka/localization.py +++ b/prewikka/localization.py @@ -19,7 +19,7 @@ # the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. import locale, gettext, __builtin__, time -from prewikka import siteconfig +from prewikka import siteconfig, utils try: from threading import local, Lock @@ -37,13 +37,13 @@ _all_locale = { _DEFAULT_LANGUAGE: None } def _safeGettext(s): try: - return _localized_thread.data.gettext(s) + return utils.toUnicode(_localized_thread.data.gettext(s)) except: return s def _safeNgettext(singular, plural, num): try: - return _localized_thread.data.ngettext(singular, plural, num) + return utils.toUnicode(_localized_thread.data.ngettext(singular, plural, num)) except: if num <= 1: return singular @@ -125,5 +125,5 @@ def getDate(): _("Saturday"), _("Sunday") ] - weekday, day, month, year = time.strftime("%A %d %B %Y").split() - return " ".join((_(weekday).lower(), day, _(month).lower(), year)) + weekday, day, month, year = utils.toUnicode(time.strftime("%A %d %B %Y")).split() + return u" ".join((_(weekday).lower(), day, _(month).lower(), year)) diff --git a/prewikka/modules/auth/cgi/cgi.py b/prewikka/modules/auth/cgi/cgi.py index e85db73..a1e12bb 100644 --- a/prewikka/modules/auth/cgi/cgi.py +++ b/prewikka/modules/auth/cgi/cgi.py @@ -40,6 +40,8 @@ class CGIAuth(Auth.Auth): if not user: raise Auth.AuthError(message=_("CGI Authentication failed: no user specified.")) + user = user.toUnicode(user) + # Create the user in the Prewikka database, so that its permission # might be modified by another administrative user. if not self.db.hasUser(user): diff --git a/prewikka/modules/auth/loginpassword/loginpassword.py b/prewikka/modules/auth/loginpassword/loginpassword.py index de81d8d..8d581f2 100644 --- a/prewikka/modules/auth/loginpassword/loginpassword.py +++ b/prewikka/modules/auth/loginpassword/loginpassword.py @@ -47,7 +47,7 @@ class MyLoginPasswordAuth(Auth.LoginPasswordAuth): self.db.setPermissions(user, User.ALL_PERMISSIONS) def _hash(self, data): - return md5.new(data).hexdigest() + return md5.new(data.encode("utf8")).hexdigest() def createUser(self, login): return self.db.createUser(login) diff --git a/prewikka/resolve.py b/prewikka/resolve.py index ca823ad..ba98367 100644 --- a/prewikka/resolve.py +++ b/prewikka/resolve.py @@ -51,10 +51,11 @@ class DNSResolver: def _resolve_cb(self, (ans, auth, add), ptr, resolve_cb): self._query -= 1 + name = unicode(ans[0].payload.name) - resolve_cb(str(ans[0].payload.name)) + resolve_cb(name) - q = dns.Query(str(ans[0].name), ans[0].type, ans[0].cls) + q = dns.Query(name, ans[0].type, ans[0].cls) self._cache.cacheResult(q, (ans, auth, add)) def _ip_reverse(self, addr): @@ -127,7 +128,7 @@ class AddressResolve: resolver.resolve(addr, self._resolve_cb) def __len__(self): - return len(str(self)) + return len(unicode(self)) def __str__(self): if resolver: @@ -136,7 +137,7 @@ class AddressResolve: return self._name or self._addr def __repr__(self): - return str(self) + return unicode(self) def process(timeout=0): diff --git a/prewikka/templates/AlertListing.tmpl b/prewikka/templates/AlertListing.tmpl index eae8ce8..d6695c8 100644 --- a/prewikka/templates/AlertListing.tmpl +++ b/prewikka/templates/AlertListing.tmpl @@ -666,12 +666,12 @@ function ${type}_set_${state}() { #else #set $protocol = "" #end if -#if $direction.service.value != None#$hstr<a class="popup_menu_toggle">$str($direction.service.value)#end if +#if $direction.service.value != None#$hstr<a class="popup_menu_toggle">$unicode($direction.service.value)#end if </a> <span class="popup_menu"> #filter Filter - $writeInlineFilter($direction.service, $_("Filter on this port/protocol"))<br/> #end filter #if $direction.port.value - - <a target="$prewikka.external_link_target" href="https://www.prelude-ids.com/port_details.php?port=$str($direction.port.value)&protocol=$protocol">$_("Port/protocol information")</a> + - <a target="$prewikka.external_link_target" href="https://www.prelude-ids.com/port_details.php?port=$unicode($direction.port.value)&protocol=$protocol">$_("Port/protocol information")</a> #else - <span>$_("Port/protocol information")</span> #end if @@ -694,7 +694,7 @@ function ${type}_set_${state}() { #if $message.aggregated and $message.aggregated_classifications_hidden > 0 #set $string = $_("%(hidden)d of %(total)d alerts not shown...") - <b>($str($string % { "hidden": $message.aggregated_classifications_hidden, "total": $message.aggregated_classifications_total }) + <b>($unicode($string % { "hidden": $message.aggregated_classifications_hidden, "total": $message.aggregated_classifications_total }) <a href="$message.aggregated_classifications_hidden_expand">$_("expand")</a>)</b> <br/> #end if @@ -756,7 +756,7 @@ $sep#if $url#<a class="popup_menu_toggle">$name.value</a>#else##filter Filter $w #if $hidden > 0 #set $string = $_("%(hidden)d of %(total)d %(name)ss not shown...") - <b>($str($string % { "hidden": $hidden, "total": $total, "name": $name }) + <b>($unicode($string % { "hidden": $hidden, "total": $total, "name": $name }) <a href="$expand">$_("expand")</a>)</b> <br/> #end if @@ -770,11 +770,11 @@ $sep#if $url#<a class="popup_menu_toggle">$name.value</a>#else##filter Filter $w #for $address in $direction.addresses <a class="popup_menu_toggle">$address.hostname</a><span class="popup_menu"> - - #filter Filter $writeInlineFilter($address, $str($_("Filter on this %s") % ($name))) #end filter + - #filter Filter $writeInlineFilter($address, $unicode($_("Filter on this %s") % ($name))) #end filter <br /> #if not $address.category or $address.category in ("ipv4-addr", "ipv4-net", "ipv6-addr", "ipv6-net") - - <a target="$prewikka.external_link_target" href="https://www.prelude-ids.com/host_details.php?host=$address.value">$str($_("%s information") % ($name)).capitalize()</a><br /> + - <a target="$prewikka.external_link_target" href="https://www.prelude-ids.com/host_details.php?host=$address.value">$unicode($_("%s information") % ($name)).capitalize()</a><br /> #for $cmdname, $link in $address.host_commands - <a href="$link">$cmdname</a><br /> diff --git a/prewikka/templates/ClassicLayout.tmpl b/prewikka/templates/ClassicLayout.tmpl index 73a15b9..ed99ba9 100644 --- a/prewikka/templates/ClassicLayout.tmpl +++ b/prewikka/templates/ClassicLayout.tmpl @@ -25,7 +25,7 @@ classic_body <div class="topmenu_item_info"> #end filter - <p>$str($_("%(username)s on %(date)s") % { "username": $prewikka.userlink, "date": $prewikka.date })</p> + <p>$unicode($_("%(username)s on %(date)s") % { "username": $prewikka.userlink, "date": $prewikka.date })</p> #filter CleanOutput </div> </div> diff --git a/prewikka/templates/SensorListing.tmpl b/prewikka/templates/SensorListing.tmpl index fbf804e..d168004 100644 --- a/prewikka/templates/SensorListing.tmpl +++ b/prewikka/templates/SensorListing.tmpl @@ -97,14 +97,14 @@ $listing.sort() <td style="background-color: white; width:150px;"> #set $nlen = $len($locations[$entry]["nodes"]) #set $alen = $locations[$entry]["total"] - $str($ngettext("%d Node", "%d Nodes", $nlen) % $nlen), $str($ngettext("%d Analyzer", "%d Analyzers", $alen) % $alen) + $unicode($ngettext("%d Node", "%d Nodes", $nlen) % $nlen), $unicode($ngettext("%d Analyzer", "%d Analyzers", $alen) % $alen) </td> #for $i in ( ("online", $ngettext("%d Online", "%d Online", $locations[$entry]["online"])), ("offline", $ngettext("%d Offline", "%d Offline", $locations[$entry]["offline"])), ("unknown", $ngettext("%d Unknown", "%d Unknown", $locations[$entry]["unknown"])), ("missing", $ngettext("%d Missing", "%d Missing", $locations[$entry]["missing"]))) #if $locations[$entry][$i[0]] > 0 - <td class="heartbeat_analyze_sensor_status_$i[0]">$str($i[1] % $locations[$entry][$i[0]])</td> + <td class="heartbeat_analyze_sensor_status_$i[0]">$unicode($i[1] % $locations[$entry][$i[0]])</td> #end if #end for </tr> diff --git a/prewikka/view.py b/prewikka/view.py index d9683b8..1e9d172 100644 --- a/prewikka/view.py +++ b/prewikka/view.py @@ -60,9 +60,15 @@ class Parameters(dict): pass def mandatory(self, name, type): + if type is str: + type = unicode + self._parameters[name] = { "type": type, "mandatory": True, "save": False } def optional(self, name, type, default=None, save=False): + if type is str: + type = unicode + if default is not None: self._default[name] = self._hard_default[name] = default diff --git a/prewikka/views/alertlisting.py b/prewikka/views/alertlisting.py index 1b2c6ee..ea60852 100644 --- a/prewikka/views/alertlisting.py +++ b/prewikka/views/alertlisting.py @@ -346,7 +346,7 @@ class SensorAlertListingParameters(AlertListingParameters): def normalize(self, view_name, user): AlertListingParameters.normalize(self, view_name, user) - self["analyzer"].insert(0, ("alert.analyzer.analyzerid", "=", str(self["analyzerid"]))) + self["analyzer"].insert(0, ("alert.analyzer.analyzerid", "=", unicode(self["analyzerid"]))) class CorrelationAlertListingParameters(AlertListingParameters): @@ -1149,7 +1149,7 @@ class AlertListing(MessageListing, view.View): else: criterion = "(! %s || %s == '')" % (path, path) else: - criterion = "%s == '%s'" % (path, utils.escape_criteria(str(value))) + criterion = "%s == '%s'" % (path, utils.escape_criteria(unicode(value))) if direction != None: message.setMessageDirectionGeneric(direction, path, value) @@ -1316,7 +1316,7 @@ class AlertListing(MessageListing, view.View): if self.parameters.has_key("filter"): filter = self.env.db.getAlertFilter(self.user.login, self.parameters["filter"]) if filter: - criteria.append("(%s)" % str(filter)) + criteria.append("(%s)" % unicode(filter)) start = end = None if self.parameters.has_key("timeline_unit") and self.parameters["timeline_unit"] != "unlimited": diff --git a/prewikka/views/sensor.py b/prewikka/views/sensor.py index 4236da7..a22331f 100644 --- a/prewikka/views/sensor.py +++ b/prewikka/views/sensor.py @@ -222,7 +222,7 @@ class HeartbeatAnalyze(view.View): analyzerid = self.parameters["analyzerid"] analyzer = self.env.idmef_db.getAnalyzer(analyzerid) - analyzer["last_heartbeat_time"] = str(analyzer["last_heartbeat_time"]) + analyzer["last_heartbeat_time"] = unicode(analyzer["last_heartbeat_time"]) analyzer["events"] = [ ] analyzer["status"] = "abnormal_offline" analyzer["status_meaning"] = "abnormal offline" _______________________________________________ Prelude-cvslog site list [email protected] http://lists.prelude-ids.org/mailman/listinfo/prelude-cvslog