prelude-correlator/master: Abstracting access to data/configuration files

commit ff0620bac4d12033e1a13cf8469d200dce2424dc
Author: Yoann Vandoorselaere <[email protected]>
Date:   Thu Jul 9 13:18:40 2009 +0200

    Abstracting access to data/configuration files
    
    This provide an abstraction layer over the way PreludeCorrelator
    access data and configuration files, depending on its installation method.
    
    When PreludeCorrelator is installed as an EGG, data/configuration
    files should be self contained within the package. We set the zip_false
    flags to False so that PreludeCorrelator can expect data written to the
    data files to be persistant accross run.
    
    However, for standard prelude-correlator installation, we keep
    using /etc/prelude-correlator as the configuration directory, and
    /var/lib/prelude-correlator as the data directory.
    
    This additionally introduce a default configuration file, which
    installation location depend on the type of installation.


========================================

 PreludeCorrelator/context.py             |   11 ++++--
 PreludeCorrelator/log.py                 |    5 ++-
 PreludeCorrelator/main.py                |   14 ++++++---
 PreludeCorrelator/pluginmanager.py       |    1 -
 PreludeCorrelator/plugins/dshield.py     |    4 +-
 PreludeCorrelator/require.py             |   44 ++++++++++++++++++++++++++++
 prelude-correlator.conf                  |   47 ++++++++++++++++++++++++++++++
 prelude_correlator.egg-info/SOURCES.txt  |    4 ++
 prelude_correlator.egg-info/not-zip-safe |    1 +
 setup.py                                 |   38 ++++++++++++++++++++++--
 10 files changed, 152 insertions(+), 17 deletions(-)

========================================

diff --git a/PreludeCorrelator/context.py b/PreludeCorrelator/context.py
index 788e164..2f21d40 100644
--- a/PreludeCorrelator/context.py
+++ b/PreludeCorrelator/context.py
@@ -18,7 +18,7 @@
 # the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 
 import os, time, StringIO, pickle
-from PreludeCorrelator import idmef, siteconfig
+from PreludeCorrelator import idmef, require
 
 _TIMER_LIST = [ ]
 _CONTEXT_TABLE = { }
@@ -129,14 +129,17 @@ def search(name):
 
     return None
 
+
+_ctxt_filename = require.get_data_filename(None, "context.dat")
+
 def save():
-        fd = open(siteconfig.lib_dir + "/context.dat", "w")
+        fd = open(_ctxt_filename, "w")
         pickle.dump(_CONTEXT_TABLE, fd)
         fd.close()
 
 def load():
-        if os.path.exists(siteconfig.lib_dir + "/context.dat"):
-                fd = open(siteconfig.lib_dir + "/context.dat", "r")
+        if os.path.exists(_ctxt_filename):
+                fd = open(_ctxt_filename, "r")
                 try:
                         _CONTEXT_TABLE.update(pickle.load(fd))
                 except EOFError:
diff --git a/PreludeCorrelator/log.py b/PreludeCorrelator/log.py
index 80f5378..168212f 100644
--- a/PreludeCorrelator/log.py
+++ b/PreludeCorrelator/log.py
@@ -18,7 +18,8 @@
 # the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 
 import PreludeEasy
-import logging, logging.config, logging.handlers, sys, os, siteconfig
+from PreludeCorrelator import require
+import logging, logging.config, logging.handlers, sys, os
 
 class Log(logging.Logger):
     def __log_callback(self, level, log):
@@ -50,7 +51,7 @@ class Log(logging.Logger):
                 pass
 
         try:
-                logging.config.fileConfig(siteconfig.conf_dir + "/prelude-correlator.conf")
+                logging.config.fileConfig(require.get_config_filename(None, "prelude-correlator.conf"))
         except Exception, e:
                 DATEFMT = "%d %b %H:%M:%S"
                 FORMAT="%(asctime)s (process:%(pid)d) %(levelname)s: %(message)s"
diff --git a/PreludeCorrelator/main.py b/PreludeCorrelator/main.py
index 32c350d..8ad0932 100644
--- a/PreludeCorrelator/main.py
+++ b/PreludeCorrelator/main.py
@@ -19,18 +19,22 @@
 # along with this program; see the file COPYING.  If not, write to
 # the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 
-import pkg_resources
+import require
 import sys, os, time, signal
 from optparse import OptionParser
 from PreludeEasy import ClientEasy, CheckVersion
 from PreludeCorrelator import __version__ as VERSION
-from PreludeCorrelator import idmef, pluginmanager, context, siteconfig, log, config
+from PreludeCorrelator import idmef, pluginmanager, context, log, config
+
+
+LIBPRELUDE_REQUIRED_VERSION = "0.9.23"
 
 
 class Env:
         def __init__(self):
                 self.logger = log.Log()
-                self.config = config.Config(siteconfig.conf_dir + '/prelude-correlator.conf')
+
+                self.config = config.Config(require.get_config_filename(None, "prelude-correlator.conf"))
                 self.pluginmanager = pluginmanager.PluginManager(self)
 
                 self.logger.info("%d plugin have been loaded." % (self.pluginmanager.getPluginCount()))
@@ -114,8 +118,8 @@ class PreludeClient:
 
 
 def main():
-        if not CheckVersion(siteconfig.libprelude_required_version):
-                raise Exception, ("Libprelude version '%s' is required" % siteconfig.libprelude_required_version)
+        if not CheckVersion(LIBPRELUDE_REQUIRED_VERSION):
+                raise Exception, ("Libprelude version '%s' is required" % LIBPRELUDE_REQUIRED_VERSION)
 
         env = Env()
 
diff --git a/PreludeCorrelator/pluginmanager.py b/PreludeCorrelator/pluginmanager.py
index 4d0b7b5..72b853e 100644
--- a/PreludeCorrelator/pluginmanager.py
+++ b/PreludeCorrelator/pluginmanager.py
@@ -18,7 +18,6 @@
 # the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 
 import pkg_resources, sys, os, traceback, ConfigParser
-from PreludeCorrelator import siteconfig
 
 
 class PluginLog:
diff --git a/PreludeCorrelator/plugins/dshield.py b/PreludeCorrelator/plugins/dshield.py
index f42454b..88908e5 100644
--- a/PreludeCorrelator/plugins/dshield.py
+++ b/PreludeCorrelator/plugins/dshield.py
@@ -19,7 +19,7 @@
 # the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
 
 import os, httplib, time
-from PreludeCorrelator import siteconfig
+from PreludeCorrelator import require
 from PreludeCorrelator.idmef import IDMEF
 from PreludeCorrelator.pluginmanager import Plugin
 from PreludeCorrelator.context import Context, Timer
@@ -30,7 +30,7 @@ class DshieldPlugin(Plugin):
     DSHIELD_SERVER = "www.dshield.org"
     DSHIELD_URI = "/ipsascii.html?limit=10000"
     DSHIELD_TIMEOUT = 10
-    DSHIELD_FILENAME = siteconfig.lib_dir + "/dshield.dat"
+    DSHIELD_FILENAME = require.get_data_filename(__name__, "dshield.dat")
 
     def __ipNormalize(self, ip):
         return ".".join([ i.lstrip("0") for i in ip.split(".") ])
diff --git a/PreludeCorrelator/require.py b/PreludeCorrelator/require.py
new file mode 100644
index 0000000..45aa1a0
--- /dev/null
+++ b/PreludeCorrelator/require.py
@@ -0,0 +1,44 @@
+# Copyright (C) 2009 PreludeIDS Technologies. All Rights Reserved.
+# Author: Yoann Vandoorselaere <[email protected]>
+#
+# This file is part of the Prelude-Correlator program.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; see the file COPYING.  If not, write to
+# the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
+
+try:
+        import os
+        from PreludeCorrelator import siteconfig
+
+        def get_config_filename(module, fname):
+                return os.path.join(siteconfig.conf_dir, fname)
+
+        def get_data_filename(module, fname):
+                return os.path.join(siteconfig.lib_dir, fname)
+
+except:
+        import pkg_resources
+
+        def get_config_filename(module, fname):
+                if module is None:
+                        module = pkg_resources.Requirement.parse("prelude-correlator")
+
+                return pkg_resources.resource_filename(module, fname)
+
+        def get_data_filename(module, fname):
+                if module is None:
+                        module = pkg_resources.Requirement.parse("prelude-correlator")
+
+                return pkg_resources.resource_filename(module, fname)
+
diff --git a/prelude-correlator.conf b/prelude-correlator.conf
new file mode 100644
index 0000000..60cb13c
--- /dev/null
+++ b/prelude-correlator.conf
@@ -0,0 +1,47 @@
+# This is a template configuration file for prelude-correlator
+#
+# [BruteForcePlugin]
+# disable = false
+#
+# [BusinessHourPlugin]
+# disable = false
+#
+# [OpenSSHAuthPlugin]
+# disable = false
+#
+# [EventScanPlugin]
+# disable = false
+#
+# [EventStormPlugin]
+# disable = false
+#
+# [EventSweepPlugin]
+# disable = false
+#
+# [WormPlugin]
+# disable = false
+#
+# [DshieldPlugin]
+# disable = false
+#
+# How often the Dshield database should be reloaded (download + reload) 
+# (default: once a week). 0 to disable reloading.
+# reload  = 604800
+#
+# The server address where the Dshield database is loaded from:
+# server  = www.dshield.org
+#
+# URI used to retrive the dshield database:
+# uri     = /ipsascii.html?limit=10000
+#
+# Define the maximum allowed time for downloading the database (only work with Python >= 2.6, default is 10 seconds) 
+# timeout = 10
+
+# Disable firewall correlation by default since it is very verbose
+[FirewallPlugin]
+disable = True
+
+
+##
+# Logging configuration might also be defined in this file:
+# http://docs.python.org/library/logging.html
diff --git a/prelude_correlator.egg-info/SOURCES.txt b/prelude_correlator.egg-info/SOURCES.txt
index ee88cac..6507113 100644
--- a/prelude_correlator.egg-info/SOURCES.txt
+++ b/prelude_correlator.egg-info/SOURCES.txt
@@ -5,6 +5,7 @@ HACKING.README
 NEWS
 README
 ez_setup.py
+prelude-correlator.conf
 setup.py
 PreludeCorrelator/__init__.py
 PreludeCorrelator/config.py
@@ -13,11 +14,13 @@ PreludeCorrelator/idmef.py
 PreludeCorrelator/log.py
 PreludeCorrelator/main.py
 PreludeCorrelator/pluginmanager.py
+PreludeCorrelator/require.py
 PreludeCorrelator/siteconfig.py
 PreludeCorrelator/utils.py
 PreludeCorrelator/plugins/__init__.py
 PreludeCorrelator/plugins/bruteforce.py
 PreludeCorrelator/plugins/businesshour.py
+PreludeCorrelator/plugins/dshield.dat
 PreludeCorrelator/plugins/dshield.py
 PreludeCorrelator/plugins/firewall.py
 PreludeCorrelator/plugins/opensshauth.py
@@ -27,4 +30,5 @@ prelude_correlator.egg-info/PKG-INFO
 prelude_correlator.egg-info/SOURCES.txt
 prelude_correlator.egg-info/dependency_links.txt
 prelude_correlator.egg-info/entry_points.txt
+prelude_correlator.egg-info/not-zip-safe
 prelude_correlator.egg-info/top_level.txt
\ No newline at end of file
diff --git a/prelude_correlator.egg-info/not-zip-safe b/prelude_correlator.egg-info/not-zip-safe
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/prelude_correlator.egg-info/not-zip-safe
@@ -0,0 +1 @@
+
diff --git a/setup.py b/setup.py
index 886fe6e..e127dbe 100644
--- a/setup.py
+++ b/setup.py
@@ -3,23 +3,39 @@
 from ez_setup import use_setuptools
 use_setuptools()
 
-import os
+import os, sys, shutil
 from setuptools import setup, find_packages
 from setuptools.command.install import install
 from setuptools.command.sdist import sdist
 
 PRELUDE_CORRELATOR_VERSION = "0.9.0-beta5"
-LIBPRELUDE_REQUIRED_VERSION = "0.9.23"
 
 
 class my_sdist(sdist):
         def __init__(self, *args, **kwargs):
+                import httplib
+
                 fin = os.popen('git log --summary --stat --no-merges --date=short', 'r')
                 fout = open('ChangeLog', 'w')
                 fout.write(fin.read())
                 fout.close()
+
+                print "Downloading DShield database, this might take a while..."
+
+                con = httplib.HTTPConnection("www.dshield.org")
+                con.request("GET", "/ipsascii.html?limit=10000")
+                r = con.getresponse()
+                if r.status != 200:
+                        raise Exception, "Could not download DShield host list, error %d" % r.status
+
+                fd = open("PreludeCorrelator/plugins/dshield.dat", "w")
+                fd.write(r.read())
+                fd.close()
+
                 sdist.__init__(self, *args)
 
+
+
 class my_install(install):
         def __install_data(self):
                 data_files = self.distribution.data_files
@@ -50,15 +66,27 @@ class my_install(install):
                 self.init_siteconfig(prefix)
                 self.__install_data()
                 install.run(self)
+                os.remove("PreludeCorrelator/siteconfig.py")
 
         def init_siteconfig(self, prefix):
                 config = open("PreludeCorrelator/siteconfig.py", "w")
                 print >> config, "conf_dir = '%s'" % os.path.abspath(prefix + "/etc/prelude-correlator")
                 print >> config, "lib_dir = '%s'" % os.path.abspath(prefix + "/var/lib/prelude-correlator")
-                print >> config, "libprelude_required_version = '%s'" % LIBPRELUDE_REQUIRED_VERSION
                 config.close()
 
 
+is_egg = "bdist_egg" in sys.argv
+if is_egg:
+        # Make sure we remove any trace of siteconfig.py
+        try: shutil.rmtree("build")
+        except: pass
+        package_data = { '': [ "*.dat"] }
+        data_files = [ ("", ["prelude-correlator.conf"]) ]
+else:
+        package_data = {}
+        data_files = [ ("etc/prelude-correlator", ["prelude-correlator.conf"]),
+                       ("var/lib/prelude-correlator", ["PreludeCorrelator/plugins/dshield.dat"]) ]
+
 setup(
         name="prelude-correlator",
         version=PRELUDE_CORRELATOR_VERSION,
@@ -113,5 +141,9 @@ suits your needs.
                 ]
         },
 
+        zip_safe = False,
+        data_files = data_files,
+        package_data = package_data,
+
         cmdclass = { 'sdist': my_sdist, 'install': my_install }
 )
_______________________________________________
Prelude-cvslog site list
[email protected]
http://lists.prelude-ids.org/mailman/listinfo/prelude-cvslog