Imported existing code

indra/lib/python/indra/util/__init__.py

@@ -0,0 +1,27 @@
+"""\
+@file __init__.py
+@brief Initialization file for the indra util module.
+
+$LicenseInfo:firstyear=2006&license=mit$
+
+Copyright (c) 2006-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""

indra/lib/python/indra/util/fastest_elementtree.py

@@ -0,0 +1,65 @@
+"""\
+@file fastest_elementtree.py
+@brief Concealing some gnarly import logic in here.  This should export the interface of elementtree.
+
+$LicenseInfo:firstyear=2008&license=mit$
+
+Copyright (c) 2008-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+# The parsing exception raised by the underlying library depends
+# on the ElementTree implementation we're using, so we provide an
+# alias here.
+#
+# Use ElementTreeError as the exception type for catching parsing
+# errors.
+
+
+# Using cElementTree might cause some unforeseen problems, so here's a
+# convenient off switch.
+
+use_celementree = True
+
+try:
+    if not use_celementree:
+        raise ImportError()
+    # Python 2.3 and 2.4.
+    from cElementTree import *
+    ElementTreeError = SyntaxError
+except ImportError:
+    try:
+        if not use_celementree:
+            raise ImportError()
+        # Python 2.5 and above.
+        from xml.etree.cElementTree import *
+        ElementTreeError = SyntaxError
+    except ImportError:
+        # Pure Python code.
+        try:
+            # Python 2.3 and 2.4.
+            from elementtree.ElementTree import *
+        except ImportError:
+            # Python 2.5 and above.
+            from xml.etree.ElementTree import *
+
+        # The pure Python ElementTree module uses Expat for parsing.
+        from xml.parsers.expat import ExpatError as ElementTreeError

indra/lib/python/indra/util/helpformatter.py

@@ -0,0 +1,52 @@
+"""\
+@file helpformatter.py
+@author Phoenix
+@brief Class for formatting optparse descriptions.
+
+$LicenseInfo:firstyear=2007&license=mit$
+
+Copyright (c) 2007-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+import optparse
+import textwrap
+
+class Formatter(optparse.IndentedHelpFormatter):
+    def __init__(
+        self, 
+        p_indentIncrement = 2, 
+        p_maxHelpPosition = 24,
+        p_width = 79,
+        p_shortFirst = 1) :
+        optparse.HelpFormatter.__init__(
+            self, 
+            p_indentIncrement,
+            p_maxHelpPosition, 
+            p_width, 
+            p_shortFirst)
+    def format_description(self, p_description):
+        t_descWidth = self.width - self.current_indent
+        t_indent = " " * (self.current_indent + 2)
+        return "\n".join(
+            [textwrap.fill(descr, t_descWidth, initial_indent = t_indent,
+                           subsequent_indent = t_indent)
+             for descr in p_description.split("\n")] )

indra/lib/python/indra/util/iterators.py

@@ -0,0 +1,63 @@
+"""\
+@file iterators.py
+@brief Useful general-purpose iterators.
+
+$LicenseInfo:firstyear=2008&license=mit$
+
+Copyright (c) 2008-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+from __future__ import nested_scopes
+
+def iter_chunks(rows, aggregate_size=100):
+    """
+    Given an iterable set of items (@p rows), produces lists of up to @p
+    aggregate_size items at a time, for example:
+    
+    iter_chunks([1,2,3,4,5,6,7,8,9,10], 3)
+
+    Values for @p aggregate_size < 1 will raise ValueError.
+
+    Will return a generator that produces, in the following order:
+    - [1, 2, 3]
+    - [4, 5, 6]
+    - [7, 8, 9]
+    - [10]
+    """
+    if aggregate_size < 1:
+        raise ValueError()
+
+    def iter_chunks_inner():
+        row_iter = iter(rows)
+        done = False
+        agg = []
+        while not done:
+            try:
+                row = row_iter.next()
+                agg.append(row)
+            except StopIteration:
+                done = True
+            if agg and (len(agg) >= aggregate_size or done):
+                yield agg
+                agg = []
+    
+    return iter_chunks_inner()

indra/lib/python/indra/util/iterators_test.py

@@ -0,0 +1,72 @@
+"""\
+@file iterators_test.py
+@brief Test cases for iterators module.
+
+$LicenseInfo:firstyear=2008&license=mit$
+
+Copyright (c) 2008-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+import unittest
+
+from indra.util.iterators import iter_chunks
+
+class TestIterChunks(unittest.TestCase):
+    """Unittests for iter_chunks"""
+    def test_bad_agg_size(self):
+        rows = [1,2,3,4]
+        self.assertRaises(ValueError, iter_chunks, rows, 0)
+        self.assertRaises(ValueError, iter_chunks, rows, -1)
+
+        try:
+            for i in iter_chunks(rows, 0):
+                pass
+        except ValueError:
+            pass
+        else:
+            self.fail()
+        
+        try:
+            result = list(iter_chunks(rows, 0))
+        except ValueError:
+            pass
+        else:
+            self.fail()
+    def test_empty(self):
+        rows = []
+        result = list(iter_chunks(rows))
+        self.assertEqual(result, [])
+    def test_small(self):
+        rows = [[1]]
+        result = list(iter_chunks(rows, 2))
+        self.assertEqual(result, [[[1]]])
+    def test_size(self):
+        rows = [[1],[2]]
+        result = list(iter_chunks(rows, 2))
+        self.assertEqual(result, [[[1],[2]]])
+    def test_multi_agg(self):
+        rows = [[1],[2],[3],[4],[5]]
+        result = list(iter_chunks(rows, 2))
+        self.assertEqual(result, [[[1],[2]],[[3],[4]],[[5]]])
+
+if __name__ == "__main__":
+    unittest.main()

indra/lib/python/indra/util/llmanifest.py

@@ -0,0 +1,668 @@
+"""\
+@file llmanifest.py
+@author Ryan Williams
+@brief Library for specifying operations on a set of files.
+
+$LicenseInfo:firstyear=2007&license=mit$
+
+Copyright (c) 2007-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+import commands
+import errno
+import filecmp
+import fnmatch
+import getopt
+import glob
+import os
+import re
+import shutil
+import sys
+import tarfile
+import errno
+
+def path_ancestors(path):
+    drive, path = os.path.splitdrive(os.path.normpath(path))
+    result = []
+    while len(path) > 0 and path != os.path.sep:
+        result.append(drive+path)
+        path, sub = os.path.split(path)
+    return result
+
+def proper_windows_path(path, current_platform = sys.platform):
+    """ This function takes an absolute Windows or Cygwin path and
+    returns a path appropriately formatted for the platform it's
+    running on (as determined by sys.platform)"""
+    path = path.strip()
+    drive_letter = None
+    rel = None
+    match = re.match("/cygdrive/([a-z])/(.*)", path)
+    if not match:
+        match = re.match('([a-zA-Z]):\\\(.*)', path)
+    if not match:
+        return None         # not an absolute path
+    drive_letter = match.group(1)
+    rel = match.group(2)
+    if current_platform == "cygwin":
+        return "/cygdrive/" + drive_letter.lower() + '/' + rel.replace('\\', '/')
+    else:
+        return drive_letter.upper() + ':\\' + rel.replace('/', '\\')
+
+def get_default_platform(dummy):
+    return {'linux2':'linux',
+            'linux1':'linux',
+            'cygwin':'windows',
+            'win32':'windows',
+            'darwin':'darwin'
+            }[sys.platform]
+
+def get_default_version(srctree):
+    # look up llversion.h and parse out the version info
+    paths = [os.path.join(srctree, x, 'llversionviewer.h') for x in ['llcommon', '../llcommon', '../../indra/llcommon.h']]
+    for p in paths:
+        if os.path.exists(p):
+            contents = open(p, 'r').read()
+            major = re.search("LL_VERSION_MAJOR\s=\s([0-9]+)", contents).group(1)
+            minor = re.search("LL_VERSION_MINOR\s=\s([0-9]+)", contents).group(1)
+            patch = re.search("LL_VERSION_PATCH\s=\s([0-9]+)", contents).group(1)
+            build = re.search("LL_VERSION_BUILD\s=\s([0-9]+)", contents).group(1)
+            return major, minor, patch, build
+
+def get_channel(srctree):
+    # look up llversionserver.h and parse out the version info
+    paths = [os.path.join(srctree, x, 'llversionviewer.h') for x in ['llcommon', '../llcommon', '../../indra/llcommon.h']]
+    for p in paths:
+        if os.path.exists(p):
+            contents = open(p, 'r').read()
+            channel = re.search("LL_CHANNEL\s=\s\"(.+)\";\s*$", contents, flags = re.M).group(1)
+            return channel
+    
+
+DEFAULT_SRCTREE = os.path.dirname(sys.argv[0])
+DEFAULT_CHANNEL = 'Second Life Release'
+DEFAULT_CHANNEL_SNOWGLOBE = 'Snowglobe Release'
+
+ARGUMENTS=[
+    dict(name='actions',
+         description="""This argument specifies the actions that are to be taken when the
+        script is run.  The meaningful actions are currently:
+          copy     - copies the files specified by the manifest into the
+                     destination directory.
+          package  - bundles up the files in the destination directory into
+                     an installer for the current platform
+          unpacked - bundles up the files in the destination directory into
+                     a simple tarball
+        Example use: %(name)s --actions="copy unpacked" """,
+         default="copy package"),
+    dict(name='arch',
+         description="""This argument is appended to the platform string for
+        determining which manifest class to run.
+        Example use: %(name)s --arch=i686
+        On Linux this would try to use Linux_i686Manifest.""",
+         default=""),
+    dict(name='build', description='Build directory.', default=DEFAULT_SRCTREE),
+    dict(name='buildtype', description="""The build type used. ('Debug', 'Release', or 'RelWithDebInfo')
+        Default is Release """,
+         default="Release"),
+    dict(name='branding_id', description="""Identifier for the branding set to 
+        use.  Currently, 'secondlife' or 'snowglobe')""", 
+         default='secondlife'),
+    dict(name='configuration',
+         description="""The build configuration used. Only used on OS X for
+        now, but it could be used for other platforms as well.""",
+         default="Universal"),
+    dict(name='dest', description='Destination directory.', default=DEFAULT_SRCTREE),
+    dict(name='grid',
+         description="""Which grid the client will try to connect to. Even
+        though it's not strictly a grid, 'firstlook' is also an acceptable
+        value for this parameter.""",
+         default=""),
+    dict(name='channel',
+         description="""The channel to use for updates, packaging, settings name, etc.""",
+         default=get_channel),
+    dict(name='login_channel',
+         description="""The channel to use for login handshake/updates only.""",
+         default=None),
+    dict(name='installer_name',
+         description=""" The name of the file that the installer should be
+        packaged up into. Only used on Linux at the moment.""",
+         default=None),
+    dict(name='login_url',
+         description="""The url that the login screen displays in the client.""",
+         default=None),
+    dict(name='platform',
+         description="""The current platform, to be used for looking up which
+        manifest class to run.""",
+         default=get_default_platform),
+    dict(name='source',
+         description='Source directory.',
+         default=DEFAULT_SRCTREE),
+    dict(name='artwork', description='Artwork directory.', default=DEFAULT_SRCTREE),
+    dict(name='touch',
+         description="""File to touch when action is finished. Touch file will
+        contain the name of the final package in a form suitable
+        for use by a .bat file.""",
+         default=None),
+    dict(name='version',
+         description="""This specifies the version of Second Life that is
+        being packaged up.""",
+         default=get_default_version)
+    ]
+
+def usage(srctree=""):
+    nd = {'name':sys.argv[0]}
+    print """Usage:
+    %(name)s [options] [destdir]
+    Options:
+    """ % nd
+    for arg in ARGUMENTS:
+        default = arg['default']
+        if hasattr(default, '__call__'):
+            default = "(computed value) \"" + str(default(srctree)) + '"'
+        elif default is not None:
+            default = '"' + default + '"'
+        print "\t--%s        Default: %s\n\t%s\n" % (
+            arg['name'],
+            default,
+            arg['description'] % nd)
+
+def main():
+    option_names = [arg['name'] + '=' for arg in ARGUMENTS]
+    option_names.append('help')
+    options, remainder = getopt.getopt(sys.argv[1:], "", option_names)
+
+    # convert options to a hash
+    args = {'source':  DEFAULT_SRCTREE,
+            'artwork': DEFAULT_SRCTREE,
+            'build':   DEFAULT_SRCTREE,
+            'dest':    DEFAULT_SRCTREE }
+    for opt in options:
+        args[opt[0].replace("--", "")] = opt[1]
+
+    for k in 'artwork build dest source'.split():
+        args[k] = os.path.normpath(args[k])
+
+    print "Source tree:", args['source']
+    print "Artwork tree:", args['artwork']
+    print "Build tree:", args['build']
+    print "Destination tree:", args['dest']
+
+    # early out for help
+    if 'help' in args:
+        # *TODO: it is a huge hack to pass around the srctree like this
+        usage(args['source'])
+        return
+
+    # defaults
+    for arg in ARGUMENTS:
+        if arg['name'] not in args:
+            default = arg['default']
+            if hasattr(default, '__call__'):
+                default = default(args['source'])
+            if default is not None:
+                args[arg['name']] = default
+
+    # fix up version
+    if isinstance(args.get('version'), str):
+        args['version'] = args['version'].split('.')
+        
+    # default and agni are default
+    if args['grid'] in ['default', 'agni']:
+        args['grid'] = ''
+
+    if 'actions' in args:
+        args['actions'] = args['actions'].split()
+
+    # debugging
+    for opt in args:
+        print "Option:", opt, "=", args[opt]
+
+    wm = LLManifest.for_platform(args['platform'], args.get('arch'))(args)
+    wm.do(*args['actions'])
+
+    # Write out the package file in this format, so that it can easily be called
+    # and used in a .bat file - yeah, it sucks, but this is the simplest...
+    touch = args.get('touch')
+    if touch:
+        fp = open(touch, 'w')
+        fp.write('set package_file=%s\n' % wm.package_file)
+        fp.close()
+        print 'touched', touch
+    return 0
+
+class LLManifestRegistry(type):
+    def __init__(cls, name, bases, dct):
+        super(LLManifestRegistry, cls).__init__(name, bases, dct)
+        match = re.match("(\w+)Manifest", name)
+        if match:
+           cls.manifests[match.group(1).lower()] = cls
+
+class LLManifest(object):
+    __metaclass__ = LLManifestRegistry
+    manifests = {}
+    def for_platform(self, platform, arch = None):
+        if arch:
+            platform = platform + '_' + arch
+        return self.manifests[platform.lower()]
+    for_platform = classmethod(for_platform)
+
+    def __init__(self, args):
+        super(LLManifest, self).__init__()
+        self.args = args
+        self.file_list = []
+        self.excludes = []
+        self.actions = []
+        self.src_prefix = [args['source']]
+        self.artwork_prefix = [args['artwork']]
+        self.build_prefix = [args['build']]
+        self.dst_prefix = [args['dest']]
+        self.created_paths = []
+        self.package_name = "Unknown"
+        
+    def default_grid(self):
+        return self.args.get('grid', None) == ''
+    def default_channel(self):
+        return self.args.get('channel', None) == DEFAULT_CHANNEL
+    
+    def default_channel_for_brand(self):
+        if self.viewer_branding_id()=='secondlife':
+            return self.args.get('channel', None) == DEFAULT_CHANNEL
+        elif self.viewer_branding_id()=="snowglobe":
+            return self.args.get('channel', None) == DEFAULT_CHANNEL_SNOWGLOBE
+        raise ValueError, "Invalid branding id: " + self.viewer_branding_id()
+
+    def construct(self):
+        """ Meant to be overriden by LLManifest implementors with code that
+        constructs the complete destination hierarchy."""
+        pass # override this method
+
+    def exclude(self, glob):
+        """ Excludes all files that match the glob from being included
+        in the file list by path()."""
+        self.excludes.append(glob)
+
+    def prefix(self, src='', build=None, dst=None):
+        """ Pushes a prefix onto the stack.  Until end_prefix is
+        called, all relevant method calls (esp. to path()) will prefix
+        paths with the entire prefix stack.  Source and destination
+        prefixes can be different, though if only one is provided they
+        are both equal.  To specify a no-op, use an empty string, not
+        None."""
+        if dst is None:
+            dst = src
+        if build is None:
+            build = src
+        self.src_prefix.append(src)
+        self.artwork_prefix.append(src)
+        self.build_prefix.append(build)
+        self.dst_prefix.append(dst)
+        return True  # so that you can wrap it in an if to get indentation
+
+    def end_prefix(self, descr=None):
+        """Pops a prefix off the stack.  If given an argument, checks
+        the argument against the top of the stack.  If the argument
+        matches neither the source or destination prefixes at the top
+        of the stack, then misnesting must have occurred and an
+        exception is raised."""
+        # as an error-prevention mechanism, check the prefix and see if it matches the source or destination prefix.  If not, improper nesting may have occurred.
+        src = self.src_prefix.pop()
+        artwork = self.artwork_prefix.pop()
+        build = self.build_prefix.pop()
+        dst = self.dst_prefix.pop()
+        if descr and not(src == descr or build == descr or dst == descr):
+            raise ValueError, "End prefix '" + descr + "' didn't match '" +src+ "' or '" +dst + "'"
+
+    def get_src_prefix(self):
+        """ Returns the current source prefix."""
+        return os.path.join(*self.src_prefix)
+
+    def get_artwork_prefix(self):
+        """ Returns the current artwork prefix."""
+        return os.path.join(*self.artwork_prefix)
+
+    def get_build_prefix(self):
+        """ Returns the current build prefix."""
+        return os.path.join(*self.build_prefix)
+
+    def get_dst_prefix(self):
+        """ Returns the current destination prefix."""
+        return os.path.join(*self.dst_prefix)
+
+    def src_path_of(self, relpath):
+        """Returns the full path to a file or directory specified
+        relative to the source directory."""
+        return os.path.join(self.get_src_prefix(), relpath)
+
+    def build_path_of(self, relpath):
+        """Returns the full path to a file or directory specified
+        relative to the build directory."""
+        return os.path.join(self.get_build_prefix(), relpath)
+
+    def dst_path_of(self, relpath):
+        """Returns the full path to a file or directory specified
+        relative to the destination directory."""
+        return os.path.join(self.get_dst_prefix(), relpath)
+
+    def ensure_src_dir(self, reldir):
+        """Construct the path for a directory relative to the
+        source path, and ensures that it exists.  Returns the
+        full path."""
+        path = os.path.join(self.get_src_prefix(), reldir)
+        self.cmakedirs(path)
+        return path
+
+    def ensure_dst_dir(self, reldir):
+        """Construct the path for a directory relative to the
+        destination path, and ensures that it exists.  Returns the
+        full path."""
+        path = os.path.join(self.get_dst_prefix(), reldir)
+        self.cmakedirs(path)
+        return path
+
+    def run_command(self, command):
+        """ Runs an external command, and returns the output.  Raises
+        an exception if the command reurns a nonzero status code.  For
+        debugging/informational purpoases, prints out the command's
+        output as it is received."""
+        print "Running command:", command
+        fd = os.popen(command, 'r')
+        lines = []
+        while True:
+            lines.append(fd.readline())
+            if lines[-1] == '':
+                break
+            else:
+                print lines[-1],
+        output = ''.join(lines)
+        status = fd.close()
+        if status:
+            raise RuntimeError(
+                "Command %s returned non-zero status (%s) \noutput:\n%s"
+                % (command, status, output) )
+        return output
+
+    def created_path(self, path):
+        """ Declare that you've created a path in order to
+          a) verify that you really have created it
+          b) schedule it for cleanup"""
+        if not os.path.exists(path):
+            raise RuntimeError, "Should be something at path " + path
+        self.created_paths.append(path)
+
+    def put_in_file(self, contents, dst):
+        # write contents as dst
+        f = open(self.dst_path_of(dst), "wb")
+        f.write(contents)
+        f.close()
+
+    def replace_in(self, src, dst=None, searchdict={}):
+        if dst == None:
+            dst = src
+        # read src
+        f = open(self.src_path_of(src), "rbU")
+        contents = f.read()
+        f.close()
+        # apply dict replacements
+        for old, new in searchdict.iteritems():
+            contents = contents.replace(old, new)
+        self.put_in_file(contents, dst)
+        self.created_paths.append(dst)
+
+    def copy_action(self, src, dst):
+        if src and (os.path.exists(src) or os.path.islink(src)):
+            # ensure that destination path exists
+            self.cmakedirs(os.path.dirname(dst))
+            self.created_paths.append(dst)
+            if not os.path.isdir(src):
+                self.ccopy(src,dst)
+            else:
+                # src is a dir
+                self.ccopytree(src,dst)
+        else:
+            print "Doesn't exist:", src
+
+    def package_action(self, src, dst):
+        pass
+
+    def copy_finish(self):
+        pass
+
+    def package_finish(self):
+        pass
+
+    def unpacked_finish(self):
+        unpacked_file_name = "unpacked_%(plat)s_%(vers)s.tar" % {
+            'plat':self.args['platform'],
+            'vers':'_'.join(self.args['version'])}
+        print "Creating unpacked file:", unpacked_file_name
+        # could add a gz here but that doubles the time it takes to do this step
+        tf = tarfile.open(self.src_path_of(unpacked_file_name), 'w:')
+        # add the entire installation package, at the very top level
+        tf.add(self.get_dst_prefix(), "")
+        tf.close()
+
+    def cleanup_finish(self):
+        """ Delete paths that were specified to have been created by this script"""
+        for c in self.created_paths:
+            # *TODO is this gonna be useful?
+            print "Cleaning up " + c
+
+    def process_file(self, src, dst):
+        if self.includes(src, dst):
+#            print src, "=>", dst
+            for action in self.actions:
+                methodname = action + "_action"
+                method = getattr(self, methodname, None)
+                if method is not None:
+                    method(src, dst)
+            self.file_list.append([src, dst])
+            return 1
+        else:
+            sys.stdout.write(" (excluding %r, %r)" % (src, dst))
+            sys.stdout.flush()
+            return 0
+
+    def process_directory(self, src, dst):
+        if not self.includes(src, dst):
+            sys.stdout.write(" (excluding %r, %r)" % (src, dst))
+            sys.stdout.flush()
+            return 0
+        names = os.listdir(src)
+        self.cmakedirs(dst)
+        errors = []
+        count = 0
+        for name in names:
+            srcname = os.path.join(src, name)
+            dstname = os.path.join(dst, name)
+            if os.path.isdir(srcname):
+                count += self.process_directory(srcname, dstname)
+            else:
+                count += self.process_file(srcname, dstname)
+        return count
+
+    def includes(self, src, dst):
+        if src:
+            for excl in self.excludes:
+                if fnmatch.fnmatch(src, excl):
+                    return False
+        return True
+
+    def remove(self, *paths):
+        for path in paths:
+            if os.path.exists(path):
+                print "Removing path", path
+                if os.path.isdir(path):
+                    shutil.rmtree(path)
+                else:
+                    os.remove(path)
+
+    def ccopy(self, src, dst):
+        """ Copy a single file or symlink.  Uses filecmp to skip copying for existing files."""
+        if os.path.islink(src):
+            linkto = os.readlink(src)
+            if os.path.islink(dst) or os.path.exists(dst):
+                os.remove(dst)  # because symlinking over an existing link fails
+            os.symlink(linkto, dst)
+        else:
+            # Don't recopy file if it's up-to-date.
+            # If we seem to be not not overwriting files that have been
+            # updated, set the last arg to False, but it will take longer.
+            if os.path.exists(dst) and filecmp.cmp(src, dst, True):
+                return
+            # only copy if it's not excluded
+            if self.includes(src, dst):
+                try:
+                    os.unlink(dst)
+                except OSError, err:
+                    if err.errno != errno.ENOENT:
+                        raise
+
+                shutil.copy2(src, dst)
+
+    def ccopytree(self, src, dst):
+        """Direct copy of shutil.copytree with the additional
+        feature that the destination directory can exist.  It
+        is so dumb that Python doesn't come with this. Also it
+        implements the excludes functionality."""
+        if not self.includes(src, dst):
+            return
+        names = os.listdir(src)
+        self.cmakedirs(dst)
+        errors = []
+        for name in names:
+            srcname = os.path.join(src, name)
+            dstname = os.path.join(dst, name)
+            try:
+                if os.path.isdir(srcname):
+                    self.ccopytree(srcname, dstname)
+                else:
+                    self.ccopy(srcname, dstname)
+                    # XXX What about devices, sockets etc.?
+            except (IOError, os.error), why:
+                errors.append((srcname, dstname, why))
+        if errors:
+            raise RuntimeError, errors
+
+
+    def cmakedirs(self, path):
+        """Ensures that a directory exists, and doesn't throw an exception
+        if you call it on an existing directory."""
+#        print "making path: ", path
+        path = os.path.normpath(path)
+        self.created_paths.append(path)
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+    def find_existing_file(self, *list):
+        for f in list:
+            if os.path.exists(f):
+                return f
+        # didn't find it, return last item in list
+        if len(list) > 0:
+            return list[-1]
+        else:
+            return None
+
+    def contents_of_tar(self, src_tar, dst_dir):
+        """ Extracts the contents of the tarfile (specified
+        relative to the source prefix) into the directory
+        specified relative to the destination directory."""
+        self.check_file_exists(src_tar)
+        tf = tarfile.open(self.src_path_of(src_tar), 'r')
+        for member in tf.getmembers():
+            tf.extract(member, self.ensure_dst_dir(dst_dir))
+            # TODO get actions working on these dudes, perhaps we should extract to a temporary directory and then process_directory on it?
+            self.file_list.append([src_tar,
+                           self.dst_path_of(os.path.join(dst_dir,member.name))])
+        tf.close()
+
+
+    def wildcard_regex(self, src_glob, dst_glob):
+        src_re = re.escape(src_glob)
+        src_re = src_re.replace('\*', '([-a-zA-Z0-9._ ]*)')
+        dst_temp = dst_glob
+        i = 1
+        while dst_temp.count("*") > 0:
+            dst_temp = dst_temp.replace('*', '\g<' + str(i) + '>', 1)
+            i = i+1
+        return re.compile(src_re), dst_temp
+
+    def check_file_exists(self, path):
+        if not os.path.exists(path) and not os.path.islink(path):
+            raise RuntimeError("Path %s doesn't exist" % (
+                os.path.normpath(os.path.join(os.getcwd(), path)),))
+
+
+    wildcard_pattern = re.compile('\*')
+    def expand_globs(self, src, dst):
+        src_list = glob.glob(src)
+        src_re, d_template = self.wildcard_regex(src.replace('\\', '/'),
+                                                 dst.replace('\\', '/'))
+        for s in src_list:
+            d = src_re.sub(d_template, s.replace('\\', '/'))
+            yield os.path.normpath(s), os.path.normpath(d)
+
+    def path(self, src, dst=None):
+        sys.stdout.write("Processing %s => %s ... " % (src, dst))
+        sys.stdout.flush()
+        if src == None:
+            raise RuntimeError("No source file, dst is " + dst)
+        if dst == None:
+            dst = src
+        dst = os.path.join(self.get_dst_prefix(), dst)
+        count = 0
+        is_glob = False
+
+        # look under each prefix for matching paths
+        paths = [os.path.join(self.get_src_prefix(), src),
+                 os.path.join(self.get_artwork_prefix(), src),
+                 os.path.join(self.get_build_prefix(), src)]
+        for path in paths:
+            if self.wildcard_pattern.search(path):
+                is_glob = True
+                for s,d in self.expand_globs(path, dst):
+                    assert(s != d)
+                    count += self.process_file(s, d)
+            else:
+                # if it's a directory, recurse through it
+                if os.path.isdir(path):
+                    count += self.process_directory(path, dst)
+                elif os.path.exists(path):
+                    count += self.process_file(path, dst)
+
+        # if we're specifying a single path (not a glob),
+        # we should error out if it doesn't exist
+        if count == 0 and not is_glob:
+            raise RuntimeError("No files match %s\n" % str(paths))
+
+        print "%d files" % count
+
+    def do(self, *actions):
+        self.actions = actions
+        self.construct()
+        # perform finish actions
+        for action in self.actions:
+            methodname = action + "_finish"
+            method = getattr(self, methodname, None)
+            if method is not None:
+                method()
+        return self.file_list

indra/lib/python/indra/util/llperformance.py

@@ -0,0 +1,158 @@
+#!/usr/bin/python
+
+# ------------------------------------------------
+# Sim metrics utility functions.
+
+import glob, os, time, sys, stat, exceptions
+
+from indra.base import llsd
+
+gBlockMap = {}              #Map of performance metric data with function hierarchy information.
+gCurrentStatPath = ""
+
+gIsLoggingEnabled=False
+
+class LLPerfStat:
+    def __init__(self,key):
+        self.mTotalTime = 0
+        self.mNumRuns = 0
+        self.mName=key
+        self.mTimeStamp = int(time.time()*1000)
+        self.mUTCTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+    def __str__(self):
+        return "%f" %  self.mTotalTime
+
+    def start(self):
+        self.mStartTime = int(time.time() * 1000000)
+        self.mNumRuns += 1
+
+    def stop(self):
+        execution_time = int(time.time() * 1000000) - self.mStartTime
+        self.mTotalTime += execution_time
+
+    def get_map(self):
+        results={}
+        results['name']=self.mName
+        results['utc_time']=self.mUTCTime
+        results['timestamp']=self.mTimeStamp
+        results['us']=self.mTotalTime
+        results['count']=self.mNumRuns
+        return results
+
+class PerfError(exceptions.Exception):
+    def __init__(self):
+        return
+
+    def __Str__(self):
+        print "","Unfinished LLPerfBlock"
+
+class LLPerfBlock:
+    def __init__( self, key ):
+        global gBlockMap
+        global gCurrentStatPath
+        global gIsLoggingEnabled
+
+        #Check to see if we're running metrics right now.
+        if gIsLoggingEnabled:
+            self.mRunning = True        #Mark myself as running.
+    
+            self.mPreviousStatPath = gCurrentStatPath
+            gCurrentStatPath += "/" + key
+            if gCurrentStatPath not in gBlockMap:
+                gBlockMap[gCurrentStatPath] = LLPerfStat(key)
+
+            self.mStat = gBlockMap[gCurrentStatPath]
+            self.mStat.start()
+    
+    def finish( self ):
+        global gBlockMap
+        global gIsLoggingEnabled
+
+        if gIsLoggingEnabled:
+            self.mStat.stop()
+            self.mRunning = False
+            gCurrentStatPath = self.mPreviousStatPath
+
+#    def __del__( self ):
+#        if self.mRunning:
+#            #SPATTERS FIXME
+#            raise PerfError
+
+class LLPerformance:
+    #--------------------------------------------------
+    # Determine whether or not we want to log statistics
+
+    def __init__( self, process_name = "python" ):
+        self.process_name = process_name
+        self.init_testing()
+        self.mTimeStamp = int(time.time()*1000)
+        self.mUTCTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+    def init_testing( self ):
+        global gIsLoggingEnabled
+
+        host_performance_file = "/dev/shm/simperf/simperf_proc_config.llsd"
+    
+        #If file exists, open
+        if os.path.exists(host_performance_file):
+            file = open (host_performance_file,'r')
+    
+            #Read serialized LLSD from file.
+            body = llsd.parse(file.read())
+    
+            #Calculate time since file last modified.
+            stats = os.stat(host_performance_file)
+            now = time.time()
+            mod = stats[stat.ST_MTIME]
+            age = now - mod
+    
+            if age < ( body['duration'] ):
+                gIsLoggingEnabled = True
+    
+
+    def get ( self ):
+        global gIsLoggingEnabled
+        return gIsLoggingEnabled
+
+    #def output(self,ptr,path):
+    #    if 'stats' in ptr:
+    #        stats = ptr['stats']
+    #        self.mOutputPtr[path] = stats.get_map()
+
+    #    if 'children' in ptr:
+    #        children=ptr['children']
+
+    #        curptr = self.mOutputPtr
+    #        curchildren={}
+    #        curptr['children'] = curchildren
+
+    #        for key in children:
+    #            curchildren[key]={}
+    #            self.mOutputPtr = curchildren[key]
+    #            self.output(children[key],path + '/' + key)
+    
+    def done(self):
+        global gBlockMap
+
+        if not self.get():
+            return
+
+        output_name = "/dev/shm/simperf/%s_proc.%d.llsd" % (self.process_name, os.getpid())
+        output_file = open(output_name, 'w')
+        process_info = {
+            "name"  :   self.process_name,
+            "pid"   :   os.getpid(),
+            "ppid"  :   os.getppid(),
+            "timestamp" :   self.mTimeStamp,
+            "utc_time"  :   self.mUTCTime,
+            }
+        output_file.write(llsd.format_notation(process_info))
+        output_file.write('\n')
+
+        for key in gBlockMap.keys():
+            gBlockMap[key] = gBlockMap[key].get_map()
+        output_file.write(llsd.format_notation(gBlockMap))
+        output_file.write('\n')
+        output_file.close()
+

indra/lib/python/indra/util/llsubprocess.py

@@ -0,0 +1,106 @@
+"""\
+@file llsubprocess.py
+@author Phoenix
+@date 2008-01-18
+@brief The simplest possible wrapper for a common sub-process paradigm.
+
+$LicenseInfo:firstyear=2007&license=mit$
+
+Copyright (c) 2007-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+import os
+import popen2
+import time
+import select
+
+class Timeout(RuntimeError):
+    "Exception raised when a subprocess times out."
+    pass
+
+def run(command, args=None, data=None, timeout=None):
+    """\
+@brief Run command with arguments
+
+This is it. This is the function I want to run all the time when doing
+subprocces, but end up copying the code everywhere. none of the
+standard commands are secure and provide a way to specify input, get
+all the output, and get the result.
+@param command A string specifying a process to launch.
+@param args Arguments to be passed to command. Must be list, tuple or None.
+@param data input to feed to the command.
+@param timeout Maximum number of many seconds to run.
+@return Returns (result, stdout, stderr) from process.
+"""
+    cmd = [command]
+    if args:
+        cmd.extend([str(arg) for arg in args])
+    #print  "cmd: ","' '".join(cmd)
+    child = popen2.Popen3(cmd, True)
+    #print child.pid
+    out = []
+    err = []
+    result = -1
+    time_left = timeout
+    tochild = [child.tochild.fileno()]
+    while True:
+        time_start = time.time()
+        #print "time:",time_left
+        p_in, p_out, p_err = select.select(
+            [child.fromchild.fileno(), child.childerr.fileno()],
+            tochild,
+            [],
+            time_left)
+        if p_in:
+            new_line = os.read(child.fromchild.fileno(), 32 * 1024)
+            if new_line:
+                #print "line:",new_line
+                out.append(new_line)
+            new_line = os.read(child.childerr.fileno(), 32 * 1024)
+            if new_line:
+                #print "error:", new_line
+                err.append(new_line)
+        if p_out:
+            if data:
+                #print "p_out"
+                bytes = os.write(child.tochild.fileno(), data)
+                data = data[bytes:]
+                if len(data) == 0:
+                    data = None
+                    tochild = []
+                    child.tochild.close()
+        result = child.poll()
+        if result != -1:
+            child.tochild.close()
+            child.fromchild.close()
+            child.childerr.close()
+            break
+        if time_left is not None:
+            time_left -= (time.time() - time_start)
+            if time_left < 0:
+                raise Timeout
+    #print "result:",result
+    out = ''.join(out)
+    #print "stdout:", out
+    err = ''.join(err)
+    #print "stderr:", err
+    return result, out, err

indra/lib/python/indra/util/llversion.py

@@ -0,0 +1,95 @@
+"""@file llversion.py
+@brief Utility for parsing llcommon/llversion${server}.h
+       for the version string and channel string
+       Utility that parses svn info for branch and revision
+
+$LicenseInfo:firstyear=2006&license=mit$
+
+Copyright (c) 2006-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+import re, sys, os, commands
+
+# Methods for gathering version information from
+# llversionviewer.h and llversionserver.h
+
+def get_src_root():
+    indra_lib_python_indra_path = os.path.dirname(__file__)
+    return os.path.abspath(os.path.realpath(indra_lib_python_indra_path + "/../../../../../"))
+
+def get_version_file_contents(version_type):
+    filepath = get_src_root() + '/indra/llcommon/llversion%s.h' % version_type
+    file = open(filepath,"r")
+    file_str = file.read()
+    file.close()
+    return file_str
+
+def get_version(version_type):
+    file_str = get_version_file_contents(version_type)
+    m = re.search('const S32 LL_VERSION_MAJOR = (\d+);', file_str)
+    VER_MAJOR = m.group(1)
+    m = re.search('const S32 LL_VERSION_MINOR = (\d+);', file_str)
+    VER_MINOR = m.group(1)
+    m = re.search('const S32 LL_VERSION_PATCH = (\d+);', file_str)
+    VER_PATCH = m.group(1)
+    m = re.search('const S32 LL_VERSION_BUILD = (\d+);', file_str)
+    VER_BUILD = m.group(1)
+    version = "%(VER_MAJOR)s.%(VER_MINOR)s.%(VER_PATCH)s.%(VER_BUILD)s" % locals()
+    return version
+
+def get_channel(version_type):
+    file_str = get_version_file_contents(version_type)
+    m = re.search('const char \* const LL_CHANNEL = "(.+)";', file_str)
+    return m.group(1)
+    
+def get_viewer_version():
+    return get_version('viewer')
+
+def get_server_version():
+    return get_version('server')
+
+def get_viewer_channel():
+    return get_channel('viewer')
+
+def get_server_channel():
+    return get_channel('server')
+
+# Methods for gathering subversion information
+def get_svn_status_matching(regular_expression):
+    # Get the subversion info from the working source tree
+    status, output = commands.getstatusoutput('svn info %s' % get_src_root())
+    m = regular_expression.search(output)
+    if not m:
+        print "Failed to parse svn info output, resultfollows:"
+        print output
+        raise Exception, "No matching svn status in "+src_root
+    return m.group(1)
+
+def get_svn_branch():
+    branch_re = re.compile('URL: (\S+)')
+    return get_svn_status_matching(branch_re)
+
+def get_svn_revision():
+    last_rev_re = re.compile('Last Changed Rev: (\d+)')
+    return get_svn_status_matching(last_rev_re)
+
+

indra/lib/python/indra/util/named_query.py

@@ -0,0 +1,572 @@
+"""\
+@file named_query.py
+@author Ryan Williams, Phoenix
+@date 2007-07-31
+@brief An API for running named queries.
+
+$LicenseInfo:firstyear=2007&license=mit$
+
+Copyright (c) 2007-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+import errno
+import MySQLdb
+import MySQLdb.cursors
+import os
+import os.path
+import re
+import time
+
+from indra.base import llsd
+from indra.base import config
+
+DEBUG = False
+
+NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq')
+NQ_FILE_SUFFIX_LEN  = len(NQ_FILE_SUFFIX)
+
+_g_named_manager = None
+
+def _init_g_named_manager(sql_dir = None):
+    """Initializes a global NamedManager object to point at a
+    specified named queries hierarchy.
+
+    This function is intended entirely for testing purposes,
+    because it's tricky to control the config from inside a test."""
+    if sql_dir is None:
+        sql_dir = config.get('named-query-base-dir')
+
+    # extra fallback directory in case config doesn't return what we want
+    if sql_dir is None:
+        sql_dir = os.path.abspath(
+            os.path.join(
+            os.path.realpath(os.path.dirname(__file__)), "..", "..", "..", "..", "web", "dataservice", "sql"))
+
+    global _g_named_manager
+    _g_named_manager = NamedQueryManager(
+        os.path.abspath(os.path.realpath(sql_dir)))
+
+def get(name):
+    "Get the named query object to be used to perform queries"
+    if _g_named_manager is None:
+        _init_g_named_manager()
+    return _g_named_manager.get(name)
+
+def sql(connection, name, params):
+    # use module-global NamedQuery object to perform default substitution
+    return get(name).sql(connection, params)
+
+def run(connection, name, params, expect_rows = None):
+    """\
+@brief given a connection, run a named query with the params
+
+Note that this function will fetch ALL rows.
+@param connection The connection to use
+@param name The name of the query to run
+@param params The parameters passed into the query
+@param expect_rows The number of rows expected. Set to 1 if return_as_map is true.  Raises ExpectationFailed if the number of returned rows doesn't exactly match.  Kind of a hack.
+@return Returns the result set as a list of dicts.
+"""
+    return get(name).run(connection, params, expect_rows)
+
+class ExpectationFailed(Exception):
+    """ Exception that is raised when an expectation for an sql query
+    is not met."""
+    def __init__(self, message):
+        Exception.__init__(self, message)
+        self.message = message
+
+class NamedQuery(object):
+    def __init__(self, name, filename):
+        """ Construct a NamedQuery object.  The name argument is an
+        arbitrary name as a handle for the query, and the filename is
+        a path to a file or a file-like object containing an llsd named
+        query document."""
+        self._stat_interval_seconds = 5  # 5 seconds
+        self._name = name
+        if (filename is not None and isinstance(filename, (str, unicode))
+            and NQ_FILE_SUFFIX != filename[-NQ_FILE_SUFFIX_LEN:]):
+            filename = filename + NQ_FILE_SUFFIX
+        self._location = filename
+        self._alternative = dict()
+        self._last_mod_time = 0
+        self._last_check_time = 0
+        self.deleted = False
+        self.load_contents()
+
+    def name(self):
+        """ The name of the query. """
+        return self._name
+
+    def get_modtime(self):
+        """ Returns the mtime (last modified time) of the named query
+        filename. For file-like objects, expect a modtime of 0"""
+        if self._location and isinstance(self._location, (str, unicode)):
+            return os.path.getmtime(self._location)
+        return 0
+
+    def load_contents(self):
+        """ Loads and parses the named query file into self.  Does
+        nothing if self.location is nonexistant."""
+        if self._location:
+            if isinstance(self._location, (str, unicode)):
+                contents = llsd.parse(open(self._location).read())
+            else:
+                # we probably have a file-like object. Godspeed!
+                contents = llsd.parse(self._location.read())
+            self._reference_contents(contents)
+            # Check for alternative implementations
+            try:
+                for name, alt in self._contents['alternative'].items():
+                    nq = NamedQuery(name, None)
+                    nq._reference_contents(alt)
+                    self._alternative[name] = nq
+            except KeyError, e:
+                pass
+            self._last_mod_time = self.get_modtime()
+            self._last_check_time = time.time()
+
+    def _reference_contents(self, contents):
+        "Helper method which builds internal structure from parsed contents"
+        self._contents = contents
+        self._ttl = int(self._contents.get('ttl', 0))
+        self._return_as_map = bool(self._contents.get('return_as_map', False))
+        self._legacy_dbname = self._contents.get('legacy_dbname', None)
+
+        # reset these before doing the sql conversion because we will
+        # read them there. reset these while loading so we pick up
+        # changes.
+        self._around = set()
+        self._append = set()
+        self._integer = set()
+        self._options = self._contents.get('dynamic_where', {})
+        for key in self._options:
+            if isinstance(self._options[key], basestring):
+                self._options[key] = self._convert_sql(self._options[key])
+            elif isinstance(self._options[key], list):
+                lines = []
+                for line in self._options[key]:
+                    lines.append(self._convert_sql(line))
+                self._options[key] = lines
+            else:
+                moreopt = {}
+                for kk in self._options[key]:
+                    moreopt[kk] = self._convert_sql(self._options[key][kk]) 
+                self._options[key] = moreopt
+        self._base_query = self._convert_sql(self._contents['base_query'])
+        self._query_suffix = self._convert_sql(
+            self._contents.get('query_suffix', ''))
+
+    def _convert_sql(self, sql):
+        """convert the parsed sql into a useful internal structure.
+
+        This function has to turn the named query format into a pyformat
+        style. It also has to look for %:name% and :name% and
+        ready them for use in LIKE statements"""
+        if sql:
+            # This first sub is to properly escape any % signs that
+            # are meant to be literally passed through to mysql in the
+            # query.  It leaves any %'s that are used for
+            # like-expressions.
+            expr = re.compile("(?<=[^a-zA-Z0-9_-])%(?=[^:])")
+            sql = expr.sub('%%', sql)
+
+            # This should tackle the rest of the %'s in the query, by
+            # converting them to LIKE clauses.
+            expr = re.compile("(%?):([a-zA-Z][a-zA-Z0-9_-]*)%")
+            sql = expr.sub(self._prepare_like, sql)
+            expr = re.compile("#:([a-zA-Z][a-zA-Z0-9_-]*)")
+            sql = expr.sub(self._prepare_integer, sql)
+            expr = re.compile(":([a-zA-Z][a-zA-Z0-9_-]*)")
+            sql = expr.sub("%(\\1)s", sql)
+        return sql
+
+    def _prepare_like(self, match):
+        """This function changes LIKE statement replace behavior
+
+        It works by turning %:name% to %(_name_around)s and :name% to
+        %(_name_append)s. Since a leading '_' is not a valid keyname
+        input (enforced via unit tests), it will never clash with
+        existing keys. Then, when building the statement, the query
+        runner will generate corrected strings."""
+        if match.group(1) == '%':
+            # there is a leading % so this is treated as prefix/suffix
+            self._around.add(match.group(2))
+            return "%(" + self._build_around_key(match.group(2)) + ")s"
+        else:
+            # there is no leading %, so this is suffix only
+            self._append.add(match.group(2))
+            return "%(" + self._build_append_key(match.group(2)) + ")s"
+
+    def _build_around_key(self, key):
+        return "_" + key + "_around"
+
+    def _build_append_key(self, key):
+        return "_" + key + "_append"
+
+    def _prepare_integer(self, match):
+        """This function adjusts the sql for #:name replacements
+
+        It works by turning #:name to %(_name_as_integer)s. Since a
+        leading '_' is not a valid keyname input (enforced via unit
+        tests), it will never clash with existing keys. Then, when
+        building the statement, the query runner will generate
+        corrected strings."""
+        self._integer.add(match.group(1))
+        return "%(" + self._build_integer_key(match.group(1)) + ")s"
+
+    def _build_integer_key(self, key):
+        return "_" + key + "_as_integer"
+
+    def _strip_wildcards_to_list(self, value):
+        """Take string, and strip out the LIKE special characters.
+
+        Technically, this is database dependant, but postgresql and
+        mysql use the same wildcards, and I am not aware of a general
+        way to handle this. I think you need a sql statement of the
+        form:
+
+        LIKE_STRING( [ANY,ONE,str]... )
+
+        which would treat ANY as their any string, and ONE as their
+        single glyph, and str as something that needs database
+        specific encoding to not allow any % or _ to affect the query.
+
+        As it stands, I believe it's impossible to write a named query
+        style interface which uses like to search the entire space of
+        text available. Imagine the query:
+
+        % of brain used by average linden
+
+        In order to search for %, it must be escaped, so once you have
+        escaped the string to not do wildcard searches, and be escaped
+        for the database, and then prepended the wildcard you come
+        back with one of:
+
+        1) %\% of brain used by average linden
+        2) %%% of brain used by average linden
+
+        Then, when passed to the database to be escaped to be database
+        safe, you get back:
+        
+        1) %\\% of brain used by average linden
+        : which means search for any character sequence, followed by a
+          backslash, followed by any sequence, followed by ' of
+          brain...'
+        2) %%% of brain used by average linden
+        : which (I believe) means search for a % followed by any
+          character sequence followed by 'of brain...'
+
+        Neither of which is what we want!
+
+        So, we need a vendor (or extention) for LIKE_STRING. Anyone
+        want to write it?"""
+        utf8_value = unicode(value, "utf-8")
+        esc_list = []
+        remove_chars = set(u"%_")
+        for glyph in utf8_value:
+            if glyph in remove_chars:
+                continue
+            esc_list.append(glyph.encode("utf-8"))
+        return esc_list
+
+    def delete(self):
+        """ Makes this query unusable by deleting all the members and
+        setting the deleted member.  This is desired when the on-disk
+        query has been deleted but the in-memory copy remains."""
+        # blow away all members except _name, _location, and deleted
+        name, location = self._name, self._location
+        for key in self.__dict__.keys():
+            del self.__dict__[key]
+        self.deleted = True
+        self._name, self._location = name, location
+
+    def ttl(self):
+        """ Estimated time to live of this query. Used for web
+        services to set the Expires header."""
+        return self._ttl
+
+    def legacy_dbname(self):
+        return self._legacy_dbname
+
+    def return_as_map(self):
+        """ Returns true if this query is configured to return its
+        results as a single map (as opposed to a list of maps, the
+        normal behavior)."""
+        
+        return self._return_as_map
+
+    def for_schema(self, db_name):
+        "Look trough the alternates and return the correct query"
+        try:
+            return self._alternative[db_name]
+        except KeyError, e:
+            pass
+        return self
+
+    def run(self, connection, params, expect_rows = None, use_dictcursor = True):
+        """given a connection, run a named query with the params
+
+        Note that this function will fetch ALL rows. We do this because it
+        opens and closes the cursor to generate the values, and this 
+        isn't a generator so the cursor has no life beyond the method call.
+
+        @param cursor The connection to use (this generates its own cursor for the query)
+        @param name The name of the query to run
+        @param params The parameters passed into the query
+        @param expect_rows The number of rows expected. Set to 1 if return_as_map is true.  Raises ExpectationFailed if the number of returned rows doesn't exactly match.  Kind of a hack.
+        @param use_dictcursor Set to false to use a normal cursor and manually convert the rows to dicts.
+        @return Returns the result set as a list of dicts, or, if the named query has return_as_map set to true, returns a single dict.
+        """
+        if use_dictcursor:
+            cursor = connection.cursor(MySQLdb.cursors.DictCursor)
+        else:
+            cursor = connection.cursor()
+        
+        statement = self.sql(connection, params)
+        if DEBUG:
+            print "SQL:", statement
+        rows = cursor.execute(statement)
+        
+        # *NOTE: the expect_rows argument is a very cheesy way to get some
+        # validation on the result set.  If you want to add more expectation
+        # logic, do something more object-oriented and flexible.  Or use an ORM.
+        if(self._return_as_map):
+            expect_rows = 1
+        if expect_rows is not None and rows != expect_rows:
+            cursor.close()
+            raise ExpectationFailed("Statement expected %s rows, got %s.  Sql: %s" % (
+                expect_rows, rows, statement))
+
+        # convert to dicts manually if we're not using a dictcursor
+        if use_dictcursor:
+            result_set = cursor.fetchall()
+        else:
+            if cursor.description is None:
+                # an insert or something
+                x = cursor.fetchall()
+                cursor.close()
+                return x
+
+            names = [x[0] for x in cursor.description]
+
+            result_set = []
+            for row in cursor.fetchall():
+                converted_row = {}
+                for idx, col_name in enumerate(names):
+                    converted_row[col_name] = row[idx]
+                result_set.append(converted_row)
+
+        cursor.close()
+        if self._return_as_map:
+            return result_set[0]
+        return result_set
+
+    def sql(self, connection, params):
+        """ Generates an SQL statement from the named query document
+        and a dictionary of parameters.
+
+        """
+        self.refresh()
+
+        # build the query from the options available and the params
+        base_query = []
+        base_query.append(self._base_query)
+        for opt, extra_where in self._options.items():
+            if type(extra_where) in (dict, list, tuple):
+                if opt in params:
+                    base_query.append(extra_where[params[opt]])
+            else:
+                if opt in params and params[opt]:
+                    base_query.append(extra_where)
+        if self._query_suffix:
+            base_query.append(self._query_suffix)
+        full_query = '\n'.join(base_query)
+
+        # Go through the query and rewrite all of the ones with the
+        # @:name syntax.
+        rewrite = _RewriteQueryForArray(params)
+        expr = re.compile("@%\(([a-zA-Z][a-zA-Z0-9_-]*)\)s")
+        full_query = expr.sub(rewrite.operate, full_query)
+        params.update(rewrite.new_params)
+
+        # build out the params for like. We only have to do this
+        # parameters which were detected to have ued the where syntax
+        # during load.
+        #
+        # * treat the incoming string as utf-8
+        # * strip wildcards
+        # * append or prepend % as appropriate
+        new_params = {}
+        for key in params:
+            if key in self._around:
+                new_value = ['%']
+                new_value.extend(self._strip_wildcards_to_list(params[key]))
+                new_value.append('%')
+                new_params[self._build_around_key(key)] = ''.join(new_value)
+            if key in self._append:
+                new_value = self._strip_wildcards_to_list(params[key])
+                new_value.append('%')
+                new_params[self._build_append_key(key)] = ''.join(new_value)
+            if key in self._integer:
+                new_params[self._build_integer_key(key)] = int(params[key])
+        params.update(new_params)
+
+        # do substitution using the mysql (non-standard) 'literal'
+        # function to do the escaping.
+        sql = full_query % connection.literal(params)
+        return sql
+
+    def refresh(self):
+        """ Refresh self from the file on the filesystem.
+
+        This is optimized to be callable as frequently as you wish,
+        without adding too much load.  It does so by only stat-ing the
+        file every N seconds, where N defaults to 5 and is
+        configurable through the member _stat_interval_seconds.  If the stat
+        reveals that the file has changed, refresh will re-parse the
+        contents of the file and use them to update the named query
+        instance.  If the stat reveals that the file has been deleted,
+        refresh will call self.delete to make the in-memory
+        representation unusable."""
+        now = time.time()
+        if(now - self._last_check_time > self._stat_interval_seconds):
+            self._last_check_time = now
+            try:
+                modtime = self.get_modtime()
+                if(modtime > self._last_mod_time):
+                    self.load_contents()
+            except OSError, e:
+                if e.errno == errno.ENOENT: # file not found
+                    self.delete() # clean up self
+                raise  # pass the exception along to the caller so they know that this query disappeared
+
+class NamedQueryManager(object):
+    """ Manages the lifespan of NamedQuery objects, drawing from a
+    directory hierarchy of named query documents.
+
+    In practice this amounts to a memory cache of NamedQuery objects."""
+    
+    def __init__(self, named_queries_dir):
+        """ Initializes a manager to look for named queries in a
+        directory."""
+        self._dir = os.path.abspath(os.path.realpath(named_queries_dir))
+        self._cached_queries = {}
+
+    def sql(self, connection, name, params):
+        nq = self.get(name)
+        return nq.sql(connection, params)
+        
+    def get(self, name):
+        """ Returns a NamedQuery instance based on the name, either
+        from memory cache, or by parsing from disk.
+
+        The name is simply a relative path to the directory associated
+        with the manager object.  Before returning the instance, the
+        NamedQuery object is cached in memory, so that subsequent
+        accesses don't have to read from disk or do any parsing.  This
+        means that NamedQuery objects returned by this method are
+        shared across all users of the manager object.
+        NamedQuery.refresh is used to bring the NamedQuery objects in
+        sync with the actual files on disk."""
+        nq = self._cached_queries.get(name)
+        if nq is None:
+            nq = NamedQuery(name, os.path.join(self._dir, name))
+            self._cached_queries[name] = nq
+        else:
+            try:
+                nq.refresh()
+            except OSError, e:
+                if e.errno == errno.ENOENT: # file not found
+                    del self._cached_queries[name]
+                raise # pass exception along to caller so they know that the query disappeared
+
+        return nq
+
+class _RewriteQueryForArray(object):
+    "Helper class for rewriting queries with the @:name syntax"
+    def __init__(self, params):
+        self.params = params
+        self.new_params = dict()
+
+    def operate(self, match):
+        "Given a match, return the string that should be in use"
+        key = match.group(1)
+        value = self.params[key]
+        if type(value) in (list,tuple):
+            rv = []
+            for idx in range(len(value)):
+                # if the value@idx is array-like, we are
+                # probably dealing with a VALUES
+                new_key = "_%s_%s"%(key, str(idx))
+                val_item = value[idx]
+                if type(val_item) in (list, tuple, dict):
+                    if type(val_item) is dict:
+                        # this is because in Python, the order of 
+                        # key, value retrieval from the dict is not
+                        # guaranteed to match what the input intended
+                        # and for VALUES, order is important.
+                        # TODO: Implemented ordered dict in LLSD parser?
+                        raise ExpectationFailed('Only lists/tuples allowed,\
+                                received dict')
+                    values_keys = []
+                    for value_idx, item in enumerate(val_item):
+                        # we want a key of the format :
+                        # key_#replacement_#value_row_#value_col
+                        # ugh... so if we are replacing 10 rows in user_note, 
+                        # the first values clause would read (for @:user_notes) :-
+                        # ( :_user_notes_0_1_1,  :_user_notes_0_1_2, :_user_notes_0_1_3 )
+                        # the input LLSD for VALUES will look like:
+                        # <llsd>...
+                        # <map>
+                        #  <key>user_notes</key>
+                        #      <array>
+                        #      <array> <!-- row 1 for VALUES -->
+                        #          <string>...</string>
+                        #          <string>...</string>
+                        #          <string>...</string>
+                        #      </array>
+                        # ...
+                        #      </array>
+                        # </map>
+                        # ... </llsd>
+                        values_key = "%s_%s"%(new_key, value_idx)
+                        self.new_params[values_key] = item
+                        values_keys.append("%%(%s)s"%values_key)
+                    # now collapse all these new place holders enclosed in ()
+                    # from [':_key_0_1_1', ':_key_0_1_2', ':_key_0_1_3,...] 
+                    # rv will have [ '(:_key_0_1_1, :_key_0_1_2, :_key_0_1_3)', ]
+                    # which is flattened a few lines below join(rv)
+                    rv.append('(%s)' % ','.join(values_keys))
+                else:
+                    self.new_params[new_key] = val_item
+                    rv.append("%%(%s)s"%new_key)
+            return ','.join(rv)
+        else:
+            # not something that can be expanded, so just drop the
+            # leading @ in the front of the match. This will mean that
+            # the single value we have, be it a string, int, whatever
+            # (other than dict) will correctly show up, eg:
+            #
+            # where foo in (@:foobar) -- foobar is a string, so we get
+            # where foo in (:foobar)
+            return match.group(0)[1:]

indra/lib/python/indra/util/shutil2.py

@@ -0,0 +1,84 @@
+'''
+@file shutil2.py
+@brief a better shutil.copytree replacement
+
+$LicenseInfo:firstyear=2007&license=mit$
+
+Copyright (c) 2007-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+'''
+
+#
+# shutil2.py
+# Taken from http://www.scons.org/wiki/AccumulateBuilder
+# the stock copytree sucks because it insists that the
+# target dir not exist
+#
+
+import os.path
+import shutil
+
+def copytree(src, dest, symlinks=False):
+    """My own copyTree which does not fail if the directory exists.
+    
+    Recursively copy a directory tree using copy2().
+
+    If the optional symlinks flag is true, symbolic links in the
+    source tree result in symbolic links in the destination tree; if
+    it is false, the contents of the files pointed to by symbolic
+    links are copied.
+    
+    Behavior is meant to be identical to GNU 'cp -R'.    
+    """
+    def copyItems(src, dest, symlinks=False):
+        """Function that does all the work.
+        
+        It is necessary to handle the two 'cp' cases:
+        - destination does exist
+        - destination does not exist
+        
+        See 'cp -R' documentation for more details
+        """
+        for item in os.listdir(src):
+           srcPath = os.path.join(src, item)
+           if os.path.isdir(srcPath):
+               srcBasename = os.path.basename(srcPath)
+               destDirPath = os.path.join(dest, srcBasename)
+               if not os.path.exists(destDirPath):
+                   os.makedirs(destDirPath)
+               copyItems(srcPath, destDirPath)
+           elif os.path.islink(item) and symlinks:
+               linkto = os.readlink(item)
+               os.symlink(linkto, dest)
+           else:
+               shutil.copy2(srcPath, dest)
+
+    # case 'cp -R src/ dest/' where dest/ already exists
+    if os.path.exists(dest):
+       destPath = os.path.join(dest, os.path.basename(src))
+       if not os.path.exists(destPath):
+           os.makedirs(destPath)
+    # case 'cp -R src/ dest/' where dest/ does not exist
+    else:
+       os.makedirs(dest)
+       destPath = dest
+    # actually copy the files
+    copyItems(src, destPath)

indra/lib/python/indra/util/simperf_host_xml_parser.py

@@ -0,0 +1,338 @@
+#!/usr/bin/env python
+"""\
+@file simperf_host_xml_parser.py
+@brief Digest collector's XML dump and convert to simple dict/list structure
+
+$LicenseInfo:firstyear=2008&license=mit$
+
+Copyright (c) 2008-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+"""
+
+import sys, os, getopt, time
+import simplejson
+from xml import sax
+
+
+def usage():
+    print "Usage:"
+    print sys.argv[0] + " [options]"
+    print "  Convert RRD's XML dump to JSON.  Script to convert the simperf_host_collector-"
+    print "  generated RRD dump into JSON.  Steps include converting selected named"
+    print "  fields from GAUGE type to COUNTER type by computing delta with preceding"
+    print "  values.  Top-level named fields are:"
+    print 
+    print "     lastupdate      Time (javascript timestamp) of last data sample"
+    print "     step            Time in seconds between samples"
+    print "     ds              Data specification (name/type) for each column"
+    print "     database        Table of data samples, one time step per row"
+    print 
+    print "Options:"
+    print "  -i, --in      Input settings filename.  (Default:  stdin)"
+    print "  -o, --out     Output settings filename.  (Default:  stdout)"
+    print "  -h, --help    Print this message and exit."
+    print
+    print "Example: %s -i rrddump.xml -o rrddump.json" % sys.argv[0]
+    print
+    print "Interfaces:"
+    print "   class SimPerfHostXMLParser()         # SAX content handler"
+    print "   def simperf_host_xml_fixup(parser)   # post-parse value fixup"
+
+class SimPerfHostXMLParser(sax.handler.ContentHandler):
+
+    def __init__(self):
+        pass
+        
+    def startDocument(self):
+        self.rrd_last_update = 0         # public
+        self.rrd_step = 0                # public
+        self.rrd_ds = []                 # public
+        self.rrd_records = []            # public
+        self._rrd_level = 0
+        self._rrd_parse_state = 0
+        self._rrd_chars = ""
+        self._rrd_capture = False
+        self._rrd_ds_val = {}
+        self._rrd_data_row = []
+        self._rrd_data_row_has_nan = False
+        
+    def endDocument(self):
+        pass
+
+    # Nasty little ad-hoc state machine to extract the elements that are
+    # necessary from the 'rrdtool dump' XML output.  The same element
+    # name '<ds>' is used for two different data sets so we need to pay
+    # some attention to the actual structure to get the ones we want
+    # and ignore the ones we don't.
+    
+    def startElement(self, name, attrs):
+        self._rrd_level = self._rrd_level + 1
+        self._rrd_capture = False
+        if self._rrd_level == 1:
+            if name == "rrd" and self._rrd_parse_state == 0:
+                self._rrd_parse_state = 1     # In <rrd>
+                self._rrd_capture = True
+                self._rrd_chars = ""
+        elif self._rrd_level == 2:
+            if self._rrd_parse_state == 1:
+                if name == "lastupdate":
+                    self._rrd_parse_state = 2         # In <rrd><lastupdate>
+                    self._rrd_capture = True
+                    self._rrd_chars = ""
+                elif name == "step":
+                    self._rrd_parse_state = 3         # In <rrd><step>
+                    self._rrd_capture = True
+                    self._rrd_chars = ""
+                elif name == "ds":
+                    self._rrd_parse_state = 4         # In <rrd><ds>
+                    self._rrd_ds_val = {}
+                    self._rrd_chars = ""
+                elif name == "rra":
+                    self._rrd_parse_state = 5         # In <rrd><rra>
+        elif self._rrd_level == 3:
+            if self._rrd_parse_state == 4:
+                if name == "name":
+                    self._rrd_parse_state = 6         # In <rrd><ds><name>
+                    self._rrd_capture = True
+                    self._rrd_chars = ""
+                elif name == "type":
+                    self._rrd_parse_state = 7         # In <rrd><ds><type>
+                    self._rrd_capture = True
+                    self._rrd_chars = ""
+            elif self._rrd_parse_state == 5:
+                if name == "database":
+                    self._rrd_parse_state = 8         # In <rrd><rra><database>
+        elif self._rrd_level == 4:
+            if self._rrd_parse_state == 8:
+                if name == "row":
+                    self._rrd_parse_state = 9         # In <rrd><rra><database><row>
+                    self._rrd_data_row = []
+                    self._rrd_data_row_has_nan = False
+        elif self._rrd_level == 5:
+            if self._rrd_parse_state == 9:
+                if name == "v":
+                    self._rrd_parse_state = 10        # In <rrd><rra><database><row><v>
+                    self._rrd_capture = True
+                    self._rrd_chars = ""
+
+    def endElement(self, name):
+        self._rrd_capture = False
+        if self._rrd_parse_state == 10:
+            self._rrd_capture = self._rrd_level == 6
+            if self._rrd_level == 5:
+                if self._rrd_chars == "NaN":
+                    self._rrd_data_row_has_nan = True
+                else:
+                    self._rrd_data_row.append(self._rrd_chars)
+                self._rrd_parse_state = 9              # In <rrd><rra><database><row>
+        elif self._rrd_parse_state == 9:
+            if self._rrd_level == 4:
+                if not self._rrd_data_row_has_nan:
+                    self.rrd_records.append(self._rrd_data_row)
+                self._rrd_parse_state = 8              # In <rrd><rra><database>
+        elif self._rrd_parse_state == 8:
+            if self._rrd_level == 3:
+                self._rrd_parse_state = 5              # In <rrd><rra>
+        elif self._rrd_parse_state == 7:
+            if self._rrd_level == 3:
+                self._rrd_ds_val["type"] = self._rrd_chars
+                self._rrd_parse_state = 4              # In <rrd><ds>
+        elif self._rrd_parse_state == 6:
+            if self._rrd_level == 3:
+                self._rrd_ds_val["name"] = self._rrd_chars
+                self._rrd_parse_state = 4              # In <rrd><ds>
+        elif self._rrd_parse_state == 5:
+            if self._rrd_level == 2:
+                self._rrd_parse_state = 1              # In <rrd>
+        elif self._rrd_parse_state == 4:
+            if self._rrd_level == 2:
+                self.rrd_ds.append(self._rrd_ds_val)
+                self._rrd_parse_state = 1              # In <rrd>
+        elif self._rrd_parse_state == 3:
+            if self._rrd_level == 2:
+                self.rrd_step = long(self._rrd_chars)
+                self._rrd_parse_state = 1              # In <rrd>
+        elif self._rrd_parse_state == 2:
+            if self._rrd_level == 2:
+                self.rrd_last_update = long(self._rrd_chars)
+                self._rrd_parse_state = 1              # In <rrd>
+        elif self._rrd_parse_state == 1:
+            if self._rrd_level == 1:
+                self._rrd_parse_state = 0              # At top
+                
+        if self._rrd_level:
+            self._rrd_level = self._rrd_level - 1
+
+    def characters(self, content):
+        if self._rrd_capture:
+            self._rrd_chars = self._rrd_chars + content.strip()
+
+def _make_numeric(value):
+    try:
+        value = float(value)
+    except:
+        value = ""
+    return value
+
+def simperf_host_xml_fixup(parser, filter_start_time = None, filter_end_time = None):
+    # Fixup for GAUGE fields that are really COUNTS.  They
+    # were forced to GAUGE to try to disable rrdtool's
+    # data interpolation/extrapolation for non-uniform time
+    # samples.
+    fixup_tags = [ "cpu_user",
+                   "cpu_nice",
+                   "cpu_sys",
+                   "cpu_idle",
+                   "cpu_waitio",
+                   "cpu_intr",
+                   # "file_active",
+                   # "file_free",
+                   # "inode_active",
+                   # "inode_free",
+                   "netif_in_kb",
+                   "netif_in_pkts",
+                   "netif_in_errs",
+                   "netif_in_drop",
+                   "netif_out_kb",
+                   "netif_out_pkts",
+                   "netif_out_errs",
+                   "netif_out_drop",
+                   "vm_page_in",
+                   "vm_page_out",
+                   "vm_swap_in",
+                   "vm_swap_out",
+                   #"vm_mem_total",
+                   #"vm_mem_used",
+                   #"vm_mem_active",
+                   #"vm_mem_inactive",
+                   #"vm_mem_free",
+                   #"vm_mem_buffer",
+                   #"vm_swap_cache",
+                   #"vm_swap_total",
+                   #"vm_swap_used",
+                   #"vm_swap_free",
+                   "cpu_interrupts",
+                   "cpu_switches",
+                   "cpu_forks" ]
+
+    col_count = len(parser.rrd_ds)
+    row_count = len(parser.rrd_records)
+
+    # Process the last row separately, just to make all values numeric.
+    for j in range(col_count):
+        parser.rrd_records[row_count - 1][j] = _make_numeric(parser.rrd_records[row_count - 1][j])
+
+    # Process all other row/columns.
+    last_different_row = row_count - 1
+    current_row = row_count - 2
+    while current_row >= 0:
+        # Check for a different value than the previous row.  If everything is the same
+        # then this is probably just a filler/bogus entry.
+        is_different = False
+        for j in range(col_count):
+            parser.rrd_records[current_row][j] = _make_numeric(parser.rrd_records[current_row][j])
+            if parser.rrd_records[current_row][j] != parser.rrd_records[last_different_row][j]:
+                # We're good.  This is a different row.
+                is_different = True
+
+        if not is_different:
+            # This is a filler/bogus entry.  Just ignore it.
+            for j in range(col_count):
+                parser.rrd_records[current_row][j] = float('nan')
+        else:
+            # Some tags need to be converted into deltas.
+            for j in range(col_count):
+                if parser.rrd_ds[j]["name"] in fixup_tags:
+                    parser.rrd_records[last_different_row][j] = \
+                        parser.rrd_records[last_different_row][j] - parser.rrd_records[current_row][j]
+            last_different_row = current_row
+
+        current_row -= 1
+
+    # Set fixup_tags in the first row to 'nan' since they aren't useful anymore.
+    for j in range(col_count):
+        if parser.rrd_ds[j]["name"] in fixup_tags:
+            parser.rrd_records[0][j] = float('nan')
+
+    # Add a timestamp to each row and to the catalog.  Format and name
+    # chosen to match other simulator logging (hopefully).
+    start_time = parser.rrd_last_update - (parser.rrd_step * (row_count - 1))
+    # Build a filtered list of rrd_records if we are limited to a time range.
+    filter_records = False
+    if filter_start_time is not None or filter_end_time is not None:
+        filter_records = True
+        filtered_rrd_records = []
+        if filter_start_time is None:
+            filter_start_time = start_time * 1000
+        if filter_end_time is None:
+            filter_end_time = parser.rrd_last_update * 1000
+        
+    for i in range(row_count):
+        record_timestamp = (start_time + (i * parser.rrd_step)) * 1000
+        parser.rrd_records[i].insert(0, record_timestamp)
+        if filter_records:
+            if filter_start_time <= record_timestamp and record_timestamp <= filter_end_time:
+                filtered_rrd_records.append(parser.rrd_records[i])
+
+    if filter_records:
+        parser.rrd_records = filtered_rrd_records
+
+    parser.rrd_ds.insert(0, {"type": "GAUGE", "name": "javascript_timestamp"})
+
+
+def main(argv=None):
+    opts, args = getopt.getopt(sys.argv[1:], "i:o:h", ["in=", "out=", "help"])
+    input_file = sys.stdin
+    output_file = sys.stdout
+    for o, a in opts:
+        if o in ("-i", "--in"):
+            input_file = open(a, 'r')
+        if o in ("-o", "--out"):
+            output_file = open(a, 'w')
+        if o in ("-h", "--help"):
+            usage()
+            sys.exit(0)
+
+    # Using the SAX parser as it is at least 4X faster and far, far
+    # smaller on this dataset than the DOM-based interface in xml.dom.minidom.
+    # With SAX and a 5.4MB xml file, this requires about seven seconds of
+    # wall-clock time and 32MB VSZ.  With the DOM interface, about 22 seconds
+    # and over 270MB VSZ.
+
+    handler = SimPerfHostXMLParser()
+    sax.parse(input_file, handler)
+    if input_file != sys.stdin:
+        input_file.close()
+
+    # Various format fixups:  string-to-num, gauge-to-counts, add
+    # a time stamp, etc.
+    simperf_host_xml_fixup(handler)
+    
+    # Create JSONable dict with interesting data and format/print it
+    print >>output_file, simplejson.dumps({ "step" : handler.rrd_step,
+                                            "lastupdate": handler.rrd_last_update * 1000,
+                                            "ds" : handler.rrd_ds,
+                                            "database" : handler.rrd_records })
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

indra/lib/python/indra/util/simperf_oprof_interface.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+"""\
+@file simperf_oprof_interface.py
+@brief Manage OProfile data collection on a host
+
+$LicenseInfo:firstyear=2008&license=internal$
+
+Copyright (c) 2008-2009, Linden Research, Inc.
+
+The following source code is PROPRIETARY AND CONFIDENTIAL. Use of
+this source code is governed by the Linden Lab Source Code Disclosure
+Agreement ("Agreement") previously entered between you and Linden
+Lab. By accessing, using, copying, modifying or distributing this
+software, you acknowledge that you have been informed of your
+obligations under the Agreement and agree to abide by those obligations.
+
+ALL LINDEN LAB SOURCE CODE IS PROVIDED "AS IS." LINDEN LAB MAKES NO
+WARRANTIES, EXPRESS, IMPLIED OR OTHERWISE, REGARDING ITS ACCURACY,
+COMPLETENESS OR PERFORMANCE.
+$/LicenseInfo$
+"""
+
+import sys, os, getopt
+import simplejson
+
+
+def usage():
+    print "Usage:"
+    print sys.argv[0] + " [options]"
+    print "  Digest the OProfile report forms that come out of the"
+    print "  simperf_oprof_ctl program's -r/--report command.  The result"
+    print "  is an array of dictionaires with the following keys:"
+    print 
+    print "     symbol        Name of sampled, calling, or called procedure"
+    print "     file          Executable or library where symbol resides"
+    print "     percentage    Percentage contribution to profile, calls or called"
+    print "     samples       Sample count"
+    print "     calls         Methods called by the method in question (full only)"
+    print "     called_by     Methods calling the method (full only)"
+    print 
+    print "  For 'full' reports the two keys 'calls' and 'called_by' are"
+    print "  themselves arrays of dictionaries based on the first four keys."
+    print
+    print "Return Codes:"
+    print "  None.  Aggressively digests everything.  Will likely mung results"
+    print "  if a program or library has whitespace in its name."
+    print
+    print "Options:"
+    print "  -i, --in      Input settings filename.  (Default:  stdin)"
+    print "  -o, --out     Output settings filename.  (Default:  stdout)"
+    print "  -h, --help    Print this message and exit."
+    print
+    print "Interfaces:"
+    print "   class SimPerfOProfileInterface()"
+    
+class SimPerfOProfileInterface:
+    def __init__(self):
+        self.isBrief = True             # public
+        self.isValid = False            # public
+        self.result = []                # public
+
+    def parse(self, input):
+        in_samples = False
+        for line in input:
+            if in_samples:
+                if line[0:6] == "------":
+                    self.isBrief = False
+                    self._parseFull(input)
+                else:
+                    self._parseBrief(input, line)
+                self.isValid = True
+                return
+            try:
+                hd1, remain = line.split(None, 1)
+                if hd1 == "samples":
+                    in_samples = True
+            except ValueError:
+                pass
+
+    def _parseBrief(self, input, line1):
+        try:
+            fld1, fld2, fld3, fld4 = line1.split(None, 3)
+            self.result.append({"samples" : fld1,
+                                "percentage" : fld2,
+                                "file" : fld3,
+                                "symbol" : fld4.strip("\n")})
+        except ValueError:
+            pass
+        for line in input:
+            try:
+                fld1, fld2, fld3, fld4 = line.split(None, 3)
+                self.result.append({"samples" : fld1,
+                                    "percentage" : fld2,
+                                    "file" : fld3,
+                                    "symbol" : fld4.strip("\n")})
+            except ValueError:
+                pass
+
+    def _parseFull(self, input):
+        state = 0       # In 'called_by' section
+        calls = []
+        called_by = []
+        current = {}
+        for line in input:
+            if line[0:6] == "------":
+                if len(current):
+                    current["calls"] = calls
+                    current["called_by"] = called_by
+                    self.result.append(current)
+                state = 0
+                calls = []
+                called_by = []
+                current = {}
+            else:
+                try:
+                    fld1, fld2, fld3, fld4 = line.split(None, 3)
+                    tmp = {"samples" : fld1,
+                           "percentage" : fld2,
+                           "file" : fld3,
+                           "symbol" : fld4.strip("\n")}
+                except ValueError:
+                    continue
+                if line[0] != " ":
+                    current = tmp
+                    state = 1       # In 'calls' section
+                elif state == 0:
+                    called_by.append(tmp)
+                else:
+                    calls.append(tmp)
+        if len(current):
+            current["calls"] = calls
+            current["called_by"] = called_by
+            self.result.append(current)
+
+
+def main(argv=None):
+    opts, args = getopt.getopt(sys.argv[1:], "i:o:h", ["in=", "out=", "help"])
+    input_file = sys.stdin
+    output_file = sys.stdout
+    for o, a in opts:
+        if o in ("-i", "--in"):
+            input_file = open(a, 'r')
+        if o in ("-o", "--out"):
+            output_file = open(a, 'w')
+        if o in ("-h", "--help"):
+            usage()
+            sys.exit(0)
+
+    oprof = SimPerfOProfileInterface()
+    oprof.parse(input_file)
+    if input_file != sys.stdin:
+        input_file.close()
+
+    # Create JSONable dict with interesting data and format/print it
+    print >>output_file, simplejson.dumps(oprof.result)
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())

indra/lib/python/indra/util/simperf_proc_interface.py

@@ -0,0 +1,164 @@
+#!/usr/bin/python
+
+# ----------------------------------------------------
+# Utility to extract log messages from *.<pid>.llsd
+# files that contain performance statistics.
+
+# ----------------------------------------------------
+import sys, os
+
+if os.path.exists("setup-path.py"):
+    execfile("setup-path.py")
+
+from indra.base import llsd
+
+DEFAULT_PATH="/dev/shm/simperf/"
+
+
+# ----------------------------------------------------
+# Pull out the stats and return a single document
+def parse_logfile(filename, target_column=None, verbose=False):
+    full_doc = []
+    # Open source temp log file.  Let exceptions percolate up.
+    sourcefile = open( filename,'r')
+        
+    if verbose:
+        print "Reading " + filename  
+    
+    # Parse and output all lines from the temp file
+    for line in sourcefile.xreadlines():
+        partial_doc = llsd.parse(line)
+        if partial_doc is not None:
+            if target_column is None:
+                full_doc.append(partial_doc)
+            else:
+                trim_doc = { target_column: partial_doc[target_column] }
+                if target_column != "fps":
+                    trim_doc[ 'fps' ] = partial_doc[ 'fps' ]
+                trim_doc[ '/total_time' ] = partial_doc[ '/total_time' ]
+                trim_doc[ 'utc_time' ] = partial_doc[ 'utc_time' ]
+                full_doc.append(trim_doc)
+
+    sourcefile.close()
+    return full_doc
+
+# Extract just the meta info line, and the timestamp of the first/last frame entry.
+def parse_logfile_info(filename, verbose=False):
+    # Open source temp log file.  Let exceptions percolate up.
+    sourcefile = open(filename, 'rU') # U is to open with Universal newline support
+        
+    if verbose:
+        print "Reading " + filename  
+
+    # The first line is the meta info line.
+    info_line = sourcefile.readline()
+    if not info_line:
+        sourcefile.close()
+        return None
+
+    # The rest of the lines are frames.  Read the first and last to get the time range.
+    info = llsd.parse( info_line )
+    info['start_time'] = None
+    info['end_time'] = None
+    first_frame = sourcefile.readline()
+    if first_frame:
+        try:
+            info['start_time'] = int(llsd.parse(first_frame)['timestamp'])
+        except:
+            pass
+
+    # Read the file backwards to find the last two lines.
+    sourcefile.seek(0, 2)
+    file_size = sourcefile.tell()
+    offset = 1024
+    num_attempts = 0
+    end_time = None
+    if file_size < offset:
+        offset = file_size
+    while 1:
+        sourcefile.seek(-1*offset, 2)
+        read_str = sourcefile.read(offset)
+        # Remove newline at the end
+        if read_str[offset - 1] == '\n':
+            read_str = read_str[0:-1]
+        lines = read_str.split('\n')
+        full_line = None
+        if len(lines) > 2:  # Got two line
+            try:
+                end_time = llsd.parse(lines[-1])['timestamp']
+            except:
+                # We couldn't parse this line.  Try once more.
+                try:
+                    end_time = llsd.parse(lines[-2])['timestamp']
+                except:
+                    # Nope.  Just move on.
+                    pass
+            break
+        if len(read_str) == file_size:   # Reached the beginning
+            break
+        offset += 1024
+
+    info['end_time'] = int(end_time)
+
+    sourcefile.close()
+    return info
+    
+
+def parse_proc_filename(filename):
+    try:
+        name_as_list = filename.split(".")
+        cur_stat_type = name_as_list[0].split("_")[0]
+        cur_pid = name_as_list[1]
+    except IndexError, ValueError:
+        return (None, None)
+    return (cur_pid, cur_stat_type)
+
+# ----------------------------------------------------
+def get_simstats_list(path=None):
+    """ Return stats (pid, type) listed in <type>_proc.<pid>.llsd """
+    if path is None:
+        path = DEFAULT_PATH
+    simstats_list = []
+    for file_name in os.listdir(path):
+        if file_name.endswith(".llsd") and file_name != "simperf_proc_config.llsd":
+            simstats_info = parse_logfile_info(path + file_name)
+            if simstats_info is not None:
+                simstats_list.append(simstats_info)
+    return simstats_list
+
+def get_log_info_list(pid=None, stat_type=None, path=None, target_column=None, verbose=False):
+    """ Return data from all llsd files matching the pid and stat type """
+    if path is None:
+        path = DEFAULT_PATH
+    log_info_list = {}
+    for file_name in os.listdir ( path ):
+        if file_name.endswith(".llsd") and file_name != "simperf_proc_config.llsd":
+            (cur_pid, cur_stat_type) = parse_proc_filename(file_name)
+            if cur_pid is None:
+                continue
+            if pid is not None and pid != cur_pid:
+                continue
+            if stat_type is not None and stat_type != cur_stat_type:
+                continue
+            log_info_list[cur_pid] = parse_logfile(path + file_name, target_column, verbose)
+    return log_info_list
+
+def delete_simstats_files(pid=None, stat_type=None, path=None):
+    """ Delete *.<pid>.llsd files """
+    if path is None:
+        path = DEFAULT_PATH
+    del_list = []
+    for file_name in os.listdir(path):
+        if file_name.endswith(".llsd") and file_name != "simperf_proc_config.llsd":
+            (cur_pid, cur_stat_type) = parse_proc_filename(file_name)
+            if cur_pid is None:
+                continue
+            if pid is not None and pid != cur_pid:
+                continue
+            if stat_type is not None and stat_type != cur_stat_type:
+                continue
+            del_list.append(cur_pid)
+            # Allow delete related exceptions to percolate up if this fails.
+            os.unlink(os.path.join(DEFAULT_PATH, file_name))
+    return del_list
+

indra/lib/python/indra/util/term.py

@@ -0,0 +1,222 @@
+'''
+@file term.py
+@brief a better shutil.copytree replacement
+
+$LicenseInfo:firstyear=2007&license=mit$
+
+Copyright (c) 2007-2009, Linden Research, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+$/LicenseInfo$
+'''
+
+#http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/475116
+
+import sys, re
+
+class TerminalController:
+    """
+    A class that can be used to portably generate formatted output to
+    a terminal.  
+    
+    `TerminalController` defines a set of instance variables whose
+    values are initialized to the control sequence necessary to
+    perform a given action.  These can be simply included in normal
+    output to the terminal:
+
+        >>> term = TerminalController()
+        >>> print 'This is '+term.GREEN+'green'+term.NORMAL
+
+    Alternatively, the `render()` method can used, which replaces
+    '${action}' with the string required to perform 'action':
+
+        >>> term = TerminalController()
+        >>> print term.render('This is ${GREEN}green${NORMAL}')
+
+    If the terminal doesn't support a given action, then the value of
+    the corresponding instance variable will be set to ''.  As a
+    result, the above code will still work on terminals that do not
+    support color, except that their output will not be colored.
+    Also, this means that you can test whether the terminal supports a
+    given action by simply testing the truth value of the
+    corresponding instance variable:
+
+        >>> term = TerminalController()
+        >>> if term.CLEAR_SCREEN:
+        ...     print 'This terminal supports clearning the screen.'
+
+    Finally, if the width and height of the terminal are known, then
+    they will be stored in the `COLS` and `LINES` attributes.
+    """
+    # Cursor movement:
+    BOL = ''             #: Move the cursor to the beginning of the line
+    UP = ''              #: Move the cursor up one line
+    DOWN = ''            #: Move the cursor down one line
+    LEFT = ''            #: Move the cursor left one char
+    RIGHT = ''           #: Move the cursor right one char
+
+    # Deletion:
+    CLEAR_SCREEN = ''    #: Clear the screen and move to home position
+    CLEAR_EOL = ''       #: Clear to the end of the line.
+    CLEAR_BOL = ''       #: Clear to the beginning of the line.
+    CLEAR_EOS = ''       #: Clear to the end of the screen
+
+    # Output modes:
+    BOLD = ''            #: Turn on bold mode
+    BLINK = ''           #: Turn on blink mode
+    DIM = ''             #: Turn on half-bright mode
+    REVERSE = ''         #: Turn on reverse-video mode
+    NORMAL = ''          #: Turn off all modes
+
+    # Cursor display:
+    HIDE_CURSOR = ''     #: Make the cursor invisible
+    SHOW_CURSOR = ''     #: Make the cursor visible
+
+    # Terminal size:
+    COLS = None          #: Width of the terminal (None for unknown)
+    LINES = None         #: Height of the terminal (None for unknown)
+
+    # Foreground colors:
+    BLACK = BLUE = GREEN = CYAN = RED = MAGENTA = YELLOW = WHITE = ''
+    
+    # Background colors:
+    BG_BLACK = BG_BLUE = BG_GREEN = BG_CYAN = ''
+    BG_RED = BG_MAGENTA = BG_YELLOW = BG_WHITE = ''
+    
+    _STRING_CAPABILITIES = """
+    BOL=cr UP=cuu1 DOWN=cud1 LEFT=cub1 RIGHT=cuf1
+    CLEAR_SCREEN=clear CLEAR_EOL=el CLEAR_BOL=el1 CLEAR_EOS=ed BOLD=bold
+    BLINK=blink DIM=dim REVERSE=rev UNDERLINE=smul NORMAL=sgr0
+    HIDE_CURSOR=cinvis SHOW_CURSOR=cnorm""".split()
+    _COLORS = """BLACK BLUE GREEN CYAN RED MAGENTA YELLOW WHITE""".split()
+    _ANSICOLORS = "BLACK RED GREEN YELLOW BLUE MAGENTA CYAN WHITE".split()
+
+    def __init__(self, term_stream=sys.stdout):
+        """
+        Create a `TerminalController` and initialize its attributes
+        with appropriate values for the current terminal.
+        `term_stream` is the stream that will be used for terminal
+        output; if this stream is not a tty, then the terminal is
+        assumed to be a dumb terminal (i.e., have no capabilities).
+        """
+        # Curses isn't available on all platforms
+        try: import curses
+        except: return
+
+        # If the stream isn't a tty, then assume it has no capabilities.
+        if not term_stream.isatty(): return
+
+        # Check the terminal type.  If we fail, then assume that the
+        # terminal has no capabilities.
+        try: curses.setupterm()
+        except: return
+
+        # Look up numeric capabilities.
+        self.COLS = curses.tigetnum('cols')
+        self.LINES = curses.tigetnum('lines')
+        
+        # Look up string capabilities.
+        for capability in self._STRING_CAPABILITIES:
+            (attrib, cap_name) = capability.split('=')
+            setattr(self, attrib, self._tigetstr(cap_name) or '')
+
+        # Colors
+        set_fg = self._tigetstr('setf')
+        if set_fg:
+            for i,color in zip(range(len(self._COLORS)), self._COLORS):
+                setattr(self, color, curses.tparm(set_fg, i) or '')
+        set_fg_ansi = self._tigetstr('setaf')
+        if set_fg_ansi:
+            for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
+                setattr(self, color, curses.tparm(set_fg_ansi, i) or '')
+        set_bg = self._tigetstr('setb')
+        if set_bg:
+            for i,color in zip(range(len(self._COLORS)), self._COLORS):
+                setattr(self, 'BG_'+color, curses.tparm(set_bg, i) or '')
+        set_bg_ansi = self._tigetstr('setab')
+        if set_bg_ansi:
+            for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
+                setattr(self, 'BG_'+color, curses.tparm(set_bg_ansi, i) or '')
+
+    def _tigetstr(self, cap_name):
+        # String capabilities can include "delays" of the form "$<2>".
+        # For any modern terminal, we should be able to just ignore
+        # these, so strip them out.
+        import curses
+        cap = curses.tigetstr(cap_name) or ''
+        return re.sub(r'\$<\d+>[/*]?', '', cap)
+
+    def render(self, template):
+        """
+        Replace each $-substitutions in the given template string with
+        the corresponding terminal control string (if it's defined) or
+        '' (if it's not).
+        """
+        return re.sub(r'\$\$|\${\w+}', self._render_sub, template)
+
+    def _render_sub(self, match):
+        s = match.group()
+        if s == '$$': return s
+        else: return getattr(self, s[2:-1])
+
+#######################################################################
+# Example use case: progress bar
+#######################################################################
+
+class ProgressBar:
+    """
+    A 3-line progress bar, which looks like::
+    
+                                Header
+        20% [===========----------------------------------]
+                           progress message
+
+    The progress bar is colored, if the terminal supports color
+    output; and adjusts to the width of the terminal.
+    """
+    BAR = '%3d%% ${GREEN}[${BOLD}%s%s${NORMAL}${GREEN}]${NORMAL}\n'
+    HEADER = '${BOLD}${CYAN}%s${NORMAL}\n\n'
+        
+    def __init__(self, term, header):
+        self.term = term
+        if not (self.term.CLEAR_EOL and self.term.UP and self.term.BOL):
+            raise ValueError("Terminal isn't capable enough -- you "
+                             "should use a simpler progress dispaly.")
+        self.width = self.term.COLS or 75
+        self.bar = term.render(self.BAR)
+        self.header = self.term.render(self.HEADER % header.center(self.width))
+        self.cleared = 1 #: true if we haven't drawn the bar yet.
+        self.update(0, '')
+
+    def update(self, percent, message):
+        if self.cleared:
+            sys.stdout.write(self.header)
+            self.cleared = 0
+        n = int((self.width-10)*percent)
+        sys.stdout.write(
+            self.term.BOL + self.term.UP + self.term.CLEAR_EOL +
+            (self.bar % (100*percent, '='*n, '-'*(self.width-10-n))) +
+            self.term.CLEAR_EOL + message.center(self.width))
+
+    def clear(self):
+        if not self.cleared:
+            sys.stdout.write(self.term.BOL + self.term.CLEAR_EOL +
+                             self.term.UP + self.term.CLEAR_EOL +
+                             self.term.UP + self.term.CLEAR_EOL)
+            self.cleared = 1