Update python stuff to latest
Probably definitely break linux
This commit is contained in:
Lirusaito
@@ -1,64 +0,0 @@
|
||||
"""\
|
||||
@file fastest_elementtree.py
|
||||
@brief Concealing some gnarly import logic in here. This should export the interface of elementtree.
|
||||
|
||||
$LicenseInfo:firstyear=2008&license=mit$
|
||||
|
||||
Copyright (c) 2008-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
# The parsing exception raised by the underlying library depends
|
||||
# on the ElementTree implementation we're using, so we provide an
|
||||
# alias here.
|
||||
#
|
||||
# Use ElementTreeError as the exception type for catching parsing
|
||||
# errors.
|
||||
|
||||
|
||||
# Using cElementTree might cause some unforeseen problems, so here's a
|
||||
# convenient off switch.
|
||||
use_celementree = True
|
||||
|
||||
try:
|
||||
if not use_celementree:
|
||||
raise ImportError()
|
||||
# Python 2.3 and 2.4.
|
||||
from cElementTree import *
|
||||
ElementTreeError = SyntaxError
|
||||
except ImportError:
|
||||
try:
|
||||
if not use_celementree:
|
||||
raise ImportError()
|
||||
# Python 2.5 and above.
|
||||
from xml.etree.cElementTree import *
|
||||
ElementTreeError = SyntaxError
|
||||
except ImportError:
|
||||
# Pure Python code.
|
||||
try:
|
||||
# Python 2.3 and 2.4.
|
||||
from elementtree.ElementTree import *
|
||||
except ImportError:
|
||||
# Python 2.5 and above.
|
||||
from xml.etree.ElementTree import *
|
||||
|
||||
# The pure Python ElementTree module uses Expat for parsing.
|
||||
from xml.parsers.expat import ExpatError as ElementTreeError
|
||||
@@ -1,52 +0,0 @@
|
||||
"""\
|
||||
@file helpformatter.py
|
||||
@author Phoenix
|
||||
@brief Class for formatting optparse descriptions.
|
||||
|
||||
$LicenseInfo:firstyear=2007&license=mit$
|
||||
|
||||
Copyright (c) 2007-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
import optparse
|
||||
import textwrap
|
||||
|
||||
class Formatter(optparse.IndentedHelpFormatter):
|
||||
def __init__(
|
||||
self,
|
||||
p_indentIncrement = 2,
|
||||
p_maxHelpPosition = 24,
|
||||
p_width = 79,
|
||||
p_shortFirst = 1) :
|
||||
optparse.HelpFormatter.__init__(
|
||||
self,
|
||||
p_indentIncrement,
|
||||
p_maxHelpPosition,
|
||||
p_width,
|
||||
p_shortFirst)
|
||||
def format_description(self, p_description):
|
||||
t_descWidth = self.width - self.current_indent
|
||||
t_indent = " " * (self.current_indent + 2)
|
||||
return "\n".join(
|
||||
[textwrap.fill(descr, t_descWidth, initial_indent = t_indent,
|
||||
subsequent_indent = t_indent)
|
||||
for descr in p_description.split("\n")] )
|
||||
@@ -1,63 +0,0 @@
|
||||
"""\
|
||||
@file iterators.py
|
||||
@brief Useful general-purpose iterators.
|
||||
|
||||
$LicenseInfo:firstyear=2008&license=mit$
|
||||
|
||||
Copyright (c) 2008-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
from __future__ import nested_scopes
|
||||
|
||||
def iter_chunks(rows, aggregate_size=100):
|
||||
"""
|
||||
Given an iterable set of items (@p rows), produces lists of up to @p
|
||||
aggregate_size items at a time, for example:
|
||||
|
||||
iter_chunks([1,2,3,4,5,6,7,8,9,10], 3)
|
||||
|
||||
Values for @p aggregate_size < 1 will raise ValueError.
|
||||
|
||||
Will return a generator that produces, in the following order:
|
||||
- [1, 2, 3]
|
||||
- [4, 5, 6]
|
||||
- [7, 8, 9]
|
||||
- [10]
|
||||
"""
|
||||
if aggregate_size < 1:
|
||||
raise ValueError()
|
||||
|
||||
def iter_chunks_inner():
|
||||
row_iter = iter(rows)
|
||||
done = False
|
||||
agg = []
|
||||
while not done:
|
||||
try:
|
||||
row = row_iter.next()
|
||||
agg.append(row)
|
||||
except StopIteration:
|
||||
done = True
|
||||
if agg and (len(agg) >= aggregate_size or done):
|
||||
yield agg
|
||||
agg = []
|
||||
|
||||
return iter_chunks_inner()
|
||||
@@ -1,72 +0,0 @@
|
||||
"""\
|
||||
@file iterators_test.py
|
||||
@brief Test cases for iterators module.
|
||||
|
||||
$LicenseInfo:firstyear=2008&license=mit$
|
||||
|
||||
Copyright (c) 2008-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from indra.util.iterators import iter_chunks
|
||||
|
||||
class TestIterChunks(unittest.TestCase):
|
||||
"""Unittests for iter_chunks"""
|
||||
def test_bad_agg_size(self):
|
||||
rows = [1,2,3,4]
|
||||
self.assertRaises(ValueError, iter_chunks, rows, 0)
|
||||
self.assertRaises(ValueError, iter_chunks, rows, -1)
|
||||
|
||||
try:
|
||||
for i in iter_chunks(rows, 0):
|
||||
pass
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
self.fail()
|
||||
|
||||
try:
|
||||
result = list(iter_chunks(rows, 0))
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
self.fail()
|
||||
def test_empty(self):
|
||||
rows = []
|
||||
result = list(iter_chunks(rows))
|
||||
self.assertEqual(result, [])
|
||||
def test_small(self):
|
||||
rows = [[1]]
|
||||
result = list(iter_chunks(rows, 2))
|
||||
self.assertEqual(result, [[[1]]])
|
||||
def test_size(self):
|
||||
rows = [[1],[2]]
|
||||
result = list(iter_chunks(rows, 2))
|
||||
self.assertEqual(result, [[[1],[2]]])
|
||||
def test_multi_agg(self):
|
||||
rows = [[1],[2],[3],[4],[5]]
|
||||
result = list(iter_chunks(rows, 2))
|
||||
self.assertEqual(result, [[[1],[2]],[[3],[4]],[[5]]])
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -33,21 +33,26 @@ import filecmp
|
||||
import fnmatch
|
||||
import getopt
|
||||
import glob
|
||||
import itertools
|
||||
import operator
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import errno
|
||||
import subprocess
|
||||
|
||||
class ManifestError(RuntimeError):
|
||||
"""Use an exception more specific than generic Python RuntimeError"""
|
||||
pass
|
||||
def __init__(self, msg):
|
||||
self.msg = msg
|
||||
super(ManifestError, self).__init__(self.msg)
|
||||
|
||||
class MissingError(ManifestError):
|
||||
"""You specified a file that doesn't exist"""
|
||||
pass
|
||||
def __init__(self, msg):
|
||||
self.msg = msg
|
||||
super(MissingError, self).__init__(self.msg)
|
||||
|
||||
def path_ancestors(path):
|
||||
drive, path = os.path.splitdrive(os.path.normpath(path))
|
||||
@@ -88,7 +93,7 @@ DEFAULT_SRCTREE = os.path.dirname(sys.argv[0])
|
||||
CHANNEL_VENDOR_BASE = 'Singularity'
|
||||
RELEASE_CHANNEL = CHANNEL_VENDOR_BASE + ' Release'
|
||||
|
||||
ARGUMENTS=[
|
||||
BASE_ARGUMENTS=[
|
||||
dict(name='actions',
|
||||
description="""This argument specifies the actions that are to be taken when the
|
||||
script is run. The meaningful actions are currently:
|
||||
@@ -106,9 +111,17 @@ ARGUMENTS=[
|
||||
Example use: %(name)s --arch=i686
|
||||
On Linux this would try to use Linux_i686Manifest.""",
|
||||
default=""),
|
||||
dict(name='artwork', description='Artwork directory.', default=DEFAULT_SRCTREE),
|
||||
dict(name='branding_id', description="Identifier for the branding set to use.", default='singularity'),
|
||||
dict(name='build', description='Build directory.', default=DEFAULT_SRCTREE),
|
||||
dict(name='buildtype', description='Build type (i.e. Debug, Release, RelWithDebInfo).', default=None),
|
||||
dict(name='channel',
|
||||
description="""The channel to use for updates, packaging, settings name, etc.""",
|
||||
default='CHANNEL UNSET'),
|
||||
dict(name='channel_suffix',
|
||||
description="""Addition to the channel for packaging and channel value,
|
||||
but not application name (used internally)""",
|
||||
default=None),
|
||||
dict(name='configuration',
|
||||
description="""The build configuration used.""",
|
||||
default="Release"),
|
||||
@@ -116,12 +129,6 @@ ARGUMENTS=[
|
||||
dict(name='grid',
|
||||
description="""Which grid the client will try to connect to.""",
|
||||
default=None),
|
||||
dict(name='channel',
|
||||
description="""The channel to use for updates, packaging, settings name, etc.""",
|
||||
default='CHANNEL UNSET'),
|
||||
dict(name='channel_suffix',
|
||||
description="""Addition to the channel for packaging and channel value, but not application name (used internally)""",
|
||||
default=None),
|
||||
dict(name='installer_name',
|
||||
description=""" The name of the file that the installer should be
|
||||
packaged up into. Only used on Linux at the moment.""",
|
||||
@@ -133,13 +140,17 @@ ARGUMENTS=[
|
||||
description="""The current platform, to be used for looking up which
|
||||
manifest class to run.""",
|
||||
default=get_default_platform),
|
||||
dict(name='signature',
|
||||
description="""This specifies an identity to sign the viewer with, if any.
|
||||
If no value is supplied, the default signature will be used, if any. Currently
|
||||
only used on Mac OS X.""",
|
||||
default=None),
|
||||
dict(name='source',
|
||||
description='Source directory.',
|
||||
default=DEFAULT_SRCTREE),
|
||||
dict(name='standalone',
|
||||
description='Set to ON if this is a standalone build.',
|
||||
default="OFF"),
|
||||
dict(name='artwork', description='Artwork directory.', default=DEFAULT_SRCTREE),
|
||||
dict(name='touch',
|
||||
description="""File to touch when action is finished. Touch file will
|
||||
contain the name of the final package in a form suitable
|
||||
@@ -147,20 +158,15 @@ ARGUMENTS=[
|
||||
default=None),
|
||||
dict(name='versionfile',
|
||||
description="""The name of a file containing the full version number."""),
|
||||
dict(name='signature',
|
||||
description="""This specifies an identity to sign the viewer with, if any.
|
||||
If no value is supplied, the default signature will be used, if any. Currently
|
||||
only used on Mac OS X.""",
|
||||
default=None)
|
||||
]
|
||||
|
||||
def usage(srctree=""):
|
||||
def usage(arguments, srctree=""):
|
||||
nd = {'name':sys.argv[0]}
|
||||
print """Usage:
|
||||
%(name)s [options] [destdir]
|
||||
Options:
|
||||
""" % nd
|
||||
for arg in ARGUMENTS:
|
||||
for arg in arguments:
|
||||
default = arg['default']
|
||||
if hasattr(default, '__call__'):
|
||||
default = "(computed value) \"" + str(default(srctree)) + '"'
|
||||
@@ -171,11 +177,15 @@ def usage(srctree=""):
|
||||
default,
|
||||
arg['description'] % nd)
|
||||
|
||||
def main():
|
||||
## import itertools
|
||||
def main(extra=[]):
|
||||
## print ' '.join((("'%s'" % item) if ' ' in item else item)
|
||||
## for item in itertools.chain([sys.executable], sys.argv))
|
||||
option_names = [arg['name'] + '=' for arg in ARGUMENTS]
|
||||
# Supplement our default command-line switches with any desired by
|
||||
# application-specific caller.
|
||||
arguments = list(itertools.chain(BASE_ARGUMENTS, extra))
|
||||
# Alphabetize them by option name in case we display usage.
|
||||
arguments.sort(key=operator.itemgetter('name'))
|
||||
option_names = [arg['name'] + '=' for arg in arguments]
|
||||
option_names.append('help')
|
||||
options, remainder = getopt.getopt(sys.argv[1:], "", option_names)
|
||||
|
||||
@@ -198,11 +208,11 @@ def main():
|
||||
# early out for help
|
||||
if 'help' in args:
|
||||
# *TODO: it is a huge hack to pass around the srctree like this
|
||||
usage(args['source'])
|
||||
usage(arguments, srctree=args['source'])
|
||||
return
|
||||
|
||||
# defaults
|
||||
for arg in ARGUMENTS:
|
||||
for arg in arguments:
|
||||
if arg['name'] not in args:
|
||||
default = arg['default']
|
||||
if hasattr(default, '__call__'):
|
||||
@@ -231,101 +241,68 @@ def main():
|
||||
print "Option:", opt, "=", args[opt]
|
||||
|
||||
# pass in sourceid as an argument now instead of an environment variable
|
||||
try:
|
||||
args['sourceid'] = os.environ["sourceid"]
|
||||
except KeyError:
|
||||
args['sourceid'] = ""
|
||||
args['sourceid'] = os.environ.get("sourceid", "")
|
||||
|
||||
# Build base package.
|
||||
touch = args.get('touch')
|
||||
if touch:
|
||||
print 'Creating base package'
|
||||
args['package_id'] = "" # base package has no package ID
|
||||
print '================ Creating base package'
|
||||
else:
|
||||
print '================ Starting base copy'
|
||||
wm = LLManifest.for_platform(args['platform'], args.get('arch'))(args)
|
||||
wm.do(*args['actions'])
|
||||
# Store package file for later if making touched file.
|
||||
base_package_file = ""
|
||||
if touch:
|
||||
print 'Created base package ', wm.package_file
|
||||
print '================ Created base package ', wm.package_file
|
||||
base_package_file = "" + wm.package_file
|
||||
else:
|
||||
print '================ Finished base copy'
|
||||
|
||||
# handle multiple packages if set
|
||||
try:
|
||||
additional_packages = os.environ["additional_packages"]
|
||||
except KeyError:
|
||||
additional_packages = ""
|
||||
# ''.split() produces empty list
|
||||
additional_packages = os.environ.get("additional_packages", "").split()
|
||||
if additional_packages:
|
||||
# Determine destination prefix / suffix for additional packages.
|
||||
base_dest_postfix = args['dest']
|
||||
base_dest_prefix = ""
|
||||
base_dest_parts = args['dest'].split(os.sep)
|
||||
if len(base_dest_parts) > 1:
|
||||
base_dest_postfix = base_dest_parts[len(base_dest_parts) - 1]
|
||||
base_dest_prefix = base_dest_parts[0]
|
||||
i = 1
|
||||
while i < len(base_dest_parts) - 1:
|
||||
base_dest_prefix = base_dest_prefix + os.sep + base_dest_parts[i]
|
||||
i = i + 1
|
||||
base_dest_parts = list(os.path.split(args['dest']))
|
||||
base_dest_parts.insert(-1, "{}")
|
||||
base_dest_template = os.path.join(*base_dest_parts)
|
||||
# Determine touched prefix / suffix for additional packages.
|
||||
base_touch_postfix = ""
|
||||
base_touch_prefix = ""
|
||||
if touch:
|
||||
base_touch_postfix = touch
|
||||
base_touch_parts = touch.split('/')
|
||||
base_touch_parts = list(os.path.split(touch))
|
||||
# Because of the special insert() logic below, we don't just want
|
||||
# [dirpath, basename]; we want [dirpath, directory, basename].
|
||||
# Further split the dirpath and replace it in the list.
|
||||
base_touch_parts[0:1] = os.path.split(base_touch_parts[0])
|
||||
if "arwin" in args['platform']:
|
||||
if len(base_touch_parts) > 1:
|
||||
base_touch_postfix = base_touch_parts[len(base_touch_parts) - 1]
|
||||
base_touch_prefix = base_touch_parts[0]
|
||||
i = 1
|
||||
while i < len(base_touch_parts) - 1:
|
||||
base_touch_prefix = base_touch_prefix + '/' + base_touch_parts[i]
|
||||
i = i + 1
|
||||
base_touch_parts.insert(-1, "{}")
|
||||
else:
|
||||
if len(base_touch_parts) > 2:
|
||||
base_touch_postfix = base_touch_parts[len(base_touch_parts) - 2] + '/' + base_touch_parts[len(base_touch_parts) - 1]
|
||||
base_touch_prefix = base_touch_parts[0]
|
||||
i = 1
|
||||
while i < len(base_touch_parts) - 2:
|
||||
base_touch_prefix = base_touch_prefix + '/' + base_touch_parts[i]
|
||||
i = i + 1
|
||||
# Store base channel name.
|
||||
base_channel_name = args['channel']
|
||||
# Build each additional package.
|
||||
package_id_list = additional_packages.split(" ")
|
||||
args['channel'] = base_channel_name
|
||||
for package_id in package_id_list:
|
||||
base_touch_parts.insert(-2, "{}")
|
||||
base_touch_template = os.path.join(*base_touch_parts)
|
||||
for package_id in additional_packages:
|
||||
args['channel_suffix'] = os.environ.get(package_id + "_viewer_channel_suffix")
|
||||
args['sourceid'] = os.environ.get(package_id + "_sourceid")
|
||||
args['dest'] = base_dest_template.format(package_id)
|
||||
if touch:
|
||||
print '================ Creating additional package for "', package_id, '" in ', args['dest']
|
||||
else:
|
||||
print '================ Starting additional copy for "', package_id, '" in ', args['dest']
|
||||
try:
|
||||
if package_id + "_viewer_channel_suffix" in os.environ:
|
||||
args['channel_suffix'] = os.environ[package_id + "_viewer_channel_suffix"]
|
||||
else:
|
||||
args['channel_suffix'] = None
|
||||
if package_id + "_sourceid" in os.environ:
|
||||
args['sourceid'] = os.environ[package_id + "_sourceid"]
|
||||
else:
|
||||
args['sourceid'] = None
|
||||
args['dest'] = base_dest_prefix + os.sep + package_id + os.sep + base_dest_postfix
|
||||
except KeyError:
|
||||
sys.stderr.write("Failed to create package for package_id: %s" % package_id)
|
||||
sys.stderr.flush()
|
||||
continue
|
||||
wm = LLManifest.for_platform(args['platform'], args.get('arch'))(args)
|
||||
wm.do(*args['actions'])
|
||||
except Exception as err:
|
||||
sys.exit(str(err))
|
||||
if touch:
|
||||
print 'Creating additional package for "', package_id, '" in ', args['dest']
|
||||
wm = LLManifest.for_platform(args['platform'], args.get('arch'))(args)
|
||||
wm.do(*args['actions'])
|
||||
if touch:
|
||||
print 'Created additional package ', wm.package_file, ' for ', package_id
|
||||
faketouch = base_touch_prefix + '/' + package_id + '/' + base_touch_postfix
|
||||
fp = open(faketouch, 'w')
|
||||
fp.write('set package_file=%s\n' % wm.package_file)
|
||||
fp.close()
|
||||
|
||||
print '================ Created additional package ', wm.package_file, ' for ', package_id
|
||||
with open(base_touch_template.format(package_id), 'w') as fp:
|
||||
fp.write('set package_file=%s\n' % wm.package_file)
|
||||
else:
|
||||
print '================ Finished additional copy "', package_id, '" in ', args['dest']
|
||||
# Write out the package file in this format, so that it can easily be called
|
||||
# and used in a .bat file - yeah, it sucks, but this is the simplest...
|
||||
touch = args.get('touch')
|
||||
if touch:
|
||||
fp = open(touch, 'w')
|
||||
fp.write('set package_file=%s\n' % base_package_file)
|
||||
fp.close()
|
||||
with open(touch, 'w') as fp:
|
||||
fp.write('set package_file=%s\n' % base_package_file)
|
||||
print 'touched', touch
|
||||
return 0
|
||||
|
||||
@@ -371,22 +348,113 @@ class LLManifest(object):
|
||||
in the file list by path()."""
|
||||
self.excludes.append(glob)
|
||||
|
||||
def prefix(self, src='', build=None, dst=None):
|
||||
""" Pushes a prefix onto the stack. Until end_prefix is
|
||||
called, all relevant method calls (esp. to path()) will prefix
|
||||
paths with the entire prefix stack. Source and destination
|
||||
prefixes can be different, though if only one is provided they
|
||||
are both equal. To specify a no-op, use an empty string, not
|
||||
None."""
|
||||
if dst is None:
|
||||
dst = src
|
||||
if build is None:
|
||||
build = src
|
||||
def prefix(self, src='', build='', dst='', src_dst=None):
|
||||
"""
|
||||
Usage:
|
||||
|
||||
with self.prefix(...args as described...):
|
||||
self.path(...)
|
||||
|
||||
For the duration of the 'with' block, pushes a prefix onto the stack.
|
||||
Within that block, all relevant method calls (esp. to path()) will
|
||||
prefix paths with the entire prefix stack. Source and destination
|
||||
prefixes are independent; if omitted (or passed as the empty string),
|
||||
the prefix has no effect. Thus:
|
||||
|
||||
with self.prefix(src='foo'):
|
||||
# no effect on dst
|
||||
|
||||
with self.prefix(dst='bar'):
|
||||
# no effect on src
|
||||
|
||||
If you want to set both at once, use src_dst:
|
||||
|
||||
with self.prefix(src_dst='subdir'):
|
||||
# same as self.prefix(src='subdir', dst='subdir')
|
||||
# Passing src_dst makes any src or dst argument in the same
|
||||
# parameter list irrelevant.
|
||||
|
||||
Also supports the older (pre-Python-2.5) syntax:
|
||||
|
||||
if self.prefix(...args as described...):
|
||||
self.path(...)
|
||||
self.end_prefix(...)
|
||||
|
||||
Before the arrival of the 'with' statement, one was required to code
|
||||
self.prefix() and self.end_prefix() in matching pairs to push and to
|
||||
pop the prefix stacks, respectively. The older prefix() method
|
||||
returned True specifically so that the caller could indent the
|
||||
relevant block of code with 'if', just for aesthetic purposes.
|
||||
"""
|
||||
if src_dst is not None:
|
||||
src = src_dst
|
||||
dst = src_dst
|
||||
self.src_prefix.append(src)
|
||||
self.artwork_prefix.append(src)
|
||||
self.build_prefix.append(build)
|
||||
self.dst_prefix.append(dst)
|
||||
return True # so that you can wrap it in an if to get indentation
|
||||
|
||||
## self.display_stacks()
|
||||
|
||||
# The above code is unchanged from the original implementation. What's
|
||||
# new is the return value. We're going to return an instance of
|
||||
# PrefixManager that binds this LLManifest instance and Does The Right
|
||||
# Thing on exit.
|
||||
return self.PrefixManager(self)
|
||||
|
||||
def display_stacks(self):
|
||||
width = 1 + max(len(stack) for stack in self.PrefixManager.stacks)
|
||||
for stack in self.PrefixManager.stacks:
|
||||
print "{} {}".format((stack + ':').ljust(width),
|
||||
os.path.join(*getattr(self, stack)))
|
||||
|
||||
class PrefixManager(object):
|
||||
# stack attributes we manage in this LLManifest (sub)class
|
||||
# instance
|
||||
stacks = ("src_prefix", "artwork_prefix", "build_prefix", "dst_prefix")
|
||||
|
||||
def __init__(self, manifest):
|
||||
self.manifest = manifest
|
||||
# If the caller wrote:
|
||||
# with self.prefix(...):
|
||||
# as intended, then bind the state of each prefix stack as it was
|
||||
# just BEFORE the call to prefix(). Since prefix() appended an
|
||||
# entry to each prefix stack, capture len()-1.
|
||||
self.prevlen = { stack: len(getattr(self.manifest, stack)) - 1
|
||||
for stack in self.stacks }
|
||||
|
||||
def __nonzero__(self):
|
||||
# If the caller wrote:
|
||||
# if self.prefix(...):
|
||||
# then a value of this class had better evaluate as 'True'.
|
||||
return True
|
||||
|
||||
def __enter__(self):
|
||||
# nobody uses 'with self.prefix(...) as variable:'
|
||||
return None
|
||||
|
||||
def __exit__(self, type, value, traceback):
|
||||
# First, if the 'with' block raised an exception, just propagate.
|
||||
# Do NOT swallow it.
|
||||
if type is not None:
|
||||
return False
|
||||
|
||||
# Okay, 'with' block completed successfully. Restore previous
|
||||
# state of each of the prefix stacks in self.stacks.
|
||||
# Note that we do NOT simply call pop() on them as end_prefix()
|
||||
# does. This is to cope with the possibility that the coder
|
||||
# changed 'if self.prefix(...):' to 'with self.prefix(...):' yet
|
||||
# forgot to remove the self.end_prefix(...) call at the bottom of
|
||||
# the block. In that case, calling pop() again would be Bad! But
|
||||
# if we restore the length of each stack to what it was before the
|
||||
# current prefix() block, it doesn't matter whether end_prefix()
|
||||
# was called or not.
|
||||
for stack, prevlen in self.prevlen.items():
|
||||
# find the attribute in 'self.manifest' named by 'stack', and
|
||||
# truncate that list back to 'prevlen'
|
||||
del getattr(self.manifest, stack)[prevlen:]
|
||||
|
||||
## self.manifest.display_stacks()
|
||||
|
||||
def end_prefix(self, descr=None):
|
||||
"""Pops a prefix off the stack. If given an argument, checks
|
||||
@@ -433,6 +501,19 @@ class LLManifest(object):
|
||||
relative to the destination directory."""
|
||||
return os.path.join(self.get_dst_prefix(), relpath)
|
||||
|
||||
def _relative_dst_path(self, dstpath):
|
||||
"""
|
||||
Returns the path to a file or directory relative to the destination directory.
|
||||
This should only be used for generating diagnostic output in the path method.
|
||||
"""
|
||||
dest_root=self.dst_prefix[0]
|
||||
if dstpath.startswith(dest_root+os.path.sep):
|
||||
return dstpath[len(dest_root)+1:]
|
||||
elif dstpath.startswith(dest_root):
|
||||
return dstpath[len(dest_root):]
|
||||
else:
|
||||
return dstpath
|
||||
|
||||
def ensure_src_dir(self, reldir):
|
||||
"""Construct the path for a directory relative to the
|
||||
source path, and ensures that it exists. Returns the
|
||||
@@ -450,29 +531,17 @@ class LLManifest(object):
|
||||
return path
|
||||
|
||||
def run_command(self, command):
|
||||
""" Runs an external command, and returns the output. Raises
|
||||
an exception if the command returns a nonzero status code. For
|
||||
debugging/informational purposes, prints out the command's
|
||||
output as it is received."""
|
||||
"""
|
||||
Runs an external command.
|
||||
Raises ManifestError exception if the command returns a nonzero status.
|
||||
"""
|
||||
print "Running command:", command
|
||||
sys.stdout.flush()
|
||||
child = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
shell=True)
|
||||
lines = []
|
||||
while True:
|
||||
lines.append(child.stdout.readline())
|
||||
if lines[-1] == '':
|
||||
break
|
||||
else:
|
||||
print lines[-1],
|
||||
output = ''.join(lines)
|
||||
child.stdout.close()
|
||||
status = child.wait()
|
||||
if status:
|
||||
raise ManifestError(
|
||||
"Command %s returned non-zero status (%s) \noutput:\n%s"
|
||||
% (command, status, output) )
|
||||
return output
|
||||
try:
|
||||
subprocess.check_call(command)
|
||||
except subprocess.CalledProcessError as err:
|
||||
raise ManifestError( "Command %s returned non-zero status (%s)"
|
||||
% (command, err.returncode) )
|
||||
|
||||
def created_path(self, path):
|
||||
""" Declare that you've created a path in order to
|
||||
@@ -485,6 +554,7 @@ class LLManifest(object):
|
||||
def put_in_file(self, contents, dst, src=None):
|
||||
# write contents as dst
|
||||
dst_path = self.dst_path_of(dst)
|
||||
self.cmakedirs(os.path.dirname(dst_path))
|
||||
f = open(dst_path, "wb")
|
||||
try:
|
||||
f.write(contents)
|
||||
@@ -546,9 +616,16 @@ class LLManifest(object):
|
||||
# *TODO is this gonna be useful?
|
||||
print "Cleaning up " + c
|
||||
|
||||
def process_either(self, src, dst):
|
||||
# If it's a real directory, recurse through it --
|
||||
# but not a symlink! Handle those like files.
|
||||
if os.path.isdir(src) and not os.path.islink(src):
|
||||
return self.process_directory(src, dst)
|
||||
else:
|
||||
return self.process_file(src, dst)
|
||||
|
||||
def process_file(self, src, dst):
|
||||
if self.includes(src, dst):
|
||||
# print src, "=>", dst
|
||||
for action in self.actions:
|
||||
methodname = action + "_action"
|
||||
method = getattr(self, methodname, None)
|
||||
@@ -573,10 +650,7 @@ class LLManifest(object):
|
||||
for name in names:
|
||||
srcname = os.path.join(src, name)
|
||||
dstname = os.path.join(dst, name)
|
||||
if os.path.isdir(srcname):
|
||||
count += self.process_directory(srcname, dstname)
|
||||
else:
|
||||
count += self.process_file(srcname, dstname)
|
||||
count += self.process_either(srcname, dstname)
|
||||
return count
|
||||
|
||||
def includes(self, src, dst):
|
||||
@@ -616,16 +690,21 @@ class LLManifest(object):
|
||||
# Don't recopy file if it's up-to-date.
|
||||
# If we seem to be not not overwriting files that have been
|
||||
# updated, set the last arg to False, but it will take longer.
|
||||
## reldst = (dst[len(self.dst_prefix[0]):]
|
||||
## if dst.startswith(self.dst_prefix[0])
|
||||
## else dst).lstrip(r'\/')
|
||||
if os.path.exists(dst) and filecmp.cmp(src, dst, True):
|
||||
## print "{} (skipping, {} exists)".format(src, reldst)
|
||||
return
|
||||
# only copy if it's not excluded
|
||||
if self.includes(src, dst):
|
||||
try:
|
||||
os.unlink(dst)
|
||||
except OSError, err:
|
||||
except OSError as err:
|
||||
if err.errno != errno.ENOENT:
|
||||
raise
|
||||
|
||||
## print "{} => {}".format(src, reldst)
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
def ccopytree(self, src, dst):
|
||||
@@ -643,7 +722,7 @@ class LLManifest(object):
|
||||
dstname = os.path.join(dst, name)
|
||||
try:
|
||||
self.ccopymumble(srcname, dstname)
|
||||
except (IOError, os.error), why:
|
||||
except (IOError, os.error) as why:
|
||||
errors.append((srcname, dstname, why))
|
||||
if errors:
|
||||
raise ManifestError, errors
|
||||
@@ -724,13 +803,13 @@ class LLManifest(object):
|
||||
return self.path(os.path.join(path, file), file)
|
||||
|
||||
def path(self, src, dst=None):
|
||||
sys.stdout.write("Processing %s => %s ... " % (src, dst))
|
||||
sys.stdout.flush()
|
||||
if src == None:
|
||||
raise ManifestError("No source file, dst is " + dst)
|
||||
if dst == None:
|
||||
dst = src
|
||||
dst = os.path.join(self.get_dst_prefix(), dst)
|
||||
sys.stdout.write("Processing %s => %s ... " % (src, self._relative_dst_path(dst)))
|
||||
|
||||
def try_path(src):
|
||||
# expand globs
|
||||
@@ -743,29 +822,21 @@ class LLManifest(object):
|
||||
# if we're specifying a single path (not a glob),
|
||||
# we should error out if it doesn't exist
|
||||
self.check_file_exists(src)
|
||||
# if it's a directory, recurse through it
|
||||
if os.path.isdir(src):
|
||||
count += self.process_directory(src, dst)
|
||||
else:
|
||||
count += self.process_file(src, dst)
|
||||
count += self.process_either(src, dst)
|
||||
return count
|
||||
|
||||
for pfx in self.get_src_prefix(), self.get_artwork_prefix(), self.get_build_prefix():
|
||||
try_prefixes = [self.get_src_prefix(), self.get_artwork_prefix(), self.get_build_prefix()]
|
||||
tried=[]
|
||||
count=0
|
||||
while not count and try_prefixes:
|
||||
pfx = try_prefixes.pop(0)
|
||||
try:
|
||||
count = try_path(os.path.join(pfx, src))
|
||||
except MissingError:
|
||||
# If src isn't a wildcard, and if that file doesn't exist in
|
||||
# this pfx, try next pfx.
|
||||
count = 0
|
||||
continue
|
||||
|
||||
# Here try_path() didn't raise MissingError. Did it process any files?
|
||||
if count:
|
||||
break
|
||||
# Even though try_path() didn't raise MissingError, it returned 0
|
||||
# files. src is probably a wildcard meant for some other pfx. Loop
|
||||
# back to try the next.
|
||||
|
||||
tried.append(pfx)
|
||||
if not try_prefixes:
|
||||
# no more prefixes left to try
|
||||
print "unable to find '%s'; looked in:\n %s" % (src, '\n '.join(tried))
|
||||
print "%d files" % count
|
||||
|
||||
# Let caller check whether we processed as many files as expected. In
|
||||
|
||||
@@ -1,182 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""\
|
||||
@file llperformance.py
|
||||
|
||||
$LicenseInfo:firstyear=2010&license=viewerlgpl$
|
||||
Second Life Viewer Source Code
|
||||
Copyright (C) 2010-2011, Linden Research, Inc.
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation;
|
||||
version 2.1 of the License only.
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
|
||||
Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
# ------------------------------------------------
|
||||
# Sim metrics utility functions.
|
||||
|
||||
import glob, os, time, sys, stat, exceptions
|
||||
|
||||
from indra.base import llsd
|
||||
|
||||
gBlockMap = {} #Map of performance metric data with function hierarchy information.
|
||||
gCurrentStatPath = ""
|
||||
|
||||
gIsLoggingEnabled=False
|
||||
|
||||
class LLPerfStat:
|
||||
def __init__(self,key):
|
||||
self.mTotalTime = 0
|
||||
self.mNumRuns = 0
|
||||
self.mName=key
|
||||
self.mTimeStamp = int(time.time()*1000)
|
||||
self.mUTCTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
def __str__(self):
|
||||
return "%f" % self.mTotalTime
|
||||
|
||||
def start(self):
|
||||
self.mStartTime = int(time.time() * 1000000)
|
||||
self.mNumRuns += 1
|
||||
|
||||
def stop(self):
|
||||
execution_time = int(time.time() * 1000000) - self.mStartTime
|
||||
self.mTotalTime += execution_time
|
||||
|
||||
def get_map(self):
|
||||
results={}
|
||||
results['name']=self.mName
|
||||
results['utc_time']=self.mUTCTime
|
||||
results['timestamp']=self.mTimeStamp
|
||||
results['us']=self.mTotalTime
|
||||
results['count']=self.mNumRuns
|
||||
return results
|
||||
|
||||
class PerfError(exceptions.Exception):
|
||||
def __init__(self):
|
||||
return
|
||||
|
||||
def __Str__(self):
|
||||
print "","Unfinished LLPerfBlock"
|
||||
|
||||
class LLPerfBlock:
|
||||
def __init__( self, key ):
|
||||
global gBlockMap
|
||||
global gCurrentStatPath
|
||||
global gIsLoggingEnabled
|
||||
|
||||
#Check to see if we're running metrics right now.
|
||||
if gIsLoggingEnabled:
|
||||
self.mRunning = True #Mark myself as running.
|
||||
|
||||
self.mPreviousStatPath = gCurrentStatPath
|
||||
gCurrentStatPath += "/" + key
|
||||
if gCurrentStatPath not in gBlockMap:
|
||||
gBlockMap[gCurrentStatPath] = LLPerfStat(key)
|
||||
|
||||
self.mStat = gBlockMap[gCurrentStatPath]
|
||||
self.mStat.start()
|
||||
|
||||
def finish( self ):
|
||||
global gBlockMap
|
||||
global gIsLoggingEnabled
|
||||
|
||||
if gIsLoggingEnabled:
|
||||
self.mStat.stop()
|
||||
self.mRunning = False
|
||||
gCurrentStatPath = self.mPreviousStatPath
|
||||
|
||||
# def __del__( self ):
|
||||
# if self.mRunning:
|
||||
# #SPATTERS FIXME
|
||||
# raise PerfError
|
||||
|
||||
class LLPerformance:
|
||||
#--------------------------------------------------
|
||||
# Determine whether or not we want to log statistics
|
||||
|
||||
def __init__( self, process_name = "python" ):
|
||||
self.process_name = process_name
|
||||
self.init_testing()
|
||||
self.mTimeStamp = int(time.time()*1000)
|
||||
self.mUTCTime = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
def init_testing( self ):
|
||||
global gIsLoggingEnabled
|
||||
|
||||
host_performance_file = "/dev/shm/simperf/simperf_proc_config.llsd"
|
||||
|
||||
#If file exists, open
|
||||
if os.path.exists(host_performance_file):
|
||||
file = open (host_performance_file,'r')
|
||||
|
||||
#Read serialized LLSD from file.
|
||||
body = llsd.parse(file.read())
|
||||
|
||||
#Calculate time since file last modified.
|
||||
stats = os.stat(host_performance_file)
|
||||
now = time.time()
|
||||
mod = stats[stat.ST_MTIME]
|
||||
age = now - mod
|
||||
|
||||
if age < ( body['duration'] ):
|
||||
gIsLoggingEnabled = True
|
||||
|
||||
|
||||
def get ( self ):
|
||||
global gIsLoggingEnabled
|
||||
return gIsLoggingEnabled
|
||||
|
||||
#def output(self,ptr,path):
|
||||
# if 'stats' in ptr:
|
||||
# stats = ptr['stats']
|
||||
# self.mOutputPtr[path] = stats.get_map()
|
||||
|
||||
# if 'children' in ptr:
|
||||
# children=ptr['children']
|
||||
|
||||
# curptr = self.mOutputPtr
|
||||
# curchildren={}
|
||||
# curptr['children'] = curchildren
|
||||
|
||||
# for key in children:
|
||||
# curchildren[key]={}
|
||||
# self.mOutputPtr = curchildren[key]
|
||||
# self.output(children[key],path + '/' + key)
|
||||
|
||||
def done(self):
|
||||
global gBlockMap
|
||||
|
||||
if not self.get():
|
||||
return
|
||||
|
||||
output_name = "/dev/shm/simperf/%s_proc.%d.llsd" % (self.process_name, os.getpid())
|
||||
output_file = open(output_name, 'w')
|
||||
process_info = {
|
||||
"name" : self.process_name,
|
||||
"pid" : os.getpid(),
|
||||
"ppid" : os.getppid(),
|
||||
"timestamp" : self.mTimeStamp,
|
||||
"utc_time" : self.mUTCTime,
|
||||
}
|
||||
output_file.write(llsd.format_notation(process_info))
|
||||
output_file.write('\n')
|
||||
|
||||
for key in gBlockMap.keys():
|
||||
gBlockMap[key] = gBlockMap[key].get_map()
|
||||
output_file.write(llsd.format_notation(gBlockMap))
|
||||
output_file.write('\n')
|
||||
output_file.close()
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
"""\
|
||||
@file llsubprocess.py
|
||||
@author Phoenix
|
||||
@date 2008-01-18
|
||||
@brief The simplest possible wrapper for a common sub-process paradigm.
|
||||
|
||||
$LicenseInfo:firstyear=2007&license=mit$
|
||||
|
||||
Copyright (c) 2007-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
import os
|
||||
import popen2
|
||||
import time
|
||||
import select
|
||||
|
||||
class Timeout(RuntimeError):
|
||||
"Exception raised when a subprocess times out."
|
||||
pass
|
||||
|
||||
def run(command, args=None, data=None, timeout=None):
|
||||
"""\
|
||||
@brief Run command with arguments
|
||||
|
||||
This is it. This is the function I want to run all the time when doing
|
||||
subprocces, but end up copying the code everywhere. none of the
|
||||
standard commands are secure and provide a way to specify input, get
|
||||
all the output, and get the result.
|
||||
@param command A string specifying a process to launch.
|
||||
@param args Arguments to be passed to command. Must be list, tuple or None.
|
||||
@param data input to feed to the command.
|
||||
@param timeout Maximum number of many seconds to run.
|
||||
@return Returns (result, stdout, stderr) from process.
|
||||
"""
|
||||
cmd = [command]
|
||||
if args:
|
||||
cmd.extend([str(arg) for arg in args])
|
||||
#print "cmd: ","' '".join(cmd)
|
||||
child = popen2.Popen3(cmd, True)
|
||||
#print child.pid
|
||||
out = []
|
||||
err = []
|
||||
result = -1
|
||||
time_left = timeout
|
||||
tochild = [child.tochild.fileno()]
|
||||
while True:
|
||||
time_start = time.time()
|
||||
#print "time:",time_left
|
||||
p_in, p_out, p_err = select.select(
|
||||
[child.fromchild.fileno(), child.childerr.fileno()],
|
||||
tochild,
|
||||
[],
|
||||
time_left)
|
||||
if p_in:
|
||||
new_line = os.read(child.fromchild.fileno(), 32 * 1024)
|
||||
if new_line:
|
||||
#print "line:",new_line
|
||||
out.append(new_line)
|
||||
new_line = os.read(child.childerr.fileno(), 32 * 1024)
|
||||
if new_line:
|
||||
#print "error:", new_line
|
||||
err.append(new_line)
|
||||
if p_out:
|
||||
if data:
|
||||
#print "p_out"
|
||||
bytes = os.write(child.tochild.fileno(), data)
|
||||
data = data[bytes:]
|
||||
if len(data) == 0:
|
||||
data = None
|
||||
tochild = []
|
||||
child.tochild.close()
|
||||
result = child.poll()
|
||||
if result != -1:
|
||||
# At this point, the child process has exited and result
|
||||
# is the return value from the process. Between the time
|
||||
# we called select() and poll() the process may have
|
||||
# exited so read all the data left on the child process
|
||||
# stdout and stderr.
|
||||
last = child.fromchild.read()
|
||||
if last:
|
||||
out.append(last)
|
||||
last = child.childerr.read()
|
||||
if last:
|
||||
err.append(last)
|
||||
child.tochild.close()
|
||||
child.fromchild.close()
|
||||
child.childerr.close()
|
||||
break
|
||||
if time_left is not None:
|
||||
time_left -= (time.time() - time_start)
|
||||
if time_left < 0:
|
||||
raise Timeout
|
||||
#print "result:",result
|
||||
out = ''.join(out)
|
||||
#print "stdout:", out
|
||||
err = ''.join(err)
|
||||
#print "stderr:", err
|
||||
return result, out, err
|
||||
@@ -1,592 +0,0 @@
|
||||
"""\
|
||||
@file named_query.py
|
||||
@author Ryan Williams, Phoenix
|
||||
@date 2007-07-31
|
||||
@brief An API for running named queries.
|
||||
|
||||
$LicenseInfo:firstyear=2007&license=mit$
|
||||
|
||||
Copyright (c) 2007-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
import errno
|
||||
import MySQLdb
|
||||
import MySQLdb.cursors
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import time
|
||||
|
||||
from indra.base import llsd
|
||||
from indra.base import config
|
||||
|
||||
DEBUG = False
|
||||
NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq')
|
||||
NQ_FILE_SUFFIX_LEN = len(NQ_FILE_SUFFIX)
|
||||
|
||||
_g_named_manager = None
|
||||
|
||||
def _init_g_named_manager(sql_dir = None):
|
||||
"""Initializes a global NamedManager object to point at a
|
||||
specified named queries hierarchy.
|
||||
|
||||
This function is intended entirely for testing purposes,
|
||||
because it's tricky to control the config from inside a test."""
|
||||
global NQ_FILE_SUFFIX
|
||||
NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq')
|
||||
global NQ_FILE_SUFFIX_LEN
|
||||
NQ_FILE_SUFFIX_LEN = len(NQ_FILE_SUFFIX)
|
||||
|
||||
if sql_dir is None:
|
||||
sql_dir = config.get('named-query-base-dir')
|
||||
|
||||
# extra fallback directory in case config doesn't return what we want
|
||||
if sql_dir is None:
|
||||
sql_dir = os.path.abspath(
|
||||
os.path.join(
|
||||
os.path.realpath(os.path.dirname(__file__)), "..", "..", "..", "..", "web", "dataservice", "sql"))
|
||||
|
||||
global _g_named_manager
|
||||
_g_named_manager = NamedQueryManager(
|
||||
os.path.abspath(os.path.realpath(sql_dir)))
|
||||
|
||||
def get(name, schema = None):
|
||||
"Get the named query object to be used to perform queries"
|
||||
if _g_named_manager is None:
|
||||
_init_g_named_manager()
|
||||
return _g_named_manager.get(name).for_schema(schema)
|
||||
|
||||
def sql(connection, name, params):
|
||||
# use module-global NamedQuery object to perform default substitution
|
||||
return get(name).sql(connection, params)
|
||||
|
||||
def run(connection, name, params, expect_rows = None):
|
||||
"""\
|
||||
@brief given a connection, run a named query with the params
|
||||
|
||||
Note that this function will fetch ALL rows.
|
||||
@param connection The connection to use
|
||||
@param name The name of the query to run
|
||||
@param params The parameters passed into the query
|
||||
@param expect_rows The number of rows expected. Set to 1 if return_as_map is true. Raises ExpectationFailed if the number of returned rows doesn't exactly match. Kind of a hack.
|
||||
@return Returns the result set as a list of dicts.
|
||||
"""
|
||||
return get(name).run(connection, params, expect_rows)
|
||||
|
||||
class ExpectationFailed(Exception):
|
||||
""" Exception that is raised when an expectation for an sql query
|
||||
is not met."""
|
||||
def __init__(self, message):
|
||||
Exception.__init__(self, message)
|
||||
self.message = message
|
||||
|
||||
class NamedQuery(object):
|
||||
def __init__(self, name, filename):
|
||||
""" Construct a NamedQuery object. The name argument is an
|
||||
arbitrary name as a handle for the query, and the filename is
|
||||
a path to a file or a file-like object containing an llsd named
|
||||
query document."""
|
||||
self._stat_interval_seconds = 5 # 5 seconds
|
||||
self._name = name
|
||||
if (filename is not None and isinstance(filename, (str, unicode))
|
||||
and NQ_FILE_SUFFIX != filename[-NQ_FILE_SUFFIX_LEN:]):
|
||||
filename = filename + NQ_FILE_SUFFIX
|
||||
self._location = filename
|
||||
self._alternative = dict()
|
||||
self._last_mod_time = 0
|
||||
self._last_check_time = 0
|
||||
self.deleted = False
|
||||
self.load_contents()
|
||||
|
||||
def name(self):
|
||||
""" The name of the query. """
|
||||
return self._name
|
||||
|
||||
def get_modtime(self):
|
||||
""" Returns the mtime (last modified time) of the named query
|
||||
filename. For file-like objects, expect a modtime of 0"""
|
||||
if self._location and isinstance(self._location, (str, unicode)):
|
||||
return os.path.getmtime(self._location)
|
||||
return 0
|
||||
|
||||
def load_contents(self):
|
||||
""" Loads and parses the named query file into self. Does
|
||||
nothing if self.location is nonexistant."""
|
||||
if self._location:
|
||||
if isinstance(self._location, (str, unicode)):
|
||||
contents = llsd.parse(open(self._location).read())
|
||||
else:
|
||||
# we probably have a file-like object. Godspeed!
|
||||
contents = llsd.parse(self._location.read())
|
||||
self._reference_contents(contents)
|
||||
# Check for alternative implementations
|
||||
try:
|
||||
for name, alt in self._contents['alternative'].items():
|
||||
nq = NamedQuery(name, None)
|
||||
nq._reference_contents(alt)
|
||||
self._alternative[name] = nq
|
||||
except KeyError, e:
|
||||
pass
|
||||
self._last_mod_time = self.get_modtime()
|
||||
self._last_check_time = time.time()
|
||||
|
||||
def _reference_contents(self, contents):
|
||||
"Helper method which builds internal structure from parsed contents"
|
||||
self._contents = contents
|
||||
self._ttl = int(self._contents.get('ttl', 0))
|
||||
self._return_as_map = bool(self._contents.get('return_as_map', False))
|
||||
self._legacy_dbname = self._contents.get('legacy_dbname', None)
|
||||
|
||||
# reset these before doing the sql conversion because we will
|
||||
# read them there. reset these while loading so we pick up
|
||||
# changes.
|
||||
self._around = set()
|
||||
self._append = set()
|
||||
self._integer = set()
|
||||
self._options = self._contents.get('dynamic_where', {})
|
||||
for key in self._options:
|
||||
if isinstance(self._options[key], basestring):
|
||||
self._options[key] = self._convert_sql(self._options[key])
|
||||
elif isinstance(self._options[key], list):
|
||||
lines = []
|
||||
for line in self._options[key]:
|
||||
lines.append(self._convert_sql(line))
|
||||
self._options[key] = lines
|
||||
else:
|
||||
moreopt = {}
|
||||
for kk in self._options[key]:
|
||||
moreopt[kk] = self._convert_sql(self._options[key][kk])
|
||||
self._options[key] = moreopt
|
||||
self._base_query = self._convert_sql(self._contents['base_query'])
|
||||
self._query_suffix = self._convert_sql(
|
||||
self._contents.get('query_suffix', ''))
|
||||
|
||||
def _convert_sql(self, sql):
|
||||
"""convert the parsed sql into a useful internal structure.
|
||||
|
||||
This function has to turn the named query format into a pyformat
|
||||
style. It also has to look for %:name% and :name% and
|
||||
ready them for use in LIKE statements"""
|
||||
if sql:
|
||||
# This first sub is to properly escape any % signs that
|
||||
# are meant to be literally passed through to mysql in the
|
||||
# query. It leaves any %'s that are used for
|
||||
# like-expressions.
|
||||
expr = re.compile("(?<=[^a-zA-Z0-9_-])%(?=[^:])")
|
||||
sql = expr.sub('%%', sql)
|
||||
|
||||
# This should tackle the rest of the %'s in the query, by
|
||||
# converting them to LIKE clauses.
|
||||
expr = re.compile("(%?):([a-zA-Z][a-zA-Z0-9_-]*)%")
|
||||
sql = expr.sub(self._prepare_like, sql)
|
||||
expr = re.compile("#:([a-zA-Z][a-zA-Z0-9_-]*)")
|
||||
sql = expr.sub(self._prepare_integer, sql)
|
||||
expr = re.compile(":([a-zA-Z][a-zA-Z0-9_-]*)")
|
||||
sql = expr.sub("%(\\1)s", sql)
|
||||
return sql
|
||||
|
||||
def _prepare_like(self, match):
|
||||
"""This function changes LIKE statement replace behavior
|
||||
|
||||
It works by turning %:name% to %(_name_around)s and :name% to
|
||||
%(_name_append)s. Since a leading '_' is not a valid keyname
|
||||
input (enforced via unit tests), it will never clash with
|
||||
existing keys. Then, when building the statement, the query
|
||||
runner will generate corrected strings."""
|
||||
if match.group(1) == '%':
|
||||
# there is a leading % so this is treated as prefix/suffix
|
||||
self._around.add(match.group(2))
|
||||
return "%(" + self._build_around_key(match.group(2)) + ")s"
|
||||
else:
|
||||
# there is no leading %, so this is suffix only
|
||||
self._append.add(match.group(2))
|
||||
return "%(" + self._build_append_key(match.group(2)) + ")s"
|
||||
|
||||
def _build_around_key(self, key):
|
||||
return "_" + key + "_around"
|
||||
|
||||
def _build_append_key(self, key):
|
||||
return "_" + key + "_append"
|
||||
|
||||
def _prepare_integer(self, match):
|
||||
"""This function adjusts the sql for #:name replacements
|
||||
|
||||
It works by turning #:name to %(_name_as_integer)s. Since a
|
||||
leading '_' is not a valid keyname input (enforced via unit
|
||||
tests), it will never clash with existing keys. Then, when
|
||||
building the statement, the query runner will generate
|
||||
corrected strings."""
|
||||
self._integer.add(match.group(1))
|
||||
return "%(" + self._build_integer_key(match.group(1)) + ")s"
|
||||
|
||||
def _build_integer_key(self, key):
|
||||
return "_" + key + "_as_integer"
|
||||
|
||||
def _strip_wildcards_to_list(self, value):
|
||||
"""Take string, and strip out the LIKE special characters.
|
||||
|
||||
Technically, this is database dependant, but postgresql and
|
||||
mysql use the same wildcards, and I am not aware of a general
|
||||
way to handle this. I think you need a sql statement of the
|
||||
form:
|
||||
|
||||
LIKE_STRING( [ANY,ONE,str]... )
|
||||
|
||||
which would treat ANY as their any string, and ONE as their
|
||||
single glyph, and str as something that needs database
|
||||
specific encoding to not allow any % or _ to affect the query.
|
||||
|
||||
As it stands, I believe it's impossible to write a named query
|
||||
style interface which uses like to search the entire space of
|
||||
text available. Imagine the query:
|
||||
|
||||
% of brain used by average linden
|
||||
|
||||
In order to search for %, it must be escaped, so once you have
|
||||
escaped the string to not do wildcard searches, and be escaped
|
||||
for the database, and then prepended the wildcard you come
|
||||
back with one of:
|
||||
|
||||
1) %\% of brain used by average linden
|
||||
2) %%% of brain used by average linden
|
||||
|
||||
Then, when passed to the database to be escaped to be database
|
||||
safe, you get back:
|
||||
|
||||
1) %\\% of brain used by average linden
|
||||
: which means search for any character sequence, followed by a
|
||||
backslash, followed by any sequence, followed by ' of
|
||||
brain...'
|
||||
2) %%% of brain used by average linden
|
||||
: which (I believe) means search for a % followed by any
|
||||
character sequence followed by 'of brain...'
|
||||
|
||||
Neither of which is what we want!
|
||||
|
||||
So, we need a vendor (or extention) for LIKE_STRING. Anyone
|
||||
want to write it?"""
|
||||
if isinstance(value, unicode):
|
||||
utf8_value = value
|
||||
else:
|
||||
utf8_value = unicode(value, "utf-8")
|
||||
esc_list = []
|
||||
remove_chars = set(u"%_")
|
||||
for glyph in utf8_value:
|
||||
if glyph in remove_chars:
|
||||
continue
|
||||
esc_list.append(glyph.encode("utf-8"))
|
||||
return esc_list
|
||||
|
||||
def delete(self):
|
||||
""" Makes this query unusable by deleting all the members and
|
||||
setting the deleted member. This is desired when the on-disk
|
||||
query has been deleted but the in-memory copy remains."""
|
||||
# blow away all members except _name, _location, and deleted
|
||||
name, location = self._name, self._location
|
||||
for key in self.__dict__.keys():
|
||||
del self.__dict__[key]
|
||||
self.deleted = True
|
||||
self._name, self._location = name, location
|
||||
|
||||
def ttl(self):
|
||||
""" Estimated time to live of this query. Used for web
|
||||
services to set the Expires header."""
|
||||
return self._ttl
|
||||
|
||||
def legacy_dbname(self):
|
||||
return self._legacy_dbname
|
||||
|
||||
def return_as_map(self):
|
||||
""" Returns true if this query is configured to return its
|
||||
results as a single map (as opposed to a list of maps, the
|
||||
normal behavior)."""
|
||||
|
||||
return self._return_as_map
|
||||
|
||||
def for_schema(self, db_name):
|
||||
"Look trough the alternates and return the correct query"
|
||||
if db_name is None:
|
||||
return self
|
||||
try:
|
||||
return self._alternative[db_name]
|
||||
except KeyError, e:
|
||||
pass
|
||||
return self
|
||||
|
||||
def run(self, connection, params, expect_rows = None, use_dictcursor = True):
|
||||
"""given a connection, run a named query with the params
|
||||
|
||||
Note that this function will fetch ALL rows. We do this because it
|
||||
opens and closes the cursor to generate the values, and this
|
||||
isn't a generator so the cursor has no life beyond the method call.
|
||||
|
||||
@param cursor The connection to use (this generates its own cursor for the query)
|
||||
@param name The name of the query to run
|
||||
@param params The parameters passed into the query
|
||||
@param expect_rows The number of rows expected. Set to 1 if return_as_map is true. Raises ExpectationFailed if the number of returned rows doesn't exactly match. Kind of a hack.
|
||||
@param use_dictcursor Set to false to use a normal cursor and manually convert the rows to dicts.
|
||||
@return Returns the result set as a list of dicts, or, if the named query has return_as_map set to true, returns a single dict.
|
||||
"""
|
||||
if use_dictcursor:
|
||||
cursor = connection.cursor(MySQLdb.cursors.DictCursor)
|
||||
else:
|
||||
cursor = connection.cursor()
|
||||
|
||||
full_query, params = self._construct_sql(params)
|
||||
if DEBUG:
|
||||
print "SQL:", self.sql(connection, params)
|
||||
rows = cursor.execute(full_query, params)
|
||||
|
||||
# *NOTE: the expect_rows argument is a very cheesy way to get some
|
||||
# validation on the result set. If you want to add more expectation
|
||||
# logic, do something more object-oriented and flexible. Or use an ORM.
|
||||
if(self._return_as_map):
|
||||
expect_rows = 1
|
||||
if expect_rows is not None and rows != expect_rows:
|
||||
cursor.close()
|
||||
raise ExpectationFailed("Statement expected %s rows, got %s. Sql: '%s' %s" % (
|
||||
expect_rows, rows, full_query, params))
|
||||
|
||||
# convert to dicts manually if we're not using a dictcursor
|
||||
if use_dictcursor:
|
||||
result_set = cursor.fetchall()
|
||||
else:
|
||||
if cursor.description is None:
|
||||
# an insert or something
|
||||
x = cursor.fetchall()
|
||||
cursor.close()
|
||||
return x
|
||||
|
||||
names = [x[0] for x in cursor.description]
|
||||
|
||||
result_set = []
|
||||
for row in cursor.fetchall():
|
||||
converted_row = {}
|
||||
for idx, col_name in enumerate(names):
|
||||
converted_row[col_name] = row[idx]
|
||||
result_set.append(converted_row)
|
||||
|
||||
cursor.close()
|
||||
if self._return_as_map:
|
||||
return result_set[0]
|
||||
return result_set
|
||||
|
||||
def _construct_sql(self, params):
|
||||
""" Returns a query string and a dictionary of parameters,
|
||||
suitable for directly passing to the execute() method."""
|
||||
self.refresh()
|
||||
|
||||
# build the query from the options available and the params
|
||||
base_query = []
|
||||
base_query.append(self._base_query)
|
||||
for opt, extra_where in self._options.items():
|
||||
if type(extra_where) in (dict, list, tuple):
|
||||
if opt in params:
|
||||
base_query.append(extra_where[params[opt]])
|
||||
else:
|
||||
if opt in params and params[opt]:
|
||||
base_query.append(extra_where)
|
||||
if self._query_suffix:
|
||||
base_query.append(self._query_suffix)
|
||||
full_query = '\n'.join(base_query)
|
||||
|
||||
# Go through the query and rewrite all of the ones with the
|
||||
# @:name syntax.
|
||||
rewrite = _RewriteQueryForArray(params)
|
||||
expr = re.compile("@%\(([a-zA-Z][a-zA-Z0-9_-]*)\)s")
|
||||
full_query = expr.sub(rewrite.operate, full_query)
|
||||
params.update(rewrite.new_params)
|
||||
|
||||
# build out the params for like. We only have to do this
|
||||
# parameters which were detected to have ued the where syntax
|
||||
# during load.
|
||||
#
|
||||
# * treat the incoming string as utf-8
|
||||
# * strip wildcards
|
||||
# * append or prepend % as appropriate
|
||||
new_params = {}
|
||||
for key in params:
|
||||
if key in self._around:
|
||||
new_value = ['%']
|
||||
new_value.extend(self._strip_wildcards_to_list(params[key]))
|
||||
new_value.append('%')
|
||||
new_params[self._build_around_key(key)] = ''.join(new_value)
|
||||
if key in self._append:
|
||||
new_value = self._strip_wildcards_to_list(params[key])
|
||||
new_value.append('%')
|
||||
new_params[self._build_append_key(key)] = ''.join(new_value)
|
||||
if key in self._integer:
|
||||
new_params[self._build_integer_key(key)] = int(params[key])
|
||||
params.update(new_params)
|
||||
|
||||
return full_query, params
|
||||
|
||||
def sql(self, connection, params):
|
||||
""" Generates an SQL statement from the named query document
|
||||
and a dictionary of parameters.
|
||||
|
||||
*NOTE: Only use for debugging, because it uses the
|
||||
non-standard MySQLdb 'literal' method.
|
||||
"""
|
||||
if not DEBUG:
|
||||
import warnings
|
||||
warnings.warn("Don't use named_query.sql() when not debugging. Used on %s" % self._location)
|
||||
# do substitution using the mysql (non-standard) 'literal'
|
||||
# function to do the escaping.
|
||||
full_query, params = self._construct_sql(params)
|
||||
return full_query % connection.literal(params)
|
||||
|
||||
|
||||
def refresh(self):
|
||||
""" Refresh self from the file on the filesystem.
|
||||
|
||||
This is optimized to be callable as frequently as you wish,
|
||||
without adding too much load. It does so by only stat-ing the
|
||||
file every N seconds, where N defaults to 5 and is
|
||||
configurable through the member _stat_interval_seconds. If the stat
|
||||
reveals that the file has changed, refresh will re-parse the
|
||||
contents of the file and use them to update the named query
|
||||
instance. If the stat reveals that the file has been deleted,
|
||||
refresh will call self.delete to make the in-memory
|
||||
representation unusable."""
|
||||
now = time.time()
|
||||
if(now - self._last_check_time > self._stat_interval_seconds):
|
||||
self._last_check_time = now
|
||||
try:
|
||||
modtime = self.get_modtime()
|
||||
if(modtime > self._last_mod_time):
|
||||
self.load_contents()
|
||||
except OSError, e:
|
||||
if e.errno == errno.ENOENT: # file not found
|
||||
self.delete() # clean up self
|
||||
raise # pass the exception along to the caller so they know that this query disappeared
|
||||
|
||||
class NamedQueryManager(object):
|
||||
""" Manages the lifespan of NamedQuery objects, drawing from a
|
||||
directory hierarchy of named query documents.
|
||||
|
||||
In practice this amounts to a memory cache of NamedQuery objects."""
|
||||
|
||||
def __init__(self, named_queries_dir):
|
||||
""" Initializes a manager to look for named queries in a
|
||||
directory."""
|
||||
self._dir = os.path.abspath(os.path.realpath(named_queries_dir))
|
||||
self._cached_queries = {}
|
||||
|
||||
def sql(self, connection, name, params):
|
||||
nq = self.get(name)
|
||||
return nq.sql(connection, params)
|
||||
|
||||
def get(self, name):
|
||||
""" Returns a NamedQuery instance based on the name, either
|
||||
from memory cache, or by parsing from disk.
|
||||
|
||||
The name is simply a relative path to the directory associated
|
||||
with the manager object. Before returning the instance, the
|
||||
NamedQuery object is cached in memory, so that subsequent
|
||||
accesses don't have to read from disk or do any parsing. This
|
||||
means that NamedQuery objects returned by this method are
|
||||
shared across all users of the manager object.
|
||||
NamedQuery.refresh is used to bring the NamedQuery objects in
|
||||
sync with the actual files on disk."""
|
||||
nq = self._cached_queries.get(name)
|
||||
if nq is None:
|
||||
nq = NamedQuery(name, os.path.join(self._dir, name))
|
||||
self._cached_queries[name] = nq
|
||||
else:
|
||||
try:
|
||||
nq.refresh()
|
||||
except OSError, e:
|
||||
if e.errno == errno.ENOENT: # file not found
|
||||
del self._cached_queries[name]
|
||||
raise # pass exception along to caller so they know that the query disappeared
|
||||
|
||||
return nq
|
||||
|
||||
class _RewriteQueryForArray(object):
|
||||
"Helper class for rewriting queries with the @:name syntax"
|
||||
def __init__(self, params):
|
||||
self.params = params
|
||||
self.new_params = dict()
|
||||
|
||||
def operate(self, match):
|
||||
"Given a match, return the string that should be in use"
|
||||
key = match.group(1)
|
||||
value = self.params[key]
|
||||
if type(value) in (list,tuple):
|
||||
rv = []
|
||||
for idx in range(len(value)):
|
||||
# if the value@idx is array-like, we are
|
||||
# probably dealing with a VALUES
|
||||
new_key = "_%s_%s"%(key, str(idx))
|
||||
val_item = value[idx]
|
||||
if type(val_item) in (list, tuple, dict):
|
||||
if type(val_item) is dict:
|
||||
# this is because in Python, the order of
|
||||
# key, value retrieval from the dict is not
|
||||
# guaranteed to match what the input intended
|
||||
# and for VALUES, order is important.
|
||||
# TODO: Implemented ordered dict in LLSD parser?
|
||||
raise ExpectationFailed('Only lists/tuples allowed,\
|
||||
received dict')
|
||||
values_keys = []
|
||||
for value_idx, item in enumerate(val_item):
|
||||
# we want a key of the format :
|
||||
# key_#replacement_#value_row_#value_col
|
||||
# ugh... so if we are replacing 10 rows in user_note,
|
||||
# the first values clause would read (for @:user_notes) :-
|
||||
# ( :_user_notes_0_1_1, :_user_notes_0_1_2, :_user_notes_0_1_3 )
|
||||
# the input LLSD for VALUES will look like:
|
||||
# <llsd>...
|
||||
# <map>
|
||||
# <key>user_notes</key>
|
||||
# <array>
|
||||
# <array> <!-- row 1 for VALUES -->
|
||||
# <string>...</string>
|
||||
# <string>...</string>
|
||||
# <string>...</string>
|
||||
# </array>
|
||||
# ...
|
||||
# </array>
|
||||
# </map>
|
||||
# ... </llsd>
|
||||
values_key = "%s_%s"%(new_key, value_idx)
|
||||
self.new_params[values_key] = item
|
||||
values_keys.append("%%(%s)s"%values_key)
|
||||
# now collapse all these new place holders enclosed in ()
|
||||
# from [':_key_0_1_1', ':_key_0_1_2', ':_key_0_1_3,...]
|
||||
# rv will have [ '(:_key_0_1_1, :_key_0_1_2, :_key_0_1_3)', ]
|
||||
# which is flattened a few lines below join(rv)
|
||||
rv.append('(%s)' % ','.join(values_keys))
|
||||
else:
|
||||
self.new_params[new_key] = val_item
|
||||
rv.append("%%(%s)s"%new_key)
|
||||
return ','.join(rv)
|
||||
else:
|
||||
# not something that can be expanded, so just drop the
|
||||
# leading @ in the front of the match. This will mean that
|
||||
# the single value we have, be it a string, int, whatever
|
||||
# (other than dict) will correctly show up, eg:
|
||||
#
|
||||
# where foo in (@:foobar) -- foobar is a string, so we get
|
||||
# where foo in (:foobar)
|
||||
return match.group(0)[1:]
|
||||
@@ -1,84 +0,0 @@
|
||||
'''
|
||||
@file shutil2.py
|
||||
@brief a better shutil.copytree replacement
|
||||
|
||||
$LicenseInfo:firstyear=2007&license=mit$
|
||||
|
||||
Copyright (c) 2007-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
'''
|
||||
|
||||
#
|
||||
# shutil2.py
|
||||
# Taken from http://www.scons.org/wiki/AccumulateBuilder
|
||||
# the stock copytree sucks because it insists that the
|
||||
# target dir not exist
|
||||
#
|
||||
|
||||
import os.path
|
||||
import shutil
|
||||
|
||||
def copytree(src, dest, symlinks=False):
|
||||
"""My own copyTree which does not fail if the directory exists.
|
||||
|
||||
Recursively copy a directory tree using copy2().
|
||||
|
||||
If the optional symlinks flag is true, symbolic links in the
|
||||
source tree result in symbolic links in the destination tree; if
|
||||
it is false, the contents of the files pointed to by symbolic
|
||||
links are copied.
|
||||
|
||||
Behavior is meant to be identical to GNU 'cp -R'.
|
||||
"""
|
||||
def copyItems(src, dest, symlinks=False):
|
||||
"""Function that does all the work.
|
||||
|
||||
It is necessary to handle the two 'cp' cases:
|
||||
- destination does exist
|
||||
- destination does not exist
|
||||
|
||||
See 'cp -R' documentation for more details
|
||||
"""
|
||||
for item in os.listdir(src):
|
||||
srcPath = os.path.join(src, item)
|
||||
if os.path.isdir(srcPath):
|
||||
srcBasename = os.path.basename(srcPath)
|
||||
destDirPath = os.path.join(dest, srcBasename)
|
||||
if not os.path.exists(destDirPath):
|
||||
os.makedirs(destDirPath)
|
||||
copyItems(srcPath, destDirPath)
|
||||
elif os.path.islink(item) and symlinks:
|
||||
linkto = os.readlink(item)
|
||||
os.symlink(linkto, dest)
|
||||
else:
|
||||
shutil.copy2(srcPath, dest)
|
||||
|
||||
# case 'cp -R src/ dest/' where dest/ already exists
|
||||
if os.path.exists(dest):
|
||||
destPath = os.path.join(dest, os.path.basename(src))
|
||||
if not os.path.exists(destPath):
|
||||
os.makedirs(destPath)
|
||||
# case 'cp -R src/ dest/' where dest/ does not exist
|
||||
else:
|
||||
os.makedirs(dest)
|
||||
destPath = dest
|
||||
# actually copy the files
|
||||
copyItems(src, destPath)
|
||||
@@ -1,338 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""\
|
||||
@file simperf_host_xml_parser.py
|
||||
@brief Digest collector's XML dump and convert to simple dict/list structure
|
||||
|
||||
$LicenseInfo:firstyear=2008&license=mit$
|
||||
|
||||
Copyright (c) 2008-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
import sys, os, getopt, time
|
||||
import simplejson
|
||||
from xml import sax
|
||||
|
||||
|
||||
def usage():
|
||||
print "Usage:"
|
||||
print sys.argv[0] + " [options]"
|
||||
print " Convert RRD's XML dump to JSON. Script to convert the simperf_host_collector-"
|
||||
print " generated RRD dump into JSON. Steps include converting selected named"
|
||||
print " fields from GAUGE type to COUNTER type by computing delta with preceding"
|
||||
print " values. Top-level named fields are:"
|
||||
print
|
||||
print " lastupdate Time (javascript timestamp) of last data sample"
|
||||
print " step Time in seconds between samples"
|
||||
print " ds Data specification (name/type) for each column"
|
||||
print " database Table of data samples, one time step per row"
|
||||
print
|
||||
print "Options:"
|
||||
print " -i, --in Input settings filename. (Default: stdin)"
|
||||
print " -o, --out Output settings filename. (Default: stdout)"
|
||||
print " -h, --help Print this message and exit."
|
||||
print
|
||||
print "Example: %s -i rrddump.xml -o rrddump.json" % sys.argv[0]
|
||||
print
|
||||
print "Interfaces:"
|
||||
print " class SimPerfHostXMLParser() # SAX content handler"
|
||||
print " def simperf_host_xml_fixup(parser) # post-parse value fixup"
|
||||
|
||||
class SimPerfHostXMLParser(sax.handler.ContentHandler):
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def startDocument(self):
|
||||
self.rrd_last_update = 0 # public
|
||||
self.rrd_step = 0 # public
|
||||
self.rrd_ds = [] # public
|
||||
self.rrd_records = [] # public
|
||||
self._rrd_level = 0
|
||||
self._rrd_parse_state = 0
|
||||
self._rrd_chars = ""
|
||||
self._rrd_capture = False
|
||||
self._rrd_ds_val = {}
|
||||
self._rrd_data_row = []
|
||||
self._rrd_data_row_has_nan = False
|
||||
|
||||
def endDocument(self):
|
||||
pass
|
||||
|
||||
# Nasty little ad-hoc state machine to extract the elements that are
|
||||
# necessary from the 'rrdtool dump' XML output. The same element
|
||||
# name '<ds>' is used for two different data sets so we need to pay
|
||||
# some attention to the actual structure to get the ones we want
|
||||
# and ignore the ones we don't.
|
||||
|
||||
def startElement(self, name, attrs):
|
||||
self._rrd_level = self._rrd_level + 1
|
||||
self._rrd_capture = False
|
||||
if self._rrd_level == 1:
|
||||
if name == "rrd" and self._rrd_parse_state == 0:
|
||||
self._rrd_parse_state = 1 # In <rrd>
|
||||
self._rrd_capture = True
|
||||
self._rrd_chars = ""
|
||||
elif self._rrd_level == 2:
|
||||
if self._rrd_parse_state == 1:
|
||||
if name == "lastupdate":
|
||||
self._rrd_parse_state = 2 # In <rrd><lastupdate>
|
||||
self._rrd_capture = True
|
||||
self._rrd_chars = ""
|
||||
elif name == "step":
|
||||
self._rrd_parse_state = 3 # In <rrd><step>
|
||||
self._rrd_capture = True
|
||||
self._rrd_chars = ""
|
||||
elif name == "ds":
|
||||
self._rrd_parse_state = 4 # In <rrd><ds>
|
||||
self._rrd_ds_val = {}
|
||||
self._rrd_chars = ""
|
||||
elif name == "rra":
|
||||
self._rrd_parse_state = 5 # In <rrd><rra>
|
||||
elif self._rrd_level == 3:
|
||||
if self._rrd_parse_state == 4:
|
||||
if name == "name":
|
||||
self._rrd_parse_state = 6 # In <rrd><ds><name>
|
||||
self._rrd_capture = True
|
||||
self._rrd_chars = ""
|
||||
elif name == "type":
|
||||
self._rrd_parse_state = 7 # In <rrd><ds><type>
|
||||
self._rrd_capture = True
|
||||
self._rrd_chars = ""
|
||||
elif self._rrd_parse_state == 5:
|
||||
if name == "database":
|
||||
self._rrd_parse_state = 8 # In <rrd><rra><database>
|
||||
elif self._rrd_level == 4:
|
||||
if self._rrd_parse_state == 8:
|
||||
if name == "row":
|
||||
self._rrd_parse_state = 9 # In <rrd><rra><database><row>
|
||||
self._rrd_data_row = []
|
||||
self._rrd_data_row_has_nan = False
|
||||
elif self._rrd_level == 5:
|
||||
if self._rrd_parse_state == 9:
|
||||
if name == "v":
|
||||
self._rrd_parse_state = 10 # In <rrd><rra><database><row><v>
|
||||
self._rrd_capture = True
|
||||
self._rrd_chars = ""
|
||||
|
||||
def endElement(self, name):
|
||||
self._rrd_capture = False
|
||||
if self._rrd_parse_state == 10:
|
||||
self._rrd_capture = self._rrd_level == 6
|
||||
if self._rrd_level == 5:
|
||||
if self._rrd_chars == "NaN":
|
||||
self._rrd_data_row_has_nan = True
|
||||
else:
|
||||
self._rrd_data_row.append(self._rrd_chars)
|
||||
self._rrd_parse_state = 9 # In <rrd><rra><database><row>
|
||||
elif self._rrd_parse_state == 9:
|
||||
if self._rrd_level == 4:
|
||||
if not self._rrd_data_row_has_nan:
|
||||
self.rrd_records.append(self._rrd_data_row)
|
||||
self._rrd_parse_state = 8 # In <rrd><rra><database>
|
||||
elif self._rrd_parse_state == 8:
|
||||
if self._rrd_level == 3:
|
||||
self._rrd_parse_state = 5 # In <rrd><rra>
|
||||
elif self._rrd_parse_state == 7:
|
||||
if self._rrd_level == 3:
|
||||
self._rrd_ds_val["type"] = self._rrd_chars
|
||||
self._rrd_parse_state = 4 # In <rrd><ds>
|
||||
elif self._rrd_parse_state == 6:
|
||||
if self._rrd_level == 3:
|
||||
self._rrd_ds_val["name"] = self._rrd_chars
|
||||
self._rrd_parse_state = 4 # In <rrd><ds>
|
||||
elif self._rrd_parse_state == 5:
|
||||
if self._rrd_level == 2:
|
||||
self._rrd_parse_state = 1 # In <rrd>
|
||||
elif self._rrd_parse_state == 4:
|
||||
if self._rrd_level == 2:
|
||||
self.rrd_ds.append(self._rrd_ds_val)
|
||||
self._rrd_parse_state = 1 # In <rrd>
|
||||
elif self._rrd_parse_state == 3:
|
||||
if self._rrd_level == 2:
|
||||
self.rrd_step = long(self._rrd_chars)
|
||||
self._rrd_parse_state = 1 # In <rrd>
|
||||
elif self._rrd_parse_state == 2:
|
||||
if self._rrd_level == 2:
|
||||
self.rrd_last_update = long(self._rrd_chars)
|
||||
self._rrd_parse_state = 1 # In <rrd>
|
||||
elif self._rrd_parse_state == 1:
|
||||
if self._rrd_level == 1:
|
||||
self._rrd_parse_state = 0 # At top
|
||||
|
||||
if self._rrd_level:
|
||||
self._rrd_level = self._rrd_level - 1
|
||||
|
||||
def characters(self, content):
|
||||
if self._rrd_capture:
|
||||
self._rrd_chars = self._rrd_chars + content.strip()
|
||||
|
||||
def _make_numeric(value):
|
||||
try:
|
||||
value = float(value)
|
||||
except:
|
||||
value = ""
|
||||
return value
|
||||
|
||||
def simperf_host_xml_fixup(parser, filter_start_time = None, filter_end_time = None):
|
||||
# Fixup for GAUGE fields that are really COUNTS. They
|
||||
# were forced to GAUGE to try to disable rrdtool's
|
||||
# data interpolation/extrapolation for non-uniform time
|
||||
# samples.
|
||||
fixup_tags = [ "cpu_user",
|
||||
"cpu_nice",
|
||||
"cpu_sys",
|
||||
"cpu_idle",
|
||||
"cpu_waitio",
|
||||
"cpu_intr",
|
||||
# "file_active",
|
||||
# "file_free",
|
||||
# "inode_active",
|
||||
# "inode_free",
|
||||
"netif_in_kb",
|
||||
"netif_in_pkts",
|
||||
"netif_in_errs",
|
||||
"netif_in_drop",
|
||||
"netif_out_kb",
|
||||
"netif_out_pkts",
|
||||
"netif_out_errs",
|
||||
"netif_out_drop",
|
||||
"vm_page_in",
|
||||
"vm_page_out",
|
||||
"vm_swap_in",
|
||||
"vm_swap_out",
|
||||
#"vm_mem_total",
|
||||
#"vm_mem_used",
|
||||
#"vm_mem_active",
|
||||
#"vm_mem_inactive",
|
||||
#"vm_mem_free",
|
||||
#"vm_mem_buffer",
|
||||
#"vm_swap_cache",
|
||||
#"vm_swap_total",
|
||||
#"vm_swap_used",
|
||||
#"vm_swap_free",
|
||||
"cpu_interrupts",
|
||||
"cpu_switches",
|
||||
"cpu_forks" ]
|
||||
|
||||
col_count = len(parser.rrd_ds)
|
||||
row_count = len(parser.rrd_records)
|
||||
|
||||
# Process the last row separately, just to make all values numeric.
|
||||
for j in range(col_count):
|
||||
parser.rrd_records[row_count - 1][j] = _make_numeric(parser.rrd_records[row_count - 1][j])
|
||||
|
||||
# Process all other row/columns.
|
||||
last_different_row = row_count - 1
|
||||
current_row = row_count - 2
|
||||
while current_row >= 0:
|
||||
# Check for a different value than the previous row. If everything is the same
|
||||
# then this is probably just a filler/bogus entry.
|
||||
is_different = False
|
||||
for j in range(col_count):
|
||||
parser.rrd_records[current_row][j] = _make_numeric(parser.rrd_records[current_row][j])
|
||||
if parser.rrd_records[current_row][j] != parser.rrd_records[last_different_row][j]:
|
||||
# We're good. This is a different row.
|
||||
is_different = True
|
||||
|
||||
if not is_different:
|
||||
# This is a filler/bogus entry. Just ignore it.
|
||||
for j in range(col_count):
|
||||
parser.rrd_records[current_row][j] = float('nan')
|
||||
else:
|
||||
# Some tags need to be converted into deltas.
|
||||
for j in range(col_count):
|
||||
if parser.rrd_ds[j]["name"] in fixup_tags:
|
||||
parser.rrd_records[last_different_row][j] = \
|
||||
parser.rrd_records[last_different_row][j] - parser.rrd_records[current_row][j]
|
||||
last_different_row = current_row
|
||||
|
||||
current_row -= 1
|
||||
|
||||
# Set fixup_tags in the first row to 'nan' since they aren't useful anymore.
|
||||
for j in range(col_count):
|
||||
if parser.rrd_ds[j]["name"] in fixup_tags:
|
||||
parser.rrd_records[0][j] = float('nan')
|
||||
|
||||
# Add a timestamp to each row and to the catalog. Format and name
|
||||
# chosen to match other simulator logging (hopefully).
|
||||
start_time = parser.rrd_last_update - (parser.rrd_step * (row_count - 1))
|
||||
# Build a filtered list of rrd_records if we are limited to a time range.
|
||||
filter_records = False
|
||||
if filter_start_time is not None or filter_end_time is not None:
|
||||
filter_records = True
|
||||
filtered_rrd_records = []
|
||||
if filter_start_time is None:
|
||||
filter_start_time = start_time * 1000
|
||||
if filter_end_time is None:
|
||||
filter_end_time = parser.rrd_last_update * 1000
|
||||
|
||||
for i in range(row_count):
|
||||
record_timestamp = (start_time + (i * parser.rrd_step)) * 1000
|
||||
parser.rrd_records[i].insert(0, record_timestamp)
|
||||
if filter_records:
|
||||
if filter_start_time <= record_timestamp and record_timestamp <= filter_end_time:
|
||||
filtered_rrd_records.append(parser.rrd_records[i])
|
||||
|
||||
if filter_records:
|
||||
parser.rrd_records = filtered_rrd_records
|
||||
|
||||
parser.rrd_ds.insert(0, {"type": "GAUGE", "name": "javascript_timestamp"})
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
opts, args = getopt.getopt(sys.argv[1:], "i:o:h", ["in=", "out=", "help"])
|
||||
input_file = sys.stdin
|
||||
output_file = sys.stdout
|
||||
for o, a in opts:
|
||||
if o in ("-i", "--in"):
|
||||
input_file = open(a, 'r')
|
||||
if o in ("-o", "--out"):
|
||||
output_file = open(a, 'w')
|
||||
if o in ("-h", "--help"):
|
||||
usage()
|
||||
sys.exit(0)
|
||||
|
||||
# Using the SAX parser as it is at least 4X faster and far, far
|
||||
# smaller on this dataset than the DOM-based interface in xml.dom.minidom.
|
||||
# With SAX and a 5.4MB xml file, this requires about seven seconds of
|
||||
# wall-clock time and 32MB VSZ. With the DOM interface, about 22 seconds
|
||||
# and over 270MB VSZ.
|
||||
|
||||
handler = SimPerfHostXMLParser()
|
||||
sax.parse(input_file, handler)
|
||||
if input_file != sys.stdin:
|
||||
input_file.close()
|
||||
|
||||
# Various format fixups: string-to-num, gauge-to-counts, add
|
||||
# a time stamp, etc.
|
||||
simperf_host_xml_fixup(handler)
|
||||
|
||||
# Create JSONable dict with interesting data and format/print it
|
||||
print >>output_file, simplejson.dumps({ "step" : handler.rrd_step,
|
||||
"lastupdate": handler.rrd_last_update * 1000,
|
||||
"ds" : handler.rrd_ds,
|
||||
"database" : handler.rrd_records })
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,167 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""\
|
||||
@file simperf_oprof_interface.py
|
||||
@brief Manage OProfile data collection on a host
|
||||
|
||||
$LicenseInfo:firstyear=2008&license=mit$
|
||||
|
||||
Copyright (c) 2008-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
import sys, os, getopt
|
||||
import simplejson
|
||||
|
||||
|
||||
def usage():
|
||||
print "Usage:"
|
||||
print sys.argv[0] + " [options]"
|
||||
print " Digest the OProfile report forms that come out of the"
|
||||
print " simperf_oprof_ctl program's -r/--report command. The result"
|
||||
print " is an array of dictionaires with the following keys:"
|
||||
print
|
||||
print " symbol Name of sampled, calling, or called procedure"
|
||||
print " file Executable or library where symbol resides"
|
||||
print " percentage Percentage contribution to profile, calls or called"
|
||||
print " samples Sample count"
|
||||
print " calls Methods called by the method in question (full only)"
|
||||
print " called_by Methods calling the method (full only)"
|
||||
print
|
||||
print " For 'full' reports the two keys 'calls' and 'called_by' are"
|
||||
print " themselves arrays of dictionaries based on the first four keys."
|
||||
print
|
||||
print "Return Codes:"
|
||||
print " None. Aggressively digests everything. Will likely mung results"
|
||||
print " if a program or library has whitespace in its name."
|
||||
print
|
||||
print "Options:"
|
||||
print " -i, --in Input settings filename. (Default: stdin)"
|
||||
print " -o, --out Output settings filename. (Default: stdout)"
|
||||
print " -h, --help Print this message and exit."
|
||||
print
|
||||
print "Interfaces:"
|
||||
print " class SimPerfOProfileInterface()"
|
||||
|
||||
class SimPerfOProfileInterface:
|
||||
def __init__(self):
|
||||
self.isBrief = True # public
|
||||
self.isValid = False # public
|
||||
self.result = [] # public
|
||||
|
||||
def parse(self, input):
|
||||
in_samples = False
|
||||
for line in input:
|
||||
if in_samples:
|
||||
if line[0:6] == "------":
|
||||
self.isBrief = False
|
||||
self._parseFull(input)
|
||||
else:
|
||||
self._parseBrief(input, line)
|
||||
self.isValid = True
|
||||
return
|
||||
try:
|
||||
hd1, remain = line.split(None, 1)
|
||||
if hd1 == "samples":
|
||||
in_samples = True
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def _parseBrief(self, input, line1):
|
||||
try:
|
||||
fld1, fld2, fld3, fld4 = line1.split(None, 3)
|
||||
self.result.append({"samples" : fld1,
|
||||
"percentage" : fld2,
|
||||
"file" : fld3,
|
||||
"symbol" : fld4.strip("\n")})
|
||||
except ValueError:
|
||||
pass
|
||||
for line in input:
|
||||
try:
|
||||
fld1, fld2, fld3, fld4 = line.split(None, 3)
|
||||
self.result.append({"samples" : fld1,
|
||||
"percentage" : fld2,
|
||||
"file" : fld3,
|
||||
"symbol" : fld4.strip("\n")})
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def _parseFull(self, input):
|
||||
state = 0 # In 'called_by' section
|
||||
calls = []
|
||||
called_by = []
|
||||
current = {}
|
||||
for line in input:
|
||||
if line[0:6] == "------":
|
||||
if len(current):
|
||||
current["calls"] = calls
|
||||
current["called_by"] = called_by
|
||||
self.result.append(current)
|
||||
state = 0
|
||||
calls = []
|
||||
called_by = []
|
||||
current = {}
|
||||
else:
|
||||
try:
|
||||
fld1, fld2, fld3, fld4 = line.split(None, 3)
|
||||
tmp = {"samples" : fld1,
|
||||
"percentage" : fld2,
|
||||
"file" : fld3,
|
||||
"symbol" : fld4.strip("\n")}
|
||||
except ValueError:
|
||||
continue
|
||||
if line[0] != " ":
|
||||
current = tmp
|
||||
state = 1 # In 'calls' section
|
||||
elif state == 0:
|
||||
called_by.append(tmp)
|
||||
else:
|
||||
calls.append(tmp)
|
||||
if len(current):
|
||||
current["calls"] = calls
|
||||
current["called_by"] = called_by
|
||||
self.result.append(current)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
opts, args = getopt.getopt(sys.argv[1:], "i:o:h", ["in=", "out=", "help"])
|
||||
input_file = sys.stdin
|
||||
output_file = sys.stdout
|
||||
for o, a in opts:
|
||||
if o in ("-i", "--in"):
|
||||
input_file = open(a, 'r')
|
||||
if o in ("-o", "--out"):
|
||||
output_file = open(a, 'w')
|
||||
if o in ("-h", "--help"):
|
||||
usage()
|
||||
sys.exit(0)
|
||||
|
||||
oprof = SimPerfOProfileInterface()
|
||||
oprof.parse(input_file)
|
||||
if input_file != sys.stdin:
|
||||
input_file.close()
|
||||
|
||||
# Create JSONable dict with interesting data and format/print it
|
||||
print >>output_file, simplejson.dumps(oprof.result)
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,191 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""\
|
||||
@file simperf_proc_interface.py
|
||||
@brief Utility to extract log messages from *.<pid>.llsd files containing performance statistics.
|
||||
|
||||
$LicenseInfo:firstyear=2008&license=mit$
|
||||
|
||||
Copyright (c) 2008-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
"""
|
||||
|
||||
# ----------------------------------------------------
|
||||
# Utility to extract log messages from *.<pid>.llsd
|
||||
# files that contain performance statistics.
|
||||
|
||||
# ----------------------------------------------------
|
||||
import sys, os
|
||||
|
||||
if os.path.exists("setup-path.py"):
|
||||
execfile("setup-path.py")
|
||||
|
||||
from indra.base import llsd
|
||||
|
||||
DEFAULT_PATH="/dev/shm/simperf/"
|
||||
|
||||
|
||||
# ----------------------------------------------------
|
||||
# Pull out the stats and return a single document
|
||||
def parse_logfile(filename, target_column=None, verbose=False):
|
||||
full_doc = []
|
||||
# Open source temp log file. Let exceptions percolate up.
|
||||
sourcefile = open( filename,'r')
|
||||
|
||||
if verbose:
|
||||
print "Reading " + filename
|
||||
|
||||
# Parse and output all lines from the temp file
|
||||
for line in sourcefile.xreadlines():
|
||||
partial_doc = llsd.parse(line)
|
||||
if partial_doc is not None:
|
||||
if target_column is None:
|
||||
full_doc.append(partial_doc)
|
||||
else:
|
||||
trim_doc = { target_column: partial_doc[target_column] }
|
||||
if target_column != "fps":
|
||||
trim_doc[ 'fps' ] = partial_doc[ 'fps' ]
|
||||
trim_doc[ '/total_time' ] = partial_doc[ '/total_time' ]
|
||||
trim_doc[ 'utc_time' ] = partial_doc[ 'utc_time' ]
|
||||
full_doc.append(trim_doc)
|
||||
|
||||
sourcefile.close()
|
||||
return full_doc
|
||||
|
||||
# Extract just the meta info line, and the timestamp of the first/last frame entry.
|
||||
def parse_logfile_info(filename, verbose=False):
|
||||
# Open source temp log file. Let exceptions percolate up.
|
||||
sourcefile = open(filename, 'rU') # U is to open with Universal newline support
|
||||
|
||||
if verbose:
|
||||
print "Reading " + filename
|
||||
|
||||
# The first line is the meta info line.
|
||||
info_line = sourcefile.readline()
|
||||
if not info_line:
|
||||
sourcefile.close()
|
||||
return None
|
||||
|
||||
# The rest of the lines are frames. Read the first and last to get the time range.
|
||||
info = llsd.parse( info_line )
|
||||
info['start_time'] = None
|
||||
info['end_time'] = None
|
||||
first_frame = sourcefile.readline()
|
||||
if first_frame:
|
||||
try:
|
||||
info['start_time'] = int(llsd.parse(first_frame)['timestamp'])
|
||||
except:
|
||||
pass
|
||||
|
||||
# Read the file backwards to find the last two lines.
|
||||
sourcefile.seek(0, 2)
|
||||
file_size = sourcefile.tell()
|
||||
offset = 1024
|
||||
num_attempts = 0
|
||||
end_time = None
|
||||
if file_size < offset:
|
||||
offset = file_size
|
||||
while 1:
|
||||
sourcefile.seek(-1*offset, 2)
|
||||
read_str = sourcefile.read(offset)
|
||||
# Remove newline at the end
|
||||
if read_str[offset - 1] == '\n':
|
||||
read_str = read_str[0:-1]
|
||||
lines = read_str.split('\n')
|
||||
full_line = None
|
||||
if len(lines) > 2: # Got two line
|
||||
try:
|
||||
end_time = llsd.parse(lines[-1])['timestamp']
|
||||
except:
|
||||
# We couldn't parse this line. Try once more.
|
||||
try:
|
||||
end_time = llsd.parse(lines[-2])['timestamp']
|
||||
except:
|
||||
# Nope. Just move on.
|
||||
pass
|
||||
break
|
||||
if len(read_str) == file_size: # Reached the beginning
|
||||
break
|
||||
offset += 1024
|
||||
|
||||
info['end_time'] = int(end_time)
|
||||
|
||||
sourcefile.close()
|
||||
return info
|
||||
|
||||
|
||||
def parse_proc_filename(filename):
|
||||
try:
|
||||
name_as_list = filename.split(".")
|
||||
cur_stat_type = name_as_list[0].split("_")[0]
|
||||
cur_pid = name_as_list[1]
|
||||
except IndexError, ValueError:
|
||||
return (None, None)
|
||||
return (cur_pid, cur_stat_type)
|
||||
|
||||
# ----------------------------------------------------
|
||||
def get_simstats_list(path=None):
|
||||
""" Return stats (pid, type) listed in <type>_proc.<pid>.llsd """
|
||||
if path is None:
|
||||
path = DEFAULT_PATH
|
||||
simstats_list = []
|
||||
for file_name in os.listdir(path):
|
||||
if file_name.endswith(".llsd") and file_name != "simperf_proc_config.llsd":
|
||||
simstats_info = parse_logfile_info(path + file_name)
|
||||
if simstats_info is not None:
|
||||
simstats_list.append(simstats_info)
|
||||
return simstats_list
|
||||
|
||||
def get_log_info_list(pid=None, stat_type=None, path=None, target_column=None, verbose=False):
|
||||
""" Return data from all llsd files matching the pid and stat type """
|
||||
if path is None:
|
||||
path = DEFAULT_PATH
|
||||
log_info_list = {}
|
||||
for file_name in os.listdir ( path ):
|
||||
if file_name.endswith(".llsd") and file_name != "simperf_proc_config.llsd":
|
||||
(cur_pid, cur_stat_type) = parse_proc_filename(file_name)
|
||||
if cur_pid is None:
|
||||
continue
|
||||
if pid is not None and pid != cur_pid:
|
||||
continue
|
||||
if stat_type is not None and stat_type != cur_stat_type:
|
||||
continue
|
||||
log_info_list[cur_pid] = parse_logfile(path + file_name, target_column, verbose)
|
||||
return log_info_list
|
||||
|
||||
def delete_simstats_files(pid=None, stat_type=None, path=None):
|
||||
""" Delete *.<pid>.llsd files """
|
||||
if path is None:
|
||||
path = DEFAULT_PATH
|
||||
del_list = []
|
||||
for file_name in os.listdir(path):
|
||||
if file_name.endswith(".llsd") and file_name != "simperf_proc_config.llsd":
|
||||
(cur_pid, cur_stat_type) = parse_proc_filename(file_name)
|
||||
if cur_pid is None:
|
||||
continue
|
||||
if pid is not None and pid != cur_pid:
|
||||
continue
|
||||
if stat_type is not None and stat_type != cur_stat_type:
|
||||
continue
|
||||
del_list.append(cur_pid)
|
||||
# Allow delete related exceptions to percolate up if this fails.
|
||||
os.unlink(os.path.join(DEFAULT_PATH, file_name))
|
||||
return del_list
|
||||
|
||||
@@ -1,222 +0,0 @@
|
||||
'''
|
||||
@file term.py
|
||||
@brief a better shutil.copytree replacement
|
||||
|
||||
$LicenseInfo:firstyear=2007&license=mit$
|
||||
|
||||
Copyright (c) 2007-2009, Linden Research, Inc.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
$/LicenseInfo$
|
||||
'''
|
||||
|
||||
#http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/475116
|
||||
|
||||
import sys, re
|
||||
|
||||
class TerminalController:
|
||||
"""
|
||||
A class that can be used to portably generate formatted output to
|
||||
a terminal.
|
||||
|
||||
`TerminalController` defines a set of instance variables whose
|
||||
values are initialized to the control sequence necessary to
|
||||
perform a given action. These can be simply included in normal
|
||||
output to the terminal:
|
||||
|
||||
>>> term = TerminalController()
|
||||
>>> print 'This is '+term.GREEN+'green'+term.NORMAL
|
||||
|
||||
Alternatively, the `render()` method can used, which replaces
|
||||
'${action}' with the string required to perform 'action':
|
||||
|
||||
>>> term = TerminalController()
|
||||
>>> print term.render('This is ${GREEN}green${NORMAL}')
|
||||
|
||||
If the terminal doesn't support a given action, then the value of
|
||||
the corresponding instance variable will be set to ''. As a
|
||||
result, the above code will still work on terminals that do not
|
||||
support color, except that their output will not be colored.
|
||||
Also, this means that you can test whether the terminal supports a
|
||||
given action by simply testing the truth value of the
|
||||
corresponding instance variable:
|
||||
|
||||
>>> term = TerminalController()
|
||||
>>> if term.CLEAR_SCREEN:
|
||||
... print 'This terminal supports clearning the screen.'
|
||||
|
||||
Finally, if the width and height of the terminal are known, then
|
||||
they will be stored in the `COLS` and `LINES` attributes.
|
||||
"""
|
||||
# Cursor movement:
|
||||
BOL = '' #: Move the cursor to the beginning of the line
|
||||
UP = '' #: Move the cursor up one line
|
||||
DOWN = '' #: Move the cursor down one line
|
||||
LEFT = '' #: Move the cursor left one char
|
||||
RIGHT = '' #: Move the cursor right one char
|
||||
|
||||
# Deletion:
|
||||
CLEAR_SCREEN = '' #: Clear the screen and move to home position
|
||||
CLEAR_EOL = '' #: Clear to the end of the line.
|
||||
CLEAR_BOL = '' #: Clear to the beginning of the line.
|
||||
CLEAR_EOS = '' #: Clear to the end of the screen
|
||||
|
||||
# Output modes:
|
||||
BOLD = '' #: Turn on bold mode
|
||||
BLINK = '' #: Turn on blink mode
|
||||
DIM = '' #: Turn on half-bright mode
|
||||
REVERSE = '' #: Turn on reverse-video mode
|
||||
NORMAL = '' #: Turn off all modes
|
||||
|
||||
# Cursor display:
|
||||
HIDE_CURSOR = '' #: Make the cursor invisible
|
||||
SHOW_CURSOR = '' #: Make the cursor visible
|
||||
|
||||
# Terminal size:
|
||||
COLS = None #: Width of the terminal (None for unknown)
|
||||
LINES = None #: Height of the terminal (None for unknown)
|
||||
|
||||
# Foreground colors:
|
||||
BLACK = BLUE = GREEN = CYAN = RED = MAGENTA = YELLOW = WHITE = ''
|
||||
|
||||
# Background colors:
|
||||
BG_BLACK = BG_BLUE = BG_GREEN = BG_CYAN = ''
|
||||
BG_RED = BG_MAGENTA = BG_YELLOW = BG_WHITE = ''
|
||||
|
||||
_STRING_CAPABILITIES = """
|
||||
BOL=cr UP=cuu1 DOWN=cud1 LEFT=cub1 RIGHT=cuf1
|
||||
CLEAR_SCREEN=clear CLEAR_EOL=el CLEAR_BOL=el1 CLEAR_EOS=ed BOLD=bold
|
||||
BLINK=blink DIM=dim REVERSE=rev UNDERLINE=smul NORMAL=sgr0
|
||||
HIDE_CURSOR=cinvis SHOW_CURSOR=cnorm""".split()
|
||||
_COLORS = """BLACK BLUE GREEN CYAN RED MAGENTA YELLOW WHITE""".split()
|
||||
_ANSICOLORS = "BLACK RED GREEN YELLOW BLUE MAGENTA CYAN WHITE".split()
|
||||
|
||||
def __init__(self, term_stream=sys.stdout):
|
||||
"""
|
||||
Create a `TerminalController` and initialize its attributes
|
||||
with appropriate values for the current terminal.
|
||||
`term_stream` is the stream that will be used for terminal
|
||||
output; if this stream is not a tty, then the terminal is
|
||||
assumed to be a dumb terminal (i.e., have no capabilities).
|
||||
"""
|
||||
# Curses isn't available on all platforms
|
||||
try: import curses
|
||||
except: return
|
||||
|
||||
# If the stream isn't a tty, then assume it has no capabilities.
|
||||
if not term_stream.isatty(): return
|
||||
|
||||
# Check the terminal type. If we fail, then assume that the
|
||||
# terminal has no capabilities.
|
||||
try: curses.setupterm()
|
||||
except: return
|
||||
|
||||
# Look up numeric capabilities.
|
||||
self.COLS = curses.tigetnum('cols')
|
||||
self.LINES = curses.tigetnum('lines')
|
||||
|
||||
# Look up string capabilities.
|
||||
for capability in self._STRING_CAPABILITIES:
|
||||
(attrib, cap_name) = capability.split('=')
|
||||
setattr(self, attrib, self._tigetstr(cap_name) or '')
|
||||
|
||||
# Colors
|
||||
set_fg = self._tigetstr('setf')
|
||||
if set_fg:
|
||||
for i,color in zip(range(len(self._COLORS)), self._COLORS):
|
||||
setattr(self, color, curses.tparm(set_fg, i) or '')
|
||||
set_fg_ansi = self._tigetstr('setaf')
|
||||
if set_fg_ansi:
|
||||
for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
|
||||
setattr(self, color, curses.tparm(set_fg_ansi, i) or '')
|
||||
set_bg = self._tigetstr('setb')
|
||||
if set_bg:
|
||||
for i,color in zip(range(len(self._COLORS)), self._COLORS):
|
||||
setattr(self, 'BG_'+color, curses.tparm(set_bg, i) or '')
|
||||
set_bg_ansi = self._tigetstr('setab')
|
||||
if set_bg_ansi:
|
||||
for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
|
||||
setattr(self, 'BG_'+color, curses.tparm(set_bg_ansi, i) or '')
|
||||
|
||||
def _tigetstr(self, cap_name):
|
||||
# String capabilities can include "delays" of the form "$<2>".
|
||||
# For any modern terminal, we should be able to just ignore
|
||||
# these, so strip them out.
|
||||
import curses
|
||||
cap = curses.tigetstr(cap_name) or ''
|
||||
return re.sub(r'\$<\d+>[/*]?', '', cap)
|
||||
|
||||
def render(self, template):
|
||||
"""
|
||||
Replace each $-substitutions in the given template string with
|
||||
the corresponding terminal control string (if it's defined) or
|
||||
'' (if it's not).
|
||||
"""
|
||||
return re.sub(r'\$\$|\${\w+}', self._render_sub, template)
|
||||
|
||||
def _render_sub(self, match):
|
||||
s = match.group()
|
||||
if s == '$$': return s
|
||||
else: return getattr(self, s[2:-1])
|
||||
|
||||
#######################################################################
|
||||
# Example use case: progress bar
|
||||
#######################################################################
|
||||
|
||||
class ProgressBar:
|
||||
"""
|
||||
A 3-line progress bar, which looks like::
|
||||
|
||||
Header
|
||||
20% [===========----------------------------------]
|
||||
progress message
|
||||
|
||||
The progress bar is colored, if the terminal supports color
|
||||
output; and adjusts to the width of the terminal.
|
||||
"""
|
||||
BAR = '%3d%% ${GREEN}[${BOLD}%s%s${NORMAL}${GREEN}]${NORMAL}\n'
|
||||
HEADER = '${BOLD}${CYAN}%s${NORMAL}\n\n'
|
||||
|
||||
def __init__(self, term, header):
|
||||
self.term = term
|
||||
if not (self.term.CLEAR_EOL and self.term.UP and self.term.BOL):
|
||||
raise ValueError("Terminal isn't capable enough -- you "
|
||||
"should use a simpler progress dispaly.")
|
||||
self.width = self.term.COLS or 75
|
||||
self.bar = term.render(self.BAR)
|
||||
self.header = self.term.render(self.HEADER % header.center(self.width))
|
||||
self.cleared = 1 #: true if we haven't drawn the bar yet.
|
||||
self.update(0, '')
|
||||
|
||||
def update(self, percent, message):
|
||||
if self.cleared:
|
||||
sys.stdout.write(self.header)
|
||||
self.cleared = 0
|
||||
n = int((self.width-10)*percent)
|
||||
sys.stdout.write(
|
||||
self.term.BOL + self.term.UP + self.term.CLEAR_EOL +
|
||||
(self.bar % (100*percent, '='*n, '-'*(self.width-10-n))) +
|
||||
self.term.CLEAR_EOL + message.center(self.width))
|
||||
|
||||
def clear(self):
|
||||
if not self.cleared:
|
||||
sys.stdout.write(self.term.BOL + self.term.CLEAR_EOL +
|
||||
self.term.UP + self.term.CLEAR_EOL +
|
||||
self.term.UP + self.term.CLEAR_EOL)
|
||||
self.cleared = 1
|
||||
Reference in New Issue
Block a user