V3 merge for most python scripts. fmod inclusion/linkage/packaging changes.

This commit is contained in:
Shyotl
2014-12-18 18:01:41 -06:00
parent 32706065ac
commit 44f8f17763
24 changed files with 1045 additions and 1735 deletions

View File

@@ -2,19 +2,24 @@
@file __init__.py
@brief Initialization file for the indra module.
$LicenseInfo:firstyear=2006&license=internal$
$LicenseInfo:firstyear=2006&license=viewerlgpl$
Second Life Viewer Source Code
Copyright (C) 2006-2010, Linden Research, Inc.
Copyright (c) 2006-2009, Linden Research, Inc.
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation;
version 2.1 of the License only.
The following source code is PROPRIETARY AND CONFIDENTIAL. Use of
this source code is governed by the Linden Lab Source Code Disclosure
Agreement ("Agreement") previously entered between you and Linden
Lab. By accessing, using, copying, modifying or distributing this
software, you acknowledge that you have been informed of your
obligations under the Agreement and agree to abide by those obligations.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
ALL LINDEN LAB SOURCE CODE IS PROVIDED "AS IS." LINDEN LAB MAKES NO
WARRANTIES, EXPRESS, IMPLIED OR OTHERWISE, REGARDING ITS ACCURACY,
COMPLETENESS OR PERFORMANCE.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
$/LicenseInfo$
"""

View File

@@ -1,3 +1,25 @@
#!/usr/bin/python
##
## $LicenseInfo:firstyear=2011&license=viewerlgpl$
## Second Life Viewer Source Code
## Copyright (C) 2011, Linden Research, Inc.
##
## This library is free software; you can redistribute it and/or
## modify it under the terms of the GNU Lesser General Public
## License as published by the Free Software Foundation;
## version 2.1 of the License only.
##
## This library is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
## Lesser General Public License for more details.
##
## You should have received a copy of the GNU Lesser General Public
## License along with this library; if not, write to the Free Software
## Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
##
## Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
## $/LicenseInfo$
from indra.base import llsd, lluuid
from datetime import datetime
import cllsd
@@ -10,7 +32,7 @@ values = (
'&<>',
u'\u81acj',
llsd.uri('http://foo<'),
lluuid.LLUUID(),
lluuid.UUID(),
llsd.LLSD(['thing']),
1,
myint(31337),

View File

@@ -1,72 +0,0 @@
"""\
@file lllog.py
@brief Logging for event processing
$LicenseInfo:firstyear=2008&license=mit$
Copyright (c) 2008-2009, Linden Research, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
$/LicenseInfo$
"""
from indra.base.llsd import format_notation
try:
import syslog
except ImportError:
# Windows
import sys
class syslog(object):
_logfp = sys.stderr
def syslog(msg):
_logfp.write(msg)
if not msg.endswith('\n'):
_logfp.write('\n')
syslog = staticmethod(syslog)
class Logger(object):
def __init__(self, name='indra'):
self._sequence = 0
try:
syslog.openlog(name, syslog.LOG_CONS | syslog.LOG_PID,
syslog.LOG_LOCAL0)
except AttributeError:
# No syslog module on Windows
pass
def next(self):
self._sequence += 1
return self._sequence
def log(self, msg, llsd):
payload = 'INFO: log: LLLOGMESSAGE (%d) %s %s' % (self.next(), msg,
format_notation(llsd))
syslog.syslog(payload)
_logger = None
def log(msg, llsd):
global _logger
if _logger is None:
_logger = Logger()
_logger.log(msg, llsd)

View File

@@ -72,8 +72,11 @@ BOOL_FALSE = ('0', '0.0', 'false', '')
def format_datestr(v):
""" Formats a datetime object into the string format shared by xml and notation serializations."""
return v.isoformat() + 'Z'
""" Formats a datetime or date object into the string format shared by xml and notation serializations."""
if hasattr(v, 'microsecond'):
return v.isoformat() + 'Z'
else:
return v.strftime('%Y-%m-%dT%H:%M:%SZ')
def parse_datestr(datestr):
"""Parses a datetime object from the string format shared by xml and notation serializations."""
@@ -183,6 +186,7 @@ class LLSDXMLFormatter(object):
unicode : self.STRING,
uri : self.URI,
datetime.datetime : self.DATE,
datetime.date : self.DATE,
list : self.ARRAY,
tuple : self.ARRAY,
types.GeneratorType : self.ARRAY,
@@ -234,7 +238,7 @@ class LLSDXMLFormatter(object):
def MAP(self, v):
return self.elt(
'map',
''.join(["%s%s" % (self.elt('key', key), self.generate(value))
''.join(["%s%s" % (self.elt('key', self.xml_esc(str(key))), self.generate(value))
for key, value in v.items()]))
typeof = type
@@ -347,6 +351,7 @@ class LLSDNotationFormatter(object):
unicode : self.STRING,
uri : self.URI,
datetime.datetime : self.DATE,
datetime.date : self.DATE,
list : self.ARRAY,
tuple : self.ARRAY,
types.GeneratorType : self.ARRAY,
@@ -924,12 +929,13 @@ def _format_binary_recurse(something):
(type(something), something))
def parse_binary(something):
header = '<?llsd/binary?>\n'
if not something.startswith(header):
raise LLSDParseError('LLSD binary encoding header not found')
return LLSDBinaryParser().parse(something[len(header):])
def parse_binary(binary):
if binary.startswith('<?llsd/binary?>'):
just_binary = binary.split('\n', 1)[1]
else:
just_binary = binary
return LLSDBinaryParser().parse(just_binary)
def parse_xml(something):
try:
return to_python(fromstring(something)[0])

View File

@@ -28,13 +28,12 @@ $/LicenseInfo$
import random, socket, string, time, re
import uuid
# *HACK: Necessary for python 2.4. Consider replacing this code wart
# after python >=2.5 has deployed everywhere. 2009-10-05
try:
# Python 2.6
from hashlib import md5
except ImportError:
from md5 import md5
# Python 2.5 and earlier
from md5 import new as md5
def _int2binstr(i,l):
s=''
@@ -73,7 +72,7 @@ class UUID(object):
ip = ''
try:
ip = socket.gethostbyname(socket.gethostname())
except(socket.gaierror):
except(socket.gaierror, socket.error):
# no ip address, so just default to somewhere in 10.x.x.x
ip = '10'
for i in range(3):
@@ -164,7 +163,7 @@ class UUID(object):
def setFromMemoryDump(self, gdb_string):
"""
We expect to get gdb_string as four hex units. eg:
0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2
0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2
Which will be translated to:
db547d14-1b3f4bc3-9b984f71-d22f890a
Returns self.
@@ -188,7 +187,7 @@ class UUID(object):
def getAsString(self):
"""
Return a different string representation of the form
AAAAAAAA-AAAABBBB-BBBBBBBB-BBCCCCCC (a 128-bit number in hex)
AAAAAAAA-AAAABBBB-BBBBBBBB-BBCCCCCC (a 128-bit number in hex)
where A=network address, B=timestamp, C=random.
"""
i1 = _binstr2int(self._bits[0:4])
@@ -234,7 +233,7 @@ NULL = UUID()
def printTranslatedMemory(four_hex_uints):
"""
We expect to get the string as four hex units. eg:
0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2
0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2
Which will be translated to:
db547d14-1b3f4bc3-9b984f71-d22f890a
"""

View File

@@ -29,25 +29,93 @@ $/LicenseInfo$
"""
import sys
from indra.base import llsd
try:
import syslog
except ImportError:
# Windows
import sys
class syslog(object):
# wrap to a lame syslog for windows
_logfp = sys.stderr
def syslog(msg):
_logfp.write(msg)
if not msg.endswith('\n'):
_logfp.write('\n')
syslog = staticmethod(syslog)
_sequence_id = 0
from indra.base.llsd import format_notation
def record_metrics(table, stats, dest=None):
def record_metrics(table, stats):
"Write a standard metrics log"
_log("LLMETRICS", table, stats, dest)
_log("LLMETRICS", table, stats)
def record_event(table, data, dest=None):
def record_event(table, data):
"Write a standard logmessage log"
_log("LLLOGMESSAGE", table, data, dest)
_log("LLLOGMESSAGE", table, data)
def _log(header, table, data, dest):
def set_destination(dest):
"""Set the destination of metrics logs for this process.
If you do not call this function prior to calling a logging
method, that function will open sys.stdout as a destination.
Attempts to set dest to None will throw a RuntimeError.
@param dest a file-like object which will be the destination for logs."""
if dest is None:
# do this check here in case sys.stdout changes at some
# point. as a default parameter, it will never be
# re-evaluated.
dest = sys.stdout
raise RuntimeError("Attempt to unset metrics destination.")
global _destination
_destination = dest
def destination():
"""Get the destination of the metrics logs for this process.
Returns None if no destination is set"""
global _destination
return _destination
class SysLogger(object):
"A file-like object which writes to syslog."
def __init__(self, ident='indra', logopt = None, facility = None):
try:
if logopt is None:
logopt = syslog.LOG_CONS | syslog.LOG_PID
if facility is None:
facility = syslog.LOG_LOCAL0
syslog.openlog(ident, logopt, facility)
import atexit
atexit.register(syslog.closelog)
except AttributeError:
# No syslog module on Windows
pass
def write(str):
syslog.syslog(str)
write = staticmethod(write)
def flush():
pass
flush = staticmethod(flush)
#
# internal API
#
_sequence_id = 0
_destination = None
def _next_id():
global _sequence_id
print >>dest, header, "(" + str(_sequence_id) + ")",
print >>dest, table, llsd.format_notation(data)
next = _sequence_id
_sequence_id += 1
return next
def _dest():
global _destination
if _destination is None:
# this default behavior is documented in the metrics functions above.
_destination = sys.stdout
return _destination
def _log(header, table, data):
log_line = "%s (%d) %s %s" \
% (header, _next_id(), table, format_notation(data))
dest = _dest()
dest.write(log_line)
dest.flush()

View File

@@ -39,6 +39,12 @@ except:
pass
_g_builder = None
def _builder():
global _g_builder
if _g_builder is None:
_g_builder = ServiceBuilder()
return _g_builder
def build(name, context={}, **kwargs):
""" Convenience method for using a global, singleton, service builder. Pass arguments either via a dict or via python keyword arguments, or both!
@@ -56,6 +62,11 @@ def build(name, context={}, **kwargs):
_g_builder = ServiceBuilder()
return _g_builder.buildServiceURL(name, context, **kwargs)
def build_path(name, context={}, **kwargs):
context = context.copy() # shouldn't modify the caller's dictionary
context.update(kwargs)
return _builder().buildPath(name, context)
class ServiceBuilder(object):
def __init__(self, services_definition = services_config):
"""\
@@ -73,12 +84,21 @@ class ServiceBuilder(object):
continue
if isinstance(service_builder, dict):
# We will be constructing several builders
for name, builder in service_builder.items():
for name, builder in service_builder.iteritems():
full_builder_name = service['name'] + '-' + name
self.builders[full_builder_name] = builder
else:
self.builders[service['name']] = service_builder
def buildPath(self, name, context):
"""\
@brief given the environment on construction, return a service path.
@param name The name of the service.
@param context A dict of name value lookups for the service.
@returns Returns the
"""
return russ.format(self.builders[name], context)
def buildServiceURL(self, name, context={}, **kwargs):
"""\
@brief given the environment on construction, return a service URL.
@@ -108,7 +128,7 @@ def on_in(query_name, host_key, schema_key):
@param schema_key Logical name of destination schema. Will
be looked up in indra.xml.
"""
host_name = config.get(host_key)
schema_name = config.get(schema_key)
return '/'.join( ('on', host_name, 'in', schema_name, query_name.lstrip('/')) )
return "on/config:%s/in/config:%s/%s" % (host_key.strip('/'),
schema_key.strip('/'),
query_name.lstrip('/'))

View File

@@ -1,3 +1,32 @@
"""\
@file siesta.py
@brief A tiny llsd based RESTful web services framework
$LicenseInfo:firstyear=2008&license=mit$
Copyright (c) 2008, Linden Research, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
$/LicenseInfo$
"""
from indra.base import config
from indra.base import llsd
from webob import exc
import webob
@@ -37,11 +66,11 @@ def mime_type(content_type):
return content_type.split(';', 1)[0].strip().lower()
class BodyLLSD(object):
'''Give a webob Request or Response an llsd property.
'''Give a webob Request or Response an llsd based "content" property.
Getting the llsd property parses the body, and caches the result.
Getting the content property parses the body, and caches the result.
Setting the llsd property formats a payload, and the body property
Setting the content property formats a payload, and the body property
is set.'''
def _llsd__get(self):
@@ -80,7 +109,7 @@ class BodyLLSD(object):
if hasattr(self, '_llsd'):
del self._llsd
llsd = property(_llsd__get, _llsd__set, _llsd__del)
content = property(_llsd__get, _llsd__set, _llsd__del)
class Response(webob.Response, BodyLLSD):
@@ -114,10 +143,10 @@ class Request(webob.Request, BodyLLSD):
Sensible content type and accept headers are used by default.
Setting the llsd property also sets the body. Getting the llsd
Setting the content property also sets the body. Getting the content
property parses the body if necessary.
If you set the body property directly, the llsd property will be
If you set the body property directly, the content property will be
deleted.'''
default_content_type = 'application/llsd+xml'
@@ -149,11 +178,11 @@ class Request(webob.Request, BodyLLSD):
body = property(webob.Request._body__get, _body__set,
webob.Request._body__del, webob.Request._body__get.__doc__)
def create_response(self, llsd=None, status='200 OK',
def create_response(self, content=None, status='200 OK',
conditional_response=webob.NoDefault):
resp = self.ResponseClass(status=status, request=self,
conditional_response=conditional_response)
resp.llsd = llsd
resp.content = content
return resp
def curl(self):
@@ -196,12 +225,18 @@ llsd_formatters = {
'application/xml': llsd.format_xml,
}
formatter_qualities = (
('application/llsd+xml', 1.0),
('application/llsd+notation', 0.5),
('application/llsd+binary', 0.4),
('application/xml', 0.3),
('application/json', 0.2),
)
def formatter_for_mime_type(mime_type):
'''Return a formatter that encodes to the given MIME type.
The result is a pair of function and MIME type.'''
try:
return llsd_formatters[mime_type], mime_type
except KeyError:
@@ -214,21 +249,19 @@ def formatter_for_request(req):
'''Return a formatter that encodes to the preferred type of the client.
The result is a pair of function and actual MIME type.'''
for ctype in req.accept.best_matches('application/llsd+xml'):
try:
return llsd_formatters[ctype], ctype
except KeyError:
pass
else:
ctype = req.accept.best_match(formatter_qualities)
try:
return llsd_formatters[ctype], ctype
except KeyError:
raise exc.HTTPNotAcceptable().exception
def wsgi_adapter(func, environ, start_response):
'''Adapt a Siesta callable to act as a WSGI application.'''
# Process the request as appropriate.
try:
req = Request(environ)
#print req.urlvars
resp = func(req, **req.urlvars)
if not isinstance(resp, webob.Response):
try:
@@ -281,7 +314,8 @@ def llsd_class(cls):
allowed = [m for m in http11_methods
if hasattr(instance, 'handle_' + m.lower())]
raise exc.HTTPMethodNotAllowed(
headers={'Allowed': ', '.join(allowed)}).exception
headers={'Allow': ', '.join(allowed)}).exception
#print "kwargs: ", kwargs
return handler(req, **kwargs)
def replacement(environ, start_response):
@@ -336,7 +370,7 @@ def curl(reqs):
route_re = re.compile(r'''
\{ # exact character "{"
(\w+) # variable name (restricted to a-z, 0-9, _)
(\w*) # "config" or variable (restricted to a-z, 0-9, _)
(?:([:~])([^}]+))? # optional :type or ~regex part
\} # exact character "}"
''', re.VERBOSE)
@@ -344,27 +378,37 @@ route_re = re.compile(r'''
predefined_regexps = {
'uuid': r'[a-f0-9][a-f0-9-]{31,35}',
'int': r'\d+',
'host': r'[a-z0-9][a-z0-9\-\.]*',
}
def compile_route(route):
fp = StringIO()
last_pos = 0
for match in route_re.finditer(route):
#print "matches: ", match.groups()
fp.write(re.escape(route[last_pos:match.start()]))
var_name = match.group(1)
sep = match.group(2)
expr = match.group(3)
if expr:
if sep == ':':
expr = predefined_regexps[expr]
# otherwise, treat what follows '~' as a regexp
if var_name == 'config':
expr = re.escape(str(config.get(var_name)))
else:
expr = '[^/]+'
expr = '(?P<%s>%s)' % (var_name, expr)
if expr:
if sep == ':':
expr = predefined_regexps[expr]
# otherwise, treat what follows '~' as a regexp
else:
expr = '[^/]+'
if var_name != '':
expr = '(?P<%s>%s)' % (var_name, expr)
else:
expr = '(%s)' % (expr,)
fp.write(expr)
last_pos = match.end()
fp.write(re.escape(route[last_pos:]))
return '^%s$' % fp.getvalue()
compiled_route = '^%s$' % fp.getvalue()
#print route, "->", compiled_route
return compiled_route
class Router(object):
'''WSGI routing class. Parses a URL and hands off a request to
@@ -372,21 +416,43 @@ class Router(object):
responds with a 404.'''
def __init__(self):
self.routes = []
self.paths = []
self._new_routes = []
self._routes = []
self._paths = []
def add(self, route, app, methods=None):
self.paths.append(route)
self.routes.append((re.compile(compile_route(route)), app,
methods and dict.fromkeys(methods)))
self._new_routes.append((route, app, methods))
def _create_routes(self):
for route, app, methods in self._new_routes:
self._paths.append(route)
self._routes.append(
(re.compile(compile_route(route)),
app,
methods and dict.fromkeys(methods)))
self._new_routes = []
def __call__(self, environ, start_response):
# load up the config from the config file. Only needs to be
# done once per interpreter. This is the entry point of all
# siesta applications, so this is where we trap it.
_conf = config.get_config()
if _conf is None:
import os.path
fname = os.path.join(
environ.get('ll.config_dir', '/local/linden/etc'),
'indra.xml')
config.load(fname)
# proceed with handling the request
self._create_routes()
path_info = environ['PATH_INFO']
request_method = environ['REQUEST_METHOD']
allowed = []
for regex, app, methods in self.routes:
for regex, app, methods in self._routes:
m = regex.match(path_info)
if m:
#print "groupdict:",m.groupdict()
if not methods or request_method in methods:
environ['paste.urlvars'] = m.groupdict()
return app(environ, start_response)
@@ -396,7 +462,7 @@ class Router(object):
allowed = dict.fromkeys(allows).keys()
allowed.sort()
resp = exc.HTTPMethodNotAllowed(
headers={'Allowed': ', '.join(allowed)})
headers={'Allow': ', '.join(allowed)})
else:
resp = exc.HTTPNotFound()
return resp(environ, start_response)

View File

@@ -41,14 +41,6 @@ import tarfile
import errno
import subprocess
class ManifestError(RuntimeError):
"""Use an exception more specific than generic Python RuntimeError"""
pass
class MissingError(ManifestError):
"""You specified a file that doesn't exist"""
pass
def path_ancestors(path):
drive, path = os.path.splitdrive(os.path.normpath(path))
result = []
@@ -253,25 +245,15 @@ def main():
for opt in args:
print "Option:", opt, "=", args[opt]
# Build base package.
touch = args.get('touch')
if touch:
print 'Creating base package'
args['package_id'] = "" # base package has no package ID
wm = LLManifest.for_platform(args['platform'], args.get('arch'))(args)
wm.do(*args['actions'])
# Store package file for later if making touched file.
base_package_file = ""
if touch:
print 'Created base package ', wm.package_file
base_package_file = "" + wm.package_file
# Write out the package file in this format, so that it can easily be called
# and used in a .bat file - yeah, it sucks, but this is the simplest...
touch = args.get('touch')
if touch:
fp = open(touch, 'w')
fp.write('set package_file=%s\n' % base_package_file)
fp.write('set package_file=%s\n' % wm.package_file)
fp.close()
print 'touched', touch
return 0
@@ -298,13 +280,14 @@ class LLManifest(object):
self.file_list = []
self.excludes = []
self.actions = []
self.src_prefix = [args['source']]
self.artwork_prefix = [args['artwork']]
self.build_prefix = [args['build']]
self.dst_prefix = [args['dest']]
self.src_prefix = list([args['source']])
self.artwork_prefix = list([args['artwork']])
self.build_prefix = list([args['build']])
self.alt_build_prefix = list([args['build']])
self.dst_prefix = list([args['dest']])
self.created_paths = []
self.package_name = "Unknown"
def default_grid(self):
return self.args.get('grid', None) == ''
def default_channel(self):
@@ -328,7 +311,7 @@ class LLManifest(object):
in the file list by path()."""
self.excludes.append(glob)
def prefix(self, src='', build=None, dst=None):
def prefix(self, src='', build=None, dst=None, alt_build=None):
""" Pushes a prefix onto the stack. Until end_prefix is
called, all relevant method calls (esp. to path()) will prefix
paths with the entire prefix stack. Source and destination
@@ -339,10 +322,15 @@ class LLManifest(object):
dst = src
if build is None:
build = src
if alt_build is None:
alt_build = build
self.src_prefix.append(src)
self.artwork_prefix.append(src)
self.build_prefix.append(build)
self.dst_prefix.append(dst)
self.alt_build_prefix.append(alt_build)
return True # so that you can wrap it in an if to get indentation
def end_prefix(self, descr=None):
@@ -355,25 +343,30 @@ class LLManifest(object):
src = self.src_prefix.pop()
artwork = self.artwork_prefix.pop()
build = self.build_prefix.pop()
alt_build_prefix = self.alt_build_prefix.pop()
dst = self.dst_prefix.pop()
if descr and not(src == descr or build == descr or dst == descr):
raise ValueError, "End prefix '" + descr + "' didn't match '" +src+ "' or '" +dst + "'"
def get_src_prefix(self):
""" Returns the current source prefix."""
return os.path.join(*self.src_prefix)
return os.path.relpath(os.path.normpath(os.path.join(*self.src_prefix)))
def get_artwork_prefix(self):
""" Returns the current artwork prefix."""
return os.path.join(*self.artwork_prefix)
return os.path.relpath(os.path.normpath(os.path.join(*self.artwork_prefix)))
def get_build_prefix(self):
""" Returns the current build prefix."""
return os.path.join(*self.build_prefix)
return os.path.relpath(os.path.normpath(os.path.join(*self.build_prefix)))
def get_alt_build_prefix(self):
""" Returns the current alternate source prefix."""
return os.path.relpath(os.path.normpath(os.path.join(*self.alt_build_prefix)))
def get_dst_prefix(self):
""" Returns the current destination prefix."""
return os.path.join(*self.dst_prefix)
return os.path.relpath(os.path.normpath(os.path.join(*self.dst_prefix)))
def src_path_of(self, relpath):
"""Returns the full path to a file or directory specified
@@ -390,26 +383,10 @@ class LLManifest(object):
relative to the destination directory."""
return os.path.join(self.get_dst_prefix(), relpath)
def ensure_src_dir(self, reldir):
"""Construct the path for a directory relative to the
source path, and ensures that it exists. Returns the
full path."""
path = os.path.join(self.get_src_prefix(), reldir)
self.cmakedirs(path)
return path
def ensure_dst_dir(self, reldir):
"""Construct the path for a directory relative to the
destination path, and ensures that it exists. Returns the
full path."""
path = os.path.join(self.get_dst_prefix(), reldir)
self.cmakedirs(path)
return path
def run_command(self, command):
""" Runs an external command, and returns the output. Raises
an exception if the command returns a nonzero status code. For
debugging/informational purposes, prints out the command's
an exception if the command reurns a nonzero status code. For
debugging/informational purpoases, prints out the command's
output as it is received."""
print "Running command:", command
fd = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
@@ -422,7 +399,7 @@ class LLManifest(object):
print lines[-1].rstrip('\n'),
output = ''.join(lines)
if fd.returncode:
raise ManifestError(
raise RuntimeError(
"Command %s returned non-zero status (%s) \noutput:\n%s"
% (command, fd.returncode, output) )
return output
@@ -432,24 +409,14 @@ class LLManifest(object):
a) verify that you really have created it
b) schedule it for cleanup"""
if not os.path.exists(path):
raise ManifestError, "Should be something at path " + path
raise RuntimeError, "Should be something at path " + path
self.created_paths.append(path)
def put_in_file(self, contents, dst, src=None):
def put_in_file(self, contents, dst):
# write contents as dst
dst_path = self.dst_path_of(dst)
f = open(dst_path, "wb")
try:
f.write(contents)
finally:
f.close()
# Why would we create a file in the destination tree if not to include
# it in the installer? The default src=None (plus the fact that the
# src param is last) is to preserve backwards compatibility.
if src:
self.file_list.append([src, dst_path])
return dst_path
f = open(self.dst_path_of(dst), "wb")
f.write(contents)
f.close()
def replace_in(self, src, dst=None, searchdict={}):
if dst == None:
@@ -512,29 +479,30 @@ class LLManifest(object):
if method is not None:
method(src, dst)
self.file_list.append([src, dst])
return 1
return [dst]
else:
sys.stdout.write(" (excluding %r, %r)" % (src, dst))
sys.stdout.flush()
return 0
return []
def process_directory(self, src, dst):
if not self.includes(src, dst):
sys.stdout.write(" (excluding %r, %r)" % (src, dst))
sys.stdout.flush()
return 0
return []
names = os.listdir(src)
self.cmakedirs(dst)
errors = []
found_files = []
count = 0
for name in names:
srcname = os.path.join(src, name)
dstname = os.path.join(dst, name)
if os.path.isdir(srcname):
count += self.process_directory(srcname, dstname)
found_files.extend(self.process_directory(srcname, dstname))
else:
count += self.process_file(srcname, dstname)
return count
found_files.extend(self.process_file(srcname, dstname))
return found_files
def includes(self, src, dst):
if src:
@@ -597,7 +565,7 @@ class LLManifest(object):
except (IOError, os.error), why:
errors.append((srcname, dstname, why))
if errors:
raise ManifestError, errors
raise RuntimeError, errors
def cmakedirs(self, path):
@@ -614,25 +582,11 @@ class LLManifest(object):
if os.path.exists(f):
return f
# didn't find it, return last item in list
if len(list) > 0:
if list:
return list[-1]
else:
return None
def contents_of_tar(self, src_tar, dst_dir):
""" Extracts the contents of the tarfile (specified
relative to the source prefix) into the directory
specified relative to the destination directory."""
self.check_file_exists(src_tar)
tf = tarfile.open(self.src_path_of(src_tar), 'r')
for member in tf.getmembers():
tf.extract(member, self.ensure_dst_dir(dst_dir))
# TODO get actions working on these dudes, perhaps we should extract to a temporary directory and then process_directory on it?
self.file_list.append([src_tar,
self.dst_path_of(os.path.join(dst_dir,member.name))])
tf.close()
def wildcard_regex(self, src_glob, dst_glob):
src_re = re.escape(src_glob)
src_re = src_re.replace('\*', '([-a-zA-Z0-9._ ]*)')
@@ -643,12 +597,7 @@ class LLManifest(object):
i = i+1
return re.compile(src_re), dst_temp
def check_file_exists(self, path):
if not os.path.exists(path) and not os.path.islink(path):
raise MissingError("Path %s doesn't exist" % (os.path.abspath(path),))
wildcard_pattern = re.compile(r'\*')
wildcard_pattern = re.compile('\*')
def expand_globs(self, src, dst):
src_list = glob.glob(src)
src_re, d_template = self.wildcard_regex(src.replace('\\', '/'),
@@ -675,53 +624,44 @@ class LLManifest(object):
return self.path(os.path.join(path, file), file)
def path(self, src, dst=None):
sys.stdout.write("Processing %s => %s ... " % (src, dst))
sys.stdout.flush()
if src == None:
raise ManifestError("No source file, dst is " + dst)
raise RuntimeError("No source file, dst is " + dst)
if dst == None:
dst = src
dst = os.path.join(self.get_dst_prefix(), dst)
sys.stdout.write("Processing %s => %s ... " % (src, dst))
count = 0
is_glob = False
found_files = []
def try_path(src):
# expand globs
count = 0
if self.wildcard_pattern.search(src):
for s,d in self.expand_globs(src, dst):
# look under each prefix for matching paths. Paths are normalized so './../blah' will match '../blah/../blah/'
paths = set([os.path.normpath(os.path.join(self.get_src_prefix(), src)),
os.path.normpath(os.path.join(self.get_artwork_prefix(), src)),
os.path.normpath(os.path.join(self.get_build_prefix(), src)),
os.path.normpath(os.path.join(self.get_alt_build_prefix(), src))]
)
for path in paths:
print path
if self.wildcard_pattern.search(path):
is_glob = True
for s,d in self.expand_globs(path, dst):
assert(s != d)
count += self.process_file(s, d)
found_files.extend(self.process_file(s, d))
else:
# if we're specifying a single path (not a glob),
# we should error out if it doesn't exist
self.check_file_exists(src)
# if it's a directory, recurse through it
if os.path.isdir(src):
count += self.process_directory(src, dst)
else:
count += self.process_file(src, dst)
return count
if os.path.isdir(path):
found_files.extend(self.process_directory(path, dst))
elif os.path.exists(path):
found_files.extend(self.process_file(path, dst))
for pfx in self.get_src_prefix(), self.get_artwork_prefix(), self.get_build_prefix():
try:
count = try_path(os.path.join(pfx, src))
except MissingError:
# If src isn't a wildcard, and if that file doesn't exist in
# this pfx, try next pfx.
count = 0
continue
# if we're specifying a single path (not a glob),
# we should error out if it doesn't exist
if not found_files and not is_glob:
raise RuntimeError("No files match %s\n" % str(paths))
# Here try_path() didn't raise MissingError. Did it process any files?
if count:
break
# Even though try_path() didn't raise MissingError, it returned 0
# files. src is probably a wildcard meant for some other pfx. Loop
# back to try the next.
print "%d files" % count
# Let caller check whether we processed as many files as expected. In
# particular, let caller notice 0.
return count
print "%d files" % len(found_files)
return found_files
def do(self, *actions):
self.actions = actions

View File

@@ -1,4 +1,28 @@
#!/usr/bin/python
#!/usr/bin/env python
"""\
@file llperformance.py
$LicenseInfo:firstyear=2010&license=viewerlgpl$
Second Life Viewer Source Code
Copyright (C) 2010-2011, Linden Research, Inc.
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation;
version 2.1 of the License only.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
$/LicenseInfo$
"""
# ------------------------------------------------
# Sim metrics utility functions.

View File

@@ -90,6 +90,17 @@ all the output, and get the result.
child.tochild.close()
result = child.poll()
if result != -1:
# At this point, the child process has exited and result
# is the return value from the process. Between the time
# we called select() and poll() the process may have
# exited so read all the data left on the child process
# stdout and stderr.
last = child.fromchild.read()
if last:
out.append(last)
last = child.childerr.read()
if last:
err.append(last)
child.tochild.close()
child.fromchild.close()
child.childerr.close()

View File

@@ -40,7 +40,6 @@ from indra.base import llsd
from indra.base import config
DEBUG = False
NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq')
NQ_FILE_SUFFIX_LEN = len(NQ_FILE_SUFFIX)
@@ -52,6 +51,11 @@ def _init_g_named_manager(sql_dir = None):
This function is intended entirely for testing purposes,
because it's tricky to control the config from inside a test."""
global NQ_FILE_SUFFIX
NQ_FILE_SUFFIX = config.get('named-query-file-suffix', '.nq')
global NQ_FILE_SUFFIX_LEN
NQ_FILE_SUFFIX_LEN = len(NQ_FILE_SUFFIX)
if sql_dir is None:
sql_dir = config.get('named-query-base-dir')
@@ -65,11 +69,11 @@ def _init_g_named_manager(sql_dir = None):
_g_named_manager = NamedQueryManager(
os.path.abspath(os.path.realpath(sql_dir)))
def get(name):
def get(name, schema = None):
"Get the named query object to be used to perform queries"
if _g_named_manager is None:
_init_g_named_manager()
return _g_named_manager.get(name)
return _g_named_manager.get(name).for_schema(schema)
def sql(connection, name, params):
# use module-global NamedQuery object to perform default substitution
@@ -280,7 +284,10 @@ class NamedQuery(object):
So, we need a vendor (or extention) for LIKE_STRING. Anyone
want to write it?"""
utf8_value = unicode(value, "utf-8")
if isinstance(value, unicode):
utf8_value = value
else:
utf8_value = unicode(value, "utf-8")
esc_list = []
remove_chars = set(u"%_")
for glyph in utf8_value:
@@ -317,6 +324,8 @@ class NamedQuery(object):
def for_schema(self, db_name):
"Look trough the alternates and return the correct query"
if db_name is None:
return self
try:
return self._alternative[db_name]
except KeyError, e:
@@ -341,21 +350,21 @@ class NamedQuery(object):
cursor = connection.cursor(MySQLdb.cursors.DictCursor)
else:
cursor = connection.cursor()
statement = self.sql(connection, params)
full_query, params = self._construct_sql(params)
if DEBUG:
print "SQL:", statement
rows = cursor.execute(statement)
print "SQL:", self.sql(connection, params)
rows = cursor.execute(full_query, params)
# *NOTE: the expect_rows argument is a very cheesy way to get some
# validation on the result set. If you want to add more expectation
# logic, do something more object-oriented and flexible. Or use an ORM.
# logic, do something more object-oriented and flexible. Or use an ORM.
if(self._return_as_map):
expect_rows = 1
if expect_rows is not None and rows != expect_rows:
cursor.close()
raise ExpectationFailed("Statement expected %s rows, got %s. Sql: %s" % (
expect_rows, rows, statement))
raise ExpectationFailed("Statement expected %s rows, got %s. Sql: '%s' %s" % (
expect_rows, rows, full_query, params))
# convert to dicts manually if we're not using a dictcursor
if use_dictcursor:
@@ -381,11 +390,9 @@ class NamedQuery(object):
return result_set[0]
return result_set
def sql(self, connection, params):
""" Generates an SQL statement from the named query document
and a dictionary of parameters.
"""
def _construct_sql(self, params):
""" Returns a query string and a dictionary of parameters,
suitable for directly passing to the execute() method."""
self.refresh()
# build the query from the options available and the params
@@ -431,10 +438,23 @@ class NamedQuery(object):
new_params[self._build_integer_key(key)] = int(params[key])
params.update(new_params)
return full_query, params
def sql(self, connection, params):
""" Generates an SQL statement from the named query document
and a dictionary of parameters.
*NOTE: Only use for debugging, because it uses the
non-standard MySQLdb 'literal' method.
"""
if not DEBUG:
import warnings
warnings.warn("Don't use named_query.sql() when not debugging. Used on %s" % self._location)
# do substitution using the mysql (non-standard) 'literal'
# function to do the escaping.
sql = full_query % connection.literal(params)
return sql
full_query, params = self._construct_sql(params)
return full_query % connection.literal(params)
def refresh(self):
""" Refresh self from the file on the filesystem.

View File

@@ -3,20 +3,27 @@
@file simperf_oprof_interface.py
@brief Manage OProfile data collection on a host
$LicenseInfo:firstyear=2008&license=internal$
$LicenseInfo:firstyear=2008&license=mit$
Copyright (c) 2008-2009, Linden Research, Inc.
The following source code is PROPRIETARY AND CONFIDENTIAL. Use of
this source code is governed by the Linden Lab Source Code Disclosure
Agreement ("Agreement") previously entered between you and Linden
Lab. By accessing, using, copying, modifying or distributing this
software, you acknowledge that you have been informed of your
obligations under the Agreement and agree to abide by those obligations.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
ALL LINDEN LAB SOURCE CODE IS PROVIDED "AS IS." LINDEN LAB MAKES NO
WARRANTIES, EXPRESS, IMPLIED OR OTHERWISE, REGARDING ITS ACCURACY,
COMPLETENESS OR PERFORMANCE.
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
$/LicenseInfo$
"""

View File

@@ -1,4 +1,31 @@
#!/usr/bin/python
#!/usr/bin/env python
"""\
@file simperf_proc_interface.py
@brief Utility to extract log messages from *.<pid>.llsd files containing performance statistics.
$LicenseInfo:firstyear=2008&license=mit$
Copyright (c) 2008-2009, Linden Research, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
$/LicenseInfo$
"""
# ----------------------------------------------------
# Utility to extract log messages from *.<pid>.llsd

View File

@@ -0,0 +1,146 @@
#!/usr/bin/env python
"""\
@file test_win32_manifest.py
@brief Test an assembly binding version and uniqueness in a windows dll or exe.
$LicenseInfo:firstyear=2009&license=viewerlgpl$
Second Life Viewer Source Code
Copyright (C) 2009-2011, Linden Research, Inc.
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation;
version 2.1 of the License only.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Linden Research, Inc., 945 Battery Street, San Francisco, CA 94111 USA
$/LicenseInfo$
"""
import sys, os
import tempfile
from xml.dom.minidom import parse
class AssemblyTestException(Exception):
pass
class NoManifestException(AssemblyTestException):
pass
class MultipleBindingsException(AssemblyTestException):
pass
class UnexpectedVersionException(AssemblyTestException):
pass
class NoMatchingAssemblyException(AssemblyTestException):
pass
def get_HKLM_registry_value(key_str, value_str):
import _winreg
reg = _winreg.ConnectRegistry(None, _winreg.HKEY_LOCAL_MACHINE)
key = _winreg.OpenKey(reg, key_str)
value = _winreg.QueryValueEx(key, value_str)[0]
#print 'Found: %s' % value
return value
def find_vc_dir():
supported_versions = (r'8.0', r'9.0')
supported_products = (r'VisualStudio', r'VCExpress')
value_str = (r'ProductDir')
for product in supported_products:
for version in supported_versions:
key_str = (r'SOFTWARE\Microsoft\%s\%s\Setup\VC' %
(product, version))
try:
return get_HKLM_registry_value(key_str, value_str)
except WindowsError, err:
x64_key_str = (r'SOFTWARE\Wow6432Node\Microsoft\VisualStudio\%s\Setup\VS' %
version)
try:
return get_HKLM_registry_value(x64_key_str, value_str)
except:
print >> sys.stderr, "Didn't find MS %s version %s " % (product,version)
raise
def find_mt_path():
vc_dir = find_vc_dir()
mt_path = '\"%sbin\\mt.exe\"' % vc_dir
return mt_path
def test_assembly_binding(src_filename, assembly_name, assembly_ver):
print "checking %s dependency %s..." % (src_filename, assembly_name)
(tmp_file_fd, tmp_file_name) = tempfile.mkstemp(suffix='.xml')
tmp_file = os.fdopen(tmp_file_fd)
tmp_file.close()
mt_path = find_mt_path()
resource_id = ""
if os.path.splitext(src_filename)[1].lower() == ".dll":
resource_id = ";#2"
system_call = '%s -nologo -inputresource:%s%s -out:%s > NUL' % (mt_path, src_filename, resource_id, tmp_file_name)
print "Executing: %s" % system_call
mt_result = os.system(system_call)
if mt_result == 31:
print "No manifest found in %s" % src_filename
raise NoManifestException()
manifest_dom = parse(tmp_file_name)
nodes = manifest_dom.getElementsByTagName('assemblyIdentity')
versions = list()
for node in nodes:
if node.getAttribute('name') == assembly_name:
versions.append(node.getAttribute('version'))
if len(versions) == 0:
print "No matching assemblies found in %s" % src_filename
raise NoMatchingAssemblyException()
elif len(versions) > 1:
print "Multiple bindings to %s found:" % assembly_name
print versions
print
raise MultipleBindingsException(versions)
elif versions[0] != assembly_ver:
print "Unexpected version found for %s:" % assembly_name
print "Wanted %s, found %s" % (assembly_ver, versions[0])
print
raise UnexpectedVersionException(assembly_ver, versions[0])
os.remove(tmp_file_name)
print "SUCCESS: %s OK!" % src_filename
print
if __name__ == '__main__':
print
print "Running test_win32_manifest.py..."
usage = 'test_win32_manfest <srcFileName> <assemblyName> <assemblyVersion>'
try:
src_filename = sys.argv[1]
assembly_name = sys.argv[2]
assembly_ver = sys.argv[3]
except:
print "Usage:"
print usage
print
raise
test_assembly_binding(src_filename, assembly_name, assembly_ver)

View File

@@ -45,13 +45,6 @@ Typical usage:
This module works with Python 2.3 or higher."""
# *HACK: Necessary for python 2.4. Consider replacing this code wart
# after python >=2.5 has deployed everywhere. 2009-10-05
try:
from hashlib import md5
except ImportError:
from md5 import md5
__author__ = 'Ka-Ping Yee <ping@zesty.ca>'
__date__ = '$Date: 2006/06/12 23:15:40 $'.split()[1].replace('/', '-')
__version__ = '$Revision: 1.30 $'.split()[1]
@@ -453,6 +446,13 @@ def uuid1(node=None, clock_seq=None):
def uuid3(namespace, name):
"""Generate a UUID from the MD5 hash of a namespace UUID and a name."""
try:
# Python 2.6
from hashlib import md5
except ImportError:
# Python 2.5 and earlier
from md5 import new as md5
hash = md5(namespace.bytes + name).digest()
return UUID(bytes=hash[:16], version=3)