Imported existing code

This commit is contained in:
Hazim Gazov
2010-04-02 02:48:44 -03:00
parent 48fbc5ae91
commit 7a86d01598
13996 changed files with 2468699 additions and 0 deletions

View File

@@ -0,0 +1,27 @@
"""\
@file __init__.py
@brief Initialization file for the indra.base module.
$LicenseInfo:firstyear=2007&license=mit$
Copyright (c) 2007-2009, Linden Research, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
$/LicenseInfo$
"""

Binary file not shown.

View File

@@ -0,0 +1,51 @@
from indra.base import llsd, lluuid
from datetime import datetime
import cllsd
import time, sys
class myint(int):
pass
values = (
'&<>',
u'\u81acj',
llsd.uri('http://foo<'),
lluuid.LLUUID(),
llsd.LLSD(['thing']),
1,
myint(31337),
sys.maxint + 10,
llsd.binary('foo'),
[],
{},
{u'f&\u1212': 3},
3.1,
True,
None,
datetime.fromtimestamp(time.time()),
)
def valuator(values):
for v in values:
yield v
longvalues = () # (values, list(values), iter(values), valuator(values))
for v in values + longvalues:
print '%r => %r' % (v, cllsd.llsd_to_xml(v))
a = [[{'a':3}]] * 1000000
s = time.time()
print hash(cllsd.llsd_to_xml(a))
e = time.time()
t1 = e - s
print t1
s = time.time()
print hash(llsd.LLSDXMLFormatter()._format(a))
e = time.time()
t2 = e - s
print t2
print 'Speedup:', t2 / t1

View File

@@ -0,0 +1,266 @@
"""\
@file config.py
@brief Utility module for parsing and accessing the indra.xml config file.
$LicenseInfo:firstyear=2006&license=mit$
Copyright (c) 2006-2009, Linden Research, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
$/LicenseInfo$
"""
import copy
import errno
import os
import traceback
import time
import types
from os.path import dirname, getmtime, join, realpath
from indra.base import llsd
_g_config = None
class IndraConfig(object):
"""
IndraConfig loads a 'indra' xml configuration file
and loads into memory. This representation in memory
can get updated to overwrite values or add new values.
The xml configuration file is considered a live file and changes
to the file are checked and reloaded periodically. If a value had
been overwritten via the update or set method, the loaded values
from the file are ignored (the values from the update/set methods
override)
"""
def __init__(self, indra_config_file):
self._indra_config_file = indra_config_file
self._reload_check_interval = 30 # seconds
self._last_check_time = 0
self._last_mod_time = 0
self._config_overrides = {}
self._config_file_dict = {}
self._combined_dict = {}
self._load()
def _load(self):
# if you initialize the IndraConfig with None, no attempt
# is made to load any files
if self._indra_config_file is None:
return
config_file = open(self._indra_config_file)
self._config_file_dict = llsd.parse(config_file.read())
self._combine_dictionaries()
config_file.close()
self._last_mod_time = self._get_last_modified_time()
self._last_check_time = time.time() # now
def _get_last_modified_time(self):
"""
Returns the mtime (last modified time) of the config file,
if such exists.
"""
if self._indra_config_file is not None:
return os.path.getmtime(self._indra_config_file)
return 0
def _combine_dictionaries(self):
self._combined_dict = {}
self._combined_dict.update(self._config_file_dict)
self._combined_dict.update(self._config_overrides)
def _reload_if_necessary(self):
now = time.time()
if (now - self._last_check_time) > self._reload_check_interval:
self._last_check_time = now
try:
modtime = self._get_last_modified_time()
if modtime > self._last_mod_time:
self._load()
except OSError, e:
if e.errno == errno.ENOENT: # file not found
# someone messed with our internal state
# or removed the file
print 'WARNING: Configuration file has been removed ' + (self._indra_config_file)
print 'Disabling reloading of configuration file.'
traceback.print_exc()
self._indra_config_file = None
self._last_check_time = 0
self._last_mod_time = 0
else:
raise # pass the exception along to the caller
def __getitem__(self, key):
self._reload_if_necessary()
return self._combined_dict[key]
def get(self, key, default = None):
try:
return self.__getitem__(key)
except KeyError:
return default
def __setitem__(self, key, value):
"""
Sets the value of the config setting of key to be newval
Once any key/value pair is changed via the set method,
that key/value pair will remain set with that value until
change via the update or set method
"""
self._config_overrides[key] = value
self._combine_dictionaries()
def set(self, key, newval):
return self.__setitem__(key, newval)
def update(self, new_conf):
"""
Load an XML file and apply its map as overrides or additions
to the existing config. Update can be a file or a dict.
Once any key/value pair is changed via the update method,
that key/value pair will remain set with that value until
change via the update or set method
"""
if isinstance(new_conf, dict):
overrides = new_conf
else:
# assuming that it is a filename
config_file = open(new_conf)
overrides = llsd.parse(config_file.read())
config_file.close()
self._config_overrides.update(overrides)
self._combine_dictionaries()
def as_dict(self):
"""
Returns immutable copy of the IndraConfig as a dictionary
"""
return copy.deepcopy(self._combined_dict)
def load(config_xml_file = None):
global _g_config
load_default_files = config_xml_file is None
if load_default_files:
## going from:
## "/opt/linden/indra/lib/python/indra/base/config.py"
## to:
## "/opt/linden/etc/indra.xml"
config_xml_file = realpath(
dirname(realpath(__file__)) + "../../../../../../etc/indra.xml")
try:
_g_config = IndraConfig(config_xml_file)
except IOError:
# Failure to load passed in file
# or indra.xml default file
if load_default_files:
try:
config_xml_file = realpath(
dirname(realpath(__file__)) + "../../../../../../etc/globals.xml")
_g_config = IndraConfig(config_xml_file)
return
except IOError:
# Failure to load globals.xml
# fall to code below
pass
# Either failed to load passed in file
# or failed to load all default files
_g_config = IndraConfig(None)
def dump(indra_xml_file, indra_cfg = None, update_in_mem=False):
'''
Dump config contents into a file
Kindof reverse of load.
Optionally takes a new config to dump.
Does NOT update global config unless requested.
'''
global _g_config
if not indra_cfg:
if _g_config is None:
return
indra_cfg = _g_config.as_dict()
if not indra_cfg:
return
config_file = open(indra_xml_file, 'w')
_config_xml = llsd.format_xml(indra_cfg)
config_file.write(_config_xml)
config_file.close()
if update_in_mem:
update(indra_cfg)
def update(new_conf):
global _g_config
if _g_config is None:
# To keep with how this function behaved
# previously, a call to update
# before the global is defined
# make a new global config which does not
# load data from a file.
_g_config = IndraConfig(None)
return _g_config.update(new_conf)
def get(key, default = None):
global _g_config
if _g_config is None:
load()
return _g_config.get(key, default)
def set(key, newval):
"""
Sets the value of the config setting of key to be newval
Once any key/value pair is changed via the set method,
that key/value pair will remain set with that value until
change via the update or set method or program termination
"""
global _g_config
if _g_config is None:
_g_config = IndraConfig(None)
_g_config.set(key, newval)
def get_config():
global _g_config
return _g_config

View File

@@ -0,0 +1,72 @@
"""\
@file lllog.py
@brief Logging for event processing
$LicenseInfo:firstyear=2008&license=mit$
Copyright (c) 2008-2009, Linden Research, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
$/LicenseInfo$
"""
from indra.base.llsd import format_notation
try:
import syslog
except ImportError:
# Windows
import sys
class syslog(object):
_logfp = sys.stderr
def syslog(msg):
_logfp.write(msg)
if not msg.endswith('\n'):
_logfp.write('\n')
syslog = staticmethod(syslog)
class Logger(object):
def __init__(self, name='indra'):
self._sequence = 0
try:
syslog.openlog(name, syslog.LOG_CONS | syslog.LOG_PID,
syslog.LOG_LOCAL0)
except AttributeError:
# No syslog module on Windows
pass
def next(self):
self._sequence += 1
return self._sequence
def log(self, msg, llsd):
payload = 'INFO: log: LLLOGMESSAGE (%d) %s %s' % (self.next(), msg,
format_notation(llsd))
syslog.syslog(payload)
_logger = None
def log(msg, llsd):
global _logger
if _logger is None:
_logger = Logger()
_logger.log(msg, llsd)

File diff suppressed because it is too large Load Diff

Binary file not shown.

View File

@@ -0,0 +1,320 @@
"""\
@file lluuid.py
@brief UUID parser/generator.
$LicenseInfo:firstyear=2004&license=mit$
Copyright (c) 2004-2009, Linden Research, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
$/LicenseInfo$
"""
import random, socket, string, time, re
import uuid
# *HACK: Necessary for python 2.4. Consider replacing this code wart
# after python >=2.5 has deployed everywhere. 2009-10-05
try:
from hashlib import md5
except ImportError:
from md5 import md5
def _int2binstr(i,l):
s=''
for a in range(l):
s=chr(i&0xFF)+s
i>>=8
return s
def _binstr2int(s):
i = long(0)
for c in s:
i = (i<<8) + ord(c)
return i
class UUID(object):
"""
A class which represents a 16 byte integer. Stored as a 16 byte 8
bit character string.
The string version is to be of the form:
AAAAAAAA-AAAA-BBBB-BBBB-BBBBBBCCCCCC (a 128-bit number in hex)
where A=network address, B=timestamp, C=random.
"""
NULL_STR = "00000000-0000-0000-0000-000000000000"
# the UUIDREGEX_STRING is helpful for parsing UUID's in text
hex_wildcard = r"[0-9a-fA-F]"
word = hex_wildcard + r"{4,4}-"
long_word = hex_wildcard + r"{8,8}-"
very_long_word = hex_wildcard + r"{12,12}"
UUID_REGEX_STRING = long_word + word + word + word + very_long_word
uuid_regex = re.compile(UUID_REGEX_STRING)
rand = random.Random()
ip = ''
try:
ip = socket.gethostbyname(socket.gethostname())
except(socket.gaierror):
# no ip address, so just default to somewhere in 10.x.x.x
ip = '10'
for i in range(3):
ip += '.' + str(rand.randrange(1,254))
hexip = ''.join(["%04x" % long(i) for i in ip.split('.')])
lastid = ''
def __init__(self, possible_uuid=None):
"""
Initialize to first valid UUID in argument (if a string),
or to null UUID if none found or argument is not supplied.
If the argument is a UUID, the constructed object will be a copy of it.
"""
self._bits = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
if possible_uuid is None:
return
if isinstance(possible_uuid, type(self)):
self.set(possible_uuid)
return
uuid_match = UUID.uuid_regex.search(possible_uuid)
if uuid_match:
uuid_string = uuid_match.group()
s = string.replace(uuid_string, '-', '')
self._bits = _int2binstr(string.atol(s[:8],16),4) + \
_int2binstr(string.atol(s[8:16],16),4) + \
_int2binstr(string.atol(s[16:24],16),4) + \
_int2binstr(string.atol(s[24:],16),4)
def __len__(self):
"""
Used by the len() builtin.
"""
return 36
def __nonzero__(self):
return self._bits != "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
def __str__(self):
uuid_string = self.toString()
return uuid_string
__repr__ = __str__
def __getitem__(self, index):
return str(self)[index]
def __eq__(self, other):
if isinstance(other, (str, unicode)):
return other == str(self)
return self._bits == getattr(other, '_bits', '')
def __ne__(self, other):
return not self.__eq__(other)
def __le__(self, other):
return self._bits <= other._bits
def __ge__(self, other):
return self._bits >= other._bits
def __lt__(self, other):
return self._bits < other._bits
def __gt__(self, other):
return self._bits > other._bits
def __hash__(self):
return hash(self._bits)
def set(self, uuid):
self._bits = uuid._bits
def setFromString(self, uuid_string):
"""
Given a string version of a uuid, set self bits
appropriately. Returns self.
"""
s = string.replace(uuid_string, '-', '')
self._bits = _int2binstr(string.atol(s[:8],16),4) + \
_int2binstr(string.atol(s[8:16],16),4) + \
_int2binstr(string.atol(s[16:24],16),4) + \
_int2binstr(string.atol(s[24:],16),4)
return self
def setFromMemoryDump(self, gdb_string):
"""
We expect to get gdb_string as four hex units. eg:
0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2
Which will be translated to:
db547d14-1b3f4bc3-9b984f71-d22f890a
Returns self.
"""
s = string.replace(gdb_string, '0x', '')
s = string.replace(s, ' ', '')
t = ''
for i in range(8,40,8):
for j in range(0,8,2):
t = t + s[i-j-2:i-j]
self.setFromString(t)
def toString(self):
"""
Return as a string matching the LL standard
AAAAAAAA-AAAA-BBBB-BBBB-BBBBBBCCCCCC (a 128-bit number in hex)
where A=network address, B=timestamp, C=random.
"""
return uuid_bits_to_string(self._bits)
def getAsString(self):
"""
Return a different string representation of the form
AAAAAAAA-AAAABBBB-BBBBBBBB-BBCCCCCC (a 128-bit number in hex)
where A=network address, B=timestamp, C=random.
"""
i1 = _binstr2int(self._bits[0:4])
i2 = _binstr2int(self._bits[4:8])
i3 = _binstr2int(self._bits[8:12])
i4 = _binstr2int(self._bits[12:16])
return '%08lx-%08lx-%08lx-%08lx' % (i1,i2,i3,i4)
def generate(self):
"""
Generate a new uuid. This algorithm is slightly different
from c++ implementation for portability reasons.
Returns self.
"""
m = md5()
m.update(uuid.uuid1().bytes)
self._bits = m.digest()
return self
def isNull(self):
"""
Returns 1 if the uuid is null - ie, equal to default uuid.
"""
return (self._bits == "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0")
def xor(self, rhs):
"""
xors self with rhs.
"""
v1 = _binstr2int(self._bits[0:4]) ^ _binstr2int(rhs._bits[0:4])
v2 = _binstr2int(self._bits[4:8]) ^ _binstr2int(rhs._bits[4:8])
v3 = _binstr2int(self._bits[8:12]) ^ _binstr2int(rhs._bits[8:12])
v4 = _binstr2int(self._bits[12:16]) ^ _binstr2int(rhs._bits[12:16])
self._bits = _int2binstr(v1,4) + \
_int2binstr(v2,4) + \
_int2binstr(v3,4) + \
_int2binstr(v4,4)
# module-level null constant
NULL = UUID()
def printTranslatedMemory(four_hex_uints):
"""
We expect to get the string as four hex units. eg:
0x147d54db 0xc34b3f1b 0x714f989b 0x0a892fd2
Which will be translated to:
db547d14-1b3f4bc3-9b984f71-d22f890a
"""
uuid = UUID()
uuid.setFromMemoryDump(four_hex_uints)
print uuid.toString()
def isUUID(id_str):
"""
This function returns:
- 1 if the string passed is a UUID
- 0 is the string passed is not a UUID
- None if it neither of the if's below is satisfied
"""
if not id_str or len(id_str) < 5 or len(id_str) > 36:
return 0
if isinstance(id_str, UUID) or UUID.uuid_regex.match(id_str):
return 1
return None
def isPossiblyID(id_str):
"""
This function returns 1 if the string passed has some uuid-like
characteristics. Otherwise returns 0.
"""
is_uuid = isUUID(id_str)
if is_uuid is not None:
return is_uuid
# build a string which matches every character.
hex_wildcard = r"[0-9a-fA-F]"
chars = len(id_str)
next = min(chars, 8)
matcher = hex_wildcard+"{"+str(next)+","+str(next)+"}"
chars = chars - next
if chars > 0:
matcher = matcher + "-"
chars = chars - 1
for block in range(3):
next = max(min(chars, 4), 0)
if next:
matcher = matcher + hex_wildcard+"{"+str(next)+","+str(next)+"}"
chars = chars - next
if chars > 0:
matcher = matcher + "-"
chars = chars - 1
if chars > 0:
next = min(chars, 12)
matcher = matcher + hex_wildcard+"{"+str(next)+","+str(next)+"}"
#print matcher
uuid_matcher = re.compile(matcher)
if uuid_matcher.match(id_str):
return 1
return 0
def uuid_bits_to_string(bits):
i1 = _binstr2int(bits[0:4])
i2 = _binstr2int(bits[4:6])
i3 = _binstr2int(bits[6:8])
i4 = _binstr2int(bits[8:10])
i5 = _binstr2int(bits[10:12])
i6 = _binstr2int(bits[12:16])
return '%08lx-%04lx-%04lx-%04lx-%04lx%08lx' % (i1,i2,i3,i4,i5,i6)
def uuid_bits_to_uuid(bits):
return UUID(uuid_bits_to_string(bits))
try:
from mulib import stacked
stacked.NoProducer() # just to exercise stacked
except:
#print "Couldn't import mulib.stacked, not registering UUID converter"
pass
else:
def convertUUID(uuid, req):
req.write(str(uuid))
stacked.add_producer(UUID, convertUUID, "*/*")
stacked.add_producer(UUID, convertUUID, "text/html")

Binary file not shown.

View File

@@ -0,0 +1,53 @@
"""\
@file metrics.py
@author Phoenix
@date 2007-11-27
@brief simple interface for logging metrics
$LicenseInfo:firstyear=2007&license=mit$
Copyright (c) 2007-2009, Linden Research, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
$/LicenseInfo$
"""
import sys
from indra.base import llsd
_sequence_id = 0
def record_metrics(table, stats, dest=None):
"Write a standard metrics log"
_log("LLMETRICS", table, stats, dest)
def record_event(table, data, dest=None):
"Write a standard logmessage log"
_log("LLLOGMESSAGE", table, data, dest)
def _log(header, table, data, dest):
if dest is None:
# do this check here in case sys.stdout changes at some
# point. as a default parameter, it will never be
# re-evaluated.
dest = sys.stdout
global _sequence_id
print >>dest, header, "(" + str(_sequence_id) + ")",
print >>dest, table, llsd.format_notation(data)
_sequence_id += 1