#!/usr/bin/env python """\ @file simperf_host_xml_parser.py @brief Digest collector's XML dump and convert to simple dict/list structure $LicenseInfo:firstyear=2008&license=mit$ Copyright (c) 2008-2009, Linden Research, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. $/LicenseInfo$ """ import sys, os, getopt, time import simplejson from xml import sax def usage(): print "Usage:" print sys.argv[0] + " [options]" print " Convert RRD's XML dump to JSON. Script to convert the simperf_host_collector-" print " generated RRD dump into JSON. Steps include converting selected named" print " fields from GAUGE type to COUNTER type by computing delta with preceding" print " values. Top-level named fields are:" print print " lastupdate Time (javascript timestamp) of last data sample" print " step Time in seconds between samples" print " ds Data specification (name/type) for each column" print " database Table of data samples, one time step per row" print print "Options:" print " -i, --in Input settings filename. (Default: stdin)" print " -o, --out Output settings filename. (Default: stdout)" print " -h, --help Print this message and exit." print print "Example: %s -i rrddump.xml -o rrddump.json" % sys.argv[0] print print "Interfaces:" print " class SimPerfHostXMLParser() # SAX content handler" print " def simperf_host_xml_fixup(parser) # post-parse value fixup" class SimPerfHostXMLParser(sax.handler.ContentHandler): def __init__(self): pass def startDocument(self): self.rrd_last_update = 0 # public self.rrd_step = 0 # public self.rrd_ds = [] # public self.rrd_records = [] # public self._rrd_level = 0 self._rrd_parse_state = 0 self._rrd_chars = "" self._rrd_capture = False self._rrd_ds_val = {} self._rrd_data_row = [] self._rrd_data_row_has_nan = False def endDocument(self): pass # Nasty little ad-hoc state machine to extract the elements that are # necessary from the 'rrdtool dump' XML output. The same element # name '' is used for two different data sets so we need to pay # some attention to the actual structure to get the ones we want # and ignore the ones we don't. def startElement(self, name, attrs): self._rrd_level = self._rrd_level + 1 self._rrd_capture = False if self._rrd_level == 1: if name == "rrd" and self._rrd_parse_state == 0: self._rrd_parse_state = 1 # In self._rrd_capture = True self._rrd_chars = "" elif self._rrd_level == 2: if self._rrd_parse_state == 1: if name == "lastupdate": self._rrd_parse_state = 2 # In self._rrd_capture = True self._rrd_chars = "" elif name == "step": self._rrd_parse_state = 3 # In self._rrd_capture = True self._rrd_chars = "" elif name == "ds": self._rrd_parse_state = 4 # In self._rrd_ds_val = {} self._rrd_chars = "" elif name == "rra": self._rrd_parse_state = 5 # In elif self._rrd_level == 3: if self._rrd_parse_state == 4: if name == "name": self._rrd_parse_state = 6 # In self._rrd_capture = True self._rrd_chars = "" elif name == "type": self._rrd_parse_state = 7 # In self._rrd_capture = True self._rrd_chars = "" elif self._rrd_parse_state == 5: if name == "database": self._rrd_parse_state = 8 # In elif self._rrd_level == 4: if self._rrd_parse_state == 8: if name == "row": self._rrd_parse_state = 9 # In self._rrd_data_row = [] self._rrd_data_row_has_nan = False elif self._rrd_level == 5: if self._rrd_parse_state == 9: if name == "v": self._rrd_parse_state = 10 # In self._rrd_capture = True self._rrd_chars = "" def endElement(self, name): self._rrd_capture = False if self._rrd_parse_state == 10: self._rrd_capture = self._rrd_level == 6 if self._rrd_level == 5: if self._rrd_chars == "NaN": self._rrd_data_row_has_nan = True else: self._rrd_data_row.append(self._rrd_chars) self._rrd_parse_state = 9 # In elif self._rrd_parse_state == 9: if self._rrd_level == 4: if not self._rrd_data_row_has_nan: self.rrd_records.append(self._rrd_data_row) self._rrd_parse_state = 8 # In elif self._rrd_parse_state == 8: if self._rrd_level == 3: self._rrd_parse_state = 5 # In elif self._rrd_parse_state == 7: if self._rrd_level == 3: self._rrd_ds_val["type"] = self._rrd_chars self._rrd_parse_state = 4 # In elif self._rrd_parse_state == 6: if self._rrd_level == 3: self._rrd_ds_val["name"] = self._rrd_chars self._rrd_parse_state = 4 # In elif self._rrd_parse_state == 5: if self._rrd_level == 2: self._rrd_parse_state = 1 # In elif self._rrd_parse_state == 4: if self._rrd_level == 2: self.rrd_ds.append(self._rrd_ds_val) self._rrd_parse_state = 1 # In elif self._rrd_parse_state == 3: if self._rrd_level == 2: self.rrd_step = long(self._rrd_chars) self._rrd_parse_state = 1 # In elif self._rrd_parse_state == 2: if self._rrd_level == 2: self.rrd_last_update = long(self._rrd_chars) self._rrd_parse_state = 1 # In elif self._rrd_parse_state == 1: if self._rrd_level == 1: self._rrd_parse_state = 0 # At top if self._rrd_level: self._rrd_level = self._rrd_level - 1 def characters(self, content): if self._rrd_capture: self._rrd_chars = self._rrd_chars + content.strip() def _make_numeric(value): try: value = float(value) except: value = "" return value def simperf_host_xml_fixup(parser, filter_start_time = None, filter_end_time = None): # Fixup for GAUGE fields that are really COUNTS. They # were forced to GAUGE to try to disable rrdtool's # data interpolation/extrapolation for non-uniform time # samples. fixup_tags = [ "cpu_user", "cpu_nice", "cpu_sys", "cpu_idle", "cpu_waitio", "cpu_intr", # "file_active", # "file_free", # "inode_active", # "inode_free", "netif_in_kb", "netif_in_pkts", "netif_in_errs", "netif_in_drop", "netif_out_kb", "netif_out_pkts", "netif_out_errs", "netif_out_drop", "vm_page_in", "vm_page_out", "vm_swap_in", "vm_swap_out", #"vm_mem_total", #"vm_mem_used", #"vm_mem_active", #"vm_mem_inactive", #"vm_mem_free", #"vm_mem_buffer", #"vm_swap_cache", #"vm_swap_total", #"vm_swap_used", #"vm_swap_free", "cpu_interrupts", "cpu_switches", "cpu_forks" ] col_count = len(parser.rrd_ds) row_count = len(parser.rrd_records) # Process the last row separately, just to make all values numeric. for j in range(col_count): parser.rrd_records[row_count - 1][j] = _make_numeric(parser.rrd_records[row_count - 1][j]) # Process all other row/columns. last_different_row = row_count - 1 current_row = row_count - 2 while current_row >= 0: # Check for a different value than the previous row. If everything is the same # then this is probably just a filler/bogus entry. is_different = False for j in range(col_count): parser.rrd_records[current_row][j] = _make_numeric(parser.rrd_records[current_row][j]) if parser.rrd_records[current_row][j] != parser.rrd_records[last_different_row][j]: # We're good. This is a different row. is_different = True if not is_different: # This is a filler/bogus entry. Just ignore it. for j in range(col_count): parser.rrd_records[current_row][j] = float('nan') else: # Some tags need to be converted into deltas. for j in range(col_count): if parser.rrd_ds[j]["name"] in fixup_tags: parser.rrd_records[last_different_row][j] = \ parser.rrd_records[last_different_row][j] - parser.rrd_records[current_row][j] last_different_row = current_row current_row -= 1 # Set fixup_tags in the first row to 'nan' since they aren't useful anymore. for j in range(col_count): if parser.rrd_ds[j]["name"] in fixup_tags: parser.rrd_records[0][j] = float('nan') # Add a timestamp to each row and to the catalog. Format and name # chosen to match other simulator logging (hopefully). start_time = parser.rrd_last_update - (parser.rrd_step * (row_count - 1)) # Build a filtered list of rrd_records if we are limited to a time range. filter_records = False if filter_start_time is not None or filter_end_time is not None: filter_records = True filtered_rrd_records = [] if filter_start_time is None: filter_start_time = start_time * 1000 if filter_end_time is None: filter_end_time = parser.rrd_last_update * 1000 for i in range(row_count): record_timestamp = (start_time + (i * parser.rrd_step)) * 1000 parser.rrd_records[i].insert(0, record_timestamp) if filter_records: if filter_start_time <= record_timestamp and record_timestamp <= filter_end_time: filtered_rrd_records.append(parser.rrd_records[i]) if filter_records: parser.rrd_records = filtered_rrd_records parser.rrd_ds.insert(0, {"type": "GAUGE", "name": "javascript_timestamp"}) def main(argv=None): opts, args = getopt.getopt(sys.argv[1:], "i:o:h", ["in=", "out=", "help"]) input_file = sys.stdin output_file = sys.stdout for o, a in opts: if o in ("-i", "--in"): input_file = open(a, 'r') if o in ("-o", "--out"): output_file = open(a, 'w') if o in ("-h", "--help"): usage() sys.exit(0) # Using the SAX parser as it is at least 4X faster and far, far # smaller on this dataset than the DOM-based interface in xml.dom.minidom. # With SAX and a 5.4MB xml file, this requires about seven seconds of # wall-clock time and 32MB VSZ. With the DOM interface, about 22 seconds # and over 270MB VSZ. handler = SimPerfHostXMLParser() sax.parse(input_file, handler) if input_file != sys.stdin: input_file.close() # Various format fixups: string-to-num, gauge-to-counts, add # a time stamp, etc. simperf_host_xml_fixup(handler) # Create JSONable dict with interesting data and format/print it print >>output_file, simplejson.dumps({ "step" : handler.rrd_step, "lastupdate": handler.rrd_last_update * 1000, "ds" : handler.rrd_ds, "database" : handler.rrd_records }) return 0 if __name__ == "__main__": sys.exit(main())