memory usage real time profile in python

簡單的 real time mem usage 範例, 利用 guppy 來 trace memory 的 utility.

explore memory utility test case "heapy.py"

from guppy import hpy
#import pickle
import gc
import sys
import time
import re
import pprint

#--------------------------------------
ppHP = hpy()
reTOTALSIZE = re.compile("Total\s+size\s+=\s+(\d+)\s+bytes",re.M)

def parser_mem_profile(funcnm=None,memprof=None):
    """ parser mem profile
    params:
        funcnm :  func name
        memprof:  table from hpy.heap() report
    return:
        print("funcnm","mem usage")
    """
    #print ("MEMPROF",funcnm,reTOTALSIZE.findall(memprof)[0])
    # interface for driveGnuPlotStreams.pl
    print  '0:%.3f' %(float(reTOTALSIZE.findall(memprof)[0])/100000)



def parser_gc_profile(funcnm=None,gcprof=None):
    """ parser gc profile
    params:
        funcnm :  func name
        gcprof :  table from gc.collect() report
    return:
        print("funcnm","num of obj to be collected")
    """
    #print ("GCPROF",funcnm, gcprof)
    # interface for driveGnuPlotStreams.pl
    #print '1:' + str(gcprof)



def decodrator_mem_profile(): #decorator **kwargs
    """ decorator mem usage profile collect """

    def inner(func):

        def wrapper(*args, **kwargs): # func *args, **kwargs

            retval = func(*args, **kwargs)

            def _store_mem_profile():
                """ store mem profile """
                parser_mem_profile(funcnm=str(func.__name__),\
                                   memprof=str(ppHP.heap()))

            def _store_gc_profile():
                """ store garbage collect """
                parser_gc_profile(funcnm=str(func.__name__),\
                                  gcprof=str(gc.collect()))

            _store_mem_profile()
            _store_gc_profile()

        return wrapper
    return inner

#---------------------------------------------

class Pattern(object):

    def __init__(self):
        self._list = []

    def run(self):
        self._list = [ i*200 for i in range(10000) ]

    def clear(self):
        self._list = None

ppPattern = None

@decodrator_mem_profile()
def run_pattern0(wait=0.1):
    """ run test pattern0 """
    global ppPattern
    ppPattern = Pattern()
    ppPattern.run()
    time.sleep(wait)

@decodrator_mem_profile()
def free_pattern0(wait=0.1):
    """ free test pattern0 """
    global ppPattern
    ppPattern.clear()
    del ppPattern
    time.sleep(wait)


def run_test():
    """ run all tests for mem profile """

    for i in xrange(100):
        run_pattern0()
        free_pattern0()


def main():
    """ gen the PIPE test pattern to heapy_gnuplot
    ex:
        os.system((python heapy.py) | (python heapy_gnuplot))
    """

    run_test()


if __name__== "__main__":
    main()

下載 driveGnuPlotStreams.pl 當成我們的 plot func.

ps: 利用 PIPE 的方式, 把 python heay.py IO 端的 Info 存到下個 driveGnuPlotStreams.pl 的 input.

run command

(python heapy.py ; read) | perl ./driveGnuPlotStreams.pl 1 1 50 0 50 500x300+0+0 'mem' 0

results:

Downloads:
https://docs.google.com/open?id=0B35jh2lwIpeKVVJQa0dMZHhPckE
https://docs.google.com/open?id=0B35jh2lwIpeKRkpVcnVTb1VKdnM

Refs:
heapy tutorial
http://www.smira.ru/wp-content/uploads/2011/08/heapy.html
http://guppy-pe.sourceforge.net/#Heapy

real time plot(Gnuplot)
http://users.softlab.ece.ntua.gr/~ttsiod/gnuplotStreaming.html
http://www.lysium.de/blog/index.php?/archives/234-Plotting-data-with-gnuplot-in-real-time.html

real time plot(matplotlib)
http://matplotlib.org/
install
http://matplotlib.org/users/installing.html#manually-installing-pre-built-packages