2013年1月27日 星期日

PiCloud + python


  • job manager
    • job create job
      • reduce the thread acquire time, try lock/mutex times
    • job queue, job schedule, job status. job new/delete
  • watch dog
    • performance monitor  
  • scientific tools
    • numpy(math array), h5py(python hash table), pandas(big data), scipy(science), scikits(machine learning)
  • multi environments support
    • ubuntu 11.04, 12.04 for user set 
  • notebook support
  • core, memory set
    • how many cores set, and memory usage limitation
  •  performance benchmark
    • ex: PI  seep up 5 times for multi process assign to multi core by mapping table set

          num_jobs = 8
          tests_per_call = total_tests/num_jobs
      
          # argument list has 8 duplicate elements
          jids = cloud.map(monte_carlo, [tests_per_call]*num_jobs, _type='c2')
      
          # get list of all counts
          num_in_circle_list = cloud.result(jids)
  • daily run(schedule run)
  • web support
    • Django

2013年1月26日 星期六

embedded python in c code

底下用 PyObject embedded c 的方式,寫了個 python sqlit3 embedded c 的範例.


  • PyObject   : PyList, Pytuple, Py_buildValue, PyCheck , assertion ...
  • standard c : struct


command:
%gcc -lpython2.7 test.c

sample code
#include 
#include 
#include 


struct Sqlite {
    PyObject* sqlite3_ptr;
    PyObject* connect_ptr;
    PyObject* cursor_ptr;
    PyObject* exec_ptr;
    PyObject* commit_ptr;
    PyObject* header_ptr;
    PyObject* body_ptr;
    PyObject* result_ptr;

} Sqlite;

struct Table {
    char* name;
    int   age;
} Table[] = {
    { "john", 18 }
};

/*
 * print macro
 */
void
print_macro(PyObject *self) {
    PyObject* repr_ptr = PyObject_Repr(self);
    const char* repr_s = PyString_AsString(repr_ptr);
    printf("%s\n",repr_s);
}

/*
 * load python modules
 * */
void
load_py_modules()
{
    PyRun_SimpleString("import sqlite3");
    Sqlite.sqlite3_ptr = PyImport_AddModule("sqlite3");
    Py_DECREF(Sqlite.sqlite3_ptr);

#ifdef DEBUG
print_macro(Sqlite.sqlite3_ptr);
#endif
}


/*
 * connect = sqlite3.connect("tt.db")
 * */
void
create_connect(char* dbname)
{

    Sqlite.connect_ptr = PyObject_GetAttrString(Sqlite.sqlite3_ptr,"connect");
    Sqlite.connect_ptr = PyObject_CallFunction(Sqlite.connect_ptr,"s",dbname);
    Py_DECREF(Sqlite.connect_ptr);

#ifdef DEBUG
print_macro(Sqlite.connect_ptr);
#endif
}

/*
 * cursor = connect.cursor()
 * */
void
create_cursor()
{

    Sqlite.cursor_ptr = PyObject_CallMethod(Sqlite.connect_ptr,"cursor",NULL,NULL);
    Py_DECREF(Sqlite.cursor_ptr);

#ifdef DEBUG
print_macro(Sqlite.cursor_ptr);
#endif
}

/*
 * cursor.execute("CREATE TABLE test (name text, age real)"
 * */
void
create_table_header(char* query)
{
    Sqlite.exec_ptr = PyObject_GetAttrString(Sqlite.cursor_ptr,"execute");
    Py_DECREF(Sqlite.exec_ptr);
    Sqlite.header_ptr = PyObject_CallFunction(Sqlite.exec_ptr,"s",query);
    Py_DECREF(Sqlite.header_ptr);

#ifdef DEBUG
print_macro(Sqlite.header_ptr);
#endif
}

/*
 * cursor.execute("INSERT INTO test VALUES ( sean, 28 )
 * */
void
create_table_body(char* query)
{
    Sqlite.exec_ptr = PyObject_GetAttrString(Sqlite.cursor_ptr,"execute");
    Py_DECREF(Sqlite.exec_ptr);
    Sqlite.body_ptr = PyObject_CallFunction(Sqlite.exec_ptr,"s",query);
    Py_DECREF(Sqlite.body_ptr);

#ifdef DEBUG
print_macro(Sqlite.body_ptr);
#endif
}

/*
 * connect.commit()
 * */
void
save_commit()
{
    Sqlite.commit_ptr = PyObject_CallMethod(Sqlite.connect_ptr,"commit",NULL,NULL);
    Py_DECREF(Sqlite.commit_ptr);
}

/*
 *  inspect_table_body
 */
void
inspect_table_body(char* query)
{
    Sqlite.exec_ptr = PyObject_GetAttrString(Sqlite.cursor_ptr,"execute");
    Sqlite.result_ptr = PyObject_CallFunction(Sqlite.exec_ptr,"s",query);

#ifdef DEBUG
print_macro(Sqlite.result_ptr);
#endif
}

/*
 * inspect results
 * */
void
inspect_table_results()
{
    PyObject* item;
    PyObject* val;
    int i;
    char* str;

    Sqlite.result_ptr = PyObject_GetAttrString(Sqlite.cursor_ptr,"fetchone");
    Py_DECREF(Sqlite.result_ptr);

    for (int i=0; i < PyList_Size(Sqlite.result_ptr); i++) {
 item = PyList_GetItem(Sqlite.result_ptr,i);

 for (int j=0; j < PyTuple_Size(item); j++) {
     val = PyTuple_GetItem(item,j);

     if (IntCheck(val)) {
  PyArg_Parse(val,"i",&i);
  printf ("%i, ",i);
     } else {
  PyArg_Parse(val,"z",&str);
  printf ("%s, ",str);
     }
 }
    }
}

/*
 * sqlite.close()
 * */
void
close_connect()
{
    Sqlite.connect_ptr = PyObject_GetAttrString(Sqlite.sqlite3_ptr,"close");

#ifdef DEBUG
print_macro(Sqlite.connect_ptr);
#endif
}


int
main(int argc, char *argv[])
{
    Py_SetProgramName(argv[0]);  /* optional but recommended */

    Py_Initialize();

    load_py_modules();
    create_connect("test.db");
    create_cursor();
    create_table_header("CREATE TABLE test (name text, age real)");
    create_table_body("INSERT INTO test VALUES (john, 18)");
    save_commit();
    inspect_table_body("SELECT * From test");
    inspect_table_results();
    close_connect();
    Py_Finalize();
    return 0;
}

Download
auto stepup tools:

ref:
object protocol

hello world 

2013年1月22日 星期二

hardare software co-simulation with parallel process + python

底下用 subprocess, 假裝透過 system call 來把底層的 virtual hardware 代起來, 做個假的 simulation. 有點蠢就是了...
virtual hardware
#!/usr/bin/python
# %parallel_proc ["--add"|"--sub"], [--A a], [--B b]]

import argparse

def decoder():
    """ decoder Add, Sub """
    parser = argparse.ArgumentParser(description='Process Proc')
    parser.add_argument('--add', action='store_true', default=False, dest='add', help='add')
    parser.add_argument('--sub', action='store_true', default=False, dest='sub', help='sub')
    parser.add_argument('--A',   action='store', type=int, default=10, dest='a',  help='A')
    parser.add_argument('--B',   action='store', type=int, default=10, dest='b',  help='B')
    parser.add_argument('--t',   action='store', type=int, default=10, dest='t',  help='time')
    args = parser.parse_args()

    if args.add:
     if args.t < 2:
         print "add %d" %(proc_add(args.a, args.b))
 else:
     print "idle %d" %(2)
    elif args.sub:
 if args.t < 3:
     print "sub %d" %(proc_sub(args.a, args.b))
 else:
     print "idle %d" %(3)

def proc_add(a, b):
    return a + b

def proc_sub(a, b):
    return a - b

def main():
    decoder()

if __name__ == '__main__':
    main()

download
#! /usr/bin/python
import subprocess
import sys
import re

reIDLE = re.compile("idle\s+(\w+)", re.M)
reADD  = re.compile("add\s+(\w+)", re.M)
reSUB  = re.compile("sub\s+(\w+)", re.M)

def spawn_add(a=0, b=0, cycle=0):
    proc = subprocess.Popen("./parallel_proc.py --add --A %d --B %d --t %d" %(a, b, cycle), \
             shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    id(proc)
    proc.wait()
    line = proc.stdout.readline()
    idle = reIDLE.findall(line)
    add  = reADD.findall(line)
    ret_idle = 0 if not idle  else idle[0]
    ret_add  = 0 if not add   else add[0]
    return (ret_idle, ret_add)


def spawn_sub(a=0, b=0, cycle=0):
    proc = subprocess.Popen("./parallel_proc.py --sub --A %d --B %d --t %d" %(a, b, cycle), \
             shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    id(proc)
    proc.wait()
    line = proc.stdout.readline()
    idle = reIDLE.findall(line)
    sub  = reADD.findall(line)
    ret_idle = 0 if not idle  else idle[0]
    ret_sub  = 0 if not sub   else sub[0]
    return (ret_idle, ret_sub)


def main():

    # (2+3)+(4-2)
    # time 1 2+3, 4-2
    # time 2 2+3, 4-2
    # time 3    , 4-2
    # time 4  5 , 2

    status = 'start'
    add = { 'a' : 2, 'b' : 3, 'cyc' : 4 }
    sub = { 'a' : 4, 'b' : 2, 'cyc' : 5 }

    def status_inspect(add_idle=0, sub_idle=0):
        if add_idle != 0 and sub_idle != 0 :
            add['cyc'] -= add_idle
     sub['cyc'] -= sub_idle
     status = 'start'
        elif add_idle != 0 and sub_idle == 0:
            add['cyc'] -= add_idle
            status = 'idle_add'
        elif add_idle == 0 and sub_idle != 0:
            sub['cyc'] -= sub_idle
            status = 'idle_sub'
        else:
            return True
        return False

    while True:
 if status == 'start':
    (add_idle, add_val) = spawn_add(add['a'], add['b'], add['cyc'])
    (sub_idle, sub_val) = spawn_sub(sub['a'], sub['b'], sub['cyc'])
    if status_inspect(int(add_idle), int(sub_idle)): break
 elif status == 'idle_add':
    (add_idle, add_val) = spawn_add(add['a'], add['b'], add['cyc'])
    if status_inspect(int(add_idle), int(sub_idle)): break
 elif status == 'idle_sub':
    (sub_idle, sub_val) = spawn_sub(sub['a'], sub['b'], sub['cyc'])
    if status_inspect(int(add_idle), int(sub_idle)): break
 else:
     break

if __name__ == '__main__':
    main()

download

2013年1月19日 星期六

MYHDL example

底下是 MyHDL 的範例, 利用 Python 建立起一個符合 hardware simulation 的 IDE, 你可以發現有許多 sensitive trigger 的 functional block. 這有點類似 Verilog blocking/no-blocking 的語法, 讓 hardware designer 能夠很快速的上手. 且提供類似 core dump的機制, 讓使用者可以透過 waveform debug, 除此之外, 也可連接 Python 現有的 physical level driver. 達到 hardware/software co-simulation 的 virtual  platform.


主要功能
1. fetch all sequence trigger to sequence queue
2. sort queue by simulation time
3. put curent trigger to work queue
4. spawn each trigger from work queue
5. code gen if simulation requirement is pass


#!/usr/bin/python

from myhdl import *
import random

# operator map table
OP = { "ADD" : 0x00,
       "SUB" : 0x01,
       "MUX" : 0x02,
       "DIV" : 0x03,
       "RSHIFT" : 0x04,
       "LSHIFT" : 0x05 }

# DUT
def alu_0(clk, srst, z, x, y, op):
    """ alu  sensitive by positive edge clk syn rst """

    @always(clk.posedge)
    def ALU():
 if srst:
     z.next = 0
 else:
     if op == OP['ADD']:
  z.next = x + y
     elif op == OP['SUB']:
      z.next = x - y
     elif op == OP['MUX']:
      z.next = x * y

    return ALU


def test_alu_0():
    """ test DUT signal create TX/RX """

    # signal create
    clk = Signal(intbv(0, min=0, max=1))
    srst = Signal(intbv(0, min=0, max=1))
    z = Signal(intbv(0, min=0, max=64))
    y = Signal(intbv(0, min=0, max=32))
    x = Signal(intbv(0, min=0, max=32))
    op = Signal(intbv(0, min=0, max=6))

    op_count = Signal(intbv(0, min=0, max=32))

    # link DUT
    ptr_alu_0 = alu_0(clk, srst, z, x, y, op)

    # test clock gen
    @always(delay(10))
    def clkgen():
     clk.next = clk

    # test pattern gen
    @always(delay(20))
    def patgen():

     def iter_op():
        op = OP[OP.keys()[op_count]]
        inc_count = op_count + 1
         op_count.next = 0 if inc_count == len(OP) else inc_count
     return op

 op.next = iter_op()
     x.next = x + 1
     y.next = y + 2

    return ptr_alu_0, clkgen, patgen


def simulate(timesteps):
    """ start simulation """
    tb = traceSignals(test_alu_0)
    sim = Simulation(tb)
    sim.run(timesteps)

simulate(100)

Download

2013年1月13日 星期日

stackless + python

ref : http://www.stackless.com/wiki/Idioms http://en.wikipedia.org/wiki/Stackless_Python http://www.learncomputer.com/stackless-python/ http://code.google.com/p/stacklessexamples/wiki/StacklessExamples

Greenlet vs yield python

Greenlet vs yield 利用 yield 或 Greenlet 來改變 process 的執行順序
import time
from greenlet import greenlet

class Recorder:

    def __init__(self):
     self._rec = []

    def push(self, data):
     self._rec.append(data)

    def pop(self):
     rst = self._rec[-1] if len(self._rec) != 0 else None
     return rst

    def clear(self):
 del self._rec[:]

    def __repr__(self):
     return repr(self._rec)

rec = Recorder()

def event(i):
    rec.append(i)

def tasklet_test_0(event):
    """ tasklet test0 """
    rec.push(0)
    event[1].switch(event)

def tasklet_test_1(event):
    """ tasklet test1 """
    rec.push(1)
    event[0].switch(event)

def tasklet_switch():
    """ tasklet switch """
    rec.clear()
    start_time = time.time()
    gr0 = greenlet(tasklet_test_0)
    gr1 = greenlet(tasklet_test_1)
    event = [gr0, gr1]
    event[0].switch(event)
    print repr(rec)
    print time.time() - start_time

def yield_test_0():
    rec.push(0)

def yield_test_1():
    rec.push(1)

def yield_test(event):
    """ list all sequence process in here """
    m = yield None

    while True:
 if m == "event0":
     print "receive send %s" %(m)
     m = yield event[0]() # to event 0
     elif m == "event1":
         print "receive send %s" %(m)
         m = yield event[1]() # to event 1
 else:
     print "receive send %s" %(m)
     m = yield None

def yield_switch():
    """ yield switch """
    rec.clear()
    start_time = time.time()
    event = [yield_test_0, yield_test_1]
    iters = yield_test(event)
    iters.next() # run event and stop iter on it
    iters.send("event2")
    iters.send("event1")
    iters.send("event0")
    iters.send("stop")
    print repr(rec)
    print time.time() - start_time

def main():
    tasklet_switch()
    yield_switch()

if __name__=='__main__':
    main()
download
ref:
http://sdiehl.github.com/gevent-tutorial/ http://stackoverflow.com/questions/8960747/stackless-in-pypy-and-pypy-greenlet-differences
http://stackoverflow.com/questions/4263059/python-erlang-whats-the-difference-between-twisted-stackless-greenlet-event

multiprocessing VS serial

底下用 how to sync IPXACT with UVM 改寫成 multiprocessing vs serial 的版本. 發現如果 parallel 的 IO access counts 越小, performance serial > multiprocessing. 如果 IO access count 越大 performance multiprocessing > serial


from StringIO import *
import sys
import os
import time
import pprint
# add third party module to PyPy path

from lxml import etree
import multiprocessing
import subprocess
import math


class Parser:

    _prefixmap = {  'spirit' : 'http://www.spiritconsortium.org/XMLSchema/SPIRIT/1.4',
      'vendorExtensions' : '$UVM_REG_GEN/XMLSchema/SPIRIT',
      'xsi' : 'http://www.w3.org/2001/XMLSchema-instance',
  }

    def __init__(self):
     self._tree  = None

    def __del__(self):
     self._tree  = None

    def open(self, ifile):
     """ open from IP-XACT file and build up etree by lxml """
 try:
     self._ifile = StringIO(open(ifile, 'r').read())
     self._tree  = etree.parse(self._ifile)
     self._ifile.close()
 except IOError as e:
     print "%s not found or build up lxml etree fail" %(ifile)

    def close(self):
     """ close """
 self._tree = None

    def findAll(self, stmt, dtype='text'):
     """ return  list in ['text', 'tag', 'tail', attrib' ]dtype
     return  list in [element ]
     """
 found = filter(lambda i: i !=None, self._tree.xpath(stmt, namespaces=self._prefixmap))
 if dtype == 'text':
     return map(lambda i: i.text, found)
 elif dtype == 'tag':
     return map(lambda i: i.tag, found)
 elif dtype == 'tail':
     return map(lambda i: i.tail, found)
 elif dtype == 'attrib':
     return map(lambda i: i.attrib, found)
 elif dtype == 'obj':
     return found
 else:
     raise valueError("dtype not support %s" %(dtype))


def test_run_lxml(ifile=None, query=None):
    """ stackless test """
    pp = Parser()
    pp.open(ifile)
    rst = pp.findAll(stmt=query, dtype='text')
    del pp
    return rst


def serial_test_lxml(nums):
    """ serial test lxml """

    start_time = time.time()
    rsts = []
    for i in nums:
     rsts.append(test_run_lxml(ifile=i, query='//spirit:vendor'))
    print "sriial nums(%d) runtime(%s)" %(len(nums), time.time() - start_time)


def multiprocess_test_lxml(nums, nprocs=4):
    """ multiprocess test lxml """

    start_time = time.time()
    procs = []

    def worker(nums, out_q):
 """
 The worker function, invoked in a process.
 """
 outdict = {}
 for i in nums:
     outdict[i] = test_run_lxml(ifile=i, query='//spirit:vendor')
 out_q.put(outdict)

    # Each process will get 'chunksize' nums and a queue to put his out
    # dict into
    out_q = multiprocessing.Queue()
    chunksize = int(math.ceil(len(nums) / float(nprocs)))
    procs = []

    for i in range(nprocs):
        p = multiprocessing.Process(
                target=worker,
  args=(nums[chunksize * i:chunksize * (i + 1)],
                      out_q))
        procs.append(p)
        p.start()

    # Collect all results into a single result dict. We know how many dicts
    # with results to expect.
    resultdict = {}
    for i in range(nprocs):
        resultdict.update(out_q.get())

    # Wait for all worker processes to finish
    for p in procs:
        p.join()

    print "multis nums(%d) procs(%d) runtime(%s)" %(len(nums), nprocs, time.time() - start_time)


def regression_test(num, npros=4):

    for i in range(1,num):
     cmd = "cp 0.xml %d.xml" %(i)
  os.system(cmd)

    nums = [ "%s.xml" %(i) for i in range(num)]

    multiprocess_test_lxml(nums, npros)
    serial_test_lxml(nums)

    for i in range(1,num):
     cmd = "rm %d.xml" %(i)
     os.system(cmd)

def main():
    regression_test(10,1)
    regression_test(10,4)
    regression_test(10,8)

    regression_test(1000,1)
    regression_test(1000,4)
    regression_test(1000,8)


if __name__ == '__main__':
    main()

Download
results
multis nums(10) procs(1) runtime(0.0143690109253)
sriial nums(10) runtime(0.00273704528809)
multis nums(10) procs(4) runtime(0.0109529495239)
sriial nums(10) runtime(0.00267791748047)
multis nums(10) procs(8) runtime(0.0175120830536)
sriial nums(10) runtime(0.00293707847595)
multis nums(1000) procs(1) runtime(0.229081869125)
sriial nums(1000) runtime(0.233665943146)
multis nums(1000) procs(4) runtime(0.116140127182)
sriial nums(1000) runtime(0.237390041351)
multis nums(1000) procs(8) runtime(0.122074127197)
sriial nums(1000) runtime(0.275545120239)

2013年1月10日 星期四

how to sync IPXACT with UVM



target:
read IP-XACT attribute value and set it to UVM conf file automatically.

environment requirements 
lP-XACT file download from UVM ref flow

flow plan
  1. using lxml to parse the IP-XACT file(.xml)  
    1. using  xpath to find all values in this .xml
      ref lxml + python + IPXACT
  2. using cog to gen UVM code


--[[[cog
    --   import cog
    --   import IParser
    --  ...
    --   pp = None
    --   def load():
    --     global pp
    --      pp = IParser()
    --     pp.open('spi_rgm.spirit')
    --
    --   def genRegfield():
    --     global pp
    --     reg_text = pp.findAll('/spirit:component/spirit:vendor', dtype='text')
    --     prefix = "uvm_reg_field::type_id::create(\"%s\")" %(req_test)
    --     cog.outl(prefix)
    --   ....
    --]]]
--[[[end]]]
results: >>> uvm_reg_field::type_id::create("5")..

lxml + python + IPXACT


底下用 lxml xPath 來 parser IPXACT file(.xml). 將來希望能透過 python template 的方式,把UVM寫成個 template file, 之後再用 code gen 的方式把 UVM 的 conf file 跟 register set, parameter set, interface set ...連接起來
from lxml import etree
from StringIO import *
import sys

class Parser:

    _prefixmap = {  'spirit' : 'http://www.spiritconsortium.org/XMLSchema/SPIRIT/1.4',
      'vendorExtensions' : '$UVM_REG_GEN/XMLSchema/SPIRIT',
      'xsi' : 'http://www.w3.org/2001/XMLSchema-instance',
  }

    def __init__(self):
     self._tree  = None

    def __del__(self):
     self._tree  = None

    def open(self, ifile):
     """ open from IP-XACT file and build up etree by lxml """
 try:
     self._ifile = StringIO(open(ifile, 'r').read())
     self._tree  = etree.parse(self._ifile)
     self._ifile.close()
 except IOError as e:
     print "%s not found or build up lxml etree fail" %(ifile)

    def close(self):
     """ close """
 self._tree = None

    def findAll(self, stmt, dtype='text'):
     """ return  list in ['text', 'tag', 'tail', attrib' ]dtype
     return  list in [element ]
     """
 found = filter(lambda i: i !=None, self._tree.xpath(stmt, namespaces=self._prefixmap))
 if dtype == 'text':
     return map(lambda i: i.text, found)
 elif dtype == 'tag':
     return map(lambda i: i.tag, found)
 elif dtype == 'tail':
     return map(lambda i: i.tail, found)
 elif dtype == 'attrib':
     return map(lambda i: i.attrib, found)
 elif dtype == 'obj':
     return found
 else:
     raise valueError("dtype not support %s" %(dtype))


def main():
    pp = Parser()
    pp.open('spi_rgm.spirit')
    print pp.findAll(stmt='/spirit:component/spirit:vendor', dtype='text')
    print pp.findAll(stmt='//spirit:vendor', dtype='text')
    print pp.findAll(stmt='//spirit:vendor', dtype='tag')
    print pp.findAll(stmt='//spirit:vendor', dtype='obj')[0].getparent().tag
    pp.close()
    del pp

if __name__ == '__main__':
    main()

results
>>> ['spiritconsortium.org'] ['spiritconsortium.org'] ['{http://www.spiritconsortium.org/XMLSchema/SPIRIT/1.4}vendor'] {http://www.spiritconsortium.org/XMLSchema/SPIRIT/1.4}component
 download
ref:
http://lxml.de/resolvers.html

2013年1月9日 星期三

syntaxhighlighter

SyntaxHighlighter is a fully functional self-contained code syntax highlighter

ref
http://alexgorbatchev.com/SyntaxHighlighter/
http://oneqonea.blogspot.tw/2012/04/how-do-i-add-syntax-highlighting-to-my.html

2013年1月5日 星期六

MongoDB is fast

table correlation search with tree map (sharing)
child, ancestors, paths...

http://www.mongodb.org/display/DOCS/Trees+in+MongoDB

index
有沒有為 big data access 而煩惱阿, 是否覺得 SQL 的 cross query 很麻煩, 一起來用 MongoDB 吧...XD, 一種類似 hash table + json + 的 format. 讓你一看就懂.

vcd file info tracer



在 IP 驗證時, 除了用 synopsys VIP 來驗證 bus 的 protocol 跟 performance 的分析. 但有時候為了 customer design 時, 會把 protocol 作些變形, 這樣就不符合 standard protocol 的方式. 底下就用大家都熟知的 VCD 擋把 bus info 錄出來轉到 data base 讓使用者可以直接用 query 的方式把 performance 跟 dynamic power 算出來. 底下用 vcdparser.py 把 vcd info 讀出來.之後再透過後端的 big data 處理把資料寫到 data base 上.
python db link
"""
conn = sqlite3.connect('example.db')
c = conn.cursor()

c.execute('''CREATE TABLE vcd
      (version, timeType(s,ns,fs...), time''')
c.execute('''CREATE TABLE _scopes_
      (scopeDomain, scopeType(fork,module,...), top(scopeNm)''')
c.execute('''CREATE TABLE _signals_
      (scopeDomain, signalID, signalType(Real,Bit,...), width, name''')
c.execute('''CREATE TABLE _signalID_ # _0_
             (time , val)''')

# ex :
#
# module top();
# #0 clk =1;
# #5 clk =0; ...
# endmodule

# header create
c.execute('''CREATE TABLE _scopes_
      (scopeDomain, scopeType, scopeNm''')
c.execute('''CREATE TABLE _signals_
      (scopeDomain, signalID, signalType, width, name''')
c.execute('''CREATE TABLE _signalID_ # _0_
             (time , val)''')

# body create
c.execute("INSERT INTO _scopes_ VALUES (0, 1(module), "top")")
c.execute("INSERT INTO _signals_ VALUES (0, 0, 0(bit), "top.clk")")

c.execute("INSERT INTO _0_ VALUES (0, 1)")
c.execute("INSERT INTO _1_ VALUES (5, 0)")

#
conn.commit()
c.close()
"""
VcdParser.py
from __future__ import with_statement
from itertools import dropwhile, takewhile, izip
from collections import defaultdict
from pprint import pprint as pp

from os import *
import io
from src.vcdDB  import *
from src.vcdErr import *
from src.vcdContext import *


class VcdParser(object):
  """ parser VCD 2 VcdDB """

  def __init__(self,vcd=None):
    self._vfile = None

    try:
      if not isinstance(vcd,VcdDB):
       raise VcdParserError("vcdDB ptr is None")
    except VcdDBError, e:
      print e._msg

    self._vcd   = vcd


  def parse_error(self,tokeniser, keyword):
    raise "Don't understand keyword: " + keyword


  def do_drop_declaration(self,tokeniser, keyword):
    dropwhile(lambda x: x != "$end", tokeniser).next()


  def do_vcd_date(self,tokeniser, keyword):
    """ parser vcd date and set 2 VcdDB::date """

    date = ' '.join(takewhile(lambda x: x != "$end", tokeniser))
    self._vcd.set_date(date)


  def do_vcd_timescale(self,tokeniser, keyword):
    """ parser vcd timescale and set 2 VcdDB::timescale """

    val, scale = tuple(takewhile(lambda x: x!= "$end", tokeniser))
    self._vcd.set_timescale(scale,val)


  def do_vcd_version(self,tokeniser, keyword):
    """ parser vcd version and set 2 VcdDB::version """

    version = ' '.join(takewhile(lambda x: x!= "$end", tokeniser))
    self._vcd.set_version(version)


  def do_vcd_enddefinitions(self,tokeniser, keyword):
    """ parser vcd enddefinitions """

    self.do_drop_declaration(tokeniser, keyword)
    self._vcd.set_enddefinition(1)


  def do_vcd_scope(self,tokeniser, keyword):
    """ parser vcd scope and set 2 VcdDB::scope """

    scope, name = tuple(takewhile(lambda x: x != "$end", tokeniser))
    self._vcd.set_scope(self._vcd,scope,name)


  def do_vcd_upscope(self,tokeniser, keyword):
    """ parser vcd upscope """

    tokeniser.next()


  def do_vcd_var(self,tokeniser, keyword):
    """ parser vcd signalInfo and set 2 vcdDB::nameMap """

    varTyp, width, uniqNm, refNm = tuple(takewhile(lambda x: x != "$end", tokeniser))
    scope = self._vcd.get_end_scope()
    maptb = self._vcd.get_mapTb()

    scope.set_signals(scope,varTyp,refNm,width)
    signal = scope.get_signal(refNm)
    maptb.set_tb(uniqNm,signal)


  def do_vcd_dumpall(tokeniser, keyword): pass
  def do_vcd_dumpoff(tokeniser, keyword): pass
  def do_vcd_dumpon(tokeniser, keyword): pass
  def do_vcd_dumpvars(tokeniser, keyword): pass
  def do_vcd_end(tokeniser, keyword): pass


  def vcd_2_VcdDB(self,vcdfile=None):
    """ parser vcd file 2 VcdDB format """

  # define key 2 proc
    keyword2handler = {
        # declaration_keyword ::=
        "$comment":        self.do_drop_declaration,
        "$date":           self.do_vcd_date,
        "$enddefinitions": self.do_vcd_enddefinitions,
        "$scope":          self.do_vcd_scope,
        "$timescale":      self.do_vcd_timescale,
        "$upscope":        self.do_vcd_upscope,
        "$var":            self.do_vcd_var,
        "$version":        self.do_vcd_version,
        # simulation_keyword ::=
        "$dumpall":        self.do_vcd_dumpall,
        "$dumpoff":        self.do_vcd_dumpoff,
        "$dumpon":         self.do_vcd_dumpon,
        "$dumpvars":       self.do_vcd_dumpvars,
        "$end":            self.do_vcd_end,
        }

    keyword2handler = defaultdict(self.parse_error, keyword2handler)

    f = io.open(vcdfile,"r")
    tokeniser = (word for line in f for word in line.split() if word)
    time = 0
    for count,token in enumerate(tokeniser):
      if self._vcd.get_enddefinition() != 1:
        keyword2handler[token](tokeniser, token)
      else:
        c, rest = token[0], token[1:]
        if c == '$':
          # skip $dump* tokens and $end tokens in sim section
          continue
        # time info
        elif c == '#':
          time = rest
        # (1)bit info
        elif c in '01xXzZ':
          uniqNm = rest
          val    = c
          maptb  = self._vcd.get_mapTb()
     signal = maptb.get_tb(uniqNm)
          signal.set_NodeInfoList(signal,time,"bit",val)
        # binary info
        elif c in 'bB':
   uniqNm = tokeniser.next()
   val    = rest
   maptb  = self._vcd.get_mapTb()
   signal = maptb.get_tb(uniqNm)
   signal.set_NodeInfoList(signal,time,"binary",val)
        # real info
        elif c in 'rR':
   uniqNm = tokeniser.next()
   val    = rest
   maptb  = self._vcd.get_mapTb()
   signal = maptb.get_tb(uniqNm)
   signal.set_NodeInfoList(signal,time,"real",val)
        else:
          raise "Don't understand: %s After %i words" % (token, count)

    f.close()

    return self._vcd

code download 主要改寫 writing-vcd-to-toggle-count-generator 成我們要跟底層溝通的api




Refs:
http://www.semiwiki.com/forum/content/753-what-changes-expect-verification-ip-landscape-after-synopsys-acquisition-nsys.html
VMM performance user guide
http://vmmcentral.org/pdfs/perf_user_guide.pdf
http://i.cmpnet.com/eetimes/news/online/2010/09/Whitepaper_Mediatek_2010.pdf


2013年1月4日 星期五

python + ctypes

最近遇到 mixed language 的問題, 除了考慮到 code 的可讀性之外, 還有安全性, 效能方面...的問題. 考慮了 swig, ctypes, boost.python, cython...來實現. 發現用 ctypes 比較簡單, 也不需額外在安裝. 而且是 standard lib. 底下就用個簡單的 sample 來實現.


wrapper DLL lib func call dpi.cc
#ifdef __cplusplus
extern "C" {

int
DPIAdd(int a, int b) {
    return a+b;
}

}
#endif



gcc command
gcc -c -fPIC dpi.cc -o dpi.o
gcc -shared -o dpi.so dpi.o


use python ctypes to call dpi
import sys
from ctypes import *

class DPIWrapper:
    """
    wrapper class for DPI example
    """

    def __init__(self, name="dpi.so"):
 """
 name : shared library path
 """
     self._dl = None
 try:
    self._dl = cdll.LoadLibrary(name)
 except IOError:
     print "load %s fail" %(name)

    def __del__(self):
     self._dl = None

    def WrapperAdd(self, a, b):
 """
 wrapper add func from dpi.so
 """
 return self._dl.DPIAdd(a,b)


def main():
    dpi = DPIWrapper(name="dpi.so")
    print "%d" %(dpi.WrapperAdd(1,2))
    del dpi

if __name__ == "__main__":
    main()


Ref: http://www.adp-gmbh.ch/cpp/gcc/create_lib.html
http://blog.ez2learn.com/2009/03/21/python-evolution-ctypes/
http://stackoverflow.com/questions/5081875/ctypes-beginner

python + multiprocessor switch swap


底下 example 利用 multiprocessor 來實現 switch swap. 當然也可以考慮用 stackless 的方式, 把 task 加載到 scheduler manager 來減少 acquire 的次數, 達到 atomic thread 更加的 independent http://www.stackless.com/

from multiprocessing import Process
import time

class Switcher(object):
    """ Switch(swap) the data stored sequence
    -0 for store in seq0, run in seq1
    -1 for store in seq1, run in seq0
    """

    def __init__(self):
     self._seq0 = []
     self._seq1 = []
     self._mode = 0

    def switch(self):
     """ switch mode """
 self._mode = 1 if self._mode == 0 else 0

    def getStoreSeq(self):
 seq = self._seq0[:] if self._mode == 0 else self._seq1[:]
 return seq

    def getRunSeq(self):
 seq = self._seq1[:] if self._mode == 0 else self._seq0[:]
 print "run switcher.seq0 %s" %(self._seq0)
     print "run switcher.seq1 %s" %(self._seq1)
     return seq

    def setStoreSeq(self,pat=[]):
 if self._mode == 0:
     self._seq0 = pat[:]
 else:
     self._seq1 = pat[:]
 print "store switcher.seq0 %s" %(self._seq0)
 print "store switcher.seq1 %s" %(self._seq1)

    def setRunSeq(self,pat=[]):
 if self._mode == 1:
     self._seq0 = pat[:]
 else:
     self._seq1 = pat[:]

    def clear(self):
 del self._seq0[:]
 del self._seq1[:]


def test(switcher):
    """ test """
    switcher.setStoreSeq([1,2,3])
    switcher.switch()
    assert(switcher.getRunSeq()==[1,2,3])
    switcher.setStoreSeq([4,5,6])
    switcher.switch()
    switcher.clear()


def sample_run(switcher):
    """ avg switcher.getRunSeq() """
    try:
     seq = switcher.getRunSeq()
     print "avg %.2f, %s" %(sum(seq)/len(seq), seq)
    except ZeroDivisionError:
     print "Error %s" %(seq)


def patten_gen(switcher,sel=0):
    """ random patten gen """
    if sel ==0:
     switcher.setStoreSeq([i for i in range(10)])
    else:
     switcher.setStoreSeq([i for i in range(10,20,1)])


def pre_load_status(switcher):
    """ pre load status """

    print "pre_load_status"
    p0 = Process(target=patten_gen, args=(switcher,0))
    p0.start()
    p0.join()

    print switcher._seq0
    print switcher._seq1


def switch_status(switcher):
    switcher.switch()


def pipe_load_status(switcher,sel=0):
    """ pipe load status """

    print "pipe_load_status"
    p0 = Process(target=patten_gen, args=(switcher,sel))
    p1 = Process(target=sample_run, args=(switcher,))
    p0.start()
    p1.start()
    p0.join()
    p1.join()


def end_load_status(switcher):
    """ end load status """

    print "end_load_status"
    p1 = Process(target=sample_run, args=(switcher,))
    p1.start()
    p1.join()


def main():

    switcher = Switcher()
    test(switcher)


    for i in range(5):
 if i == 0:
     pre_load_status(switcher)
     switch_status(switcher)
 elif i in [1,2,3]:
     pipe_load_status(switcher, i%2)
     switch_status(switcher)
 else:
     end_load_status(switcher)
     switch_status(switcher)

if __name__ == "__main__":
    main()

2013年1月1日 星期二

jenkins + python

http://bhfsteve.blogspot.tw/2012/04/automated-python-unit-testing-code_20.html

https://wiki.jenkins-ci.org/display/JENKINS/Extend+Jenkins