andr3/scyther
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Better reporting, better analysis.

Browse Source 
- End result now weighed according to undecided^2 * states, making the
  decidability a more important factor.
This commit is contained in:
ccremers 2006-02-24 12:50:04 +00:00
parent 1d3d154a2f
commit 8995bc4d28
1 changed files with 42 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
test/compareheuristics.py
View File

@ -18,7 +18,7 @@ def parse(scout):
out: out:
ra: number of failed claims ra: number of failed claims
rb: number of bounded proofs of claims rb: number of bounded proofs of claims
rc: number of complete proofs of claims rp: number of complete proofs of claims
nc: number of processed claims (should be the sum of the previous) nc: number of processed claims (should be the sum of the previous)
st: number of states traversed st: number of states traversed
""" """
@ -28,29 +28,36 @@ def parse(scout):
rp = 0 rp = 0
nc = 0 nc = 0
st = 0 st = 0
timeout = False to = 0
for l in scout.splitlines(): for l in scout.splitlines():
data = l.split() data = l.split()
if len(data) > 4 and data[0] == 'claim': if len(data) > 4 and data[0] == 'claim':
# determine claim status
tag = data[4] # Determine timeout, count states
if tag == 'Fail': nc += 1
ra = ra + 1 timeout = False
nc = nc + 1
elif tag == 'Ok':
nc = nc + 1
if l.rfind("proof of correctness") != -1:
rp = rp + 1
else:
rb = rb + 1
# now count the states
for d in data: for d in data:
if d.startswith("states="): if d.startswith("states="):
st = st + int(d[7:]) st = st + int(d[7:])
if d.startswith("time="): if d.startswith("time="):
timeout = True timeout = True
return (ra,rb,rp,nc,st,timeout) # Determine claim status
if not timeout:
tag = data[4]
if tag == 'Fail':
ra += 1
elif tag == 'Ok':
if l.rfind("proof of correctness") != -1:
rp += 1
else:
rb += 1
else:
print "Weird tag [%s] in line [%s]." % (tag, l)
else:
to += 1
return (ra,rb,rp,nc,st,to)
def test_goal_selector(goalselector, options,branchbound): def test_goal_selector(goalselector, options,branchbound):
@ -87,15 +94,13 @@ def test_goal_selector(goalselector, options,branchbound):
(status,scout) = scythertest.default_test([p], \ (status,scout) = scythertest.default_test([p], \
int(options.match), \ int(options.match), \
int(options.bounds)) int(options.bounds))
(ra,rb,rp,nc,st,timeout) = parse(scout) (ra,rb,rp,nc,st,to) = parse(scout)
attacks = attacks + ra claims += nc
bounds = bounds + rb states += st
proofs = proofs + rp timeouts += to
claims = claims + nc attacks += ra
states = states + st bounds += rb
if timeout: proofs += rp
timeouts += 1
if hurry and (bounds * states) > branchbound: if hurry and (bounds * states) > branchbound:
return (-1,0,0,0,0,0) return (-1,0,0,0,0,0)
@ -107,7 +112,7 @@ class maxor:
"""Class for a dynamic maximum determination and corresponding formatting """Class for a dynamic maximum determination and corresponding formatting
""" """
def __init__(self,dir=0,mymin=99999999, mymax=-99999999): def __init__(self,dir=0,mymin=9999999999, mymax=-9999999999):
"""Init """Init
in: in:
@ -161,8 +166,7 @@ def main():
(options, args) = parser.parse_args() (options, args) = parser.parse_args()
scythertest.process_default_options(options) scythertest.process_default_options(options)
print print "G-sel\tDecide\tAttack\tProof\tBound\tTOuts\tClaims\tProts\tStates\tBndTo*Sts"
"G-sel\tAttack\tBound\tProof\tClaims\tTOuts\tProts\tStates\tBnd*Sts"
print print
ramax = maxor(1) ramax = maxor(1)
@ -171,8 +175,9 @@ def main():
statesmax = maxor(2) statesmax = maxor(2)
boundstatesmax = maxor(2) boundstatesmax = maxor(2)
timeoutsmax = maxor(2) timeoutsmax = maxor(2)
decidemax = maxor(1)
for g in range(1,15): for g in range(1,16):
(ra,rb,rp,nc,np,st,timeouts) = test_goal_selector(g, options, (ra,rb,rp,nc,np,st,timeouts) = test_goal_selector(g, options,
boundstatesmax.get()) boundstatesmax.get())
@ -181,17 +186,22 @@ def main():
# Error: not well bounded # Error: not well bounded
res += "\tWent over bound, stopped investigation." res += "\tWent over bound, stopped investigation."
else: else:
boundstates = rb * st undecided = rb + timeouts
boundstates = undecided * undecided * st
def shows (res, mx, data): def shows (res, mx, data):
return res + "\t" + str(data) + mx.reg(data) return res + "\t" + str(data) + mx.reg(data)
decide = (100 * (ra + rp)) / nc
res = shows (res, decidemax, decide)
res += "%"
res = shows (res, ramax, ra) res = shows (res, ramax, ra)
res = shows (res, rbmax, rb)
res = shows (res, rpmax, rp) res = shows (res, rpmax, rp)
res = res + "\t" + str(nc) res = shows (res, rbmax, rb)
res = shows (res, timeoutsmax, timeouts) res = shows (res, timeoutsmax, timeouts)
res += "\t<%i>" % np res = res + "\t%i" % nc
res += "\t%i" % np
res = shows (res, statesmax, st) res = shows (res, statesmax, st)
res = shows (res, boundstatesmax, boundstates) res = shows (res, boundstatesmax, boundstates)