summaryrefslogtreecommitdiffstats
path: root/tool/eval.rb
blob: 906ba9c23ca847c8dcd27ce7c615e50e1cc26e90 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161

require 'rbconfig'
require 'fileutils'
require 'pp'

Ruby = ENV['RUBY'] ||
  File.join(Config::CONFIG["bindir"],
            Config::CONFIG["ruby_install_name"] + Config::CONFIG["EXEEXT"])
#

OPTIONS = %w{
 opt-direct-threaded-code
 opt-basic-operations
 opt-operands-unification
 opt-instructions-unification
 opt-inline-method-cache
 opt-stack-caching
}.map{|opt|
  '--disable-' + opt
}

opts = OPTIONS.dup
Configs = OPTIONS.map{|opt|
  o = opts.dup
  opts.delete(opt)
  o
} + [[]]

pp Configs if $DEBUG


def exec_cmd(cmd)
  puts cmd
  unless system(cmd)
    p cmd
    raise "error"
  end
end

def dirname idx
  "ev-#{idx}"
end

def build
  Configs.each_with_index{|config, idx|
    dir = dirname(idx)
    FileUtils.rm_rf(dir) if FileTest.exist?(dir)
    Dir.mkdir(dir)
    FileUtils.cd(dir){
      exec_cmd("#{Ruby} ../extconf.rb " + config.join(" "))
      exec_cmd("make clean test-all")
    }
  }
end

def check
  Configs.each_with_index{|c, idx|
    puts "= #{idx}"
    system("#{Ruby} -r ev-#{idx}/yarvcore -e 'puts YARVCore::OPTS'")
  }
end

def bench_each idx
  puts "= #{idx}"
  5.times{|count|
    print count
    FileUtils.cd(dirname(idx)){
      exec_cmd("make benchmark OPT=-y ITEMS=#{ENV['ITEMS']} > ../b#{idx}-#{count}")
    }
  }
  puts
end

def bench
  # return bench_each(6)
  Configs.each_with_index{|c, idx|
    bench_each idx
  }
end

def parse_result data
  flag = false
  stat = []
  data.each{|line|
    if flag
      if /(\w+)\t([\d\.]+)/ =~ line
        stat << [$1, $2.to_f]
      else
        raise "not a data"
      end
      
    end
    if /benchmark summary/ =~ line
      flag = true
    end
  }
  stat
end

def calc_each data
  data.sort!
  data.pop   # remove max
  data.shift # remove min

  data.inject(0.0){|res, e|
    res += e
  } / data.size
end

def calc_stat stats
  stat = []
  stats[0].each_with_index{|e, idx|
    bm = e[0]
    vals = stats.map{|st|
      st[idx][1]
    }
    [bm, calc_each(vals)]
  }
end

def stat
  total = []
  Configs.each_with_index{|c, idx|
    stats = []
    5.times{|count|
      file = "b#{idx}-#{count}"
      # p file
      open(file){|f|
        stats << parse_result(f.read)
      }
    }
    # merge stats
    total << calc_stat(stats)
    total
  }
  # pp total
  total[0].each_with_index{|e, idx|
    bm = e[0]
    # print "#{bm}\t"
    total.each{|st|
      print st[idx][1], "\t"
    }
    puts
  }
end

ARGV.each{|cmd|
  case cmd
  when 'build'
    build
  when 'check'
    check
  when 'bench'
    bench
  when 'stat'
    stat
  else
    raise
  end
}