2005-11-23 Graydon Hoare <graydon@redhat.com>

	* elaborate.h (get_symbol_within_expression): Make visible.
	* elaborate.cxx (get_symbol_within_expression): Make non-static.
	(stat_decl_collector): New struct.
	(semantic_pass_stats): New semantic pass.
	(semantic_pass): Call it.
	(semantic_pass_symbols): Remove collection of statistic_decls from files.
	(visit_stat_op): Only fail if inferred type is not pe_long.

	* parse.cxx (parser::parse): Don't pass per-file statistic_decl
	into parse_global.
	(parser::parse_global): Don't parse global statistic_decls,
	they're obsolete.
	* parse.hh (parser::parse_global): Adjust signature to match.

	* session.h (statistic_decl::operator==): New method.

	* staptree.h (print_format::is_empty): New method.
	(stapfile::stat_decls): Remove field.
	* staptree.cxx (string_to_components): Fix bugs in format-string
	parser.

	* translate.cxx (var): Make private fields protected.
	(var::init): Support HIST_NONE stats.
	(aggvar): New struct.
	(mapvar::is_parallel): New method.
	(mapvar::call_prefix): Use it.
	(mapvar::calculate_aggregate): New method.
	(mapvar::fetch_existing_aggregate): New method.
	(mapvar::get): Support pe_stats.
	(mapvar::init): Use is_parallel(), and support HIST_NONE.
	(itervar::itervar): Only fault on pe_unknown.
	(itervar::start): Use mapvar::is_parallel and
	mapvar::fetch_existing_aggregate.
	(emit_map_type_instantiations): Include alloc.c before pmap-gen.c.
	Include pmap-gen.c for pe_stats maps.
	(c_unparser::gensym_aggregate): New method.
	(c_unparser::visit_foreach_loop): Handle mapvar::is_parallel case.
	(arrayindex_downcaster): New struct.
	(expression_is_arrayindex): New function.
	(c_tmpcounter::visit_stat_op): New method.
	(c_unparser::visit_stat_op): Implement.
	(c_unparser::visit_hist_op): Add commentary, still not implemented.

	* testsuite/buildok/stat_{insert,extract}.stp: New tests.
	* testsuite/semok/ten.stp: Correct for changes to global declarations.
	* testsuite/semko/*.stp: Likewise.

19 files changed, 679 insertions, 295 deletions

diff --git a/ChangeLog b/ChangeLog
index 9a21f058..9062e793 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,52 @@
+2005-11-23  Graydon Hoare  <graydon@redhat.com>
+
+	* elaborate.h (get_symbol_within_expression): Make visible.
+	* elaborate.cxx (get_symbol_within_expression): Make non-static.
+	(stat_decl_collector): New struct.
+	(semantic_pass_stats): New semantic pass.
+	(semantic_pass): Call it.
+	(semantic_pass_symbols): Remove collection of statistic_decls from files.
+	(visit_stat_op): Only fail if inferred type is not pe_long.
+
+	* parse.cxx (parser::parse): Don't pass per-file statistic_decl
+	into parse_global.	
+	(parser::parse_global): Don't parse global statistic_decls,
+	they're obsolete.  
+	* parse.hh (parser::parse_global): Adjust signature to match.
+
+	* session.h (statistic_decl::operator==): New method.
+
+	* staptree.h (print_format::is_empty): New method.
+	(stapfile::stat_decls): Remove field.
+	* staptree.cxx (string_to_components): Fix bugs in format-string
+	parser.
+
+	* translate.cxx (var): Make private fields protected.
+	(var::init): Support HIST_NONE stats.
+	(aggvar): New struct.
+	(mapvar::is_parallel): New method.
+	(mapvar::call_prefix): Use it.
+	(mapvar::calculate_aggregate): New method.
+	(mapvar::fetch_existing_aggregate): New method.
+	(mapvar::get): Support pe_stats.
+	(mapvar::init): Use is_parallel(), and support HIST_NONE.
+	(itervar::itervar): Only fault on pe_unknown.
+	(itervar::start): Use mapvar::is_parallel and
+	mapvar::fetch_existing_aggregate.
+	(emit_map_type_instantiations): Include alloc.c before pmap-gen.c.
+	Include pmap-gen.c for pe_stats maps.
+	(c_unparser::gensym_aggregate): New method.
+	(c_unparser::visit_foreach_loop): Handle mapvar::is_parallel case.
+	(arrayindex_downcaster): New struct.
+	(expression_is_arrayindex): New function.
+	(c_tmpcounter::visit_stat_op): New method.
+	(c_unparser::visit_stat_op): Implement.
+	(c_unparser::visit_hist_op): Add commentary, still not implemented.
+
+	* testsuite/buildok/stat_{insert,extract}.stp: New tests.
+	* testsuite/semok/ten.stp: Correct for changes to global declarations.
+	* testsuite/semko/*.stp: Likewise.
+
 2005-11-21  Roland McGrath  <roland@redhat.com>
 
 	* loc2c.c (c_translate_location): Take Dwarf_Op vector as argument
diff --git a/elaborate.cxx b/elaborate.cxx
index 280fd92f..de469a4f 100644
--- a/elaborate.cxx
+++ b/elaborate.cxx
@@ -456,7 +456,7 @@ derive_probes (systemtap_session& s,
 //
 
 struct symbol_fetcher
-  : virtual public throwing_visitor
+  : public throwing_visitor
 {
   symbol *&sym;
 
@@ -480,7 +480,7 @@ struct symbol_fetcher
   }
 };
 
-static symbol *
+symbol *
 get_symbol_within_expression (expression *e)
 {
   symbol *sym = NULL;
@@ -504,7 +504,7 @@ get_symbol_within_indexable (indexable *ix)
 }
 
 struct mutated_var_collector
-  : virtual public traversing_visitor
+  : public traversing_visitor
 {
   set<vardecl *> * mutated_vars;
 
@@ -539,7 +539,7 @@ struct mutated_var_collector
 
 
 struct no_var_mutation_during_iteration_check
-  : virtual public traversing_visitor
+  : public traversing_visitor
 {
   systemtap_session & session;
   map<functiondecl *,set<vardecl *> *> & function_mutates_vars;
@@ -648,10 +648,113 @@ semantic_pass_vars (systemtap_session & sess)
 
 // ------------------------------------------------------------------------
 
+struct stat_decl_collector
+  : public traversing_visitor
+{
+  systemtap_session & session;
+  
+  stat_decl_collector(systemtap_session & sess)
+    : session(sess)
+  {}
+
+  void visit_stat_op (stat_op* e)
+  {
+    symbol *sym = get_symbol_within_expression (e->stat);
+    if (session.stat_decls.find(sym->name) == session.stat_decls.end())
+      session.stat_decls[sym->name] = statistic_decl();
+  }
+
+  void visit_assignment (assignment* e)
+  {
+    if (e->op == "<<<")
+      {
+	symbol *sym = get_symbol_within_expression (e->left);
+	if (session.stat_decls.find(sym->name) == session.stat_decls.end())
+	  session.stat_decls[sym->name] = statistic_decl();
+      }
+    else
+      traversing_visitor::visit_assignment(e);
+  }
+
+  void visit_hist_op (hist_op* e)
+  {
+    symbol *sym = get_symbol_within_expression (e->stat);
+    statistic_decl new_stat;
+
+    if (e->htype == hist_linear)
+      {
+	new_stat.type = statistic_decl::linear;
+	assert (e->params.size() == 3);
+	new_stat.linear_low = e->params[0];
+	new_stat.linear_high = e->params[1];
+	new_stat.linear_step = e->params[2];
+      }
+    else
+      {
+	assert (e->htype == hist_log);
+	new_stat.type = statistic_decl::logarithmic;
+	assert (e->params.size() == 1);
+	new_stat.logarithmic_buckets = e->params[0];
+      }
+
+    map<string, statistic_decl>::iterator i = session.stat_decls.find(sym->name);
+    if (i == session.stat_decls.end())
+      session.stat_decls[sym->name] = new_stat;
+    else
+      {
+	statistic_decl & old_stat = i->second;
+	if (!(old_stat == new_stat))
+	  {
+	    if (old_stat.type == statistic_decl::none)
+	      i->second = new_stat;
+	    else
+	      {
+		// FIXME: Support multiple co-declared histogram types
+		semantic_error se("multiple histogram types declared on '" + sym->name + "'", 
+				  e->tok);
+		session.print_error (se);
+	      }
+	  }
+      }    
+  }
+
+};
+
+static int
+semantic_pass_stats (systemtap_session & sess)
+{
+  stat_decl_collector sdc(sess);
+
+  for (unsigned i = 0; i < sess.functions.size(); ++i)    
+    sess.functions[i]->body->visit (&sdc);
+
+  for (unsigned i = 0; i < sess.probes.size(); ++i)    
+    sess.probes[i]->body->visit (&sdc);
+
+  for (unsigned i = 0; i < sess.globals.size(); ++i)
+    {
+      vardecl *v = sess.globals[i];
+      if (v->type == pe_stats)
+	{
+	  
+	  if (sess.stat_decls.find(v->name) == sess.stat_decls.end())
+	    {
+	      semantic_error se("unable to infer statistic parameters for global '" + v->name + "'");
+	      sess.print_error (se);
+	    }
+	}
+    }
+  
+  return sess.num_errors;
+}
+
+// ------------------------------------------------------------------------
+
 
 static int semantic_pass_symbols (systemtap_session&);
 static int semantic_pass_types (systemtap_session&);
 static int semantic_pass_vars (systemtap_session&);
+static int semantic_pass_stats (systemtap_session&);
 
 
 
@@ -684,23 +787,6 @@ semantic_pass_symbols (systemtap_session& s)
       for (unsigned i=0; i<dome->embeds.size(); i++)
         s.embeds.push_back (dome->embeds[i]);
 
-      for (std::map<std::string, statistic_decl>::const_iterator i = 
-	     dome->stat_decls.begin(); i != dome->stat_decls.end(); ++i)
-	{
-	  try 
-	    {
-	      
-	      if (s.stat_decls.find(i->first) != s.stat_decls.end())
-		throw semantic_error("multiple statistic declarations for " + i->first);
-	      s.stat_decls.insert(std::make_pair(i->first,
-						 i->second));
-	    }
-          catch (const semantic_error& e)
-            {
-              s.print_error (e);
-            }	  
-	}
-
       // Pass 2: process functions
 
       for (unsigned i=0; i<dome->functions.size(); i++)
@@ -768,6 +854,7 @@ semantic_pass (systemtap_session& s)
       rc = semantic_pass_symbols (s);
       if (rc == 0) rc = semantic_pass_types (s);
       if (rc == 0) rc = semantic_pass_vars (s);
+      if (rc == 0) rc = semantic_pass_stats (s);
     }
   catch (const semantic_error& e)
     {
@@ -1941,7 +2028,7 @@ typeresolution_info::visit_stat_op (stat_op* e)
       e->type = pe_long;
       resolved (e->tok, e->type);
     }
-  else
+  else if (e->type != pe_long)
     mismatch (e->tok, e->type, pe_long);
 }
 
diff --git a/elaborate.h b/elaborate.h
index 0d2da63b..f2a4bd56 100644
--- a/elaborate.h
+++ b/elaborate.h
@@ -201,5 +201,9 @@ void derive_probes (systemtap_session& s,
                     probe *p, std::vector<derived_probe*>& dps,
                     bool exc_outermost = true);
 
+// A helper we use here and in translate, for pulling symbols out of lvalue
+// expressions.
+symbol * get_symbol_within_expression (expression *e);
+
 
 #endif // ELABORATE_H
diff --git a/parse.cxx b/parse.cxx
index b7de56e2..2ceb6f5f 100644
--- a/parse.cxx
+++ b/parse.cxx
@@ -672,7 +672,7 @@ parser::parse ()
 	  if (t->type == tok_identifier && t->content == "probe")
             parse_probe (f->probes, f->aliases);
 	  else if (t->type == tok_identifier && t->content == "global")
-	    parse_global (f->globals, f->stat_decls);
+	    parse_global (f->globals);
 	  else if (t->type == tok_identifier && t->content == "function")
             parse_functiondecl (f->functions);
           else if (t->type == tok_embedded)
@@ -886,8 +886,7 @@ parser::parse_statement ()
 
 
 void
-parser::parse_global (vector <vardecl*>& globals,
-		      std::map<std::string, statistic_decl> &stat_decls)
+parser::parse_global (vector <vardecl*>& globals)
 {
   const token* t0 = next ();
   if (! (t0->type == tok_identifier && t0->content == "global"))
@@ -899,33 +898,6 @@ parser::parse_global (vector <vardecl*>& globals,
       if (! (t->type == tok_identifier))
         throw parse_error ("expected identifier");
 
-      statistic_decl sd;
-
-      if (t->content == "log_hist")
-	{
-	  std::string tmp;
-	  expect_op ("(");
-	  t = expect_ident (tmp);
-	  expect_op (",");
-	  expect_number (sd.logarithmic_buckets);
-	  expect_op (")");
-	  sd.type = statistic_decl::logarithmic;
-	}
-      else if (t->content == "linear_hist")
-	{
-	  std::string tmp;
-	  expect_op ("(");
-	  t = expect_ident (tmp);
-	  expect_op (",");
-	  expect_number (sd.linear_low);
-	  expect_op (",");
-	  expect_number (sd.linear_high);
-	  expect_op (",");
-	  expect_number (sd.linear_step);
-	  expect_op (")");
-	  sd.type = statistic_decl::linear;
-	}
-
       for (unsigned i=0; i<globals.size(); i++)
 	if (globals[i]->name == t->content)
 	  throw parse_error ("duplicate global name");
@@ -935,12 +907,6 @@ parser::parse_global (vector <vardecl*>& globals,
       d->tok = t;
       globals.push_back (d);
 
-      if (sd.type != statistic_decl::none)
-	{
-	  d->type = pe_stats;
-	  stat_decls[d->name] = sd;
-	}
-
       t = peek ();
       if (t && t->type == tok_operator && t->content == ",")
 	{
diff --git a/parse.h b/parse.h
index 5ebdcef4..81544eb9 100644
--- a/parse.h
+++ b/parse.h
@@ -118,8 +118,7 @@ private:
 
 private: // nonterminals
   void parse_probe (std::vector<probe*>&, std::vector<probe_alias*>&);
-  void parse_global (std::vector<vardecl*>&,
-		     std::map<std::string, statistic_decl> &stat_decls);
+  void parse_global (std::vector<vardecl*>&);
   void parse_functiondecl (std::vector<functiondecl*>&);
   embeddedcode* parse_embeddedcode ();
   probe_point* parse_probe_point ();
diff --git a/session.h b/session.h
index 666d8ce2..6e507dd7 100644
--- a/session.h
+++ b/session.h
@@ -42,6 +42,14 @@ struct statistic_decl
   int64_t linear_low;
   int64_t linear_high;
   int64_t linear_step;
+  bool operator==(statistic_decl const & other)
+  {
+    return type == other.type 
+      && logarithmic_buckets == other.logarithmic_buckets
+      && linear_low == other.linear_low
+      && linear_high == other.linear_high
+      && linear_step == other.linear_step;
+  }
 };
 
 
diff --git a/staptree.cxx b/staptree.cxx
index 6ea1fc70..6f75e09b 100644
--- a/staptree.cxx
+++ b/staptree.cxx
@@ -404,206 +404,158 @@ print_format::string_to_components(string const & str)
   format_component curr;
   vector<format_component> res;
 
-  enum 
-    {
-      parsing_plain_data,
-      parsing_flags,
-      parsing_width,
-      parsing_precision,
-      parsing_conversion_specifier
-    } 
-  state = parsing_plain_data;
-
   curr.clear();
 
   string::const_iterator i = str.begin();
-
+  
   while (i != str.end())
     {
-      switch (state)
+      if (*i != '%')
 	{
-	case parsing_plain_data:
-
-	  if (*i != '%')
-	    {
-	      assert (curr.type == conv_unspecified || curr.type == conv_literal);
-	      curr.type = conv_literal;
-	      curr.literal_string += *i;	      
-	    }
-	  else if (i+1 == str.end() || *(i+1) == '%')
-	    {
-	      assert(*i == '%');
-	      // *i == '%' and *(i+1) == '%'; append only one '%' to the literal string
-	      assert (curr.type == conv_unspecified || curr.type == conv_literal);
-	      curr.type = conv_literal;
-	      curr.literal_string += '%';
-	    }
-	  else 
-	    {
-	      assert(*i == '%');
-	      state = parsing_flags;
-	      if (curr.type != conv_unspecified)
-		{
-		  assert (curr.type == conv_literal);
-		  res.push_back(curr);
-		  curr.clear();
-		}
-	    }
+	  assert (curr.type == conv_unspecified || curr.type == conv_literal);
+	  curr.type = conv_literal;
+	  curr.literal_string += *i;	      
 	  ++i;
-	  break;
-
-	case parsing_flags:
-	  switch (*i)
+	  continue;
+	}
+      else if (i+1 == str.end() || *(i+1) == '%')
+	{
+	  assert(*i == '%');
+	  // *i == '%' and *(i+1) == '%'; append only one '%' to the literal string
+	  assert (curr.type == conv_unspecified || curr.type == conv_literal);
+	  curr.type = conv_literal;
+	  curr.literal_string += '%';
+	  ++i;
+	  continue;
+	}
+      else 
+	{
+	  assert(*i == '%');
+	  if (curr.type != conv_unspecified)
 	    {
-	    case '0':
-	      curr.flags |= static_cast<unsigned long>(fmt_flag_zeropad);
-	      ++i;
-	      break;
-
-	    case '+':
-	      curr.flags |= static_cast<unsigned long>(fmt_flag_plus);
-	      ++i;
-	      break;
-
-	    case '-':
-	      curr.flags |= static_cast<unsigned long>(fmt_flag_left);
-	      ++i;
-	      break;
-
-	    case ' ':
-	      curr.flags |= static_cast<unsigned long>(fmt_flag_space);
-	      ++i;
-	      break;
+	      // Flush any component we were previously accumulating
+	      assert (curr.type == conv_literal);
+	      res.push_back(curr);
+	      curr.clear();
+	    }
+	}
+      ++i;
+      
+      if (i == str.end())
+	break;
 
-	    case '#':
-	      curr.flags |= static_cast<unsigned long>(fmt_flag_special);
-	      ++i;
-	      break;
+      // Now we are definitely parsing a conversion. 
+      // Begin by parsing flags (whicih are optional).
 
-	    default:
-	      state = parsing_width;
-	      break;
-	    }
+      switch (*i)
+	{
+	case '0':
+	  curr.flags |= static_cast<unsigned long>(fmt_flag_zeropad);
+	  ++i;
 	  break;
-
-	case parsing_width:
-	  while (isdigit(*i))
-	    {
-	      curr.width *= 10;
-	      curr.width += (*i - '0');
-	      ++i;
-	    }
-	  state = parsing_precision;
+	  
+	case '+':
+	  curr.flags |= static_cast<unsigned long>(fmt_flag_plus);
+	  ++i;
 	  break;
 	  
-	case parsing_precision:
-	  if (*i == '.')
-	    {
-	      ++i;
-	      while (isdigit(*i))
-		{
-		  curr.precision *= 10;
-		  curr.precision += (*i - '0');
-		  ++i;
-		}
-	    }
-	  state = parsing_conversion_specifier;
+	case '-':
+	  curr.flags |= static_cast<unsigned long>(fmt_flag_left);
+	  ++i;
 	  break;
+	  
+	case ' ':
+	  curr.flags |= static_cast<unsigned long>(fmt_flag_space);
+	  ++i;
+	  break;
+	  
+	case '#':
+	  curr.flags |= static_cast<unsigned long>(fmt_flag_special);
+	  ++i;
+	  break;
+	  
+	default:
+	  break;
+	}
 
-	case parsing_conversion_specifier:
-	  switch (*i)
-	    {
-
-	    default:
-	      if (curr.type == conv_unspecified)
-		throw semantic_error("no conversion specifier provided");
-	      
-	      res.push_back(curr);
-	      curr.clear();
-	      state = parsing_plain_data;
-	      break;
-	      
-	      // Valid conversion types
-	    case 's':
-	      if (curr.type != conv_unspecified)
-		throw semantic_error("multiple conversion types supplied");
-	      curr.type = conv_string;
-	      ++i;
-	      break;
-
-	    case 'd':
-	    case 'i':
-	      if (curr.type != conv_unspecified)
-		throw semantic_error("multiple conversion types supplied");
-	      curr.type = conv_signed_decimal;
-	      ++i;
-	      break;
-
-	    case 'o':
-	      if (curr.type != conv_unspecified)
-		throw semantic_error("multiple conversion types supplied");
-	      curr.type = conv_unsigned_octal;
-	      ++i;
-	      break;
+      // Parse optional width
+	  
+      while (i != str.end() && isdigit(*i))
+	{
+	  curr.width *= 10;
+	  curr.width += (*i - '0');
+	  ++i;
+	}
 
-	    case 'u':
-	      if (curr.type != conv_unspecified)
-		throw semantic_error("multiple conversion types supplied");
-	      curr.type = conv_unsigned_decimal;
-	      ++i;
-	      break;
+      if (i == str.end())
+	break;
 
-	    case 'X':
-	      if (curr.type != conv_unspecified)
-		throw semantic_error("multiple conversion types supplied");
-	      curr.type = conv_unsigned_uppercase_hex;
+      // Parse optional precision
+      if (*i == '.')
+	{
+	  ++i;
+	  if (i == str.end())
+	    break;
+	  while (i != str.end() && isdigit(*i))
+	    {
+	      curr.precision *= 10;
+	      curr.precision += (*i - '0');
 	      ++i;
+	    }
+	}
 
-	    case 'x':
-	      if (curr.type != conv_unspecified)
-		throw semantic_error("multiple conversion types supplied");
-	      curr.type = conv_unsigned_lowercase_hex;
-	      ++i;
-	      break;
+      if (i == str.end())
+	break;
 
-	      // We prohibit users passing any funny stuff through which might
-	      // make linux's printf function do naughty things.
-	    case 'p':
-	    case 'n':
-	    case 'c':
-	    case 'q':
-	    case 'j':
-	    case 't':
-
-	    case ',':
-	    case '.':
-	    case '*':
-
-	    case 'e':
-	    case 'E':
-	    case 'f':
-	    case 'F':
-	    case 'g':
-	    case 'G':
-	    case 'h':
-	    case 'H':
-	    case 'I':
-	    case 'l':
-	    case 'L':
-	    case 'z':
-	    case 'Z':
-	      string err("prohibited conversion character '");
-	      err += *i;
-	      err += '"';
-	      throw parse_error(err);
-	    }
+      // Parse the actual conversion specifier (sdiouxX)
+      switch (*i)
+	{
+	  // Valid conversion types
+	case 's':
+	  curr.type = conv_string;
+	  break;
+	  
+	case 'd':
+	case 'i':
+	  curr.type = conv_signed_decimal;
+	  break;
+	  
+	case 'o':
+	  curr.type = conv_unsigned_octal;
+	  break;
+	  
+	case 'u':
+	  curr.type = conv_unsigned_decimal;
+	  break;
+	  
+	case 'X':
+	  curr.type = conv_unsigned_uppercase_hex;
+	  break;
+	  
+	case 'x':
+	  curr.type = conv_unsigned_lowercase_hex;
+	  break;
+	  
+	default:
 	  break;
 	}
+      
+      if (curr.type == conv_unspecified)
+	throw semantic_error("invalid or missing conversion specifier");
+      
+      ++i;
+      res.push_back(curr);      
+      curr.clear();      
     }
 
-  // Flush final component
-  if (curr.type != conv_unspecified)
-    res.push_back(curr);
+  // If there's a remaining partly-composed conversion, fail.
+  if (!curr.is_empty())
+    {
+      if (curr.type == conv_literal)
+	res.push_back(curr);      
+      else
+	throw semantic_error("trailing incomplete print format conversion");
+    }
 
   return res;
 }
diff --git a/staptree.h b/staptree.h
index b509ad7a..5c98507a 100644
--- a/staptree.h
+++ b/staptree.h
@@ -288,6 +288,14 @@ struct print_format: public expression
     unsigned precision;
     conversion_type type;
     std::string literal_string;
+    bool is_empty() const
+    {
+      return flags == 0 
+	&& width == 0
+	&& precision == 0
+	&& type == conv_unspecified
+	&& literal_string.empty();
+    }
     void clear()
     {
       flags = 0;
@@ -522,7 +530,6 @@ struct stapfile
   std::vector<functiondecl*> functions;
   std::vector<vardecl*> globals;
   std::vector<embeddedcode*> embeds;
-  std::map<std::string, statistic_decl> stat_decls;
   bool privileged;
   stapfile (): privileged (false) {}
   void print (std::ostream& o) const;
diff --git a/testsuite/buildok/stat_extract.stp b/testsuite/buildok/stat_extract.stp
new file mode 100755
index 00000000..82b37545
--- /dev/null
+++ b/testsuite/buildok/stat_extract.stp
@@ -0,0 +1,27 @@
+#! stap -p4
+
+# test the translatability of the statistic extraction operators
+
+global foo
+global bar
+
+global i
+
+probe timer.ms(100)
+{
+	foo <<< i++
+	bar[10,"hello"] <<< (12 * i)
+	if (@count(foo) > 1000)
+		exit()
+}
+
+probe begin
+{
+	i = 1
+}
+
+probe end
+{
+	printf("foo: min %d, max %d, count %d, sum %d, avg %d\n",
+		@min(foo), @max(foo), @count(foo), @sum(foo), @avg(foo)) 
+}
diff --git a/testsuite/buildok/stat_insert.stp b/testsuite/buildok/stat_insert.stp
new file mode 100755
index 00000000..4039a190
--- /dev/null
+++ b/testsuite/buildok/stat_insert.stp
@@ -0,0 +1,62 @@
+#! stap -p4
+
+# test the translatability of the statistic insertion operator
+
+global loggy
+global logmap
+global liney
+global linemap
+
+global numbers
+global strings
+
+function wibble()
+{
+	i = 0
+	logmap[i++, "stewed"] <<< 1
+	logmap[i++, "boiled"] <<< 1 + 2
+	logmap[i++, "baked"] <<< x
+	logmap[i++, "fried"] <<< (x * y) + 3
+}
+
+function wobble()
+{
+	foreach (i in numbers)
+	{
+		foreach (j in strings)
+		{
+			linemap[strings[j],2*numbers[i],numbers[2*i]] <<< x * (y + i)
+		}
+	}
+}
+
+
+probe end {
+
+	numbers[1] = 2
+	numbers[2] = 5
+	numbers[3] = 98279
+	numbers[4] = 8739287
+
+	strings[1] = "sun"
+	strings[2] = "moon"
+	strings[3] = "saturn"
+	strings[4] = "venus"
+
+	p = 4
+	q = 5
+
+	liney <<< 1
+	liney <<< 1 + 2
+	liney <<< p
+	liney <<< (p * q) + 3
+
+	loggy <<< 1
+	loggy <<< 1 + 2
+	loggy <<< p
+	loggy <<< (p * q) + 3
+
+	wibble()
+	wobble()
+
+}
diff --git a/testsuite/semko/thirty.stp b/testsuite/semko/thirty.stp
index b8f0ab4b..b145f826 100755
--- a/testsuite/semko/thirty.stp
+++ b/testsuite/semko/thirty.stp
@@ -2,10 +2,11 @@
 
 # need one of these for each prohibited statistic operation
 
-global log_hist(x, 10)
+global x
 
 probe end
 {
+	x <<< 1
 	x = 10
 }
 
diff --git a/testsuite/semko/thirtyone.stp b/testsuite/semko/thirtyone.stp
index 4b6e98d5..0462d4d5 100755
--- a/testsuite/semko/thirtyone.stp
+++ b/testsuite/semko/thirtyone.stp
@@ -2,10 +2,11 @@
 
 # need one of these for each prohibited statistic operation
 
-global log_hist(x, 10)
+global x
 
 probe end
 {
+	x[10] <<< 1
 	x[10] = 10
 }
 
diff --git a/testsuite/semko/twentyeight.stp b/testsuite/semko/twentyeight.stp
index 68590f86..ea214596 100755
--- a/testsuite/semko/twentyeight.stp
+++ b/testsuite/semko/twentyeight.stp
@@ -2,7 +2,7 @@
 
 # need one of these for each prohibited statistic operation
 
-global log_hist(x, 10)
+global x
 
 probe end
 {
diff --git a/testsuite/semko/twentyfive.stp b/testsuite/semko/twentyfive.stp
index 7ac7e099..8758bb67 100755
--- a/testsuite/semko/twentyfive.stp
+++ b/testsuite/semko/twentyfive.stp
@@ -2,7 +2,7 @@
 
 # need one of these for each prohibited statistic operation
 
-global log_hist(x, 10), log_hist(y, 10)
+global x, y
 
 
 probe end
diff --git a/testsuite/semko/twentyfour.stp b/testsuite/semko/twentyfour.stp
index 81a69902..ae75bf31 100755
--- a/testsuite/semko/twentyfour.stp
+++ b/testsuite/semko/twentyfour.stp
@@ -2,7 +2,7 @@
 
 # need one of these for each prohibited statistic operation
 
-global log_hist(x, 10), log_hist(y, 10)
+global x, y
 
 
 probe end
diff --git a/testsuite/semko/twentynine.stp b/testsuite/semko/twentynine.stp
index 8cc2e2d5..b26e9872 100755
--- a/testsuite/semko/twentynine.stp
+++ b/testsuite/semko/twentynine.stp
@@ -2,7 +2,7 @@
 
 # need one of these for each prohibited statistic operation
 
-global log_hist(x, 10)
+global x
 
 probe end
 {
diff --git a/testsuite/semko/twentyseven.stp b/testsuite/semko/twentyseven.stp
index a678bcd2..6c5bd4f9 100755
--- a/testsuite/semko/twentyseven.stp
+++ b/testsuite/semko/twentyseven.stp
@@ -2,7 +2,7 @@
 
 # need one of these for each prohibited statistic operation
 
-global log_hist(x, 10)
+global x
 
 function foo(bar)
 {
diff --git a/testsuite/semko/twentysix.stp b/testsuite/semko/twentysix.stp
index 34d6d1d3..9ebffa4a 100755
--- a/testsuite/semko/twentysix.stp
+++ b/testsuite/semko/twentysix.stp
@@ -2,11 +2,12 @@
 
 # need one of these for each prohibited statistic operation
 
-global log_hist(x, 10)
+global x
 
 
 probe end
 {
+	x[10] <<< 1
 	x[10] = 10
 }
 
diff --git a/translate.cxx b/translate.cxx
index ef401c76..31899120 100644
--- a/translate.cxx
+++ b/translate.cxx
@@ -54,6 +54,7 @@ lex_cast_qstring(IN const & in)
 
 struct var;
 struct tmpvar;
+struct aggvar;
 struct mapvar;
 struct itervar;
 
@@ -106,6 +107,8 @@ struct c_unparser: public unparser, public visitor
   bool is_local (vardecl const* r, token const* tok);
 
   tmpvar gensym(exp_type ty);
+  aggvar gensym_aggregate();
+
   var getvar(vardecl* v, token const* tok = NULL);
   itervar getiter(symbol* s);
   mapvar getmap(vardecl* v, token const* tok = NULL);
@@ -184,6 +187,7 @@ struct c_tmpcounter:
   void visit_arrayindex (arrayindex* e);
   void visit_functioncall (functioncall* e);
   void visit_print_format (print_format* e);
+  void visit_stat_op (stat_op* e);
 };
 
 struct c_unparser_assignment: 
@@ -233,8 +237,43 @@ struct c_tmpcounter_assignment:
 
 ostream & operator<<(ostream & o, var const & v);
 
+
+/*
+  Some clarification on the runtime structures involved in statistics:
+  
+  The basic type for collecting statistics in the runtime is struct
+  stat_data. This contains the count, min, max, sum, and possibly
+  histogram fields.
+  
+  There are two places struct stat_data shows up.
+  
+  1. If you declare a statistic variable of any sort, you want to make
+  a struct _Stat. A struct _Stat* is also called a Stat. Struct _Stat
+  contains a per-CPU array of struct stat_data values, as well as a
+  struct stat_data which it aggregates into. Writes into a Struct
+  _Stat go into the per-CPU struct stat. Reads involve write-locking
+  the struct _Stat, aggregating into its aggregate struct stat_data,
+  unlocking, read-locking the struct _Stat, then reading values out of
+  the aggregate and unlocking.
+
+  2. If you declare a statistic-valued map, you want to make a
+  pmap. This is a per-CPU array of maps, each of which holds struct
+  stat_data values, as well as an aggregate *map*. Writes into a pmap
+  go into the per-CPU map. Reads involve write-locking the pmap,
+  aggregating into its aggregate map, unlocking, read-locking the
+  pmap, then reading values out of its aggregate (which is a normal
+  map) and unlocking.
+
+  Because, at the moment, the runtime does not support the concept of
+  a statistic which collects multiple histogram types, we may need to
+  instantiate one pmap or struct _Stat for each histogram variation
+  the user wants to track.  
+ */
+
 class var
 {
+
+protected:
   bool local;
   exp_type ty;
   statistic_decl sd;
@@ -285,7 +324,8 @@ public:
 	switch (sd.type)
 	  {
 	  case statistic_decl::none:
-	    assert(false);
+	    return (qname() 
+		    + " = _stp_stat_init (HIST_NONE);");
 	    break;
 	    
 	  case statistic_decl::linear:
@@ -406,6 +446,26 @@ struct tmpvar
   {}
 };
 
+struct aggvar
+  : public var
+{
+  aggvar(unsigned & counter) 
+    : var(true, pe_stats, ("__tmp" + stringify(counter++)))
+  {}
+
+  string init() const
+  {
+    assert (type() == pe_stats);
+    return qname() + " = NULL;";
+  }
+
+  void declare(c_unparser &c) const
+  {
+    assert (type() == pe_stats);
+    c.o->newline() << "struct stat_data *" << name << ";";
+  }
+};
+
 struct mapvar
   : public var
 {
@@ -450,7 +510,8 @@ struct mapvar
 
   string call_prefix (string const & fname, vector<tmpvar> const & indices) const
   {
-    string result = "_stp_map_" + fname + "_" + keysym() + " (" + qname();
+    string mtype = is_parallel() ? "pmap" : "map";
+    string result = "_stp_" + mtype + "_" + fname + "_" + keysym() + " (" + qname();
     for (unsigned i = 0; i < indices.size(); ++i)
       {
 	if (indices[i].type() != index_types[i])
@@ -462,6 +523,26 @@ struct mapvar
     return result;
   }
 
+  bool is_parallel() const
+  {
+    return type() == pe_stats;
+  }
+
+  string calculate_aggregate() const
+  {
+    if (!is_parallel())
+      throw semantic_error("aggregating non-parallel map type");
+    
+    return "_stp_pmap_agg (" + qname() + ")";
+  }
+
+  string fetch_existing_aggregate() const
+  {
+    if (!is_parallel())
+      throw semantic_error("fetching aggregate of non-parallel map type");
+    
+    return "_stp_pmap_get_agg(" + qname() + ")";
+  }
 
   string del (vector<tmpvar> const & indices) const
   { 
@@ -485,7 +566,7 @@ struct mapvar
         // impedance matching: NULL -> empty strings
       return ("({ char *v = " + call_prefix("get", indices) + ");"
 	      + "if (!v) v = \"\"; v; })");
-    else if (type() == pe_long)
+    else if (type() == pe_long || type() == pe_stats)
       return call_prefix("get", indices) + ")";
     else
       throw semantic_error("getting a value from an unsupported map type");
@@ -514,14 +595,16 @@ struct mapvar
 		
   string init () const
   {
-    string prefix = qname() + " = _stp_map_new_" + keysym() + " (MAXMAPENTRIES" ;
+    string mtype = is_parallel() ? "pmap" : "map";
+    string prefix = qname() + " = _stp_" + mtype + "_new_" + keysym() + " (MAXMAPENTRIES" ;
 
     if (type() == pe_stats)
       {
 	switch (sdecl().type)
 	  {
 	  case statistic_decl::none:
-	    assert(false);
+	    return (prefix 
+		    + ", HIST_NONE);");
 	    break;
 
 	  case statistic_decl::linear:
@@ -551,10 +634,10 @@ struct mapvar
   string fini () const
   {
     return "_stp_map_del (" + qname() + ");";
-  }
-  
+  }  
 };
 
+
 class itervar
 {
   exp_type referent_ty;
@@ -566,21 +649,26 @@ public:
     : referent_ty(e->referent->type), 
       name("__tmp" + stringify(counter++))
   {
-    if (referent_ty != pe_long && referent_ty != pe_string)
-      throw semantic_error("iterating over illegal reference type", e->tok);
+    if (referent_ty == pe_unknown)
+      throw semantic_error("iterating over unknown reference type", e->tok);
   }
   
   string declare () const
   {
     return "struct map_node *" + name + ";";
   }
-
+  
   string start (mapvar const & mv) const
   {
+    string res;
+
     if (mv.type() != referent_ty)
       throw semantic_error("inconsistent iterator type in itervar::start()");
-
-    return "_stp_map_start (" + mv.qname() + ")";
+    
+    if (mv.is_parallel())
+      return "_stp_map_start (" + mv.fetch_existing_aggregate() + ")";
+    else
+      return "_stp_map_start (" + mv.qname() + ")";
   }
 
   string next (mapvar const & mv) const
@@ -1125,6 +1213,9 @@ c_unparser::emit_map_type_instantiations ()
   for (unsigned i = 0; i < session->functions.size(); ++i)
     collect_map_index_types(session->functions[i]->locals, types);
 
+  if (!types.empty())
+    o->newline() << "#include \"alloc.c\"";
+
   for (set< pair<vector<exp_type>, exp_type> >::const_iterator i = types.begin();
        i != types.end(); ++i)
     {
@@ -1134,7 +1225,10 @@ c_unparser::emit_map_type_instantiations ()
 	  string ktype = mapvar::key_typename(i->first.at(j));
 	  o->newline() << "#define KEY" << (j+1) << "_TYPE " << ktype;
 	}
-      o->newline() << "#include \"map-gen.c\"";
+      if (i->second == pe_stats)
+	o->newline() << "#include \"pmap-gen.c\"";
+      else
+	o->newline() << "#include \"map-gen.c\"";
       o->newline() << "#undef VALUE_TYPE";
       for (unsigned j = 0; j < i->first.size(); ++j)
 	{
@@ -1144,6 +1238,7 @@ c_unparser::emit_map_type_instantiations ()
 
   if (!types.empty())
     o->newline() << "#include \"map.c\"";
+
 };
 
 
@@ -1432,6 +1527,12 @@ c_unparser::gensym(exp_type ty)
   return tmpvar (ty, tmpvar_counter); 
 }
 
+aggvar 
+c_unparser::gensym_aggregate() 
+{ 
+  return aggvar (tmpvar_counter); 
+}
+
 
 var 
 c_unparser::getvar(vardecl *v, token const *tok) 
@@ -1669,14 +1770,28 @@ c_unparser::visit_foreach_loop (foreach_loop *s)
       // NB: structure parallels for_loop
       
       // initialization
-      
-      // sort array if desired
-      if (s->sort_direction)
+
+      // aggregate array if required
+      if (mv.is_parallel())
 	{
-	  varlock_w sort_guard (*this, mv);
-	  o->newline() << "_stp_map_sort (" << mv.qname() << ", "
-		       << s->sort_column << ", " << - s->sort_direction << ");";
+	  varlock_w agg_and_maybe_sort_guard(*this, mv);
+	  o->newline() << mv.calculate_aggregate() << ";";
+	  // sort array if desired
+	  if (s->sort_direction)
+	    o->newline() << "_stp_map_sort (" << mv.fetch_existing_aggregate() << ", "
+			 << s->sort_column << ", " << - s->sort_direction << ");";
+	}
+      else
+	{      
+	  // sort array if desired
+	  if (s->sort_direction)
+	    {
+	      varlock_w sort_guard (*this, mv);
+	      o->newline() << "_stp_map_sort (" << mv.qname() << ", "
+			   << s->sort_column << ", " << - s->sort_direction << ");";
+	    }
 	}
+
       // NB: sort direction sense is opposite in runtime, thus the negation
       
       // XXX: There is a race condition here.  Since we can't convert a
@@ -2657,7 +2772,7 @@ c_unparser::visit_functioncall (functioncall* e)
 }
 
 struct hist_op_downcaster
-  : virtual public traversing_visitor
+  : public traversing_visitor
 {
   hist_op *& hist;
 
@@ -2809,56 +2924,161 @@ c_unparser::visit_print_format (print_format* e)
     }
 }
 
+struct arrayindex_downcaster
+  : public traversing_visitor
+{
+  arrayindex *& arr;
+  
+  arrayindex_downcaster (arrayindex *& arr)
+    : arr(arr) 
+  {}
+
+  void visit_arrayindex (arrayindex* e)
+  {
+    arr = e;
+  }
+};
+
+static bool
+expression_is_arrayindex (expression *e, 
+			  arrayindex *& hist)
+{
+  arrayindex *h = NULL;
+  arrayindex_downcaster d(h);
+  e->visit (&d);
+  if (static_cast<void*>(h) == static_cast<void*>(e))
+    {
+      hist = h;
+      return true;
+    }
+  return false;
+}
+
+void 
+c_tmpcounter::visit_stat_op (stat_op* e)
+{
+  symbol *sym = get_symbol_within_expression (e->stat);
+  var v = parent->getvar(sym->referent, e->tok);
+  aggvar agg = parent->gensym_aggregate ();
+  tmpvar res = parent->gensym (pe_long);
+
+  agg.declare(*(this->parent));
+  res.declare(*(this->parent));
+
+  if (sym->referent->arity != 0)
+    {
+      // One temporary per index dimension.
+      for (unsigned i=0; i<sym->referent->index_types.size(); i++)
+	{
+	  // Sorry about this, but with no dynamic_cast<> and no
+	  // constructor patterns, this is how things work.
+	  arrayindex *arr = NULL;
+	  if (!expression_is_arrayindex (e->stat, arr))
+	    throw semantic_error("expected arrayindex expression in stat_op of array", e->tok);
+
+	  tmpvar ix = parent->gensym (sym->referent->index_types[i]);
+	  ix.declare (*parent);
+	  arr->indexes[i]->visit(this);
+	}
+    }
+}
+
 void 
 c_unparser::visit_stat_op (stat_op* e)
 {
-  //
   // Stat ops can be *applied* to two types of expression:
   //
   //  1. An arrayindex expression on a pe_stats-valued array. 
   //
   //  2. A symbol of type pe_stats. 
-  //
-  // Stat ops can only *occur* in a limited set of circumstances:
-  //
-  //  1. Inside an arrayindex expression, as the base referent, when
-  //     the stat_component_type is a histogram type. See
-  //     c_unparser::visit_arrayindex for handling of this case.
-  //
-  //  2. Inside a foreach statement, as the base referent, when the
-  //     stat_component_type is a histogram type. See
-  //     c_unparser::visit_foreach_loop for handling this case.
-  //
-  //  3. Inside a print_format expression, as the sole argument, when
-  //     the stat_component_type is a histogram type. See
-  //     c_unparser::visit_print_format for handling this case.
-  //
-  //  4. Inside a normal rvalue context, when the stat_component_type
-  //     is a scalar. That's this case.
-  //
 
   // FIXME: classify the expression the stat_op is being applied to,
   // call appropriate stp_get_stat() / stp_pmap_get_stat() helper,
   // then reach into resultant struct stat_data.
 
-  switch (e->type)
-    {
-    case sc_average:
-    case sc_count:
-    case sc_sum:
-    case sc_min:
-    case sc_max:
+  // FIXME: also note that summarizing anything is expensive, and we
+  // really ought to pass a timeout handler into the summary routine,
+  // check its response, possibly exit if it ran out of cycles.
 
-    default:
-      assert(false);
-      break;
-    }  
+  symbol *sym = get_symbol_within_expression (e->stat);
+
+  if (sym->referent->type != pe_stats)
+    throw semantic_error ("non-statistic value in statistic operator context", sym->tok);
+  
+  {
+    stmt_expr block(*this);  
+    aggvar agg = gensym_aggregate ();
+    tmpvar res = gensym (pe_long);      
+    
+    {
+      var v = getvar(sym->referent, e->tok);
+      varlock_w guard(*this, v);
+      
+      if (sym->referent->arity == 0)
+	{
+	  o->newline() << "c->last_stmt = " << lex_cast_qstring(*sym->tok) << ";";
+	  o->newline() << agg << " = _stp_stat_get (" << v << ", 0);";	  
+	}
+      else
+	{
+	  arrayindex *arr = NULL;
+	  if (!expression_is_arrayindex (e->stat, arr))
+	    throw semantic_error("expected arrayindex expression in stat_op of array", e->tok);
+	  
+	  vector<tmpvar> idx;
+	  load_map_indices (arr, idx);
+	  mapvar mvar = getmap (sym->referent, sym->tok);
+	  o->newline() << "c->last_stmt = " << lex_cast_qstring(*sym->tok) << ";";
+	  o->newline() << agg << " = " << mvar.get(idx) << ";";
+	}
+    
+      switch (e->ctype)
+	{
+	case sc_average:
+	  // impedance matching: Have to compupte average ourselves
+	  c_assign(res, ("_stp_div64(&c->last_error, " + agg.qname() + "->sum, " + agg.qname() + "->count)"),
+		   e->tok);
+	  break;
+
+	case sc_count:
+	  c_assign(res, agg.qname() + "->count", e->tok);
+	  break;
+
+	case sc_sum:
+	  c_assign(res, agg.qname() + "->sum", e->tok);
+	  break;
+
+	case sc_min:
+	  c_assign(res, agg.qname() + "->min", e->tok);
+	  break;
+
+	case sc_max:
+	  c_assign(res, agg.qname() + "->max", e->tok);
+	  break;
+	}
+    }    
+    o->newline() << res << ";";
+  }
 }
 
 
 void 
 c_unparser::visit_hist_op (hist_op* e)
 {
+  // Hist ops can only occur in a limited set of circumstances:
+  //
+  //  1. Inside an arrayindex expression, as the base referent. See
+  //     c_unparser::visit_arrayindex for handling of this case.
+  //
+  //  2. Inside a foreach statement, as the base referent. See
+  //     c_unparser::visit_foreach_loop for handling this case.
+  //
+  //  3. Inside a print_format expression, as the sole argument. See
+  //     c_unparser::visit_print_format for handling this case.
+  //
+  // Note that none of these cases involves the c_unparser ever
+  // visiting this node. We should not get here.
+
   assert(false);
 }