cgo: create skeleton of a Pratt parser

This converts the existing const parser to the basics of a Pratt parser,
following the book "Writing An Interpreter In Go" by Thorsten Ball. It
doesn't really do anything interesting yet, it simply converts the
existing code (with existing tests) to the new structure.

cgo/const.go

@@ -11,14 +11,30 @@ import (
 	"strings"
 )
 
+var prefixParseFns map[token.Token]func(*tokenizer) (ast.Expr, *scanner.Error)
+
+func init() {
+	// This must be done in an init function to avoid an initialization order
+	// failure.
+	prefixParseFns = map[token.Token]func(*tokenizer) (ast.Expr, *scanner.Error){
+		token.IDENT:  parseIdent,
+		token.INT:    parseBasicLit,
+		token.FLOAT:  parseBasicLit,
+		token.STRING: parseBasicLit,
+		token.CHAR:   parseBasicLit,
+		token.LPAREN: parseParenExpr,
+	}
+}
+
 // parseConst parses the given string as a C constant.
 func parseConst(pos token.Pos, fset *token.FileSet, value string) (ast.Expr, *scanner.Error) {
 	t := newTokenizer(pos, fset, value)
 	expr, err := parseConstExpr(t)
+	t.Next()
 	if t.token != token.EOF {
 		return nil, &scanner.Error{
 			Pos: t.fset.Position(t.pos),
-			Msg: "unexpected token " + t.token.String(),
+			Msg: "unexpected token " + t.token.String() + ", expected end of expression",
 		}
 	}
 	return expr, err
@@ -26,14 +42,46 @@ func parseConst(pos token.Pos, fset *token.FileSet, value string) (ast.Expr, *sc
 
 // parseConstExpr parses a stream of C tokens to a Go expression.
 func parseConstExpr(t *tokenizer) (ast.Expr, *scanner.Error) {
-	switch t.token {
+	if t.token == token.EOF {
-	case token.LPAREN:
+		return nil, &scanner.Error{
+			Pos: t.fset.Position(t.pos),
+			Msg: "empty constant",
+		}
+	}
+	prefix := prefixParseFns[t.token]
+	if prefix == nil {
+		return nil, &scanner.Error{
+			Pos: t.fset.Position(t.pos),
+			Msg: fmt.Sprintf("unexpected token %s", t.token),
+		}
+	}
+	leftExpr, err := prefix(t)
+	return leftExpr, err
+}
+
+func parseIdent(t *tokenizer) (ast.Expr, *scanner.Error) {
+	return &ast.Ident{
+		NamePos: t.pos,
+		Name:    "C." + t.value,
+	}, nil
+}
+
+func parseBasicLit(t *tokenizer) (ast.Expr, *scanner.Error) {
+	return &ast.BasicLit{
+		ValuePos: t.pos,
+		Kind:     t.token,
+		Value:    t.value,
+	}, nil
+}
+
+func parseParenExpr(t *tokenizer) (ast.Expr, *scanner.Error) {
 	lparen := t.pos
 	t.Next()
 	x, err := parseConstExpr(t)
 	if err != nil {
 		return nil, err
 	}
+	t.Next()
 	if t.token != token.RPAREN {
 		return nil, unexpectedToken(t, token.RPAREN)
 	}
@@ -42,34 +90,7 @@ func parseConstExpr(t *tokenizer) (ast.Expr, *scanner.Error) {
 		X:      x,
 		Rparen: t.pos,
 	}
-		t.Next()
 	return expr, nil
-	case token.INT, token.FLOAT, token.STRING, token.CHAR:
-		expr := &ast.BasicLit{
-			ValuePos: t.pos,
-			Kind:     t.token,
-			Value:    t.value,
-		}
-		t.Next()
-		return expr, nil
-	case token.IDENT:
-		expr := &ast.Ident{
-			NamePos: t.pos,
-			Name:    "C." + t.value,
-		}
-		t.Next()
-		return expr, nil
-	case token.EOF:
-		return nil, &scanner.Error{
-			Pos: t.fset.Position(t.pos),
-			Msg: "empty constant",
-		}
-	default:
-		return nil, &scanner.Error{
-			Pos: t.fset.Position(t.pos),
-			Msg: fmt.Sprintf("unexpected token %s", t.token),
-		}
-	}
 }
 
 // unexpectedToken returns an error of the form "unexpected token FOO, expected

cgo/const_test.go

@@ -18,7 +18,7 @@ func TestParseConst(t *testing.T) {
 		{`(5)`, `(5)`},
 		{`(((5)))`, `(5)`},
 		{`)`, `error: 1:1: unexpected token )`},
-		{`5)`, `error: 1:2: unexpected token )`},
+		{`5)`, `error: 1:2: unexpected token ), expected end of expression`},
 		{"  \t)", `error: 1:4: unexpected token )`},
 		{`5.8f`, `5.8`},
 		{`foo`, `C.foo`},
@@ -30,7 +30,7 @@ func TestParseConst(t *testing.T) {
 		{`'a'`, `'a'`},
 		{`0b10`, `0b10`},
 		{`0x1234_5678`, `0x1234_5678`},
-		{`5 5`, `error: 1:3: unexpected token INT`}, // test for a bugfix
+		{`5 5`, `error: 1:3: unexpected token INT, expected end of expression`}, // test for a bugfix
 	} {
 		fset := token.NewFileSet()
 		startPos := fset.AddFile("", -1, 1000).Pos(0)