cgo: support preprocessor macros passed on the command line

Go code might sometimes want to use preprocessor macros that were passed
on the command line. This wasn't working before and resulted in the
following error:

    internal error: could not find file where macro is defined

This is now supported, though location information isn't available
(which makes sense: the command line is not a file).

I had to use the `clang_tokenize` API for this and reconstruct the
original source location. Apparently this is the only way to do it:
https://stackoverflow.com/a/19074846/559350
In the future we could consider replacing our own tokenization with the
tokenizer that's built into Clang directly. This should reduce the
possibility of bugs a bit.

cgo/testdata/errors.go

@@ -13,10 +13,14 @@ typedef someType noType; // undefined type
 #define SOME_CONST_1 5) // invalid const syntax
 #define SOME_CONST_2 6) // const not used (so no error)
 #define SOME_CONST_3 1234 // const too large for byte
+#define   SOME_CONST_b      3   ) // const with lots of weird whitespace (to test error locations)
+#  define SOME_CONST_startspace 3)
 */
 //
 //
 // #define SOME_CONST_4 8) // after some empty lines
+// #cgo CFLAGS: -DSOME_PARAM_CONST_invalid=3/+3
+// #cgo CFLAGS: -DSOME_PARAM_CONST_valid=3+4
 import "C"
 
 // #warning another warning
@@ -24,6 +28,7 @@ import "C"
 
 // Make sure that errors for the following lines won't change with future
 // additions to the CGo preamble.
+//
 //line errors.go:100
 var (
 	// constant too large
@@ -38,4 +43,12 @@ var (
 	_ byte = C.SOME_CONST_3
 
 	_ = C.SOME_CONST_4
+
+	_ = C.SOME_CONST_b
+
+	_ = C.SOME_CONST_startspace
+
+	// constants passed by a command line parameter
+	_ = C.SOME_PARAM_CONST_invalid
+	_ = C.SOME_PARAM_CONST_valid
 )

cgo/testdata/errors.out.go

@@ -1,9 +1,12 @@
 // CGo errors:
 //     testdata/errors.go:4:2: warning: some warning
 //     testdata/errors.go:11:9: error: unknown type name 'someType'
-//     testdata/errors.go:22:5: warning: another warning
+//     testdata/errors.go:26:5: warning: another warning
 //     testdata/errors.go:13:23: unexpected token ), expected end of expression
-//     testdata/errors.go:19:26: unexpected token ), expected end of expression
+//     testdata/errors.go:21:26: unexpected token ), expected end of expression
+//     testdata/errors.go:16:33: unexpected token ), expected end of expression
+//     testdata/errors.go:17:34: unexpected token ), expected end of expression
+//     -: unexpected token INT, expected end of expression
 
 // Type checking errors after CGo processing:
 //     testdata/errors.go:102: cannot use 2 << 10 (untyped int constant 2048) as C.char value in variable declaration (overflows)
@@ -11,6 +14,9 @@
 //     testdata/errors.go:108: undefined: C.SOME_CONST_1
 //     testdata/errors.go:110: cannot use C.SOME_CONST_3 (untyped int constant 1234) as byte value in variable declaration (overflows)
 //     testdata/errors.go:112: undefined: C.SOME_CONST_4
+//     testdata/errors.go:114: undefined: C.SOME_CONST_b
+//     testdata/errors.go:116: undefined: C.SOME_CONST_startspace
+//     testdata/errors.go:119: undefined: C.SOME_PARAM_CONST_invalid
 
 package main
 
@@ -58,3 +64,4 @@ type C.struct_point_t struct {
 type C.point_t = C.struct_point_t
 
 const C.SOME_CONST_3 = 1234
+const C.SOME_PARAM_CONST_valid = 3 + 4