Adding extra files for testing multiline, globbing, etc.

These two heaps conflict with each other, so that if any function uses
the dlmalloc heap implementation it will eventually result in memory
corruption.

This commit fixes this by implementing all heap-related functions. This
overrides the functions that are implemented in wasi-libc. That's why
all of them are implemented (even if they just panic): to make sure no
program accidentally uses the wrong one.

builder/build.go

@@ -470,33 +470,10 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		linkerDependencies = append(linkerDependencies, job)
 	}
 
-	// Add libc dependency if needed.
-	root := goenv.Get("TINYGOROOT")
-	switch config.Target.Libc {
-	case "picolibc":
-		job, err := Picolibc.load(config.Triple(), config.CPU(), dir)
-		if err != nil {
-			return err
-		}
-		// The library needs to be compiled (cache miss).
-		jobs = append(jobs, job.dependencies...)
-		jobs = append(jobs, job)
-		linkerDependencies = append(linkerDependencies, job)
-	case "wasi-libc":
-		path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a")
-		if _, err := os.Stat(path); os.IsNotExist(err) {
-			return errors.New("could not find wasi-libc, perhaps you need to run `make wasi-libc`?")
-		}
-		ldflags = append(ldflags, path)
-	case "":
-		// no library specified, so nothing to do
-	default:
-		return fmt.Errorf("unknown libc: %s", config.Target.Libc)
-	}
-
 	// Add jobs to compile extra files. These files are in C or assembly and
 	// contain things like the interrupt vector table and low level operations
 	// such as stack switching.
+	root := goenv.Get("TINYGOROOT")
 	for _, path := range config.ExtraFiles() {
 		abspath := filepath.Join(root, path)
 		job := &compileJob{
@@ -537,6 +514,31 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		ldflags = append(ldflags, lprogram.LDFlags...)
 	}
 
+	// Add libc dependency if needed.
+	switch config.Target.Libc {
+	case "picolibc":
+		job, err := Picolibc.load(config.Triple(), config.CPU(), dir)
+		if err != nil {
+			return err
+		}
+		// The library needs to be compiled (cache miss).
+		jobs = append(jobs, job.dependencies...)
+		jobs = append(jobs, job)
+		linkerDependencies = append(linkerDependencies, job)
+	case "wasi-libc":
+		path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a")
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			return errors.New("could not find wasi-libc, perhaps you need to run `make wasi-libc`?")
+		}
+		job := dummyCompileJob(path)
+		jobs = append(jobs, job)
+		linkerDependencies = append(linkerDependencies, job)
+	case "":
+		// no library specified, so nothing to do
+	default:
+		return fmt.Errorf("unknown libc: %s", config.Target.Libc)
+	}
+
 	// Create a linker job, which links all object files together and does some
 	// extra stuff that can only be done after linking.
 	jobs = append(jobs, &compileJob{

compiler/compiler.go

@@ -693,6 +693,7 @@ func (c *compilerContext) getDIFile(filename string) llvm.Metadata {
 // createPackage builds the LLVM IR for all types, methods, and global variables
 // in the given package.
 func (c *compilerContext) createPackage(irbuilder llvm.Builder, pkg *ssa.Package) {
+	println("called create package:", pkg.Pkg.Name())
 	// Sort by position, so that the order of the functions in the IR matches
 	// the order of functions in the source file. This is useful for testing,
 	// for example.
@@ -758,6 +759,35 @@ func (c *compilerContext) createPackage(irbuilder llvm.Builder, pkg *ssa.Package
 		case *ssa.Global:
 			// Global variable.
 			info := c.getGlobalInfo(member)
+			// process go:embed pragmas
+			// TODO: skip this if not Go 1.16 or higher
+			if strings.TrimSpace(info.embeds) != "" {
+				embeds, err := parseGoEmbed(info.embeds)
+				if err != nil {
+					c.addError(member.Pos(), `invalid go:embed pragma: `+err.Error())
+					goto getGlobal
+				}
+				importsEmbedPkg := false
+				for _, imprt := range pkg.Pkg.Imports() {
+					if imprt.Name() == "embed" {
+						importsEmbedPkg = true
+						break
+					}
+				}
+				if !importsEmbedPkg {
+					// FIXME: technically the "embed" package needs to be imported in every
+					// *source file* that has a //go:embed pragma... that will always be the
+					// case when referencing embed.FS but not necessarily when a go:embed
+					// pragma is added to a string or []byte.  For now, as long as the embed
+					// package is imported in at least one source file, TinyGo will process
+					// any go:embed pragmas in the entire package.
+					c.addError(member.Pos(), `//go:embed only allowed in Go files that import "embed"`)
+					goto getGlobal
+				}
+				fmt.Printf("file embed found at %v (import: %v): %v\n", info.linkName, importsEmbedPkg, embeds)
+			}
+
+		getGlobal:
 			global := c.getGlobal(member)
 			if !info.extern {
 				global.SetInitializer(llvm.ConstNull(global.Type().ElementType()))
@@ -766,6 +796,7 @@ func (c *compilerContext) createPackage(irbuilder llvm.Builder, pkg *ssa.Package
 					global.SetSection(info.section)
 				}
 			}
+
 		}
 	}
 }

compiler/embed.go

@@ -0,0 +1,70 @@
+package compiler
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// parseGoEmbed parses the text following "//go:embed" to extract the glob patterns.
+// It accepts unquoted space-separated patterns as well as double-quoted and back-quoted Go strings.
+// Copied from https://github.com/golang/go/blob/219fe9d547d09d3de1b024c6c8b8314dd0bf12e4/src/cmd/compile/internal/noder/noder.go#L1717-L1776
+func parseGoEmbed(args string) ([]string, error) {
+	var list []string
+	for args = strings.TrimSpace(args); args != ""; args = strings.TrimSpace(args) {
+		var path string
+	Switch:
+		switch args[0] {
+		default:
+			i := len(args)
+			for j, c := range args {
+				if unicode.IsSpace(c) {
+					i = j
+					break
+				}
+			}
+			path = args[:i]
+			args = args[i:]
+
+		case '`':
+			i := strings.Index(args[1:], "`")
+			if i < 0 {
+				return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args)
+			}
+			path = args[1 : 1+i]
+			args = args[1+i+1:]
+
+		case '"':
+			i := 1
+			for ; i < len(args); i++ {
+				if args[i] == '\\' {
+					i++
+					continue
+				}
+				if args[i] == '"' {
+					q, err := strconv.Unquote(args[:i+1])
+					if err != nil {
+						return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args[:i+1])
+					}
+					path = q
+					args = args[i+1:]
+					break Switch
+				}
+			}
+			if i >= len(args) {
+				return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args)
+			}
+		}
+
+		if args != "" {
+			r, _ := utf8.DecodeRuneInString(args)
+			if !unicode.IsSpace(r) {
+				return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args)
+			}
+		}
+		list = append(list, path)
+	}
+	return list, nil
+}

compiler/symbol.go

@@ -331,6 +331,7 @@ type globalInfo struct {
 	extern   bool   // go:extern
 	align    int    // go:align
 	section  string // go:section
+	embeds   string // go:embed
 }
 
 // loadASTComments loads comments on globals from the AST, for use later in the
@@ -448,6 +449,9 @@ func (info *globalInfo) parsePragmas(doc *ast.CommentGroup) {
 			if len(parts) == 2 {
 				info.section = parts[1]
 			}
+		case "//go:embed":
+			raw := strings.TrimPrefix(comment.Text, "//go:embed")
+			info.embeds += " " + raw
 		}
 	}
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/tinygo-org/tinygo
 
-go 1.13
+go 1.16
 
 require (
 	github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2

interp/errors.go

@@ -57,6 +57,7 @@ func (r *runner) errorAt(inst instruction, err error) *Error {
 	pos := getPosition(inst.llvmInst)
 	return &Error{
 		ImportPath: r.pkgName,
+		Inst:       inst.llvmInst,
 		Pos:        pos,
 		Err:        err,
 		Traceback:  []ErrorLine{{pos, inst.llvmInst}},

interp/interp.go

@@ -15,7 +15,7 @@ import (
 // package is changed in a way that affects the output so that cached package
 // builds will be invalidated.
 // This version is independent of the TinyGo version number.
-const Version = 1
+const Version = 2 // last change: fix GEP on untyped pointers
 
 // Enable extra checks, which should be disabled by default.
 // This may help track down bugs by adding a few more sanity checks.
@@ -110,17 +110,26 @@ func Run(mod llvm.Module, debug bool) error {
 			fmt.Fprintln(os.Stderr, "call:", fn.Name())
 		}
 		_, mem, callErr := r.run(r.getFunction(fn), nil, nil, "    ")
+		call.EraseFromParentAsInstruction()
 		if callErr != nil {
 			if isRecoverableError(callErr.Err) {
 				if r.debug {
 					fmt.Fprintln(os.Stderr, "not interpreting", r.pkgName, "because of error:", callErr.Error())
 				}
+				// Remove instructions that were created as part of interpreting
+				// the package.
 				mem.revert()
+				// Create a call to the package initializer (which was
+				// previously deleted).
+				i8undef := llvm.Undef(r.i8ptrType)
+				r.builder.CreateCall(fn, []llvm.Value{i8undef, i8undef}, "")
+				// Make sure that any globals touched by the package
+				// initializer, won't be accessed by later package initializers.
+				r.markExternalLoad(fn)
 				continue
 			}
 			return callErr
 		}
-		call.EraseFromParentAsInstruction()
 		for index, obj := range mem.objects {
 			r.objects[index] = obj
 		}
@@ -270,3 +279,19 @@ func (r *runner) getFunction(llvmFn llvm.Value) *function {
 	r.functionCache[llvmFn] = fn
 	return fn
 }
+
+// markExternalLoad marks the given llvmValue as being loaded externally. This
+// is primarily used to mark package initializers that could not be run at
+// compile time. As an example, a package initialize might store to a global
+// variable. Another package initializer might read from the same global
+// variable. By marking this function as being run at runtime, that load
+// instruction will need to be run at runtime instead of at compile time.
+func (r *runner) markExternalLoad(llvmValue llvm.Value) {
+	mem := memoryView{r: r}
+	mem.markExternalLoad(llvmValue)
+	for index, obj := range mem.objects {
+		if obj.marked > r.objects[index].marked {
+			r.objects[index].marked = obj.marked
+		}
+	}
+}

interp/interp_test.go

@@ -17,6 +17,7 @@ func TestInterp(t *testing.T) {
 		"slice-copy",
 		"consteval",
 		"interface",
+		"revert",
 	} {
 		name := name // make tc local to this closure
 		t.Run(name, func(t *testing.T) {

interp/memory.go

@@ -572,6 +572,17 @@ func (v pointerValue) toLLVMValue(llvmType llvm.Type, mem *memoryView) (llvm.Val
 	}
 
 	if llvmType.IsNil() {
+		if v.offset() != 0 {
+			// If there is an offset, make sure to use a GEP to index into the
+			// pointer. Because there is no expected type, we use whatever is
+			// most convenient: an *i8 type. It is trivial to index byte-wise.
+			if llvmValue.Type() != mem.r.i8ptrType {
+				llvmValue = llvm.ConstBitCast(llvmValue, mem.r.i8ptrType)
+			}
+			llvmValue = llvm.ConstInBoundsGEP(llvmValue, []llvm.Value{
+				llvm.ConstInt(llvmValue.Type().Context().Int32Type(), uint64(v.offset()), false),
+			})
+		}
 		return llvmValue, nil
 	}
 

interp/testdata/revert.ll

@@ -0,0 +1,32 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64--linux"
+
+declare void @externalCall(i64)
+
+@foo.knownAtRuntime = global i64 0
+@bar.knownAtRuntime = global i64 0
+
+define void @runtime.initAll() unnamed_addr {
+entry:
+  call void @foo.init(i8* undef, i8* undef)
+  call void @bar.init(i8* undef, i8* undef)
+  call void @main.init(i8* undef, i8* undef)
+  ret void
+}
+
+define internal void @foo.init(i8* %context, i8* %parentHandle) unnamed_addr {
+  store i64 5, i64* @foo.knownAtRuntime
+  unreachable ; this triggers a revert of @foo.init.
+}
+
+define internal void @bar.init(i8* %context, i8* %parentHandle) unnamed_addr {
+  %val = load i64, i64* @foo.knownAtRuntime
+  store i64 %val, i64* @bar.knownAtRuntime
+  ret void
+}
+
+define internal void @main.init(i8* %context, i8* %parentHandle) unnamed_addr {
+entry:
+  call void @externalCall(i64 3)
+  ret void
+}

interp/testdata/revert.out.ll

@@ -0,0 +1,21 @@
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64--linux"
+
+@foo.knownAtRuntime = local_unnamed_addr global i64 0
+@bar.knownAtRuntime = local_unnamed_addr global i64 0
+
+declare void @externalCall(i64) local_unnamed_addr
+
+define void @runtime.initAll() unnamed_addr {
+entry:
+  call fastcc void @foo.init(i8* undef, i8* undef)
+  %val = load i64, i64* @foo.knownAtRuntime, align 8
+  store i64 %val, i64* @bar.knownAtRuntime, align 8
+  call void @externalCall(i64 3)
+  ret void
+}
+
+define internal fastcc void @foo.init(i8* %context, i8* %parentHandle) unnamed_addr {
+  store i64 5, i64* @foo.knownAtRuntime, align 8
+  unreachable
+}

main_test.go

@@ -125,7 +125,7 @@ func TestCompiler(t *testing.T) {
 		// Test with few optimizations enabled (no inlining, etc).
 		t.Run("opt=1", func(t *testing.T) {
 			t.Parallel()
-			runTestWithConfig("stdlib.go", "", t, &compileopts.Options{
+			runTestWithConfig("stdlib.go", "", t, compileopts.Options{
 				Opt: "1",
 			}, nil, nil)
 		})
@@ -134,15 +134,14 @@ func TestCompiler(t *testing.T) {
 		// TODO: fix this for stdlib.go, which currently fails.
 		t.Run("opt=0", func(t *testing.T) {
 			t.Parallel()
-			runTestWithConfig("print.go", "", t, &compileopts.Options{
+			runTestWithConfig("print.go", "", t, compileopts.Options{
 				Opt: "0",
 			}, nil, nil)
 		})
 
 		t.Run("ldflags", func(t *testing.T) {
 			t.Parallel()
-			runTestWithConfig("ldflags.go", "", t, &compileopts.Options{
-				Opt: "z",
+			runTestWithConfig("ldflags.go", "", t, compileopts.Options{
 				GlobalValues: map[string]map[string]string{
 					"main": {
 						"someGlobal": "foobar",
@@ -188,20 +187,20 @@ func runBuild(src, out string, opts *compileopts.Options) error {
 }
 
 func runTest(name, target string, t *testing.T, cmdArgs, environmentVars []string) {
-	options := &compileopts.Options{
-		Target:     target,
-		Opt:        "z",
-		PrintIR:    false,
-		DumpSSA:    false,
-		VerifyIR:   true,
-		Debug:      true,
-		PrintSizes: "",
-		WasmAbi:    "",
+	options := compileopts.Options{
+		Target: target,
 	}
 	runTestWithConfig(name, target, t, options, cmdArgs, environmentVars)
 }
 
-func runTestWithConfig(name, target string, t *testing.T, options *compileopts.Options, cmdArgs, environmentVars []string) {
+func runTestWithConfig(name, target string, t *testing.T, options compileopts.Options, cmdArgs, environmentVars []string) {
+	// Set default config.
+	options.Debug = true
+	options.VerifyIR = true
+	if options.Opt == "" {
+		options.Opt = "z"
+	}
+
 	// Get the expected output for this test.
 	// Note: not using filepath.Join as it strips the path separator at the end
 	// of the path.
@@ -230,7 +229,7 @@ func runTestWithConfig(name, target string, t *testing.T, options *compileopts.O
 
 	// Build the test binary.
 	binary := filepath.Join(tmpdir, "test")
-	err = runBuild("./"+path, binary, options)
+	err = runBuild("./"+path, binary, &options)
 	if err != nil {
 		printCompilerError(t.Log, err)
 		t.Fail()

src/examples/embed/embed.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"embed"
+	"log"
+)
+
+//go:embed file*.txt
+//go:embed index.html styles.css
+var files embed.FS
+
+func main() {
+	println(".....")
+	println(msg)
+	contents, err := files.ReadDir(".")
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, c := range contents {
+		println("file:", c.Name())
+	}
+}

src/examples/embed/file1.txt

@@ -0,0 +1 @@
+file 1

src/examples/embed/file2.txt

@@ -0,0 +1 @@
+file 2

src/examples/embed/file3.txt

@@ -0,0 +1 @@
+file 3

src/examples/embed/message.txt

@@ -0,0 +1 @@
+hello world!

src/examples/embed/test.go

@@ -0,0 +1,6 @@
+package main
+
+import _ "embed"
+
+//go:embed message.txt
+var msg string

src/runtime/arch_tinygowasm.go

@@ -55,3 +55,51 @@ func growHeap() bool {
 	// Heap has grown successfully.
 	return true
 }
+
+// The below functions override the default allocator of wasi-libc.
+// Most functions are defined but unimplemented to make sure that if there is
+// any code using them, they will get an error instead of (incorrectly) using
+// the wasi-libc dlmalloc heap implementation instead. If they are needed by any
+// program, they can certainly be implemented.
+
+//export malloc
+func libc_malloc(size uintptr) unsafe.Pointer {
+	return alloc(size)
+}
+
+//export free
+func libc_free(ptr unsafe.Pointer) {
+	free(ptr)
+}
+
+//export calloc
+func libc_calloc(nmemb, size uintptr) unsafe.Pointer {
+	// Note: we could be even more correct here and check that nmemb * size
+	// doesn't overflow. However the current implementation should normally work
+	// fine.
+	return alloc(nmemb * size)
+}
+
+//export realloc
+func libc_realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer {
+	runtimePanic("unimplemented: realloc")
+	return nil
+}
+
+//export posix_memalign
+func libc_posix_memalign(memptr *unsafe.Pointer, alignment, size uintptr) int {
+	runtimePanic("unimplemented: posix_memalign")
+	return 0
+}
+
+//export aligned_alloc
+func libc_aligned_alloc(alignment, bytes uintptr) unsafe.Pointer {
+	runtimePanic("unimplemented: aligned_alloc")
+	return nil
+}
+
+//export malloc_usable_size
+func libc_malloc_usable_size(ptr unsafe.Pointer) uintptr {
+	runtimePanic("unimplemented: malloc_usable_size")
+	return 0
+}

testdata/init.go

@@ -13,6 +13,8 @@ func main() {
 	println("v5:", len(v5), v5 == nil)
 	println("v6:", v6)
 	println("v7:", cap(v7), string(v7))
+	println("v8:", v8)
+	println("v9:", len(v9), v9[0], v9[1], v9[2])
 
 	println(uint8SliceSrc[0])
 	println(uint8SliceDst[0])
@@ -35,6 +37,8 @@ var (
 	v5 = map[string]int{}
 	v6 = float64(v1) < 2.6
 	v7 = []byte("foo")
+	v8 string
+	v9 []int
 
 	uint8SliceSrc = []uint8{3, 100}
 	uint8SliceDst []uint8
@@ -48,4 +52,15 @@ func init() {
 
 	intSliceDst = make([]int16, len(intSliceSrc))
 	copy(intSliceDst, intSliceSrc)
+
+	v8 = sliceString("foobarbaz", 3, 8)
+	v9 = sliceSlice([]int{0, 1, 2, 3, 4, 5}, 2, 5)
+}
+
+func sliceString(s string, start, end int) string {
+	return s[start:end]
+}
+
+func sliceSlice(s []int, start, end int) []int {
+	return s[start:end]
 }

testdata/init.txt

@@ -7,6 +7,8 @@ v4: 0 true
 v5: 0 false
 v6: false
 v7: 3 foo
+v8: barba
+v9: 3 2 3 4
 3
 3
 5