compiler: implement copy directly

The compiler now implements the copy builtin directly instead of calling sliceCopy.
The length is calculated with the llvm.umax.* intrinsics, and the move is performed by llvm.memmove.*.
Both of these operations are easily understood by LLVM's optimization passes.
The type's alignment is also provided to llvm.memmove.*, which is useful when rewriting the move.

Interp no longer needs to reimplement sliceCopy.
Some edge case handling was implemented by sliceCopy but not llvm.memmove.*/llvm.memcpy.*.
I copied this over, so copies of external slices should work now.
Volatile moves/copies are now run at runtime by interp.

There is a 4-byte size increase due to some confusing length logic in sendUSBPacket.
I will look at sendUSBPacket in a future PR.
This commit is contained in:
Nia Waldvogel
2025-12-26 19:19:21 -05:00
committed by Nia
parent bc9708f51a
commit a40586d092
17 changed files with 353 additions and 358 deletions
+2 -11
View File
@@ -2,7 +2,6 @@ package interp
import (
"os"
"strconv"
"strings"
"testing"
"time"
@@ -11,25 +10,17 @@ import (
)
func TestInterp(t *testing.T) {
llvmVersion, err := strconv.Atoi(strings.Split(llvm.Version, ".")[0])
if err != nil {
// Note: this should never happen and if it does, it will always happen
// for a particular build because llvm.Version is a constant.
panic(err)
}
for _, name := range []string{
"basic",
"phi",
"slice-copy",
"consteval",
"intrinsics",
"copy",
"interface",
"revert",
"alloc",
} {
name := name // make local to this closure
if name == "slice-copy" && llvmVersion < 14 {
continue
}
t.Run(name, func(t *testing.T) {
t.Parallel()
runTest(t, "testdata/"+name)
+22 -51
View File
@@ -312,33 +312,28 @@ func (r *runner) run(fn *function, params []value, parentMem *memoryView, indent
fmt.Fprintln(os.Stderr, indent+"runtime.alloc:", size, "->", ptr)
}
locals[inst.localIndex] = ptr
case callFn.name == "runtime.sliceCopy":
// sliceCopy implements the built-in copy function for slices.
// It is implemented here so that it can be used even if the
// runtime implementation is not available. Doing it this way
// may also be faster.
// Code:
// func sliceCopy(dst, src unsafe.Pointer, dstLen, srcLen uintptr, elemSize uintptr) int {
// n := srcLen
// if n > dstLen {
// n = dstLen
// }
// memmove(dst, src, n*elemSize)
// return int(n)
// }
dstLen := operands[3].Uint(r)
srcLen := operands[4].Uint(r)
elemSize := operands[5].Uint(r)
n := srcLen
if n > dstLen {
n = dstLen
case strings.HasPrefix(callFn.name, "llvm.umin."):
locals[inst.localIndex] = makeLiteralInt(min(operands[1].Uint(r), operands[2].Uint(r)), inst.llvmInst.Type().IntTypeWidth())
case strings.HasPrefix(callFn.name, "llvm.smin."):
locals[inst.localIndex] = makeLiteralInt(uint64(min(operands[1].Int(r), operands[2].Int(r))), inst.llvmInst.Type().IntTypeWidth())
case strings.HasPrefix(callFn.name, "llvm.umax."):
locals[inst.localIndex] = makeLiteralInt(max(operands[1].Uint(r), operands[2].Uint(r)), inst.llvmInst.Type().IntTypeWidth())
case strings.HasPrefix(callFn.name, "llvm.smax."):
locals[inst.localIndex] = makeLiteralInt(uint64(max(operands[1].Int(r), operands[2].Int(r))), inst.llvmInst.Type().IntTypeWidth())
case strings.HasPrefix(callFn.name, "llvm.memcpy.p0") || strings.HasPrefix(callFn.name, "llvm.memmove.p0"):
// Copy a block of memory from one pointer to another.
if operands[4].Uint(r) != 0 {
// This is a volatile copy/move.
err := r.runAtRuntime(fn, inst, locals, &mem, indent)
if err != nil {
return nil, mem, err
}
continue
}
if r.debug {
fmt.Fprintln(os.Stderr, indent+"copy:", operands[1], operands[2], n)
}
if n != 0 {
nBytes := operands[3].Uint(r)
if nBytes != 0 {
// Only try to copy bytes when there are any bytes to copy.
// This is not just an optimization. If one of the slices
// This is not just an optimization. If one of the pointers
// (or both) are nil, the asPointer method call will fail
// even though copying a nil slice is allowed.
dst, err := operands[1].asPointer(r)
@@ -363,11 +358,10 @@ func (r *runner) run(fn *function, params []value, parentMem *memoryView, indent
}
continue
}
nBytes := uint32(n * elemSize)
srcObj := mem.get(src.index())
dstObj := mem.getWritable(dst.index())
if srcObj.buffer == nil || dstObj.buffer == nil {
// If the buffer is nil, it means the slice is external.
// If the buffer is nil, it means the memory is external.
// This can happen for example when copying data out of
// a //go:embed slice, which is not available at interp
// time.
@@ -380,33 +374,10 @@ func (r *runner) run(fn *function, params []value, parentMem *memoryView, indent
}
dstBuf := dstObj.buffer.asRawValue(r)
srcBuf := srcObj.buffer.asRawValue(r)
copy(dstBuf.buf[dst.offset():dst.offset()+nBytes], srcBuf.buf[src.offset():])
copy(dstBuf.buf[dst.offset():][:nBytes], srcBuf.buf[src.offset():][:nBytes])
dstObj.buffer = dstBuf
mem.put(dst.index(), dstObj)
}
locals[inst.localIndex] = makeLiteralInt(n, inst.llvmInst.Type().IntTypeWidth())
case strings.HasPrefix(callFn.name, "llvm.memcpy.p0") || strings.HasPrefix(callFn.name, "llvm.memmove.p0"):
// Copy a block of memory from one pointer to another.
dst, err := operands[1].asPointer(r)
if err != nil {
return nil, mem, r.errorAt(inst, err)
}
src, err := operands[2].asPointer(r)
if err != nil {
return nil, mem, r.errorAt(inst, err)
}
nBytes := uint32(operands[3].Uint(r))
dstObj := mem.getWritable(dst.index())
dstBuf := dstObj.buffer.asRawValue(r)
if mem.get(src.index()).buffer == nil {
// Looks like the source buffer is not defined.
// This can happen with //extern or //go:embed.
return nil, mem, r.errorAt(inst, errUnsupportedRuntimeInst)
}
srcBuf := mem.get(src.index()).buffer.asRawValue(r)
copy(dstBuf.buf[dst.offset():dst.offset()+nBytes], srcBuf.buf[src.offset():])
dstObj.buffer = dstBuf
mem.put(dst.index(), dstObj)
case callFn.name == "runtime.typeAssert":
// This function must be implemented manually as it is normally
// implemented by the interface lowering pass.
+68
View File
@@ -0,0 +1,68 @@
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64--linux"
@string = internal unnamed_addr constant [3 x i8] c"foo"
@moveDst = global [3 x i8] zeroinitializer
@copyDst = global [3 x i8] zeroinitializer
@externalSrc = external global [2 x i8]
@moveExternalDst = global [2 x i8] zeroinitializer
@moveEscapedSrc = global [4 x i8] c"abcd"
@moveEscapedDst = global [4 x i8] zeroinitializer
@volatileSrc = global [2 x i8] c"xy"
@volatileDst = global [2 x i8] zeroinitializer
declare void @use(ptr)
define void @runtime.initAll() {
call void @main.init()
ret void
}
define internal void @main.init() {
call void @testMove()
call void @testCopy()
call void @testMoveExternal()
call void @testMoveEscaped()
call void @testVolatileCopy()
ret void
}
; Test a simple memmove between globals.
define internal void @testMove() {
call void @llvm.memmove.p0.p0.i64(ptr @moveDst, ptr @string, i64 3, i1 false)
ret void
}
; Test a simple memcpy between globals.
define internal void @testCopy() {
call void @llvm.memcpy.p0.p0.i64(ptr @copyDst, ptr @string, i64 3, i1 false)
ret void
}
; Test a memmove from an external global.
; This should be run at runtime.
define internal void @testMoveExternal() {
call void @llvm.memmove.p0.p0.i64(ptr @moveExternalDst, ptr @externalSrc, i64 2, i1 false)
ret void
}
; Test a memmove from an escaped (and potentially modified) source buffer.
define internal void @testMoveEscaped() {
call void @use(ptr @moveEscapedSrc)
call void @llvm.memmove.p0.p0.i64(ptr @moveEscapedDst, ptr @moveEscapedSrc, i64 4, i1 false)
ret void
}
; Test a volatile memcpy.
; This should always be run at runtime.
define internal void @testVolatileCopy() {
call void @llvm.memcpy.p0.p0.i64(ptr @volatileDst, ptr @volatileSrc, i64 2, i1 true)
ret void
}
declare void @llvm.memmove.p0.p0.i64(ptr, ptr, i64, i1)
declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
+27
View File
@@ -0,0 +1,27 @@
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64--linux"
@moveDst = local_unnamed_addr global [3 x i8] c"foo"
@copyDst = local_unnamed_addr global [3 x i8] c"foo"
@externalSrc = external local_unnamed_addr global [2 x i8]
@moveExternalDst = local_unnamed_addr global [2 x i8] zeroinitializer
@moveEscapedSrc = global [4 x i8] c"abcd"
@moveEscapedDst = local_unnamed_addr global [4 x i8] zeroinitializer
@volatileSrc = global [2 x i8] c"xy"
@volatileDst = global [2 x i8] zeroinitializer
declare void @use(ptr) local_unnamed_addr
define void @runtime.initAll() local_unnamed_addr {
call void @llvm.memmove.p0.p0.i64(ptr @moveExternalDst, ptr @externalSrc, i64 2, i1 false)
call void @use(ptr @moveEscapedSrc)
call void @llvm.memmove.p0.p0.i64(ptr @moveEscapedDst, ptr @moveEscapedSrc, i64 4, i1 false)
call void @llvm.memcpy.p0.p0.i64(ptr @volatileDst, ptr @volatileSrc, i64 2, i1 true)
ret void
}
declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1 immarg) #0
declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #0
attributes #0 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
+52
View File
@@ -0,0 +1,52 @@
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64--linux"
@uminResult = global i32 0
@sminResult = global i32 0
@umaxResult = global i32 0
@smaxResult = global i32 0
define void @runtime.initAll() {
call void @main.init()
ret void
}
define internal void @main.init() {
call void @testUMin()
call void @testSMin()
call void @testUMax()
call void @testSMax()
ret void
}
define internal void @testUMin() {
%umin = call i32 @llvm.umin.i32(i32 12, i32 -1)
store i32 %umin, ptr @uminResult
ret void
}
declare i32 @llvm.umin.i32(i32, i32)
define internal void @testSMin() {
%smin = call i32 @llvm.smin.i32(i32 12, i32 -1)
store i32 %smin, ptr @sminResult
ret void
}
declare i32 @llvm.smin.i32(i32, i32)
define internal void @testUMax() {
%umax = call i32 @llvm.umax.i32(i32 12, i32 -1)
store i32 %umax, ptr @umaxResult
ret void
}
declare i32 @llvm.umax.i32(i32, i32)
define internal void @testSMax() {
%smax = call i32 @llvm.smax.i32(i32 12, i32 -1)
store i32 %smax, ptr @smaxResult
ret void
}
declare i32 @llvm.smax.i32(i32, i32)
+11
View File
@@ -0,0 +1,11 @@
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64--linux"
@uminResult = local_unnamed_addr global i32 12
@sminResult = local_unnamed_addr global i32 -1
@umaxResult = local_unnamed_addr global i32 -1
@smaxResult = local_unnamed_addr global i32 12
define void @runtime.initAll() local_unnamed_addr {
ret void
}
-124
View File
@@ -1,124 +0,0 @@
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64--linux"
@main.uint8SliceSrc.buf = internal global [2 x i8] c"\03d"
@main.uint8SliceSrc = internal unnamed_addr global { ptr, i64, i64 } { ptr @main.uint8SliceSrc.buf, i64 2, i64 2 }
@main.uint8SliceDst = internal unnamed_addr global { ptr, i64, i64 } zeroinitializer
@main.int16SliceSrc.buf = internal global [3 x i16] [i16 5, i16 123, i16 1024]
@main.int16SliceSrc = internal unnamed_addr global { ptr, i64, i64 } { ptr @main.int16SliceSrc.buf, i64 3, i64 3 }
@main.int16SliceDst = internal unnamed_addr global { ptr, i64, i64 } zeroinitializer
@main.sliceSrcUntaint.buf = internal global [2 x i8] c"ab"
@main.sliceDstUntaint.buf = internal global [2 x i8] zeroinitializer
@main.sliceSrcTaint.buf = internal global [2 x i8] c"cd"
@main.sliceDstTaint.buf = internal global [2 x i8] zeroinitializer
@main.sliceSrcExternal1.buf = external global [2 x i8]
@main.sliceDstExternal1.buf = internal global [2 x i8] zeroinitializer
@main.sliceSrcExternal2.buf = internal global [2 x i8] zeroinitializer
@main.sliceDstExternal2.buf = external global [2 x i8]
declare i64 @runtime.sliceCopy(ptr %dst, ptr %src, i64 %dstLen, i64 %srcLen, i64 %elemSize) unnamed_addr
declare ptr @runtime.alloc(i64, ptr) unnamed_addr
declare void @runtime.printuint8(i8)
declare void @runtime.printint16(i16)
declare void @use(ptr)
define void @runtime.initAll() unnamed_addr {
entry:
call void @main.init()
ret void
}
define void @main() unnamed_addr {
entry:
; print(uintSliceSrc[0])
%uint8SliceSrc.buf = load ptr, ptr @main.uint8SliceSrc
%uint8SliceSrc.val = load i8, ptr %uint8SliceSrc.buf
call void @runtime.printuint8(i8 %uint8SliceSrc.val)
; print(uintSliceDst[0])
%uint8SliceDst.buf = load ptr, ptr @main.uint8SliceDst
%uint8SliceDst.val = load i8, ptr %uint8SliceDst.buf
call void @runtime.printuint8(i8 %uint8SliceDst.val)
; print(int16SliceSrc[0])
%int16SliceSrc.buf = load ptr, ptr @main.int16SliceSrc
%int16SliceSrc.val = load i16, ptr %int16SliceSrc.buf
call void @runtime.printint16(i16 %int16SliceSrc.val)
; print(int16SliceDst[0])
%int16SliceDst.buf = load ptr, ptr @main.int16SliceDst
%int16SliceDst.val = load i16, ptr %int16SliceDst.buf
call void @runtime.printint16(i16 %int16SliceDst.val)
; print(sliceDstUntaint[0])
%sliceDstUntaint.val = load i8, ptr getelementptr inbounds (i8, ptr @main.sliceDstUntaint.buf, i32 0)
call void @runtime.printuint8(i8 %sliceDstUntaint.val)
; print(sliceDstTaint[0])
%sliceDstTaint.val = load i8, ptr getelementptr inbounds (i8, ptr @main.sliceDstTaint.buf, i32 0)
call void @runtime.printuint8(i8 %sliceDstTaint.val)
; print(sliceDstExternal1[0])
%sliceDstExternal1.val = load i8, ptr getelementptr inbounds (i8, ptr @main.sliceDstExternal1.buf, i32 0)
call void @runtime.printuint8(i8 %sliceDstExternal1.val)
; print(sliceDstExternal2[0])
%sliceDstExternal2.val = load i8, ptr getelementptr inbounds (i8, ptr @main.sliceDstExternal2.buf, i32 0)
call void @runtime.printuint8(i8 %sliceDstExternal2.val)
ret void
}
define internal void @main.init() unnamed_addr {
entry:
; equivalent of:
; uint8SliceDst = make([]uint8, len(uint8SliceSrc))
%uint8SliceSrc = load { ptr, i64, i64 }, ptr @main.uint8SliceSrc
%uint8SliceSrc.len = extractvalue { ptr, i64, i64 } %uint8SliceSrc, 1
%uint8SliceDst.buf = call ptr @runtime.alloc(i64 %uint8SliceSrc.len, ptr null)
%0 = insertvalue { ptr, i64, i64 } undef, ptr %uint8SliceDst.buf, 0
%1 = insertvalue { ptr, i64, i64 } %0, i64 %uint8SliceSrc.len, 1
%2 = insertvalue { ptr, i64, i64 } %1, i64 %uint8SliceSrc.len, 2
store { ptr, i64, i64 } %2, ptr @main.uint8SliceDst
; equivalent of:
; copy(uint8SliceDst, uint8SliceSrc)
%uint8SliceSrc.buf = extractvalue { ptr, i64, i64 } %uint8SliceSrc, 0
%copy.n = call i64 @runtime.sliceCopy(ptr %uint8SliceDst.buf, ptr %uint8SliceSrc.buf, i64 %uint8SliceSrc.len, i64 %uint8SliceSrc.len, i64 1)
; equivalent of:
; int16SliceDst = make([]int16, len(int16SliceSrc))
%int16SliceSrc = load { ptr, i64, i64 }, ptr @main.int16SliceSrc
%int16SliceSrc.len = extractvalue { ptr, i64, i64 } %int16SliceSrc, 1
%int16SliceSrc.len.bytes = mul i64 %int16SliceSrc.len, 2
%int16SliceDst.buf = call ptr @runtime.alloc(i64 %int16SliceSrc.len.bytes, ptr null)
%3 = insertvalue { ptr, i64, i64 } undef, ptr %int16SliceDst.buf, 0
%4 = insertvalue { ptr, i64, i64 } %3, i64 %int16SliceSrc.len, 1
%5 = insertvalue { ptr, i64, i64 } %4, i64 %int16SliceSrc.len, 2
store { ptr, i64, i64 } %5, ptr @main.int16SliceDst
; equivalent of:
; copy(int16SliceDst, int16SliceSrc)
%int16SliceSrc.buf = extractvalue { ptr, i64, i64 } %int16SliceSrc, 0
%copy.n2 = call i64 @runtime.sliceCopy(ptr %int16SliceDst.buf, ptr %int16SliceSrc.buf, i64 %int16SliceSrc.len, i64 %int16SliceSrc.len, i64 2)
; Copy slice that has a known value.
%copy.n3 = call i64 @runtime.sliceCopy(ptr @main.sliceDstUntaint.buf, ptr @main.sliceSrcUntaint.buf, i64 2, i64 2, i64 1)
; Copy slice that might have been modified by the external @use call.
; This is a fix for https://github.com/tinygo-org/tinygo/issues/3890.
call void @use(ptr @main.sliceSrcTaint.buf)
%copy.n4 = call i64 @runtime.sliceCopy(ptr @main.sliceDstTaint.buf, ptr @main.sliceSrcTaint.buf, i64 2, i64 2, i64 1)
; Test that copying from or into external buffers works correctly.
; These copy operations must be done at runtime.
; https://github.com/tinygo-org/tinygo/issues/4895
%copy.n5 = call i64 @runtime.sliceCopy(ptr @main.sliceDstExternal1.buf, ptr @main.sliceSrcExternal1.buf, i64 2, i64 2, i64 1)
%copy.n6 = call i64 @runtime.sliceCopy(ptr @main.sliceDstExternal2.buf, ptr @main.sliceSrcExternal2.buf, i64 2, i64 2, i64 1)
ret void
}
-42
View File
@@ -1,42 +0,0 @@
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64--linux"
@main.sliceSrcTaint.buf = internal global [2 x i8] c"cd"
@main.sliceDstTaint.buf = internal global [2 x i8] zeroinitializer
@main.sliceSrcExternal1.buf = external global [2 x i8]
@main.sliceDstExternal1.buf = internal global [2 x i8] zeroinitializer
@main.sliceSrcExternal2.buf = internal global [2 x i8] zeroinitializer
@main.sliceDstExternal2.buf = external global [2 x i8]
declare i64 @runtime.sliceCopy(ptr, ptr, i64, i64, i64) unnamed_addr
declare void @runtime.printuint8(i8) local_unnamed_addr
declare void @runtime.printint16(i16) local_unnamed_addr
declare void @use(ptr) local_unnamed_addr
define void @runtime.initAll() unnamed_addr {
entry:
call void @use(ptr @main.sliceSrcTaint.buf)
%copy.n4 = call i64 @runtime.sliceCopy(ptr @main.sliceDstTaint.buf, ptr @main.sliceSrcTaint.buf, i64 2, i64 2, i64 1)
%copy.n5 = call i64 @runtime.sliceCopy(ptr @main.sliceDstExternal1.buf, ptr @main.sliceSrcExternal1.buf, i64 2, i64 2, i64 1)
%copy.n6 = call i64 @runtime.sliceCopy(ptr @main.sliceDstExternal2.buf, ptr @main.sliceSrcExternal2.buf, i64 2, i64 2, i64 1)
ret void
}
define void @main() unnamed_addr {
entry:
call void @runtime.printuint8(i8 3)
call void @runtime.printuint8(i8 3)
call void @runtime.printint16(i16 5)
call void @runtime.printint16(i16 5)
call void @runtime.printuint8(i8 97)
%sliceDstTaint.val = load i8, ptr @main.sliceDstTaint.buf, align 1
call void @runtime.printuint8(i8 %sliceDstTaint.val)
%sliceDstExternal1.val = load i8, ptr @main.sliceDstExternal1.buf, align 1
call void @runtime.printuint8(i8 %sliceDstExternal1.val)
%sliceDstExternal2.val = load i8, ptr @main.sliceDstExternal2.buf, align 1
call void @runtime.printuint8(i8 %sliceDstExternal2.val)
ret void
}