all: introduce a temperature type

This type should be used whenever a sensor (or actuator?) works with a
temperature. For example, this commit changes the signature:

    ReadTemperature() (int32, error)

to the following:

    ReadTemperature() (drivers.Temperature, error)

I believe this is much clearer in intent. It also makes it trivial to
introduce common conversions. For example, there are already Celsius()
and Fahrenheit() methods to convert to the given units, as a floating
point. More units could be added as needed, for example a CelsiusInt().

CHANGELOG.md

@@ -1,3 +1,28 @@
+0.17.1
+---
+- To correct an error in the release process. Same as 0.17.0.
+
+0.17.0
+---
+- **new devices**
+    - rtl8720dn: add support for rtl8720dn
+    - sdcard: add support for spi sdcard driver, along with fatfs
+- **enhancements**
+    - apa102: use 4-byte buffer to improve speed
+    - bmi160: avoid heap allocations
+    - ili9341: add standard SPI driver
+    - wifinina
+        - avoid fmt package
+        - Fix RSSI command for WiFiNINA + Print current SSID + Wait for correct time before printing it out + Cleanup
+    - ws2812
+        - rename the pin to ws2812
+        - add tag for nrf52833
+        - Disable interrupts before sending ws2812 data
+        - add support for qtpy and atsame5x
+- **core**
+    - modules: switch to use tinygo-org version of tinyfs package
+    - all: use machine.Serial as the default output
+
 0.16.0
 ---
 - **new devices**

Makefile

@@ -2,7 +2,7 @@
 clean:
 	@rm -rf build
 
-FMT_PATHS = ./*.go ./examples/**/*.go
+FMT_PATHS = ./
 
 fmt-check:
 	@unformatted=$$(gofmt -l $(FMT_PATHS)); [ -z "$$unformatted" ] && exit 0; echo "Unformatted:"; for fn in $$unformatted; do echo "  $$fn"; done; exit 1
@@ -75,6 +75,8 @@ smoke-test:
 	@md5sum ./build/test.hex
 	tinygo build -size short -o ./build/test.hex -target=xiao ./examples/ili9341/scroll
 	@md5sum ./build/test.hex
+	tinygo build -size short -o ./build/test.hex -target=pyportal ./examples/ili9341/slideshow
+	@md5sum ./build/test.hex
 	tinygo build -size short -o ./build/test.hex -target=circuitplay-express ./examples/lis3dh/main.go
 	@md5sum ./build/test.hex
 	tinygo build -size short -o ./build/test.hex -target=microbit ./examples/lsm303agr/main.go
@@ -197,15 +199,21 @@ endif
 	@md5sum ./build/test.hex
 	tinygo build -size short -o ./build/test.hex -target=feather-m4 ./examples/sdcard/tinyfs/
 	@md5sum ./build/test.hex
+	tinygo build -size short -o ./build/test.hex -target=wioterminal ./examples/rtl8720dn/webclient/
+	@md5sum ./build/test.hex
+	tinygo build -size short -o ./build/test.hex -target=wioterminal ./examples/rtl8720dn/webserver/
+	@md5sum ./build/test.hex
+	tinygo build -size short -o ./build/test.hex -target=wioterminal ./examples/rtl8720dn/mqttsub/
+	@md5sum ./build/test.hex
 
 DRIVERS = $(wildcard */)
 NOTESTS = build examples flash semihosting pcd8544 shiftregister st7789 microphone mcp3008 gps microbitmatrix \
 		hcsr04 ssd1331 ws2812 thermistor apa102 easystepper ssd1351 ili9341 wifinina shifter hub75 \
 		hd44780 buzzer ssd1306 espat l9110x st7735 bmi160 l293x dht keypad4x4 max72xx p1am tone tm1637 \
-		pcf8563 mcp2515 servo sdcard
+		pcf8563 mcp2515 servo sdcard rtl8720dn image cmd
 TESTS = $(filter-out $(addsuffix /%,$(NOTESTS)),$(DRIVERS))
 
 unit-test:
-	@go test -v $(addprefix ./,$(TESTS)) 
+	@go test -v . $(addprefix ./,$(TESTS)) 
 
 test: clean fmt-check unit-test smoke-test

README.md

@@ -52,7 +52,7 @@ func main() {
 
 ## Currently supported devices
 
-The following 66 devices are supported.
+The following 67 devices are supported.
 
 | Device Name | Interface Type |
 |----------|-------------|
@@ -100,6 +100,7 @@ The following 66 devices are supported.
 | [PCD8544 display](http://eia.udg.edu/~forest/PCD8544_1.pdf) | SPI |
 | [PCF8563 real time clock](https://www.nxp.com/docs/en/data-sheet/PCF8563.pdf) | I2C |
 | [Resistive Touchscreen (4-wire)](http://ww1.microchip.com/downloads/en/Appnotes/doc8091.pdf) | GPIO |
+| [RTL8720DN 2.4G/5G Dual Bands Wireless and BLE5.0](https://www.seeedstudio.com/Realtek8720DN-2-4G-5G-Dual-Bands-Wireless-and-BLE5-0-Combo-Module-p-4442.html) | UART |
 | [Semihosting](https://wiki.segger.com/Semihosting) | Debug |
 | [Servo](https://learn.sparkfun.com/tutorials/hobby-servo-tutorial/all) | PWM |
 | [Shift register (PISO)](https://en.wikipedia.org/wiki/Shift_register#Parallel-in_serial-out_\(PISO\)) | GPIO |

adt7410/adt7410.go

@@ -60,19 +60,8 @@ func (d *Device) Connected() bool {
 }
 
 // ReadTemperature returns the temperature in celsius milli degrees (°C/1000)
-func (d *Device) ReadTemperature() (temperature int32, err error) {
-	return (int32(d.readUint16(RegTempValueMSB)) * 1000) / 128, nil
-}
-
-// ReadTempC returns the value in the temperature value register, in Celsius.
-func (d *Device) ReadTempC() float32 {
-	t := d.readUint16(RegTempValueMSB)
-	return float32(int(t)) / 128.0
-}
-
-// ReadTempF returns the value in the temperature value register, in Fahrenheit.
-func (d *Device) ReadTempF() float32 {
-	return d.ReadTempC()*1.8 + 32.0
+func (d *Device) ReadTemperature() (temperature drivers.Temperature, err error) {
+	return (drivers.Temperature(d.readUint16(RegTempValueMSB)) * 1000) / 128, nil
 }
 
 func (d *Device) writeByte(reg uint8, data byte) {

bme280/bme280.go

@@ -114,14 +114,14 @@ func (d *Device) Reset() {
 }
 
 // ReadTemperature returns the temperature in celsius milli degrees (°C/1000)
-func (d *Device) ReadTemperature() (int32, error) {
+func (d *Device) ReadTemperature() (drivers.Temperature, error) {
 	data, err := d.readData()
 	if err != nil {
 		return 0, err
 	}
 
 	temp, _ := d.calculateTemp(data)
-	return temp, nil
+	return drivers.Temperature(temp), nil
 }
 
 // ReadPressure returns the pressure in milli pascals mPa

bmi160/bmi160.go

@@ -81,7 +81,7 @@ func (d *DeviceSPI) Reset() error {
 }
 
 // ReadTemperature returns the temperature in celsius milli degrees (°C/1000).
-func (d *DeviceSPI) ReadTemperature() (temperature int32, err error) {
+func (d *DeviceSPI) ReadTemperature() (temperature drivers.Temperature, err error) {
 	data := d.buf[:3]
 	data[0] = 0x80 | reg_TEMPERATURE_0
 	data[1] = 0
@@ -109,7 +109,7 @@ func (d *DeviceSPI) ReadTemperature() (temperature int32, err error) {
 	//    rawTemperature * 1000 * 64 / 0x8000 + 23000
 	//    rawTemperature * 64000 / 0x8000 + 23000
 	//    rawTemperature * 125 / 64 + 23000
-	temperature = int32(rawTemperature)*125/64 + 23000
+	temperature = drivers.Temperature(rawTemperature)*125/64 + 23000
 	return
 }
 

bmp180/bmp180.go

@@ -81,14 +81,14 @@ func (d *Device) Configure() {
 }
 
 // ReadTemperature returns the temperature in celsius milli degrees (°C/1000).
-func (d *Device) ReadTemperature() (temperature int32, err error) {
+func (d *Device) ReadTemperature() (temperature drivers.Temperature, err error) {
 	rawTemp, err := d.rawTemp()
 	if err != nil {
 		return
 	}
 	b5 := d.calculateB5(rawTemp)
 	t := (b5 + 8) >> 4
-	return 100 * t, nil
+	return drivers.Temperature(100 * t), nil
 }
 
 // ReadPressure returns the pressure in milli pascals (mPa).

bmp280/bmp280.go

@@ -132,7 +132,7 @@ func (d *Device) PrintCali() {
 }
 
 // ReadTemperature returns the temperature in celsius milli degrees (°C/1000).
-func (d *Device) ReadTemperature() (temperature int32, err error) {
+func (d *Device) ReadTemperature() (temperature drivers.Temperature, err error) {
 	data, err := d.readData(REG_TEMP, 3)
 	if err != nil {
 		return
@@ -150,7 +150,7 @@ func (d *Device) ReadTemperature() (temperature int32, err error) {
 
 	// Convert from degrees to milli degrees by multiplying by 10.
 	// Will output 30250 milli degrees celsius for 30.25 degrees celsius
-	temperature = 10 * ((tFine*5 + 128) >> 8)
+	temperature = drivers.Temperature(10 * ((tFine*5 + 128) >> 8))
 	return
 }
 

bmp388/bmp388.go

@@ -133,16 +133,16 @@ func (d *Device) tlinCompensate() (int64, error) {
 
 }
 
-// ReadTemperature returns the temperature in centicelsius, i.e 2426 / 100 = 24.26 C
-func (d *Device) ReadTemperature() (int32, error) {
+// ReadTemperature returns the temperature in milli degrees Celsius, i.e 24260 / 1000 = 24.26°C.
+func (d *Device) ReadTemperature() (drivers.Temperature, error) {
 
 	tlin, err := d.tlinCompensate()
 	if err != nil {
 		return 0, err
 	}
 
-	temp := (tlin * 25) / 16384
-	return int32(temp), nil
+	temp := (tlin * 125) / 8192
+	return drivers.Temperature(temp), nil
 }
 
 // ReadPressure returns the pressure in centipascals, i.e 10132520 / 100 = 101325.20 Pa

cmd/convert2bin/convert2bin.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+)
+
+// See ../../image/README.md for the usage.
+
+func main() {
+	err := run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run(args []string) error {
+	if len(args) < 2 {
+		return fmt.Errorf("usage: %s FILE")
+	}
+
+	b, err := ioutil.ReadFile(args[1])
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("const %s = \"\" +\n", strings.Replace(args[1], ".", "_", -1))
+
+	i := 0
+	max := 32
+	for i = 0; i < len(b); i++ {
+		bb := b[i]
+		if (i % max) == 0 {
+			fmt.Printf("	\"")
+		}
+		fmt.Printf("\\x%02X", bb)
+		if (i%max) == max-1 && i != len(b)-1 {
+			fmt.Printf("\" + \n")
+		}
+	}
+	if (i % max) < max-1 {
+		fmt.Printf("\"\n")
+	}
+
+	return nil
+}

dht/constants.go

@@ -35,18 +35,6 @@ func (d DeviceType) extractData(buf []byte) (temp int16, hum uint16) {
 	return
 }
 
-// Celsius and Fahrenheit temperature scales
-type TemperatureScale uint8
-
-func (t TemperatureScale) convertToFloat(temp int16) float32 {
-	if t == C {
-		return float32(temp) / 10
-	} else {
-		// Fahrenheit
-		return float32(temp)*(9.0/50.) + 32.
-	}
-}
-
 // All functions return ErrorCode instance as error. This class can be used for more efficient error processing
 type ErrorCode uint8
 
@@ -57,9 +45,6 @@ const (
 	DHT11 DeviceType = iota
 	DHT22
 
-	C TemperatureScale = iota
-	F
-
 	ChecksumError ErrorCode = iota
 	NoSignalError
 	NoDataError

dht/highfreq.go

@@ -1,4 +1,5 @@
-// +build mimxrt1062  stm32f405  atsamd51  stm32f103xx  k210  stm32f407
+//go:build mimxrt1062 || stm32f405 || atsamd51 || stm32f103xx || k210 || stm32f407
+// +build mimxrt1062 stm32f405 atsamd51 stm32f103xx k210 stm32f407
 
 package dht // import "tinygo.org/x/drivers/dht"
 

dht/lowfreq.go

@@ -1,4 +1,5 @@
-// +build arduino atmega1284p nrf52840 digispark nrf52 arduino_nano nrf51 atsamd21 fe310 arduino_nano33 circuitplay_express arduino_mega2560
+//go:build !mimxrt1062 && !stm32f405 && !atsamd51 && !stm32f103xx && !k210 && !stm32f407
+// +build !mimxrt1062,!stm32f405,!atsamd51,!stm32f103xx,!k210,!stm32f407
 
 package dht // import "tinygo.org/x/drivers/dht"
 

dht/thermometer.go

@@ -9,14 +9,15 @@ package dht // import "tinygo.org/x/drivers/dht"
 import (
 	"machine"
 	"time"
+
+	"tinygo.org/x/drivers"
 )
 
 // DummyDevice provides a basic interface for DHT devices.
 type DummyDevice interface {
 	ReadMeasurements() error
 	Measurements() (temperature int16, humidity uint16, err error)
-	Temperature() (int16, error)
-	TemperatureFloat(scale TemperatureScale) (float32, error)
+	Temperature() (drivers.Temperature, error)
 	Humidity() (uint16, error)
 	HumidityFloat() (float32, error)
 }
@@ -49,23 +50,13 @@ func (t *device) ReadMeasurements() error {
 	return err
 }
 
-// Getter for temperature. Temperature method returns temperature as it is sent by device.
-// The temperature is measured temperature in Celsius multiplied by 10.
+// Getter for temperature. The temperature is returned in milli degrees Celsius.
 // If no successful measurements for this device was performed, returns UninitializedDataError.
-func (t *device) Temperature() (int16, error) {
+func (t *device) Temperature() (drivers.Temperature, error) {
 	if !t.initialized {
 		return 0, UninitializedDataError
 	}
-	return t.temperature, nil
-}
-
-// Getter for temperature. TemperatureFloat returns temperature in a given scale.
-// If no successful measurements for this device was performed, returns UninitializedDataError.
-func (t *device) TemperatureFloat(scale TemperatureScale) (float32, error) {
-	if !t.initialized {
-		return 0, UninitializedDataError
-	}
-	return scale.convertToFloat(t.temperature), nil
+	return drivers.Temperature(t.temperature) * 100, nil
 }
 
 // Getter for humidity. Humidity returns humidity as it is sent by device.

dht/timesafethermometer.go

@@ -9,6 +9,8 @@ package dht // import "tinygo.org/x/drivers/dht"
 import (
 	"machine"
 	"time"
+
+	"tinygo.org/x/drivers"
 )
 
 // Device interface provides main functionality of the DHTXX sensors.
@@ -35,10 +37,9 @@ func (m *managedDevice) Measurements() (temperature int16, humidity uint16, err
 	return m.t.Measurements()
 }
 
-// Getter for temperature. Temperature method returns temperature as it is sent by device.
-// The temperature is measured temperature in Celsius multiplied by 10.
+// Getter for temperature. The temperature is returned in milli degrees Celsius.
 // Depending on the UpdatePolicy of the device may update cached measurements.
-func (m *managedDevice) Temperature() (temp int16, err error) {
+func (m *managedDevice) Temperature() (temp drivers.Temperature, err error) {
 	err = m.checkForUpdateOnDataRequest()
 	if err != nil {
 		return 0, err
@@ -64,16 +65,6 @@ func (m *managedDevice) checkForUpdateOnDataRequest() (err error) {
 	return err
 }
 
-// Getter for temperature. TemperatureFloat returns temperature in a given scale.
-// Depending on the UpdatePolicy of the device may update cached measurements.
-func (m *managedDevice) TemperatureFloat(scale TemperatureScale) (float32, error) {
-	err := m.checkForUpdateOnDataRequest()
-	if err != nil {
-		return 0, err
-	}
-	return m.t.TemperatureFloat(scale)
-}
-
 // Getter for humidity. Humidity returns humidity as it is sent by device.
 // The humidity is measured in percentages multiplied by 10.
 // Depending on the UpdatePolicy of the device may update cached measurements.

ds3231/ds3231.go

@@ -134,13 +134,13 @@ func (d *Device) ReadTime() (dt time.Time, err error) {
 }
 
 // ReadTemperature returns the temperature in millicelsius (mC)
-func (d *Device) ReadTemperature() (int32, error) {
+func (d *Device) ReadTemperature() (drivers.Temperature, error) {
 	data := make([]uint8, 2)
 	err := d.bus.ReadRegister(uint8(d.Address), REG_TEMP, data)
 	if err != nil {
 		return 0, err
 	}
-	return int32(data[0])*1000 + int32((data[1]>>6)*25)*10, nil
+	return drivers.Temperature(int32(data[0])*1000 + int32((data[1]>>6)*25)*10), nil
 }
 
 // uint8ToBCD converts a byte to BCD for the DS3231

examples/adt7410/main.go

@@ -19,8 +19,8 @@ func main() {
 	sensor.Configure()
 
 	for {
-		temp := sensor.ReadTempF()
-		fmt.Printf("temperature: %f\r\n", temp)
+		temp, _ := sensor.ReadTemperature()
+		fmt.Printf("temperature: %f°C\r\n", temp.Celsius())
 		time.Sleep(time.Second)
 	}
 

examples/bme280/main.go

@@ -22,7 +22,7 @@ func main() {
 
 	for {
 		temp, _ := sensor.ReadTemperature()
-		println("Temperature:", strconv.FormatFloat(float64(temp)/1000, 'f', 2, 64), "°C")
+		println("Temperature:", strconv.FormatFloat(float64(temp.Celsius()), 'f', 2, 64), "°C")
 		press, _ := sensor.ReadPressure()
 		println("Pressure:", strconv.FormatFloat(float64(press)/100000, 'f', 2, 64), "hPa")
 		hum, _ := sensor.ReadHumidity()

examples/bmi160/main.go

@@ -27,7 +27,7 @@ func main() {
 			println("Error reading temperature", err)
 			continue
 		}
-		fmt.Printf("Temperature: %.2f °C\n", float32(t)/1000)
+		fmt.Printf("Temperature: %.2f °C\n", t.Celsius())
 
 		accelX, accelY, accelZ, err := sensor.ReadAcceleration()
 		if err != nil {

examples/bmp180/main.go

@@ -22,7 +22,7 @@ func main() {
 
 	for {
 		temp, _ := sensor.ReadTemperature()
-		println("Temperature:", float32(temp)/1000, "°C")
+		println("Temperature:", temp.Celsius(), "°C")
 
 		pressure, _ := sensor.ReadPressure()
 		println("Pressure", float32(pressure)/100000, "hPa")

examples/bmp280/main.go

@@ -30,7 +30,7 @@ func main() {
 			println("Error reading temperature")
 		}
 		// Temperature in degrees Celsius
-		fmt.Printf("Temperature: %.2f °C\n", float32(t)/1000)
+		fmt.Printf("Temperature: %.2f °C\n", t.Celsius())
 
 		p, err := sensor.ReadPressure()
 		if err != nil {

examples/bmp388/main.go

@@ -38,13 +38,13 @@ func main() {
 	}
 
 	for {
-		temp, err := sensor.ReadTemperature() // returns the temperature in centicelsius
+		temp, err := sensor.ReadTemperature() // returns the temperature in millicelsius
 		press, err := sensor.ReadPressure()   // returns the pressure in centipascals
 
 		if err != nil {
 			println(err)
 		} else {
-			println("Temperature: " + strconv.FormatInt(int64(temp), 10) + " cC")
+			println("Temperature:", temp/1000, "C")
 			println("Pressure:    " + strconv.FormatInt(int64(press), 10) + " cPa\n")
 		}
 

examples/ds3231/main.go

@@ -38,7 +38,7 @@ func main() {
 			fmt.Printf("Date: %d/%s/%02d %02d:%02d:%02d \r\n", dt.Year(), dt.Month(), dt.Day(), dt.Hour(), dt.Minute(), dt.Second())
 		}
 		temp, _ := rtc.ReadTemperature()
-		fmt.Printf("Temperature: %.2f °C \r\n", float32(temp)/1000)
+		fmt.Printf("Temperature: %.2f °C \r\n", temp.Celsius())
 
 		time.Sleep(time.Second * 1)
 	}

examples/ili9341/basic/atsamd21.go

@@ -1,3 +1,4 @@
+//go:build atsamd21
 // +build atsamd21
 
 package main

examples/ili9341/basic/pyportal.go

@@ -1,3 +1,4 @@
+//go:build pyportal
 // +build pyportal
 
 package main

examples/ili9341/basic/wioterminal.go

@@ -1,3 +1,4 @@
+//go:build wioterminal
 // +build wioterminal
 
 package main

examples/ili9341/pyportal_boing/main.go

@@ -10,25 +10,6 @@ import (
 	"tinygo.org/x/drivers/ili9341"
 )
 
-const (
-	BALLWIDTH  = 136
-	BALLHEIGHT = 100
-)
-
-const (
-	SCREENHEIGHT = 240
-	SCREENWIDTH  = 320
-)
-
-const (
-	invBGCOLOR    = 0x75AD
-	invGRIDCOLOR  = 0x15A8
-	invBGSHADOW   = 0x8552
-	invGRIDSHADOW = 0x0C60
-	invRED        = 0x00F8
-	invWHITE      = 0xFFFF
-)
-
 const (
 	BGCOLOR    = 0xAD75
 	GRIDCOLOR  = 0xA815
@@ -44,12 +25,7 @@ const (
 )
 
 var (
-	dbg5 = machine.D5
-	dbg6 = machine.D6
-)
-
-var (
-	frameBuffer = [2][(BALLHEIGHT + 8) * (BALLWIDTH + 8)]uint16{}
+	frameBuffer = [(graphics.BALLHEIGHT + 8) * (graphics.BALLWIDTH + 8) * 2]uint8{}
 
 	startTime int64
 	frame     int64
@@ -95,18 +71,14 @@ func main() {
 	balloldx = ballx
 	balloldy = bally // Prior ball position
 
-	var bufIdx int8 = 0
-
 	for {
-		dbg5.High()
-		bufIdx = 1 - bufIdx
 
 		balloldx = ballx // Save prior position
 		balloldy = bally
 		ballx += ballvx // Update position
 		bally += ballvy
 		ballvy += 0.06 // Update Y velocity
-		if (ballx <= 15) || (ballx >= SCREENWIDTH-BALLWIDTH) {
+		if (ballx <= 15) || (ballx >= graphics.SCREENWIDTH-graphics.BALLWIDTH) {
 			ballvx *= -1 // Left/right bounce
 		}
 		if bally >= YBOTTOM { // Hit ground?
@@ -128,13 +100,13 @@ func main() {
 		if int16(balloldy) < miny {
 			miny = int16(balloldy)
 		}
-		maxx = int16(ballx + BALLWIDTH - 1)
-		if int16(balloldx+BALLWIDTH-1) > maxx {
-			maxx = int16(balloldx + BALLWIDTH - 1)
+		maxx = int16(ballx + graphics.BALLWIDTH - 1)
+		if int16(balloldx+graphics.BALLWIDTH-1) > maxx {
+			maxx = int16(balloldx + graphics.BALLWIDTH - 1)
 		}
-		maxy = int16(bally + BALLHEIGHT - 1)
-		if int16(balloldy+BALLHEIGHT-1) > maxy {
-			maxy = int16(balloldy + BALLHEIGHT - 1)
+		maxy = int16(bally + graphics.BALLHEIGHT - 1)
+		if int16(balloldy+graphics.BALLHEIGHT-1) > maxy {
+			maxy = int16(balloldy + graphics.BALLHEIGHT - 1)
 		}
 
 		width = maxx - minx + 1
@@ -148,13 +120,13 @@ func main() {
 			ballframe -= 14
 		}
 
-		//// Set 7 palette entries to white, 7 to red, based on frame number.
-		//// This makes the ball spin
+		// Set 7 palette entries to white, 7 to red, based on frame number.
+		// This makes the ball spin
 		for i := 0; i < 14; i++ {
 			if (int(ballframe)+i)%14 < 7 {
-				palette[i+2] = invWHITE
+				palette[i+2] = WHITE
 			} else {
-				palette[i+2] = invRED
+				palette[i+2] = RED
 			} // Palette entries 0 and 1 aren't used (clear and shadow, respectively)
 		}
 
@@ -164,100 +136,62 @@ func main() {
 		by := miny - int16(bally) // Y relative to ball bitmap (can be negative)
 		bgx := minx               // X relative to background bitmap (>= 0)
 		bgy := miny               // Y relative to background bitmap (>= 0)
-		//var bufIdx int8 = 0
+		var bx1, bgx1 int16       // Loop counters and working vars
+		var p uint8               // 'packed' value of 2 ball pixels
+		var bufIdx int8 = 0
 
 		//tft.setAddrWindow(minx, miny, width, height)
-		//dbg5.Low()
-		dbg6.High()
-		//fmt.Printf("%d < %d < %d < %d\r\n", by, 0, BALLHEIGHT, height)
 
-		y := 0
-		if by < 0 {
-			max := -1 * int(by)
-			for y = 0; y < max; y++ { // For each row...
-				var bgidxBase = int(bgy)*(SCREENWIDTH) + int(bgx)
-				var yBase = y * int(width)
-				for x := 0; x < int(width); x++ {
-					frameBuffer[bufIdx][yBase+x] = graphics.Background[bgidxBase+x]
-				}
-				bgy++
-			}
-		}
-
-		y2 := y
-		max := 0
-		if bx < 0 {
-			max = -1 * int(bx)
-			bgy2 := bgy
-			for y = y2; y < y2+int(BALLHEIGHT); y++ { // For each row...
-				var bgidxBase = int(bgy2)*(SCREENWIDTH) + int(bgx)
-				var yBase = y * int(width)
-				//fmt.Printf("- %d %d %d %d %d %d\r\n", bgy, y, bgx, max, yBase, bgidxBase)
-				for x := 0; x < int(max); x++ {
-					//fmt.Printf("  %d %d\r\n", yBase+x, bgidxBase+x)
-					frameBuffer[bufIdx][yBase+x] = graphics.Background[bgidxBase+x]
-				}
-				bgy2++
-			}
-			//fmt.Printf("(%d, %d) - (%d, %d)\r\n", bx, 0, -1, BALLHEIGHT-1)
-		}
-
-		{
-			bgy2 := bgy
-			//fmt.Printf("(%d, %d) - (%d, %d)\r\n", 0, 0, BALLWIDTH-1, BALLHEIGHT-1)
-			for y = y2; y < y2+int(BALLHEIGHT); y++ { // For each row...
-				var bgidxBase = int(bgy2)*(SCREENWIDTH) + int(bgx)
-				var byBase = (y - y2) * BALLWIDTH
-				var yBase = y * int(width)
-				for x := max; x < int(BALLWIDTH)+max; x++ {
-					//fmt.Printf("%d %d %d %d\r\n", byBase, x, bgidxBase, yBase)
-					//time.Sleep(1 * time.Millisecond)
+		for y := 0; y < int(height); y++ { // For each row...
+			//destPtr = &renderbuf[bufIdx][0];
+			bx1 = bx   // Need to keep the original bx and bgx values,
+			bgx1 = bgx // so copies of them are made here (and changed in loop below)
+			for x := 0; x < int(width); x++ {
+				var bgidx = int(bgy)*(graphics.SCREENWIDTH/8) + int(bgx1/8)
+				if (bx1 >= 0) && (bx1 < graphics.BALLWIDTH) && // Is current pixel row/column
+					(by >= 0) && (by < graphics.BALLHEIGHT) { // inside the ball bitmap area?
 					// Yes, do ball compositing math...
-					c = uint16(graphics.Ball[int(byBase)+x-max]) // Get packed value (2 pixels)
-
+					p = graphics.Ball[int(by*(graphics.BALLWIDTH/2))+int(bx1/2)] // Get packed value (2 pixels)
+					if (bx1 & 1) != 0 {
+						c = uint16(p & 0xF)
+					} else {
+						c = uint16(p >> 4)
+					} // Unpack high or low nybble
 					if c == 0 { // Outside ball - just draw grid
-						c = graphics.Background[bgidxBase+x]
+						if graphics.Background[bgidx]&(0x80>>(bgx1&7)) != 0 {
+							c = GRIDCOLOR
+						} else {
+							c = BGCOLOR
+						}
 					} else if c > 1 { // In ball area...
 						c = palette[c]
 					} else { // In shadow area...
-						c = graphics.BackgroundShadow[bgidxBase+x]
+						if graphics.Background[bgidx]&(0x80>>(bgx1&7)) != 0 {
+							c = GRIDSHADOW
+						} else {
+							c = BGSHADOW
+						}
+					}
+				} else { // Outside ball bitmap, just draw background bitmap...
+					if graphics.Background[bgidx]&(0x80>>(bgx1&7)) != 0 {
+						c = GRIDCOLOR
+					} else {
+						c = BGCOLOR
 					}
-					frameBuffer[bufIdx][yBase+x] = c
 				}
-				bgy2++
+				frameBuffer[(y*int(width)+x)*2] = byte(c >> 8)
+				frameBuffer[(y*int(width)+x)*2+1] = byte(c)
+				bx1++ // Increment bitmap position counters (X axis)
+				bgx1++
 			}
+			//tft.dmaWait(); // Wait for prior line to complete
+			//tft.writePixels(&renderbuf[bufIdx][0], width, false); // Non-blocking write
+			bufIdx = 1 - bufIdx
+			by++ // Increment bitmap position counters (Y axis)
+			bgy++
 		}
 
-		{
-			bgy2 := bgy
-			for y = y2; y < y2+int(BALLHEIGHT); y++ { // For each row...
-				var bgidxBase = int(bgy2)*(SCREENWIDTH) + int(bgx)
-				var yBase = y * int(width)
-				//fmt.Printf("+ %d %d %d %d %d\r\n", bgy, y, bgx, yBase, bgidxBase)
-				for x := int(BALLWIDTH) + max; x < int(width); x++ {
-					frameBuffer[bufIdx][yBase+x] = graphics.Background[bgidxBase+x]
-				}
-				bgy2++
-			}
-		}
-
-		y = y2 + int(BALLHEIGHT)
-		bgy += BALLHEIGHT
-		{
-			for ; y < int(height); y++ { // For each row...
-				//destPtr = &renderbuf[bufIdx][0];
-				var bgidxBase = int(bgy)*(SCREENWIDTH) + int(bgx)
-				var yBase = y * int(width)
-				for x := 0; x < int(width); x++ {
-					frameBuffer[bufIdx][yBase+x] = graphics.Background[bgidxBase+x]
-				}
-				bgy++
-			}
-		}
-		dbg6.Low()
-
-		display.DrawRGBBitmap(minx, miny, frameBuffer[bufIdx][:width*height], width, height)
-		//time.Sleep(10 * time.Millisecond)
+		display.DrawRGBBitmap8(minx, miny, frameBuffer[:width*height*2], width, height)
 
 		// Show approximate frame rate
 		frame++
@@ -272,13 +206,23 @@ func main() {
 
 func DrawBackground() {
 	w, h := display.Size()
-	var bufIdx int8 = 0
-	for j := 0; j < int(h); j++ {
-		bufIdx = 1 - bufIdx
-		for k := 0; k < int(w); k++ {
-			frameBuffer[bufIdx][k] = graphics.Background[j*int(w)+k]
+	byteWidth := (w + 7) / 8 // Bitmap scanline pad = whole byte
+	var b uint8
+	for j := int16(0); j < h; j++ {
+		for k := int16(0); k < w; k++ {
+			if k&7 > 0 {
+				b <<= 1
+			} else {
+				b = graphics.Background[j*byteWidth+k/8]
+			}
+			if b&0x80 == 0 {
+				frameBuffer[2*k] = byte(BGCOLOR >> 8)
+				frameBuffer[2*k+1] = byte(BGCOLOR & 0xFF)
+			} else {
+				frameBuffer[2*k] = byte(GRIDCOLOR >> 8)
+				frameBuffer[2*k+1] = byte(GRIDCOLOR & 0xFF)
+			}
 		}
-		display.DrawRGBBitmap(0, int16(j), frameBuffer[bufIdx][0:w], w, 1)
-		time.Sleep(1 * time.Millisecond)
+		display.DrawRGBBitmap8(0, j, frameBuffer[0:w*2], w, 1)
 	}
 }

examples/ili9341/pyportal_boing/pyportal.go

@@ -1,3 +1,4 @@
+//go:build pyportal
 // +build pyportal
 
 package main

examples/ili9341/pyportal_boing/wioterminal.go

@@ -1,3 +1,4 @@
+//go:build wioterminal
 // +build wioterminal
 
 package main

examples/ili9341/scroll/atsamd21.go

@@ -1,3 +1,4 @@
+//go:build atsamd21
 // +build atsamd21
 
 package main

examples/ili9341/scroll/pyportal.go

@@ -1,3 +1,4 @@
+//go:build pyportal
 // +build pyportal
 
 package main

examples/ili9341/scroll/wioterminal.go

@@ -1,3 +1,4 @@
+//go:build wioterminal
 // +build wioterminal
 
 package main

examples/ili9341/slideshow/README.md

@@ -0,0 +1,18 @@
+# examples/ili9341/slideshow
+
+![](./slideshow.jpg)
+
+This example uses the image package for TinyGo to display png and jpeg images.  
+
+## How to create an image
+
+The following program will output an image binary like the one in [images.go](./images.go).  
+
+```
+go run ./examples/ili9341/slideshow/convert2bin ./path/to/png_or_jpg.png
+```
+
+## Notes
+
+Displaying a 320x240 png or jpeg often requires more than 50KB of memory.  
+The examples include samd21 settings, but if you run them as is, you will get a memory size error.  

examples/ili9341/slideshow/atsamd21.go

@@ -0,0 +1,30 @@
+//go:build atsamd21
+// +build atsamd21
+
+package main
+
+import (
+	"machine"
+
+	"tinygo.org/x/drivers/ili9341"
+)
+
+var (
+	display = ili9341.NewSPI(
+		machine.SPI0,
+		machine.D0,
+		machine.D1,
+		machine.D2,
+	)
+
+	backlight = machine.D3
+)
+
+func init() {
+	machine.SPI0.Configure(machine.SPIConfig{
+		SCK:       machine.SPI0_SCK_PIN,
+		SDO:       machine.SPI0_SDO_PIN,
+		SDI:       machine.SPI0_SDI_PIN,
+		Frequency: 24000000,
+	})
+}

examples/ili9341/slideshow/main.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"fmt"
+	"image/color"
+	"machine"
+	"strings"
+	"time"
+
+	"tinygo.org/x/drivers/ili9341"
+	"tinygo.org/x/drivers/image/jpeg"
+	"tinygo.org/x/drivers/image/png"
+)
+
+var (
+	black = color.RGBA{0, 0, 0, 255}
+	white = color.RGBA{255, 255, 255, 255}
+	red   = color.RGBA{255, 0, 0, 255}
+	blue  = color.RGBA{0, 0, 255, 255}
+	green = color.RGBA{0, 255, 0, 255}
+)
+
+func main() {
+	err := run()
+	for err != nil {
+		errorMessage(err)
+	}
+}
+
+func run() error {
+	backlight.Configure(machine.PinConfig{machine.PinOutput})
+
+	display.Configure(ili9341.Config{})
+
+	width, height := display.Size()
+	if width < 320 || height < 240 {
+		display.SetRotation(ili9341.Rotation270)
+	}
+
+	display.FillScreen(black)
+	backlight.High()
+
+	for {
+		err := drawJpeg(display)
+		if err != nil {
+			return err
+		}
+		time.Sleep(time.Second)
+
+		err = drawPng(display)
+		if err != nil {
+			return err
+		}
+		time.Sleep(time.Second)
+	}
+
+	return nil
+}
+
+// Define the buffer required for the callback. In most cases, this setting
+// should be sufficient.  For jpeg, the callback will always be called every
+// 3*8*8*4 pix. png will be called every line, i.e. every width pix.
+var buffer [3 * 8 * 8 * 4]uint16
+
+func drawPng(display *ili9341.Device) error {
+	p := strings.NewReader(pngImage)
+	png.SetCallback(buffer[:], func(data []uint16, x, y, w, h, width, height int16) {
+		err := display.DrawRGBBitmap(x, y, data[:w*h], w, h)
+		if err != nil {
+			errorMessage(fmt.Errorf("error drawPng: %s", err))
+		}
+	})
+
+	return png.Decode(p)
+}
+
+func drawJpeg(display *ili9341.Device) error {
+	p := strings.NewReader(jpegImage)
+	jpeg.SetCallback(buffer[:], func(data []uint16, x, y, w, h, width, height int16) {
+		err := display.DrawRGBBitmap(x, y, data[:w*h], w, h)
+		if err != nil {
+			errorMessage(fmt.Errorf("error drawJpeg: %s", err))
+		}
+	})
+
+	return jpeg.Decode(p)
+}
+
+func errorMessage(err error) {
+	for {
+		fmt.Printf("%s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}

examples/ili9341/slideshow/pyportal.go

@@ -0,0 +1,23 @@
+//go:build pyportal
+// +build pyportal
+
+package main
+
+import (
+	"machine"
+
+	"tinygo.org/x/drivers/ili9341"
+)
+
+var (
+	display = ili9341.NewParallel(
+		machine.LCD_DATA0,
+		machine.TFT_WR,
+		machine.TFT_DC,
+		machine.TFT_CS,
+		machine.TFT_RESET,
+		machine.TFT_RD,
+	)
+
+	backlight = machine.TFT_BACKLIGHT
+)

examples/ili9341/slideshow/wioterminal.go

@@ -0,0 +1,30 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"machine"
+
+	"tinygo.org/x/drivers/ili9341"
+)
+
+var (
+	display = ili9341.NewSPI(
+		machine.SPI3,
+		machine.LCD_DC,
+		machine.LCD_SS_PIN,
+		machine.LCD_RESET,
+	)
+
+	backlight = machine.LCD_BACKLIGHT
+)
+
+func init() {
+	machine.SPI3.Configure(machine.SPIConfig{
+		SCK:       machine.LCD_SCK_PIN,
+		SDO:       machine.LCD_SDO_PIN,
+		SDI:       machine.LCD_SDI_PIN,
+		Frequency: 40000000,
+	})
+}

examples/lsm6ds3/main.go

@@ -23,8 +23,8 @@ func main() {
 		println("Acceleration:", float32(x)/1000000, float32(y)/1000000, float32(z)/1000000)
 		x, y, z = accel.ReadRotation()
 		println("Gyroscope:", float32(x)/1000000, float32(y)/1000000, float32(z)/1000000)
-		x, _ = accel.ReadTemperature()
-		println("Degrees C", float32(x)/1000, "\n\n")
+		t, _ := accel.ReadTemperature()
+		println("Degrees C", t.Celsius(), "\n\n")
 		time.Sleep(time.Millisecond * 1000)
 	}
 }

examples/lsm6dsox/main.go

@@ -0,0 +1,109 @@
+// Connects to an LSM6DSOX I2C a 6 axis Inertial Measurement Unit (IMU)
+package main
+
+import (
+	"fmt"
+	"machine"
+	"time"
+
+	"tinygo.org/x/drivers"
+	"tinygo.org/x/drivers/lsm6dsox"
+)
+
+const (
+	PLOTTER           = true
+	SHOW_ACCELERATION = false
+	SHOW_ROTATION     = true
+	SHOW_TEMPERATURE  = false
+
+	// calibration is naive, good enough for a demo: do not shake a second after flashing
+	GYRO_CALIBRATION = true
+)
+
+var cal = [...]float32{0, 0, 0}
+
+func main() {
+
+	machine.I2C0.Configure(machine.I2CConfig{})
+
+	device := lsm6dsox.New(machine.I2C0)
+	device.Configure(lsm6dsox.Configuration{
+		AccelRange:      lsm6dsox.ACCEL_2G,
+		AccelSampleRate: lsm6dsox.ACCEL_SR_104,
+		GyroRange:       lsm6dsox.GYRO_250DPS,
+		GyroSampleRate:  lsm6dsox.GYRO_SR_104,
+	})
+
+	for {
+
+		if !device.Connected() {
+			println("LSM6DSOX not connected")
+			time.Sleep(time.Second)
+			continue
+		}
+
+		// heuristic: after successful calibration the value can't be 0
+		if GYRO_CALIBRATION && cal[0] == 0 {
+			calibrateGyro(device)
+		}
+
+		ax, ay, az := device.ReadAcceleration()
+		gx, gy, gz := device.ReadRotation()
+		t, _ := device.ReadTemperature()
+
+		if PLOTTER {
+			printPlotter(ax, ay, az, gx, gy, gz, t)
+			time.Sleep(time.Millisecond * 100)
+		} else {
+			printMonitor(ax, ay, az, gx, gy, gz, t)
+			time.Sleep(time.Millisecond * 1000)
+		}
+
+	}
+
+}
+
+func calibrateGyro(device *lsm6dsox.Device) {
+	for i := 0; i < 100; i++ {
+		gx, gy, gz := device.ReadRotation()
+		cal[0] += float32(gx) / 1000000
+		cal[1] += float32(gy) / 1000000
+		cal[2] += float32(gz) / 1000000
+		time.Sleep(time.Millisecond * 10)
+	}
+	cal[0] /= 100
+	cal[1] /= 100
+	cal[2] /= 100
+}
+
+// Arduino IDE's Serial Plotter
+func printPlotter(ax, ay, az, gx, gy, gz int32, t drivers.Temperature) {
+	if SHOW_ACCELERATION {
+		fmt.Printf("AX:%f, AY:%f, AZ:%f,", axis(ax, 0), axis(ay, 0), axis(az, 0))
+	}
+	if SHOW_ROTATION {
+		fmt.Printf("GX:%f, GY:%f, GZ:%f,", axis(gx, cal[0]), axis(gy, cal[1]), axis(gz, cal[2]))
+	}
+	if SHOW_TEMPERATURE {
+		fmt.Printf("T:%f", t.Celsius())
+	}
+	println()
+}
+
+// Any Serial Monitor
+func printMonitor(ax, ay, az, gx, gy, gz, t int32) {
+	if SHOW_ACCELERATION {
+		fmt.Printf("Acceleration: %f, %f, %f\r\n", axis(ax, 0), axis(ay, 0), axis(az, 0))
+	}
+	if SHOW_ROTATION {
+		fmt.Printf("Rotation: %f, %f, %f\r\n", axis(gx, cal[0]), axis(gy, cal[1]), axis(gz, cal[2]))
+	}
+	if SHOW_TEMPERATURE {
+		fmt.Printf("Temperature C: %f\r\n", float32(t)/1000)
+	}
+	println()
+}
+
+func axis(raw int32, cal float32) float32 {
+	return float32(raw)/1000000 - cal
+}

examples/mag3110/main.go

@@ -19,7 +19,7 @@ func main() {
 		println("Magnetic readings:", x, y, z)
 
 		c, _ := mag.ReadTemperature()
-		println("Temperature:", float32(c)/1000, "°C")
+		println("Temperature:", c.Celsius(), "°C")
 
 		time.Sleep(time.Millisecond * 100)
 	}

examples/rtl8720dn/mqttclient/main.go

@@ -0,0 +1,131 @@
+// This is a sensor station that uses a RTL8720DN running on the device UART2.
+// It creates an MQTT connection that publishes a message every second
+// to an MQTT broker.
+//
+// In other words:
+// Your computer <--> USB-CDC <--> MCU <--> UART2 <--> RTL8720DN <--> Internet <--> MQTT broker.
+//
+// You must install the Paho MQTT package to build this program:
+//
+// 		go get -u github.com/eclipse/paho.mqtt.golang
+//
+// You can check that mqttpub is running successfully with the following command.
+//
+// 		mosquitto_sub -h test.mosquitto.org -t tinygo
+//
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+
+	"tinygo.org/x/drivers/net"
+	"tinygo.org/x/drivers/net/mqtt"
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+// You can override the setting with the init() in another source code.
+// func init() {
+//    ssid = "your-ssid"
+//    password = "your-password"
+//    debug = true
+//    server = "tinygo.org"
+// }
+
+var (
+	ssid     string
+	password string
+	server   string = "tcp://test.mosquitto.org:1883"
+	debug           = false
+)
+
+var buf [0x400]byte
+
+var lastRequestTime time.Time
+var conn net.Conn
+var adaptor *rtl8720dn.RTL8720DN
+
+func main() {
+	err := run()
+	for err != nil {
+		fmt.Printf("error: %s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}
+
+var (
+	topic = "tinygo"
+)
+
+func run() error {
+	rtl, err := setupRTL8720DN()
+	if err != nil {
+		return err
+	}
+	net.UseDriver(rtl)
+
+	err = rtl.ConnectToAP(ssid, password)
+	if err != nil {
+		return err
+	}
+
+	ip, subnet, gateway, err := rtl.GetIP()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("IP Address : %s\r\n", ip)
+	fmt.Printf("Mask       : %s\r\n", subnet)
+	fmt.Printf("Gateway    : %s\r\n", gateway)
+
+	rand.Seed(time.Now().UnixNano())
+
+	opts := mqtt.NewClientOptions()
+	opts.AddBroker(server).SetClientID("tinygo-client-" + randomString(10))
+
+	println("Connectng to MQTT...")
+	cl := mqtt.NewClient(opts)
+	if token := cl.Connect(); token.Wait() && token.Error() != nil {
+		failMessage(token.Error().Error())
+	}
+
+	for i := 0; ; i++ {
+		println("Publishing MQTT message...")
+		data := []byte(fmt.Sprintf(`{"e":[{"n":"hello %d","v":101}]}`, i))
+		token := cl.Publish(topic, 0, false, data)
+		token.Wait()
+		if err := token.Error(); err != nil {
+			return err
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	// Right now this code is never reached. Need a way to trigger it...
+	println("Disconnecting MQTT...")
+	cl.Disconnect(100)
+
+	println("Done.")
+
+	return nil
+}
+
+// Returns an int >= min, < max
+func randomInt(min, max int) int {
+	return min + rand.Intn(max-min)
+}
+
+// Generate a random string of A-Z chars with len = l
+func randomString(len int) string {
+	bytes := make([]byte, len)
+	for i := 0; i < len; i++ {
+		bytes[i] = byte(randomInt(65, 90))
+	}
+	return string(bytes)
+}
+
+func failMessage(msg string) {
+	for {
+		println(msg)
+		time.Sleep(1 * time.Second)
+	}
+}

examples/rtl8720dn/mqttclient/wioterminal.go

@@ -0,0 +1,74 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"device/sam"
+	"machine"
+	"runtime/interrupt"
+	"time"
+
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+var (
+	uart UARTx
+)
+
+func handleInterrupt(interrupt.Interrupt) {
+	// should reset IRQ
+	uart.Receive(byte((uart.Bus.DATA.Get() & 0xFF)))
+	uart.Bus.INTFLAG.SetBits(sam.SERCOM_USART_INT_INTFLAG_RXC)
+}
+
+func setupRTL8720DN() (*rtl8720dn.RTL8720DN, error) {
+	machine.RTL8720D_CHIP_PU.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	machine.RTL8720D_CHIP_PU.Low()
+	time.Sleep(100 * time.Millisecond)
+	machine.RTL8720D_CHIP_PU.High()
+	time.Sleep(1000 * time.Millisecond)
+	if debug {
+		waitSerial()
+	}
+
+	uart = UARTx{
+		UART: &machine.UART{
+			Buffer: machine.NewRingBuffer(),
+			Bus:    sam.SERCOM0_USART_INT,
+			SERCOM: 0,
+		},
+	}
+
+	uart.Interrupt = interrupt.New(sam.IRQ_SERCOM0_2, handleInterrupt)
+	uart.Configure(machine.UARTConfig{TX: machine.PB24, RX: machine.PC24, BaudRate: 614400})
+
+	rtl := rtl8720dn.New(uart)
+	rtl.Debug(debug)
+
+	_, err := rtl.Rpc_tcpip_adapter_init()
+	if err != nil {
+		return nil, err
+	}
+
+	return rtl, nil
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+type UARTx struct {
+	*machine.UART
+}
+
+func (u UARTx) Read(p []byte) (n int, err error) {
+	if u.Buffered() == 0 {
+		time.Sleep(1 * time.Millisecond)
+		return 0, nil
+	}
+	return u.UART.Read(p)
+}

examples/rtl8720dn/mqttsub/main.go

@@ -0,0 +1,154 @@
+// This is a sensor station that uses a RTL8720DN running on the device UART2.
+// It creates an MQTT connection that publishes a message every second
+// to an MQTT broker.
+//
+// In other words:
+// Your computer <--> USB-CDC <--> MCU <--> UART2 <--> RTL8720DN <--> Internet <--> MQTT broker.
+//
+// You must also install the Paho MQTT package to build this program:
+//
+// 		go get -u github.com/eclipse/paho.mqtt.golang
+//
+// You can check that mqttpub/mqttsub is running successfully with the following command.
+//
+// 		mosquitto_sub -h test.mosquitto.org -t tinygo/tx
+// 		mosquitto_pub -h test.mosquitto.org -t tinygo/rx -m "hello world"
+//
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"time"
+
+	"tinygo.org/x/drivers/net"
+	"tinygo.org/x/drivers/net/mqtt"
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+// You can override the setting with the init() in another source code.
+// func init() {
+//    ssid = "your-ssid"
+//    password = "your-password"
+//    debug = true
+//    server = "tinygo.org"
+// }
+
+var (
+	ssid     string
+	password string
+	server   string = "tcp://test.mosquitto.org:1883"
+	debug           = false
+)
+
+var buf [0x400]byte
+
+var lastRequestTime time.Time
+var conn net.Conn
+var adaptor *rtl8720dn.RTL8720DN
+
+func main() {
+	err := run()
+	for err != nil {
+		fmt.Printf("error: %s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}
+
+// change these to connect to a different UART or pins for the ESP8266/ESP32
+var (
+	cl      mqtt.Client
+	topicTx = "tinygo/tx"
+	topicRx = "tinygo/rx"
+)
+
+func subHandler(client mqtt.Client, msg mqtt.Message) {
+	fmt.Printf("[%s]  ", msg.Topic())
+	fmt.Printf("%s\r\n", msg.Payload())
+}
+
+func run() error {
+	rtl, err := setupRTL8720DN()
+	if err != nil {
+		return err
+	}
+	net.UseDriver(rtl)
+
+	err = rtl.ConnectToAP(ssid, password)
+	if err != nil {
+		return err
+	}
+
+	ip, subnet, gateway, err := rtl.GetIP()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("IP Address : %s\r\n", ip)
+	fmt.Printf("Mask       : %s\r\n", subnet)
+	fmt.Printf("Gateway    : %s\r\n", gateway)
+
+	rand.Seed(time.Now().UnixNano())
+
+	opts := mqtt.NewClientOptions()
+	opts.AddBroker(server).SetClientID("tinygo-client-" + randomString(10))
+
+	println("Connecting to MQTT broker at", server)
+	cl = mqtt.NewClient(opts)
+	if token := cl.Connect(); token.Wait() && token.Error() != nil {
+		failMessage(token.Error().Error())
+	}
+
+	// subscribe
+	token := cl.Subscribe(topicRx, 0, subHandler)
+	token.Wait()
+	if token.Error() != nil {
+		failMessage(token.Error().Error())
+	}
+
+	go publishing()
+
+	select {}
+
+	// Right now this code is never reached. Need a way to trigger it...
+	println("Disconnecting MQTT...")
+	cl.Disconnect(100)
+
+	println("Done.")
+
+	return nil
+}
+
+func publishing() {
+	for i := 0; ; i++ {
+		println("Publishing MQTT message...")
+		data := []byte(fmt.Sprintf(`{"e":[{"n":"hello %d","v":101}]}`, i))
+		token := cl.Publish(topicTx, 0, false, data)
+		token.Wait()
+		if token.Error() != nil {
+			println(token.Error().Error())
+		}
+
+		time.Sleep(20 * 100 * time.Millisecond)
+	}
+}
+
+// Returns an int >= min, < max
+func randomInt(min, max int) int {
+	return min + rand.Intn(max-min)
+}
+
+// Generate a random string of A-Z chars with len = l
+func randomString(len int) string {
+	bytes := make([]byte, len)
+	for i := 0; i < len; i++ {
+		bytes[i] = byte(randomInt(65, 90))
+	}
+	return string(bytes)
+}
+
+func failMessage(msg string) {
+	for {
+		println(msg)
+		time.Sleep(1 * time.Second)
+	}
+}

examples/rtl8720dn/mqttsub/wioterminal.go

@@ -0,0 +1,74 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"device/sam"
+	"machine"
+	"runtime/interrupt"
+	"time"
+
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+var (
+	uart UARTx
+)
+
+func handleInterrupt(interrupt.Interrupt) {
+	// should reset IRQ
+	uart.Receive(byte((uart.Bus.DATA.Get() & 0xFF)))
+	uart.Bus.INTFLAG.SetBits(sam.SERCOM_USART_INT_INTFLAG_RXC)
+}
+
+func setupRTL8720DN() (*rtl8720dn.RTL8720DN, error) {
+	machine.RTL8720D_CHIP_PU.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	machine.RTL8720D_CHIP_PU.Low()
+	time.Sleep(100 * time.Millisecond)
+	machine.RTL8720D_CHIP_PU.High()
+	time.Sleep(1000 * time.Millisecond)
+	if debug {
+		waitSerial()
+	}
+
+	uart = UARTx{
+		UART: &machine.UART{
+			Buffer: machine.NewRingBuffer(),
+			Bus:    sam.SERCOM0_USART_INT,
+			SERCOM: 0,
+		},
+	}
+
+	uart.Interrupt = interrupt.New(sam.IRQ_SERCOM0_2, handleInterrupt)
+	uart.Configure(machine.UARTConfig{TX: machine.PB24, RX: machine.PC24, BaudRate: 614400})
+
+	rtl := rtl8720dn.New(uart)
+	rtl.Debug(debug)
+
+	_, err := rtl.Rpc_tcpip_adapter_init()
+	if err != nil {
+		return nil, err
+	}
+
+	return rtl, nil
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+type UARTx struct {
+	*machine.UART
+}
+
+func (u UARTx) Read(p []byte) (n int, err error) {
+	if u.Buffered() == 0 {
+		time.Sleep(1 * time.Millisecond)
+		return 0, nil
+	}
+	return u.UART.Read(p)
+}

examples/rtl8720dn/ntpclient/main.go

@@ -0,0 +1,144 @@
+// This is an example of using the rtl8720dn driver to implement a NTP client.
+// It creates a UDP connection to request the current time and parse the
+// response from a NTP server.
+package main
+
+import (
+	"errors"
+	"fmt"
+	"runtime"
+	"time"
+
+	"tinygo.org/x/drivers/net"
+)
+
+// IP address of the server aka "hub". Replace with your own info.
+// You can override the setting with the init() in another source code.
+// func init() {
+//    ssid = "your-ssid"
+//    password = "your-password"
+//    ntpHost = "129.6.15.29"
+//    debug = true
+// }
+
+var (
+	ssid     string
+	password string
+	ntpHost  = "129.6.15.29"
+	debug    = false
+)
+
+const NTP_PACKET_SIZE = 48
+
+var b = make([]byte, NTP_PACKET_SIZE)
+
+func main() {
+	err := run()
+	for err != nil {
+		fmt.Printf("error: %s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func run() error {
+	rtl, err := setupRTL8720DN()
+	if err != nil {
+		return err
+	}
+	net.UseDriver(rtl)
+
+	err = rtl.ConnectToAP(ssid, password)
+	if err != nil {
+		return err
+	}
+
+	ip, subnet, gateway, err := rtl.GetIP()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("IP Address : %s\r\n", ip)
+	fmt.Printf("Mask       : %s\r\n", subnet)
+	fmt.Printf("Gateway    : %s\r\n", gateway)
+
+	// now make UDP connection
+	hip := net.ParseIP(ntpHost)
+	raddr := &net.UDPAddr{IP: hip, Port: 123}
+	laddr := &net.UDPAddr{Port: 2390}
+	conn, err := net.DialUDP("udp", laddr, raddr)
+	if err != nil {
+		return err
+	}
+
+	for {
+		// send data
+		println("Requesting NTP time...")
+		t, err := getCurrentTime(conn)
+		if err != nil {
+			message("Error getting current time: %v", err)
+		} else {
+			message("NTP time: %v", t)
+		}
+		runtime.AdjustTimeOffset(-1 * int64(time.Since(t)))
+		for i := 0; i < 10; i++ {
+			message("Current time: %v", time.Now())
+			time.Sleep(1 * time.Second)
+		}
+	}
+
+}
+
+func getCurrentTime(conn *net.UDPSerialConn) (time.Time, error) {
+	if err := sendNTPpacket(conn); err != nil {
+		return time.Time{}, err
+	}
+	clearBuffer()
+	for now := time.Now(); time.Since(now) < time.Second; {
+		time.Sleep(5 * time.Millisecond)
+		if n, err := conn.Read(b); err != nil {
+			return time.Time{}, fmt.Errorf("error reading UDP packet: %w", err)
+		} else if n == 0 {
+			continue // no packet received yet
+		} else if n != NTP_PACKET_SIZE {
+			if n != NTP_PACKET_SIZE {
+				return time.Time{}, fmt.Errorf("expected NTP packet size of %d: %d", NTP_PACKET_SIZE, n)
+			}
+		}
+		return parseNTPpacket(), nil
+	}
+	return time.Time{}, errors.New("no packet received after 1 second")
+}
+
+func sendNTPpacket(conn *net.UDPSerialConn) error {
+	clearBuffer()
+	b[0] = 0b11100011 // LI, Version, Mode
+	b[1] = 0          // Stratum, or type of clock
+	b[2] = 6          // Polling Interval
+	b[3] = 0xEC       // Peer Clock Precision
+	// 8 bytes of zero for Root Delay & Root Dispersion
+	b[12] = 49
+	b[13] = 0x4E
+	b[14] = 49
+	b[15] = 52
+	if _, err := conn.Write(b); err != nil {
+		return err
+	}
+	return nil
+}
+
+func parseNTPpacket() time.Time {
+	// the timestamp starts at byte 40 of the received packet and is four bytes,
+	// this is NTP time (seconds since Jan 1 1900):
+	t := uint32(b[40])<<24 | uint32(b[41])<<16 | uint32(b[42])<<8 | uint32(b[43])
+	const seventyYears = 2208988800
+	return time.Unix(int64(t-seventyYears), 0)
+}
+
+func clearBuffer() {
+	for i := range b {
+		b[i] = 0
+	}
+}
+
+func message(format string, args ...interface{}) {
+	println(fmt.Sprintf(format, args...), "\r")
+}

examples/rtl8720dn/ntpclient/wioterminal.go

@@ -0,0 +1,72 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"device/sam"
+	"machine"
+	"runtime/interrupt"
+	"time"
+
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+var (
+	uart UARTx
+)
+
+func handleInterrupt(interrupt.Interrupt) {
+	// should reset IRQ
+	uart.Receive(byte((uart.Bus.DATA.Get() & 0xFF)))
+	uart.Bus.INTFLAG.SetBits(sam.SERCOM_USART_INT_INTFLAG_RXC)
+}
+
+func setupRTL8720DN() (*rtl8720dn.RTL8720DN, error) {
+	machine.RTL8720D_CHIP_PU.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	machine.RTL8720D_CHIP_PU.Low()
+	time.Sleep(100 * time.Millisecond)
+	machine.RTL8720D_CHIP_PU.High()
+	time.Sleep(1000 * time.Millisecond)
+	waitSerial()
+
+	uart = UARTx{
+		UART: &machine.UART{
+			Buffer: machine.NewRingBuffer(),
+			Bus:    sam.SERCOM0_USART_INT,
+			SERCOM: 0,
+		},
+	}
+
+	uart.Interrupt = interrupt.New(sam.IRQ_SERCOM0_2, handleInterrupt)
+	uart.Configure(machine.UARTConfig{TX: machine.PB24, RX: machine.PC24, BaudRate: 614400})
+
+	rtl := rtl8720dn.New(uart)
+	rtl.Debug(debug)
+
+	_, err := rtl.Rpc_tcpip_adapter_init()
+	if err != nil {
+		return nil, err
+	}
+
+	return rtl, nil
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+type UARTx struct {
+	*machine.UART
+}
+
+func (u UARTx) Read(p []byte) (n int, err error) {
+	if u.Buffered() == 0 {
+		time.Sleep(1 * time.Millisecond)
+		return 0, nil
+	}
+	return u.UART.Read(p)
+}

examples/rtl8720dn/tcpclient/main.go

@@ -0,0 +1,121 @@
+// This example opens a TCP connection using a device with RTL8720DN firmware
+// and sends some data, for the purpose of testing speed and connectivity.
+//
+// You can open a server to accept connections from this program using:
+//
+// nc -w 5 -lk 8080
+//
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"time"
+
+	"tinygo.org/x/drivers/net"
+)
+
+// You can override the setting with the init() in another source code.
+// func init() {
+//    ssid = "sghome-gw"
+//    password = "3af25537b4524"
+//    serverIP = "192.168.1.119"
+//    debug = true
+// }
+
+var (
+	ssid     string
+	password string
+	serverIP = ""
+	debug    = false
+)
+
+var buf = &bytes.Buffer{}
+
+func main() {
+	err := run()
+	for err != nil {
+		fmt.Printf("error: %s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func run() error {
+	rtl, err := setupRTL8720DN()
+	if err != nil {
+		return err
+	}
+	net.UseDriver(rtl)
+
+	err = rtl.ConnectToAP(ssid, password)
+	if err != nil {
+		return err
+	}
+
+	ip, subnet, gateway, err := rtl.GetIP()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("IP Address : %s\r\n", ip)
+	fmt.Printf("Mask       : %s\r\n", subnet)
+	fmt.Printf("Gateway    : %s\r\n", gateway)
+
+	for {
+		sendBatch()
+		time.Sleep(500 * time.Millisecond)
+	}
+	println("Done.")
+
+	return nil
+}
+
+func sendBatch() {
+
+	// make TCP connection
+	ip := net.ParseIP(serverIP)
+	raddr := &net.TCPAddr{IP: ip, Port: 8080}
+	laddr := &net.TCPAddr{Port: 8080}
+
+	message("---------------\r\nDialing TCP connection")
+	conn, err := net.DialTCP("tcp", laddr, raddr)
+	for ; err != nil; conn, err = net.DialTCP("tcp", laddr, raddr) {
+		message(err.Error())
+		time.Sleep(5 * time.Second)
+	}
+
+	n := 0
+	w := 0
+	start := time.Now()
+
+	// send data
+	message("Sending data")
+
+	for i := 0; i < 1000; i++ {
+		buf.Reset()
+		fmt.Fprint(buf,
+			"\r---------------------------- i == ", i, " ----------------------------"+
+				"\r---------------------------- i == ", i, " ----------------------------")
+		if w, err = conn.Write(buf.Bytes()); err != nil {
+			println("error:", err.Error(), "\r")
+			continue
+		}
+		n += w
+	}
+
+	buf.Reset()
+	ms := time.Now().Sub(start).Milliseconds()
+	fmt.Fprint(buf, "\nWrote ", n, " bytes in ", ms, " ms\r\n")
+	message(buf.String())
+
+	if _, err := conn.Write(buf.Bytes()); err != nil {
+		println("error:", err.Error(), "\r")
+	}
+
+	// Right now this code is never reached. Need a way to trigger it...
+	println("Disconnecting TCP...")
+	conn.Close()
+}
+
+func message(msg string) {
+	println(msg, "\r")
+}

examples/rtl8720dn/tcpclient/wioterminal.go

@@ -0,0 +1,72 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"device/sam"
+	"machine"
+	"runtime/interrupt"
+	"time"
+
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+var (
+	uart UARTx
+)
+
+func handleInterrupt(interrupt.Interrupt) {
+	// should reset IRQ
+	uart.Receive(byte((uart.Bus.DATA.Get() & 0xFF)))
+	uart.Bus.INTFLAG.SetBits(sam.SERCOM_USART_INT_INTFLAG_RXC)
+}
+
+func setupRTL8720DN() (*rtl8720dn.RTL8720DN, error) {
+	machine.RTL8720D_CHIP_PU.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	machine.RTL8720D_CHIP_PU.Low()
+	time.Sleep(100 * time.Millisecond)
+	machine.RTL8720D_CHIP_PU.High()
+	time.Sleep(1000 * time.Millisecond)
+	waitSerial()
+
+	uart = UARTx{
+		UART: &machine.UART{
+			Buffer: machine.NewRingBuffer(),
+			Bus:    sam.SERCOM0_USART_INT,
+			SERCOM: 0,
+		},
+	}
+
+	uart.Interrupt = interrupt.New(sam.IRQ_SERCOM0_2, handleInterrupt)
+	uart.Configure(machine.UARTConfig{TX: machine.PB24, RX: machine.PC24, BaudRate: 614400})
+
+	rtl := rtl8720dn.New(uart)
+	rtl.Debug(debug)
+
+	_, err := rtl.Rpc_tcpip_adapter_init()
+	if err != nil {
+		return nil, err
+	}
+
+	return rtl, nil
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+type UARTx struct {
+	*machine.UART
+}
+
+func (u UARTx) Read(p []byte) (n int, err error) {
+	if u.Buffered() == 0 {
+		time.Sleep(1 * time.Millisecond)
+		return 0, nil
+	}
+	return u.UART.Read(p)
+}

examples/rtl8720dn/tlsclient/main.go

@@ -0,0 +1,137 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"strings"
+	"time"
+
+	"tinygo.org/x/drivers/net"
+	"tinygo.org/x/drivers/net/http"
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+// You can override the setting with the init() in another source code.
+// func init() {
+//    ssid = "your-ssid"
+//    password = "your-password"
+//    debug = true
+//    url = "https://www.example.com"
+//    test_root_ca = "..."
+// }
+
+var (
+	ssid     string
+	password string
+	url      string = "https://www.example.com"
+	debug           = false
+)
+
+// Set the test_root_ca created by the following command
+// $ openssl s_client -showcerts -verify 5 -connect www.example.com:443 < /dev/null
+var test_root_ca = `-----BEGIN CERTIFICATE-----
+MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD
+QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
+b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB
+CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97
+nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt
+43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P
+T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4
+gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO
+BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR
+TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw
+DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr
+hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg
+06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF
+PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls
+YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk
+CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4=
+-----END CERTIFICATE-----
+`
+
+var buf [0x1000]byte
+
+var lastRequestTime time.Time
+var conn net.Conn
+var adaptor *rtl8720dn.RTL8720DN
+
+func main() {
+	err := run()
+	for err != nil {
+		fmt.Printf("error: %s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func run() error {
+	rtl, err := setupRTL8720DN()
+	if err != nil {
+		return err
+	}
+	rtl.SetRootCA(&test_root_ca)
+	net.UseDriver(rtl)
+	http.SetBuf(buf[:])
+
+	err = rtl.ConnectToAP(ssid, password)
+	if err != nil {
+		return err
+	}
+
+	ip, subnet, gateway, err := rtl.GetIP()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("IP Address : %s\r\n", ip)
+	fmt.Printf("Mask       : %s\r\n", subnet)
+	fmt.Printf("Gateway    : %s\r\n", gateway)
+
+	// You can send and receive cookies in the following way
+	// 	import "tinygo.org/x/drivers/net/http/cookiejar"
+	// 	jar, err := cookiejar.New(nil)
+	// 	if err != nil {
+	// 		return err
+	// 	}
+	// 	client := &http.Client{Jar: jar}
+	// 	http.DefaultClient = client
+
+	cnt := 0
+	for {
+		// Various examples are as follows
+		//
+		// -- Get
+		// 	resp, err := http.Get(url)
+		//
+		// -- Post
+		// 	body := `cnt=12`
+		// 	resp, err = http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(body))
+		//
+		// -- Post with JSON
+		// 	body := `{"msg": "hello"}`
+		// 	resp, err := http.Post(url, "application/json", strings.NewReader(body))
+
+		resp, err := http.Get(url)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("%s %s\r\n", resp.Proto, resp.Status)
+		for k, v := range resp.Header {
+			fmt.Printf("%s: %s\r\n", k, strings.Join(v, " "))
+		}
+		fmt.Printf("\r\n")
+
+		scanner := bufio.NewScanner(resp.Body)
+		for scanner.Scan() {
+			fmt.Printf("%s\r\n", scanner.Text())
+		}
+		resp.Body.Close()
+
+		cnt++
+		fmt.Printf("-------- %d --------\r\n", cnt)
+		time.Sleep(10 * time.Second)
+	}
+}

examples/rtl8720dn/tlsclient/wioterminal.go

@@ -0,0 +1,72 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"device/sam"
+	"machine"
+	"runtime/interrupt"
+	"time"
+
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+var (
+	uart UARTx
+)
+
+func handleInterrupt(interrupt.Interrupt) {
+	// should reset IRQ
+	uart.Receive(byte((uart.Bus.DATA.Get() & 0xFF)))
+	uart.Bus.INTFLAG.SetBits(sam.SERCOM_USART_INT_INTFLAG_RXC)
+}
+
+func setupRTL8720DN() (*rtl8720dn.RTL8720DN, error) {
+	machine.RTL8720D_CHIP_PU.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	machine.RTL8720D_CHIP_PU.Low()
+	time.Sleep(100 * time.Millisecond)
+	machine.RTL8720D_CHIP_PU.High()
+	time.Sleep(1000 * time.Millisecond)
+	waitSerial()
+
+	uart = UARTx{
+		UART: &machine.UART{
+			Buffer: machine.NewRingBuffer(),
+			Bus:    sam.SERCOM0_USART_INT,
+			SERCOM: 0,
+		},
+	}
+
+	uart.Interrupt = interrupt.New(sam.IRQ_SERCOM0_2, handleInterrupt)
+	uart.Configure(machine.UARTConfig{TX: machine.PB24, RX: machine.PC24, BaudRate: 614400})
+
+	rtl := rtl8720dn.New(uart)
+	rtl.Debug(debug)
+
+	_, err := rtl.Rpc_tcpip_adapter_init()
+	if err != nil {
+		return nil, err
+	}
+
+	return rtl, nil
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+type UARTx struct {
+	*machine.UART
+}
+
+func (u UARTx) Read(p []byte) (n int, err error) {
+	if u.Buffered() == 0 {
+		time.Sleep(1 * time.Millisecond)
+		return 0, nil
+	}
+	return u.UART.Read(p)
+}

examples/rtl8720dn/udpstation/main.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	"tinygo.org/x/drivers/net"
+	"tinygo.org/x/drivers/net/http"
+)
+
+// IP address of the server aka "hub". Replace with your own info.
+// You can override the setting with the init() in another source code.
+// func init() {
+//    ssid = "your-ssid"
+//    password = "your-password"
+//    hubIP = "192.168.1.118"
+//    debug = true
+// }
+
+var (
+	ssid     string
+	password string
+	hubIP    = ""
+	debug    = false
+)
+
+var buf [0x400]byte
+
+func main() {
+	err := run()
+	for err != nil {
+		fmt.Printf("error: %s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func run() error {
+	rtl, err := setupRTL8720DN()
+	if err != nil {
+		return err
+	}
+	net.UseDriver(rtl)
+	http.SetBuf(buf[:])
+
+	err = rtl.ConnectToAP(ssid, password)
+	if err != nil {
+		return err
+	}
+
+	ip, subnet, gateway, err := rtl.GetIP()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("IP Address : %s\r\n", ip)
+	fmt.Printf("Mask       : %s\r\n", subnet)
+	fmt.Printf("Gateway    : %s\r\n", gateway)
+
+	// now make UDP connection
+	hip := net.ParseIP(hubIP)
+	raddr := &net.UDPAddr{IP: hip, Port: 2222}
+	laddr := &net.UDPAddr{Port: 2222}
+
+	println("Dialing UDP connection...")
+	conn, err := net.DialUDP("udp", laddr, raddr)
+	if err != nil {
+		return err
+	}
+
+	for {
+		// send data
+		println("Sending data...")
+		for i := 0; i < 25; i++ {
+			conn.Write([]byte("hello " + strconv.Itoa(i) + "\r\n"))
+		}
+		time.Sleep(1000 * time.Millisecond)
+	}
+
+	// Right now this code is never reached. Need a way to trigger it...
+	println("Disconnecting UDP...")
+	conn.Close()
+	println("Done.")
+
+	return nil
+}
+
+func message(msg string) {
+	println(msg, "\r")
+}

examples/rtl8720dn/udpstation/wioterminal.go

@@ -0,0 +1,72 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"device/sam"
+	"machine"
+	"runtime/interrupt"
+	"time"
+
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+var (
+	uart UARTx
+)
+
+func handleInterrupt(interrupt.Interrupt) {
+	// should reset IRQ
+	uart.Receive(byte((uart.Bus.DATA.Get() & 0xFF)))
+	uart.Bus.INTFLAG.SetBits(sam.SERCOM_USART_INT_INTFLAG_RXC)
+}
+
+func setupRTL8720DN() (*rtl8720dn.RTL8720DN, error) {
+	machine.RTL8720D_CHIP_PU.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	machine.RTL8720D_CHIP_PU.Low()
+	time.Sleep(100 * time.Millisecond)
+	machine.RTL8720D_CHIP_PU.High()
+	time.Sleep(1000 * time.Millisecond)
+	waitSerial()
+
+	uart = UARTx{
+		UART: &machine.UART{
+			Buffer: machine.NewRingBuffer(),
+			Bus:    sam.SERCOM0_USART_INT,
+			SERCOM: 0,
+		},
+	}
+
+	uart.Interrupt = interrupt.New(sam.IRQ_SERCOM0_2, handleInterrupt)
+	uart.Configure(machine.UARTConfig{TX: machine.PB24, RX: machine.PC24, BaudRate: 614400})
+
+	rtl := rtl8720dn.New(uart)
+	rtl.Debug(debug)
+
+	_, err := rtl.Rpc_tcpip_adapter_init()
+	if err != nil {
+		return nil, err
+	}
+
+	return rtl, nil
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+type UARTx struct {
+	*machine.UART
+}
+
+func (u UARTx) Read(p []byte) (n int, err error) {
+	if u.Buffered() == 0 {
+		time.Sleep(1 * time.Millisecond)
+		return 0, nil
+	}
+	return u.UART.Read(p)
+}

examples/rtl8720dn/webclient-tinyterm/main.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"image/color"
+	"strings"
+	"time"
+
+	"tinygo.org/x/drivers/net"
+	"tinygo.org/x/drivers/net/http"
+	"tinygo.org/x/tinyfont/proggy"
+	"tinygo.org/x/tinyterm"
+)
+
+// You can override the setting with the init() in another source code.
+// If debug is enabled, a serial connection is required.
+// func init() {
+//    ssid = "your-ssid"
+//    password = "your-password"
+//    debug = false // true
+//    server = "tinygo.org"
+// }
+
+var (
+	ssid     string
+	password string
+	url      = "http://tinygo.org/"
+	debug    = false
+)
+
+var (
+	terminal = tinyterm.NewTerminal(display)
+
+	black = color.RGBA{0, 0, 0, 255}
+	white = color.RGBA{255, 255, 255, 255}
+	red   = color.RGBA{255, 0, 0, 255}
+	blue  = color.RGBA{0, 0, 255, 255}
+	green = color.RGBA{0, 255, 0, 255}
+
+	font = &proggy.TinySZ8pt7b
+)
+
+var buf [0x400]byte
+
+func main() {
+	display.FillScreen(black)
+	backlight.High()
+
+	terminal.Configure(&tinyterm.Config{
+		Font:       font,
+		FontHeight: 10,
+		FontOffset: 6,
+	})
+
+	err := run()
+	for err != nil {
+		fmt.Fprintf(terminal, "error: %s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func run() error {
+	fmt.Fprintf(terminal, "setupRTL8720DN()\r\n")
+	if debug {
+		fmt.Fprintf(terminal, "Running in debug mode.\r\n")
+		fmt.Fprintf(terminal, "A serial connection is required to continue execution.\r\n")
+	}
+	rtl, err := setupRTL8720DN()
+	if err != nil {
+		return err
+	}
+	net.UseDriver(rtl)
+	http.SetBuf(buf[:])
+
+	fmt.Fprintf(terminal, "ConnectToAP()\r\n")
+	err = rtl.ConnectToAP(ssid, password)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(terminal, "connected\r\n\r\n")
+
+	ip, subnet, gateway, err := rtl.GetIP()
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(terminal, "IP Address : %s\r\n", ip)
+	fmt.Fprintf(terminal, "Mask       : %s\r\n", subnet)
+	fmt.Fprintf(terminal, "Gateway    : %s\r\n", gateway)
+
+	// You can send and receive cookies in the following way
+	// 	import "tinygo.org/x/drivers/net/http/cookiejar"
+	// 	jar, err := cookiejar.New(nil)
+	// 	if err != nil {
+	// 		return err
+	// 	}
+	// 	client := &http.Client{Jar: jar}
+	// 	http.DefaultClient = client
+
+	cnt := 0
+	for {
+		// Various examples are as follows
+		//
+		// -- Get
+		// 	resp, err := http.Get(url)
+		//
+		// -- Post
+		// 	body := `cnt=12`
+		// 	resp, err = http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(body))
+		//
+		// -- Post with JSON
+		// 	body := `{"msg": "hello"}`
+		// 	resp, err := http.Post(url, "application/json", strings.NewReader(body))
+
+		resp, err := http.Get(url)
+		if err != nil {
+			return err
+		}
+
+		fmt.Fprintf(terminal, "%s %s\r\n", resp.Proto, resp.Status)
+		for k, v := range resp.Header {
+			fmt.Fprintf(terminal, "%s: %s\r\n", k, strings.Join(v, " "))
+		}
+		fmt.Printf("\r\n")
+
+		scanner := bufio.NewScanner(resp.Body)
+		for scanner.Scan() {
+			fmt.Fprintf(terminal, "%s\r\n", scanner.Text())
+		}
+		resp.Body.Close()
+
+		cnt++
+		fmt.Fprintf(terminal, "-------- %d --------\r\n", cnt)
+		time.Sleep(10 * time.Second)
+	}
+}

examples/rtl8720dn/webclient-tinyterm/wioterminal.go

@@ -0,0 +1,98 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"device/sam"
+	"machine"
+	"runtime/interrupt"
+	"time"
+
+	"tinygo.org/x/drivers/ili9341"
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+var (
+	uart UARTx
+)
+
+var (
+	display = ili9341.NewSPI(
+		machine.SPI3,
+		machine.LCD_DC,
+		machine.LCD_SS_PIN,
+		machine.LCD_RESET,
+	)
+
+	backlight = machine.LCD_BACKLIGHT
+)
+
+func init() {
+	machine.SPI3.Configure(machine.SPIConfig{
+		SCK:       machine.LCD_SCK_PIN,
+		SDO:       machine.LCD_SDO_PIN,
+		SDI:       machine.LCD_SDI_PIN,
+		Frequency: 40000000,
+	})
+	display.Configure(ili9341.Config{})
+
+	backlight.Configure(machine.PinConfig{machine.PinOutput})
+}
+
+func handleInterrupt(interrupt.Interrupt) {
+	// should reset IRQ
+	uart.Receive(byte((uart.Bus.DATA.Get() & 0xFF)))
+	uart.Bus.INTFLAG.SetBits(sam.SERCOM_USART_INT_INTFLAG_RXC)
+}
+
+func setupRTL8720DN() (*rtl8720dn.RTL8720DN, error) {
+	machine.RTL8720D_CHIP_PU.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	machine.RTL8720D_CHIP_PU.Low()
+	time.Sleep(100 * time.Millisecond)
+	machine.RTL8720D_CHIP_PU.High()
+	time.Sleep(1000 * time.Millisecond)
+	if debug {
+		waitSerial()
+	}
+
+	uart = UARTx{
+		UART: &machine.UART{
+			Buffer: machine.NewRingBuffer(),
+			Bus:    sam.SERCOM0_USART_INT,
+			SERCOM: 0,
+		},
+	}
+
+	uart.Interrupt = interrupt.New(sam.IRQ_SERCOM0_2, handleInterrupt)
+	uart.Configure(machine.UARTConfig{TX: machine.PB24, RX: machine.PC24, BaudRate: 614400})
+
+	rtl := rtl8720dn.New(uart)
+	rtl.Debug(debug)
+
+	_, err := rtl.Rpc_tcpip_adapter_init()
+	if err != nil {
+		return nil, err
+	}
+
+	return rtl, nil
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+type UARTx struct {
+	*machine.UART
+}
+
+func (u UARTx) Read(p []byte) (n int, err error) {
+	if u.Buffered() == 0 {
+		time.Sleep(1 * time.Millisecond)
+		return 0, nil
+	}
+	return u.UART.Read(p)
+}

examples/rtl8720dn/webclient/main.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"strings"
+	"time"
+
+	"tinygo.org/x/drivers/net"
+	"tinygo.org/x/drivers/net/http"
+)
+
+// You can override the setting with the init() in another source code.
+// func init() {
+//    ssid = "your-ssid"
+//    password = "your-password"
+//    url = "http://tinygo.org/"
+//    debug = true
+// }
+
+var (
+	ssid     string
+	password string
+	url      = "http://tinygo.org/"
+	debug    = false
+)
+
+var buf [0x400]byte
+
+func main() {
+	err := run()
+	for err != nil {
+		fmt.Printf("error: %s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func run() error {
+	rtl, err := setupRTL8720DN()
+	if err != nil {
+		return err
+	}
+	net.UseDriver(rtl)
+	http.SetBuf(buf[:])
+
+	err = rtl.ConnectToAP(ssid, password)
+	if err != nil {
+		return err
+	}
+
+	ip, subnet, gateway, err := rtl.GetIP()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("IP Address : %s\r\n", ip)
+	fmt.Printf("Mask       : %s\r\n", subnet)
+	fmt.Printf("Gateway    : %s\r\n", gateway)
+
+	// You can send and receive cookies in the following way
+	// 	import "tinygo.org/x/drivers/net/http/cookiejar"
+	// 	jar, err := cookiejar.New(nil)
+	// 	if err != nil {
+	// 		return err
+	// 	}
+	// 	client := &http.Client{Jar: jar}
+	// 	http.DefaultClient = client
+
+	cnt := 0
+	for {
+		// Various examples are as follows
+		//
+		// -- Get
+		// 	resp, err := http.Get(url)
+		//
+		// -- Post
+		// 	body := `cnt=12`
+		// 	resp, err = http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(body))
+		//
+		// -- Post with JSON
+		// 	body := `{"msg": "hello"}`
+		// 	resp, err := http.Post(url, "application/json", strings.NewReader(body))
+
+		resp, err := http.Get(url)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("%s %s\r\n", resp.Proto, resp.Status)
+		for k, v := range resp.Header {
+			fmt.Printf("%s: %s\r\n", k, strings.Join(v, " "))
+		}
+		fmt.Printf("\r\n")
+
+		scanner := bufio.NewScanner(resp.Body)
+		for scanner.Scan() {
+			fmt.Printf("%s\r\n", scanner.Text())
+		}
+		resp.Body.Close()
+
+		cnt++
+		fmt.Printf("-------- %d --------\r\n", cnt)
+		time.Sleep(10 * time.Second)
+	}
+}

examples/rtl8720dn/webclient/wioterminal.go

@@ -0,0 +1,72 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"device/sam"
+	"machine"
+	"runtime/interrupt"
+	"time"
+
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+var (
+	uart UARTx
+)
+
+func handleInterrupt(interrupt.Interrupt) {
+	// should reset IRQ
+	uart.Receive(byte((uart.Bus.DATA.Get() & 0xFF)))
+	uart.Bus.INTFLAG.SetBits(sam.SERCOM_USART_INT_INTFLAG_RXC)
+}
+
+func setupRTL8720DN() (*rtl8720dn.RTL8720DN, error) {
+	machine.RTL8720D_CHIP_PU.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	machine.RTL8720D_CHIP_PU.Low()
+	time.Sleep(100 * time.Millisecond)
+	machine.RTL8720D_CHIP_PU.High()
+	time.Sleep(1000 * time.Millisecond)
+	waitSerial()
+
+	uart = UARTx{
+		UART: &machine.UART{
+			Buffer: machine.NewRingBuffer(),
+			Bus:    sam.SERCOM0_USART_INT,
+			SERCOM: 0,
+		},
+	}
+
+	uart.Interrupt = interrupt.New(sam.IRQ_SERCOM0_2, handleInterrupt)
+	uart.Configure(machine.UARTConfig{TX: machine.PB24, RX: machine.PC24, BaudRate: 614400})
+
+	rtl := rtl8720dn.New(uart)
+	rtl.Debug(debug)
+
+	_, err := rtl.Rpc_tcpip_adapter_init()
+	if err != nil {
+		return nil, err
+	}
+
+	return rtl, nil
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+type UARTx struct {
+	*machine.UART
+}
+
+func (u UARTx) Read(p []byte) (n int, err error) {
+	if u.Buffered() == 0 {
+		time.Sleep(1 * time.Millisecond)
+		return 0, nil
+	}
+	return u.UART.Read(p)
+}

examples/rtl8720dn/webserver/main.go

@@ -0,0 +1,192 @@
+package main
+
+import (
+	"fmt"
+	"machine"
+	"strconv"
+	"time"
+
+	"tinygo.org/x/drivers/net/http"
+)
+
+// You can override the setting with the init() in another source code.
+// func init() {
+//    ssid = "your-ssid"
+//    password = "your-password"
+//    debug = true
+// }
+
+var (
+	ssid     string
+	password string
+	debug    = false
+)
+
+var led = machine.LED
+var backlight = machine.LCD_BACKLIGHT
+
+func main() {
+	led.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	backlight.Configure(machine.PinConfig{Mode: machine.PinOutput})
+
+	err := run()
+	for err != nil {
+		fmt.Printf("error: %s\r\n", err.Error())
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func run() error {
+	rtl, err := setupRTL8720DN()
+	if err != nil {
+		return err
+	}
+	http.UseDriver(rtl)
+
+	err = rtl.ConnectToAP(ssid, password)
+	if err != nil {
+		return err
+	}
+
+	ip, subnet, gateway, err := rtl.GetIP()
+	if err != nil {
+		return err
+	}
+	fmt.Printf("IP Address : %s\r\n", ip)
+	fmt.Printf("Mask       : %s\r\n", subnet)
+	fmt.Printf("Gateway    : %s\r\n", gateway)
+
+	http.HandleFunc("/", root)
+	http.HandleFunc("/hello", hello)
+	http.HandleFunc("/cnt", cnt)
+	http.HandleFunc("/6", sixlines)
+	http.HandleFunc("/off", LED_OFF)
+	http.HandleFunc("/on", LED_ON)
+	if err := http.ListenAndServe(":80", nil); err != nil {
+		message(err.Error())
+	}
+	return nil
+}
+
+func root(w http.ResponseWriter, r *http.Request) {
+	access := 1
+
+	cookie, err := r.Cookie("access")
+	if err != nil {
+		if err == http.ErrNoCookie {
+			cookie = &http.Cookie{
+				Name:  "access",
+				Value: "1",
+			}
+		} else {
+			http.Error(w, fmt.Sprintf("%s", err.Error()), http.StatusBadRequest)
+			return
+		}
+	} else {
+		v, err := strconv.ParseInt(cookie.Value, 10, 0)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("invalid cookie.Value : %s", cookie.Value), http.StatusBadRequest)
+			return
+		}
+		cookie.Value = fmt.Sprintf("%d", v+1)
+		access = int(v) + 1
+	}
+	http.SetCookie(w, cookie)
+	w.WriteHeader(http.StatusOK)
+
+	fmt.Fprintf(w, `
+<html>
+<head>
+    <title>TinyGo HTTP Server</title>
+    <script language="javascript" type="text/javascript">
+        var counter = 0
+        function ledOn() { fetch("/on").then(response => response.text()).then(text => { led.innerHTML = "<p>on</p>"; }); }
+        function ledOff() { fetch("/off").then(response => response.text()).then(text => { led.innerHTML = "<p>off</p>"; }); }
+        function fetchCnt() { fetch("/cnt").then(response => response.json()).then(json => { counter = json.cnt; cnt.innerHTML = counter; }); }
+        function incrCnt() { counter = counter + 1; fetch("/cnt?cnt=" + counter, { method: 'POST' }).then(response => response.json()).then(json => { counter = json.cnt; cnt.innerHTML = counter; }); }
+        function setCnt() { fetch("/cnt", {
+            method: "POST",
+            body: "cnt=" + document.getElementsByName("cnt")[0].value,
+            headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+        }).then(response => response.json()).then(json => { counter = json.cnt; cnt.innerHTML = counter; }); return false; }
+        function onLoad() { fetchCnt(); }
+    </script>
+</head>
+<body onLoad="onLoad()">
+    <h5>TinyGo HTTP Server</h5>
+
+    <p>
+        access: %d
+    </p>
+
+    <a href="/hello">/hello</a><br>
+    <a href="/6">/6</a><br>
+
+    <p>
+        LED<br>
+        <a href="javascript:ledOn();">/on</a><br>
+        <a href="javascript:ledOff();">/off</a><br>
+    </p>
+
+
+    <p>
+        <a href="/cnt">/cnt</a><br>
+        cnt: <span id="cnt"></span><br>
+        <a href="javascript:incrCnt()">incrCnt()</a><br>
+        <form id="form1" style="display: inline" onSubmit="return setCnt()">
+        <input type="text" name="cnt">
+        <input type="button" value="set cnt", onClick="setCnt()">
+        </form>
+    </p>
+</body>
+</html>
+    `, access)
+}
+
+func sixlines(w http.ResponseWriter, r *http.Request) {
+	// https://fukuno.jig.jp/3267
+	fmt.Fprint(w, `<body onload='onkeydown=e=>K=parseInt(e.key[5]||6,28)/3-8;Z=X=[B=A=12];Y=_=>`+
+		`{for(C=[q=c=i=4];f=i--*K;c-=!Z[h+(K+6?p+K:C[i]=p*A-(p/9|0)*145)])p=B[i];for(c?0:K+6?h+=K:B=C;`+
+		`i=K=q--;f+=Z[A+p])X[p=h+B[q]]=1;h+=A;if(f|B)for(Z=X,X=[l=228],B=[[-7,-20,6,h=17,-9,3,3][t=++t%7]-4,0,1,t-6?-A:2];l--;)`+
+		`for(l%A?l-=l%A*!Z[l]:(P++,c=l+=A);--c>A;)Z[c]=Z[c-A];for(S="";i<240;S+=X[i]|(X[i]=Z[i]|=++i%A<2|i>228)?i%A?"■":"■<br>":"　");`+
+		`D.innerHTML=S+P;setTimeout(Y,i-P)};Y(h=K=t=P=0)'id=D>`)
+}
+
+func LED_ON(w http.ResponseWriter, r *http.Request) {
+	led.High()
+	backlight.High()
+	w.Header().Set(`Content-Type`, `text/plain; charset=UTF-8`)
+	fmt.Fprintf(w, "led.High()")
+}
+
+func LED_OFF(w http.ResponseWriter, r *http.Request) {
+	led.Low()
+	backlight.Low()
+	w.Header().Set(`Content-Type`, `text/plain; charset=UTF-8`)
+	fmt.Fprintf(w, "led.Low()")
+}
+
+func hello(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set(`Content-Type`, `text/plain; charset=UTF-8`)
+	fmt.Fprintf(w, "hello")
+}
+
+var counter int
+
+func cnt(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	if r.Method == "POST" {
+		c := r.Form.Get("cnt")
+		if c != "" {
+			i64, _ := strconv.ParseInt(c, 0, 0)
+			counter = int(i64)
+		}
+	}
+
+	w.Header().Set(`Content-Type`, `application/json`)
+	fmt.Fprintf(w, `{"cnt": %d}`, counter)
+}
+
+func message(msg string) {
+	println(msg, "\r")
+}

examples/rtl8720dn/webserver/wioterminal.go

@@ -0,0 +1,74 @@
+//go:build wioterminal
+// +build wioterminal
+
+package main
+
+import (
+	"device/sam"
+	"machine"
+	"runtime/interrupt"
+	"time"
+
+	"tinygo.org/x/drivers/rtl8720dn"
+)
+
+var (
+	uart UARTx
+)
+
+func handleInterrupt(interrupt.Interrupt) {
+	// should reset IRQ
+	uart.Receive(byte((uart.Bus.DATA.Get() & 0xFF)))
+	uart.Bus.INTFLAG.SetBits(sam.SERCOM_USART_INT_INTFLAG_RXC)
+}
+
+func setupRTL8720DN() (*rtl8720dn.RTL8720DN, error) {
+	machine.RTL8720D_CHIP_PU.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	machine.RTL8720D_CHIP_PU.Low()
+	time.Sleep(100 * time.Millisecond)
+	machine.RTL8720D_CHIP_PU.High()
+	time.Sleep(1000 * time.Millisecond)
+	if debug {
+		waitSerial()
+	}
+
+	uart = UARTx{
+		UART: &machine.UART{
+			Buffer: machine.NewRingBuffer(),
+			Bus:    sam.SERCOM0_USART_INT,
+			SERCOM: 0,
+		},
+	}
+
+	uart.Interrupt = interrupt.New(sam.IRQ_SERCOM0_2, handleInterrupt)
+	uart.Configure(machine.UARTConfig{TX: machine.PB24, RX: machine.PC24, BaudRate: 614400})
+
+	rtl := rtl8720dn.New(uart)
+	rtl.Debug(debug)
+
+	_, err := rtl.Rpc_tcpip_adapter_init()
+	if err != nil {
+		return nil, err
+	}
+
+	return rtl, nil
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+type UARTx struct {
+	*machine.UART
+}
+
+func (u UARTx) Read(p []byte) (n int, err error) {
+	if u.Buffered() == 0 {
+		time.Sleep(1 * time.Millisecond)
+		return 0, nil
+	}
+	return u.UART.Read(p)
+}

examples/sdcard/console/feather-m4.go

@@ -1,3 +1,4 @@
+//go:build feather_m4 || feather_m4_can || feather_nrf52840
 // +build feather_m4 feather_m4_can feather_nrf52840
 
 package main

examples/sdcard/console/grandcentral-m4.go

@@ -1,4 +1,5 @@
-// +build grandcentral-m4
+//go:build grandcentral_m4
+// +build grandcentral_m4
 
 package main
 

examples/sdcard/console/m0.go

@@ -1,3 +1,4 @@
+//go:build atsamd21 && !p1am_100
 // +build atsamd21,!p1am_100
 
 package main

examples/sdcard/console/p1am-100.go

@@ -1,3 +1,4 @@
+//go:build p1am_100
 // +build p1am_100
 
 package main

examples/sdcard/console/pygamer.go

@@ -1,3 +1,4 @@
+//go:build pygamer
 // +build pygamer
 
 package main

examples/sdcard/console/pyportal.go

@@ -1,3 +1,4 @@
+//go:build pyportal
 // +build pyportal
 
 package main

examples/sdcard/console/wioterminal.go

@@ -1,3 +1,4 @@
+//go:build wioterminal
 // +build wioterminal
 
 package main

examples/sdcard/tinyfs/console/console.go

@@ -1,3 +1,4 @@
+//go:build tinygo
 // +build tinygo
 
 package console
@@ -11,7 +12,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/tinygo-org/tinyfs"
+	"tinygo.org/x/tinyfs"
 )
 
 const consoleBufLen = 64
@@ -35,6 +36,7 @@ var (
 	commands = map[string]cmdfunc{
 		"":        noop,
 		"dbg":     dbg,
+		"help":    help,
 		"lsblk":   lsblk,
 		"mount":   mount,
 		"umount":  umount,
@@ -160,6 +162,28 @@ func runCommand(line string) {
 
 func noop(argv []string) {}
 
+func help(argv []string) {
+	fmt.Printf("help\r\n")
+	fmt.Printf("  show help\r\n")
+	fmt.Printf("dbg\r\n")
+	fmt.Printf("  toggle debug mode\r\n")
+	fmt.Printf("xxd <hex address, ex: 0xA0> <size of hexdump in bytes>\r\n")
+	fmt.Printf("  hexdump the specified address\r\n")
+	fmt.Printf("ls <target file>\r\n")
+	fmt.Printf("  list information\r\n")
+	fmt.Printf("samples\r\n")
+	fmt.Printf("  write some files in the root directory\r\n")
+	fmt.Printf("mkdir <target dir>\r\n")
+	fmt.Printf("  create directory\r\n")
+	fmt.Printf("cat <target file>\r\n")
+	fmt.Printf("  print the contents of file\r\n")
+	fmt.Printf("create <target file>\r\n")
+	fmt.Printf("  create file\r\n")
+	fmt.Printf("write <target file>\r\n")
+	fmt.Printf("  write to file (press CTRL-D to exit)\r\n")
+	fmt.Printf("rm\r\n")
+}
+
 func dbg(argv []string) {
 	if debug {
 		debug = false
@@ -421,7 +445,7 @@ func cat(argv []string) {
 		println("Trying to cat to " + tgt)
 	}
 	if tgt == "" {
-		println("Usage: cat <target dir>")
+		println("Usage: cat <target file>")
 		return
 	}
 	if debug {

examples/sdcard/tinyfs/feather-m4.go

@@ -1,3 +1,4 @@
+//go:build feather_m4 || feather_m4_can || feather_nrf52840
 // +build feather_m4 feather_m4_can feather_nrf52840
 
 package main

examples/sdcard/tinyfs/grandcentral-m4.go

@@ -1,4 +1,5 @@
-// +build grandcentral-m4
+//go:build grandcentral_m4
+// +build grandcentral_m4
 
 package main
 

examples/sdcard/tinyfs/m0.go

@@ -1,3 +1,4 @@
+//go:build atsamd21 && !p1am_100
 // +build atsamd21,!p1am_100
 
 package main

examples/sdcard/tinyfs/main.go

@@ -5,9 +5,9 @@ import (
 	"machine"
 	"time"
 
-	"github.com/tinygo-org/tinyfs/fatfs"
 	"tinygo.org/x/drivers/examples/sdcard/tinyfs/console"
 	"tinygo.org/x/drivers/sdcard"
+	"tinygo.org/x/tinyfs/fatfs"
 )
 
 var (

examples/sdcard/tinyfs/p1am-100.go

@@ -1,3 +1,4 @@
+//go:build p1am_100
 // +build p1am_100
 
 package main

examples/sdcard/tinyfs/pygamer.go

@@ -1,3 +1,4 @@
+//go:build pygamer
 // +build pygamer
 
 package main

examples/sdcard/tinyfs/pyportal.go

@@ -1,3 +1,4 @@
+//go:build pyportal
 // +build pyportal
 
 package main

examples/sdcard/tinyfs/wioterminal.go

@@ -1,3 +1,4 @@
+//go:build wioterminal
 // +build wioterminal
 
 package main

examples/tmp102/main.go

@@ -20,7 +20,7 @@ func main() {
 
 		temp, _ := thermo.ReadTemperature()
 
-		print(fmt.Sprintf("%.2f°C\r\n", float32(temp)/1000.0))
+		print(fmt.Sprintf("%.2f°C\r\n", temp.Celsius()))
 
 		time.Sleep(time.Millisecond * 1000)
 	}

examples/wifinina/http-get/main.go

@@ -0,0 +1,161 @@
+// This example opens a TCP connection using a device with WiFiNINA firmware
+// and sends a HTTP request to retrieve a webpage, based on the following
+// Arduino example:
+//
+// https://github.com/arduino-libraries/WiFiNINA/blob/master/examples/WiFiWebClientRepeating/
+//
+// This example will not work with samd21 or other systems with less than 32KB
+// of RAM.  Use the following if you want to run wifinina on samd21, etc.
+//
+// examples/wifinina/webclient
+// examples/wifinina/tlsclient
+//
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"machine"
+	"strings"
+	"time"
+
+	"tinygo.org/x/drivers/net"
+	"tinygo.org/x/drivers/net/http"
+	"tinygo.org/x/drivers/wifinina"
+)
+
+// access point info
+const ssid = ""
+const pass = ""
+
+// IP address of the server aka "hub". Replace with your own info.
+// Can specify a URL starting with http or https
+const url = "http://tinygo.org/"
+
+// these are the default pins for the Arduino Nano33 IoT.
+// change these to connect to a different UART or pins for the ESP8266/ESP32
+var (
+
+	// these are the default pins for the Arduino Nano33 IoT.
+	spi = machine.NINA_SPI
+
+	// this is the ESP chip that has the WIFININA firmware flashed on it
+	adaptor *wifinina.Device
+)
+
+var buf [0x400]byte
+
+var lastRequestTime time.Time
+var conn net.Conn
+
+func setup() {
+	// Configure SPI for 8Mhz, Mode 0, MSB First
+	spi.Configure(machine.SPIConfig{
+		Frequency: 8 * 1e6,
+		SDO:       machine.NINA_SDO,
+		SDI:       machine.NINA_SDI,
+		SCK:       machine.NINA_SCK,
+	})
+
+	adaptor = wifinina.New(spi,
+		machine.NINA_CS,
+		machine.NINA_ACK,
+		machine.NINA_GPIO0,
+		machine.NINA_RESETN)
+	adaptor.Configure()
+}
+
+func main() {
+
+	setup()
+	http.SetBuf(buf[:])
+
+	waitSerial()
+
+	connectToAP()
+
+	// You can send and receive cookies in the following way
+	// 	import "tinygo.org/x/drivers/net/http/cookiejar"
+	// 	jar, err := cookiejar.New(nil)
+	// 	if err != nil {
+	// 		return err
+	// 	}
+	// 	client := &http.Client{Jar: jar}
+	// 	http.DefaultClient = client
+
+	cnt := 0
+	for {
+		// Various examples are as follows
+		//
+		// -- Get
+		// 	resp, err := http.Get(url)
+		//
+		// -- Post
+		// 	body := `cnt=12`
+		// 	resp, err = http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(body))
+		//
+		// -- Post with JSON
+		// 	body := `{"msg": "hello"}`
+		// 	resp, err := http.Post(url, "application/json", strings.NewReader(body))
+
+		resp, err := http.Get(url)
+		if err != nil {
+			fmt.Printf("%s\r\n", err.Error())
+			continue
+		}
+
+		fmt.Printf("%s %s\r\n", resp.Proto, resp.Status)
+		for k, v := range resp.Header {
+			fmt.Printf("%s: %s\r\n", k, strings.Join(v, " "))
+		}
+		fmt.Printf("\r\n")
+
+		scanner := bufio.NewScanner(resp.Body)
+		for scanner.Scan() {
+			fmt.Printf("%s\r\n", scanner.Text())
+		}
+		resp.Body.Close()
+
+		cnt++
+		fmt.Printf("-------- %d --------\r\n", cnt)
+		time.Sleep(10 * time.Second)
+	}
+
+}
+
+// Wait for user to open serial console
+func waitSerial() {
+	for !machine.Serial.DTR() {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+// connect to access point
+func connectToAP() {
+	if len(ssid) == 0 || len(pass) == 0 {
+		for {
+			println("Connection failed: Either ssid or password not set")
+			time.Sleep(10 * time.Second)
+		}
+	}
+	time.Sleep(2 * time.Second)
+	message("Connecting to " + ssid)
+	adaptor.SetPassphrase(ssid, pass)
+	for st, _ := adaptor.GetConnectionStatus(); st != wifinina.StatusConnected; {
+		message("Connection status: " + st.String())
+		time.Sleep(1 * time.Second)
+		st, _ = adaptor.GetConnectionStatus()
+	}
+	message("Connected.")
+	time.Sleep(2 * time.Second)
+	ip, _, _, err := adaptor.GetIP()
+	for ; err != nil; ip, _, _, err = adaptor.GetIP() {
+		message(err.Error())
+		time.Sleep(1 * time.Second)
+	}
+	message(ip.String())
+}
+
+func message(msg string) {
+	println(msg, "\r")
+}

examples/ws2812/arduino.go

@@ -1,3 +1,4 @@
+//go:build arduino
 // +build arduino
 
 package main

examples/ws2812/digispark.go

@@ -1,3 +1,4 @@
+//go:build digispark
 // +build digispark
 
 package main

examples/ws2812/others.go

@@ -1,3 +1,4 @@
+//go:build !digispark && !arduino && !qtpy
 // +build !digispark,!arduino,!qtpy
 
 package main
@@ -6,5 +7,5 @@ import "machine"
 
 // Replace neo and led in the code below to match the pin
 // that you are using if different.
-var neo machine.Pin = machine.NEOPIXELS
+var neo machine.Pin = machine.WS2812
 var led = machine.LED

examples/ws2812/qtpy.go

@@ -1,3 +1,4 @@
+//go:build qtpy
 // +build qtpy
 
 package main

flash/transport_qspi_samd.go

@@ -1,3 +1,4 @@
+//go:build atsamd51
 // +build atsamd51
 
 package flash

go.mod

@@ -5,10 +5,8 @@ go 1.15
 require (
 	github.com/eclipse/paho.mqtt.golang v1.2.0
 	github.com/frankban/quicktest v1.10.2
-	github.com/sago35/tinygo-dma v0.0.0-20210610020721-297675ab9b23 // indirect
-	github.com/tinygo-org/tinyfs v0.0.0-20210514090915-924e60a7bcf8
-)
-
-replace (
-    github.com/sago35/tinygo-dma => ../../sago35/tinygo-dma
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
+	tinygo.org/x/tinyfont v0.2.1
+	tinygo.org/x/tinyfs v0.1.0
+	tinygo.org/x/tinyterm v0.1.0
 )

go.sum

@@ -1,3 +1,4 @@
+github.com/bgould/http v0.0.0-20190627042742-d268792bdee7/go.mod h1:BTqvVegvwifopl4KTEDth6Zezs9eR+lCWhvGKvkxJHE=
 github.com/eclipse/paho.mqtt.golang v1.2.0 h1:1F8mhG9+aO5/xpdtFkW4SxOJB67ukuDC3t2y2qayIX0=
 github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts=
 github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk=
@@ -9,12 +10,23 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/sago35/tinygo-dma v0.0.0-20210610020721-297675ab9b23 h1:ESMYhe6RQJ2RbBCH2OK1SNyYq3gKoBpI7g07aLjQgJI=
-github.com/sago35/tinygo-dma v0.0.0-20210610020721-297675ab9b23/go.mod h1:gXCCsg4cKOJ6NJl7teGN597XgN0hzQJz0Wekayj4k+M=
-github.com/tinygo-org/tinyfs v0.0.0-20210514004344-b02c062d12c9 h1:66or7MohOph3xr3gMGCXceLkd+MBoUbV/rPPjl8iQOU=
-github.com/tinygo-org/tinyfs v0.0.0-20210514004344-b02c062d12c9/go.mod h1:HGuyo42bGd1zWSuS1gwgyyjN36ZZMlEpwM0vSrglmXc=
-github.com/tinygo-org/tinyfs v0.0.0-20210514090915-924e60a7bcf8 h1:pYwuKe1XuNeJnGV1UjeZ0FhuZ5+lapIq1NUmGacbBwo=
-github.com/tinygo-org/tinyfs v0.0.0-20210514090915-924e60a7bcf8/go.mod h1:HGuyo42bGd1zWSuS1gwgyyjN36ZZMlEpwM0vSrglmXc=
+github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+tinygo.org/x/drivers v0.14.0/go.mod h1:uT2svMq3EpBZpKkGO+NQHjxjGf1f42ra4OnMMwQL2aI=
+tinygo.org/x/drivers v0.15.1/go.mod h1:uT2svMq3EpBZpKkGO+NQHjxjGf1f42ra4OnMMwQL2aI=
 tinygo.org/x/drivers v0.16.0/go.mod h1:uT2svMq3EpBZpKkGO+NQHjxjGf1f42ra4OnMMwQL2aI=
+tinygo.org/x/tinyfont v0.2.1 h1:FAaemBzw8wsfhAtG6fWW+QjyWw/K8YqEeiWo4N1pv4o=
+tinygo.org/x/tinyfont v0.2.1/go.mod h1:eLqnYSrFRjt5STxWaMeOWJTzrKhXqpWw7nU3bPfKOAM=
+tinygo.org/x/tinyfs v0.1.0 h1:yx1Tq9L60rpCm6HURo45x+Tnag+O9RGSbQfgeCb6XYU=
+tinygo.org/x/tinyfs v0.1.0/go.mod h1:ysc8Y92iHfhTXeyEM9+c7zviUQ4fN9UCFgSOFfMWv20=
+tinygo.org/x/tinyterm v0.1.0 h1:80i+j+KWoxCFa/Xfp6pWbh79x+8zUdMXC1vaKj2QhkY=
+tinygo.org/x/tinyterm v0.1.0/go.mod h1:/DDhNnGwNF2/tNgHywvyZuCGnbH3ov49Z/6e8LPLRR4=

ili9341/ili9341.go

@@ -28,35 +28,6 @@ type Device struct {
 	rd  machine.Pin
 }
 
-var cmdBuf [4]byte
-var initCmd = []byte{
-	0xEF, 3, 0x03, 0x80, 0x02,
-	0xCF, 3, 0x00, 0xC1, 0x30,
-	0xED, 4, 0x64, 0x03, 0x12, 0x81,
-	0xE8, 3, 0x85, 0x00, 0x78,
-	0xCB, 5, 0x39, 0x2C, 0x00, 0x34, 0x02,
-	0xF7, 1, 0x20,
-	0xEA, 2, 0x00, 0x00,
-	PWCTR1, 1, 0x23, // Power control VRH[5:0]
-	PWCTR2, 1, 0x10, // Power control SAP[2:0];BT[3:0]
-	VMCTR1, 2, 0x3e, 0x28, // VCM control
-	VMCTR2, 1, 0x86, // VCM control2
-	MADCTL, 1, 0x48, // Memory Access Control
-	VSCRSADD, 1, 0x00, // Vertical scroll zero
-	PIXFMT, 1, 0x55,
-	FRMCTR1, 2, 0x00, 0x18,
-	DFUNCTR, 3, 0x08, 0x82, 0x27, // Display Function Control
-	0xF2, 1, 0x00, // 3Gamma Function Disable
-	GAMMASET, 1, 0x01, // Gamma curve selected
-	GMCTRP1, 15, 0x0F, 0x31, 0x2B, 0x0C, 0x0E, 0x08, // Set Gamma
-	0x4E, 0xF1, 0x37, 0x07, 0x10, 0x03, 0x0E, 0x09, 0x00,
-	GMCTRN1, 15, 0x00, 0x0E, 0x14, 0x03, 0x11, 0x07, // Set Gamma
-	0x31, 0xC1, 0x48, 0x08, 0x0F, 0x0C, 0x31, 0x36, 0x0F,
-	SLPOUT, 0x80, // Exit Sleep
-	DISPON, 0x80, // Display on
-	0x00, // End of list
-}
-
 // Configure prepares display for use
 func (d *Device) Configure(config Config) {
 
@@ -109,6 +80,33 @@ func (d *Device) Configure(config Config) {
 		delay(150)
 	}
 
+	initCmd := []byte{
+		0xEF, 3, 0x03, 0x80, 0x02,
+		0xCF, 3, 0x00, 0xC1, 0x30,
+		0xED, 4, 0x64, 0x03, 0x12, 0x81,
+		0xE8, 3, 0x85, 0x00, 0x78,
+		0xCB, 5, 0x39, 0x2C, 0x00, 0x34, 0x02,
+		0xF7, 1, 0x20,
+		0xEA, 2, 0x00, 0x00,
+		PWCTR1, 1, 0x23, // Power control VRH[5:0]
+		PWCTR2, 1, 0x10, // Power control SAP[2:0];BT[3:0]
+		VMCTR1, 2, 0x3e, 0x28, // VCM control
+		VMCTR2, 1, 0x86, // VCM control2
+		MADCTL, 1, 0x48, // Memory Access Control
+		VSCRSADD, 1, 0x00, // Vertical scroll zero
+		PIXFMT, 1, 0x55,
+		FRMCTR1, 2, 0x00, 0x18,
+		DFUNCTR, 3, 0x08, 0x82, 0x27, // Display Function Control
+		0xF2, 1, 0x00, // 3Gamma Function Disable
+		GAMMASET, 1, 0x01, // Gamma curve selected
+		GMCTRP1, 15, 0x0F, 0x31, 0x2B, 0x0C, 0x0E, 0x08, // Set Gamma
+		0x4E, 0xF1, 0x37, 0x07, 0x10, 0x03, 0x0E, 0x09, 0x00,
+		GMCTRN1, 15, 0x00, 0x0E, 0x14, 0x03, 0x11, 0x07, // Set Gamma
+		0x31, 0xC1, 0x48, 0x08, 0x0F, 0x0C, 0x31, 0x36, 0x0F,
+		SLPOUT, 0x80, // Exit Sleep
+		DISPON, 0x80, // Display on
+		0x00, // End of list
+	}
 	for i, c := 0, len(initCmd); i < c; {
 		cmd := initCmd[i]
 		if cmd == 0x00 {
@@ -148,14 +146,12 @@ func (d *Device) Display() error {
 	return nil
 }
 
-var err1 = errors.New("rectangle coordinates outside display area")
-
 // DrawRGBBitmap copies an RGB bitmap to the internal buffer at given coordinates
 func (d *Device) DrawRGBBitmap(x, y int16, data []uint16, w, h int16) error {
 	k, i := d.Size()
 	if x < 0 || y < 0 || w <= 0 || h <= 0 ||
 		x >= k || (x+w) > k || y >= i || (y+h) > i {
-		return err1
+		return errors.New("rectangle coordinates outside display area")
 	}
 	d.setWindow(x, y, w, h)
 	d.startWrite()
@@ -253,8 +249,7 @@ func (d *Device) SetRotation(rotation Rotation) {
 	case 3:
 		madctl = MADCTL_MX | MADCTL_MY | MADCTL_MV | MADCTL_BGR
 	}
-	cmdBuf[0] = madctl
-	d.sendCommand(MADCTL, cmdBuf[:1])
+	d.sendCommand(MADCTL, []uint8{madctl})
 	d.rotation = rotation
 }
 
@@ -285,14 +280,16 @@ func (d *Device) setWindow(x, y, w, h int16) {
 	//y += d.rowOffset
 	x1 := x + w - 1
 	if x != d.x0 || x1 != d.x1 {
-		cmdBuf[0], cmdBuf[1], cmdBuf[2], cmdBuf[3] = uint8(x>>8), uint8(x), uint8(x1>>8), uint8(x1)
-		d.sendCommand(CASET, cmdBuf[:4])
+		d.sendCommand(CASET, []uint8{
+			uint8(x >> 8), uint8(x), uint8(x1 >> 8), uint8(x1),
+		})
 		d.x0, d.x1 = x, x1
 	}
 	y1 := y + h - 1
 	if y != d.y0 || y1 != d.y1 {
-		cmdBuf[0], cmdBuf[1], cmdBuf[2], cmdBuf[3] = uint8(y>>8), uint8(y), uint8(y1>>8), uint8(y1)
-		d.sendCommand(PASET, cmdBuf[:4])
+		d.sendCommand(PASET, []uint8{
+			uint8(y >> 8), uint8(y), uint8(y1 >> 8), uint8(y1),
+		})
 		d.y0, d.y1 = y, y1
 	}
 	d.sendCommand(RAMWR, nil)

ili9341/parallel_atsamd51.go

@@ -1,3 +1,4 @@
+//go:build atsamd51
 // +build atsamd51
 
 package ili9341

ili9341/spi.go

@@ -1,50 +1,21 @@
+//go:build !atsamd51 && !atsamd21
 // +build !atsamd51,!atsamd21
 
 package ili9341
 
 import (
-	"device/sam"
 	"machine"
-	"unsafe"
 
-	dma "github.com/sago35/tinygo-dma"
+	"tinygo.org/x/drivers"
 )
 
-var (
-	dbg5 = machine.D5
-	dbg6 = machine.D6
-)
-
-func init() {
-	dbg5.Configure(machine.PinConfig{Mode: machine.PinOutput})
-	dbg6.Configure(machine.PinConfig{Mode: machine.PinOutput})
-}
-
 var buf [64]byte
 
 type spiDriver struct {
-	bus machine.SPI
+	bus drivers.SPI
 }
 
-var (
-	dmatx *dma.DMA
-	desc  *dma.DMADescriptor
-)
-
-func NewSPI(bus machine.SPI, dc, cs, rst machine.Pin) *Device {
-	from := make([]byte, 256)
-	for i := range from {
-		from[i] = byte(i)
-	}
-
-	dmatx = dma.NewDMA(func(d *dma.DMA) {
-		d.Wait()
-		return
-	})
-	dmatx.SetTrigger(dma.DMAC_CHANNEL_CHCTRLA_TRIGSRC_SERCOM1_TX)
-	dmatx.SetTriggerAction(sam.DMAC_CHANNEL_CHCTRLA_TRIGACT_BURST)
-	desc = dmatx.GetDescriptor()
-
+func NewSPI(bus drivers.SPI, dc, cs, rst machine.Pin) *Device {
 	return &Device{
 		dc:  dc,
 		cs:  cs,
@@ -72,19 +43,6 @@ func (pd *spiDriver) write8n(b byte, n int) {
 }
 
 func (pd *spiDriver) write8sl(b []byte) {
-	if len(b) > 64 {
-		desc.UpdateDescriptor(dma.DescriptorConfig{
-			SRC:      unsafe.Pointer(&b[0]),
-			DST:      unsafe.Pointer(&pd.bus.Bus.DATA.Reg),
-			SRCINC:   dma.DMAC_SRAM_BTCTRL_SRCINC_ENABLE,
-			DSTINC:   dma.DMAC_SRAM_BTCTRL_DSTINC_DISABLE,
-			SIZE:     uint32(len(b)), // Total size of DMA transfer
-			BLOCKACT: 1,
-		})
-		dmatx.Start()
-		return
-	}
-
 	pd.bus.Tx(b, nil)
 }
 
@@ -108,18 +66,6 @@ func (pd *spiDriver) write16n(data uint16, n int) {
 }
 
 func (pd *spiDriver) write16sl(data []uint16) {
-	if len(data) > 64 {
-		desc.UpdateDescriptor(dma.DescriptorConfig{
-			SRC:      unsafe.Pointer(&data[0]),
-			DST:      unsafe.Pointer(&pd.bus.Bus.DATA.Reg),
-			SRCINC:   dma.DMAC_SRAM_BTCTRL_SRCINC_ENABLE,
-			DSTINC:   dma.DMAC_SRAM_BTCTRL_DSTINC_DISABLE,
-			SIZE:     uint32(len(data) * 2), // Total size of DMA transfer
-			BLOCKACT: 1,
-		})
-		dmatx.Start()
-		return
-	}
 	for i, c := 0, len(data); i < c; i++ {
 		buf[0] = uint8(data[i] >> 8)
 		buf[1] = uint8(data[i])

ili9341/spi_atsamd21.go

@@ -1,3 +1,4 @@
+//go:build atsamd21
 // +build atsamd21
 
 package ili9341

ili9341/spi_atsamd51.go

@@ -1,3 +1,4 @@
+//go:build atsamd51
 // +build atsamd51
 
 package ili9341

image/README.md

@@ -0,0 +1,60 @@
+# tinygo.org/x/drivers/image
+
+This is an image package that uses less RAM to run on a microcontroller.
+Unlike Go's original image package, `image.Decode()` does not return `image.Image`.
+
+Instead, a callback can be set to process the data corresponding to the image.
+
+## How to use
+
+First, use `SetCallback()` to set the callback.
+Then call `png.Decode()` or `jpeg.Decode()`.
+The callback will be called as many times as necessary to load the image.
+
+`SetCallback()` needs to be given a Buffer to handle the callback and the actual function to be called.
+The `data []uint16` in the callback is in RGB565 format.
+
+The `io.Reader` to pass to `Decode()` specifies the binary data of the image.
+
+```go
+func drawPng(display *ili9341.Device) error {
+	p := strings.NewReader(pngImage)
+	png.SetCallback(buffer[:], func(data []uint16, x, y, w, h, width, height int16) {
+		err := display.DrawRGBBitmap(x, y, data[:w*h], w, h)
+		if err != nil {
+			errorMessage(fmt.Errorf("error drawPng: %s", err))
+		}
+	})
+
+	return png.Decode(p)
+}
+```
+
+```go
+func drawJpeg(display *ili9341.Device) error {
+	p := strings.NewReader(jpegImage)
+	jpeg.SetCallback(buffer[:], func(data []uint16, x, y, w, h, width, height int16) {
+		err := display.DrawRGBBitmap(x, y, data[:w*h], w, h)
+		if err != nil {
+			errorMessage(fmt.Errorf("error drawJpeg: %s", err))
+		}
+	})
+
+	return jpeg.Decode(p)
+}
+```
+
+## How to create an image
+
+The following program will output an image binary like the one in [images.go](./examples/ili9341/slideshow/images.go).  
+
+```
+go run ./cmd/convert2bin ./path/to/png_or_jpg.png
+```
+
+## Examples
+
+An example can be found below.
+Processing jpegs requires a minimum of 32KB of RAM.
+
+* [./examples/ili9341/slideshow](./examples/ili9341/slideshow)

image/internal/compress/flate/deflate.go

@@ -0,0 +1,748 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package flate
+
+import (
+	"fmt"
+	"io"
+	"math"
+)
+
+const (
+	NoCompression      = 0
+	BestSpeed          = 1
+	BestCompression    = 9
+	DefaultCompression = -1
+
+	// HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
+	// entropy encoding. This mode is useful in compressing data that has
+	// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
+	// that lacks an entropy encoder. Compression gains are achieved when
+	// certain bytes in the input stream occur more frequently than others.
+	//
+	// Note that HuffmanOnly produces a compressed output that is
+	// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
+	// continue to be able to decompress this output.
+	HuffmanOnly = -2
+)
+
+const (
+	logWindowSize = 15
+	windowSize    = 1 << logWindowSize
+	windowMask    = windowSize - 1
+
+	// The LZ77 step produces a sequence of literal tokens and <length, offset>
+	// pair tokens. The offset is also known as distance. The underlying wire
+	// format limits the range of lengths and offsets. For example, there are
+	// 256 legitimate lengths: those in the range [3, 258]. This package's
+	// compressor uses a higher minimum match length, enabling optimizations
+	// such as finding matches via 32-bit loads and compares.
+	baseMatchLength = 3       // The smallest match length per the RFC section 3.2.5
+	minMatchLength  = 4       // The smallest match length that the compressor actually emits
+	maxMatchLength  = 258     // The largest match length
+	baseMatchOffset = 1       // The smallest match offset
+	maxMatchOffset  = 1 << 15 // The largest match offset
+
+	// The maximum number of tokens we put into a single flate block, just to
+	// stop things from getting too large.
+	maxFlateBlockTokens = 1 << 14
+	maxStoreBlockSize   = 65535
+	hashBits            = 17 // After 17 performance degrades
+	hashSize            = 1 << hashBits
+	hashMask            = (1 << hashBits) - 1
+	maxHashOffset       = 1 << 24
+
+	skipNever = math.MaxInt32
+)
+
+type compressionLevel struct {
+	level, good, lazy, nice, chain, fastSkipHashing int
+}
+
+var levels = []compressionLevel{
+	{0, 0, 0, 0, 0, 0}, // NoCompression.
+	{1, 0, 0, 0, 0, 0}, // BestSpeed uses a custom algorithm; see deflatefast.go.
+	// For levels 2-3 we don't bother trying with lazy matches.
+	{2, 4, 0, 16, 8, 5},
+	{3, 4, 0, 32, 32, 6},
+	// Levels 4-9 use increasingly more lazy matching
+	// and increasingly stringent conditions for "good enough".
+	{4, 4, 4, 16, 16, skipNever},
+	{5, 8, 16, 32, 32, skipNever},
+	{6, 8, 16, 128, 128, skipNever},
+	{7, 8, 32, 128, 256, skipNever},
+	{8, 32, 128, 258, 1024, skipNever},
+	{9, 32, 258, 258, 4096, skipNever},
+}
+
+type compressor struct {
+	compressionLevel
+
+	w          *huffmanBitWriter
+	bulkHasher func([]byte, []uint32)
+
+	// compression algorithm
+	fill      func(*compressor, []byte) int // copy data to window
+	step      func(*compressor)             // process window
+	sync      bool                          // requesting flush
+	bestSpeed *deflateFast                  // Encoder for BestSpeed
+
+	// Input hash chains
+	// hashHead[hashValue] contains the largest inputIndex with the specified hash value
+	// If hashHead[hashValue] is within the current window, then
+	// hashPrev[hashHead[hashValue] & windowMask] contains the previous index
+	// with the same hash value.
+	chainHead  int
+	hashHead   [hashSize]uint32
+	hashPrev   [windowSize]uint32
+	hashOffset int
+
+	// input window: unprocessed data is window[index:windowEnd]
+	index         int
+	window        []byte
+	windowEnd     int
+	blockStart    int  // window index where current tokens start
+	byteAvailable bool // if true, still need to process window[index-1].
+
+	// queued output tokens
+	tokens []token
+
+	// deflate state
+	length         int
+	offset         int
+	hash           uint32
+	maxInsertIndex int
+	err            error
+
+	// hashMatch must be able to contain hashes for the maximum match length.
+	hashMatch [maxMatchLength - 1]uint32
+}
+
+func (d *compressor) fillDeflate(b []byte) int {
+	if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
+		// shift the window by windowSize
+		copy(d.window, d.window[windowSize:2*windowSize])
+		d.index -= windowSize
+		d.windowEnd -= windowSize
+		if d.blockStart >= windowSize {
+			d.blockStart -= windowSize
+		} else {
+			d.blockStart = math.MaxInt32
+		}
+		d.hashOffset += windowSize
+		if d.hashOffset > maxHashOffset {
+			delta := d.hashOffset - 1
+			d.hashOffset -= delta
+			d.chainHead -= delta
+
+			// Iterate over slices instead of arrays to avoid copying
+			// the entire table onto the stack (Issue #18625).
+			for i, v := range d.hashPrev[:] {
+				if int(v) > delta {
+					d.hashPrev[i] = uint32(int(v) - delta)
+				} else {
+					d.hashPrev[i] = 0
+				}
+			}
+			for i, v := range d.hashHead[:] {
+				if int(v) > delta {
+					d.hashHead[i] = uint32(int(v) - delta)
+				} else {
+					d.hashHead[i] = 0
+				}
+			}
+		}
+	}
+	n := copy(d.window[d.windowEnd:], b)
+	d.windowEnd += n
+	return n
+}
+
+func (d *compressor) writeBlock(tokens []token, index int) error {
+	if index > 0 {
+		var window []byte
+		if d.blockStart <= index {
+			window = d.window[d.blockStart:index]
+		}
+		d.blockStart = index
+		d.w.writeBlock(tokens, false, window)
+		return d.w.err
+	}
+	return nil
+}
+
+// fillWindow will fill the current window with the supplied
+// dictionary and calculate all hashes.
+// This is much faster than doing a full encode.
+// Should only be used after a reset.
+func (d *compressor) fillWindow(b []byte) {
+	// Do not fill window if we are in store-only mode.
+	if d.compressionLevel.level < 2 {
+		return
+	}
+	if d.index != 0 || d.windowEnd != 0 {
+		panic("internal error: fillWindow called with stale data")
+	}
+
+	// If we are given too much, cut it.
+	if len(b) > windowSize {
+		b = b[len(b)-windowSize:]
+	}
+	// Add all to window.
+	n := copy(d.window, b)
+
+	// Calculate 256 hashes at the time (more L1 cache hits)
+	loops := (n + 256 - minMatchLength) / 256
+	for j := 0; j < loops; j++ {
+		index := j * 256
+		end := index + 256 + minMatchLength - 1
+		if end > n {
+			end = n
+		}
+		toCheck := d.window[index:end]
+		dstSize := len(toCheck) - minMatchLength + 1
+
+		if dstSize <= 0 {
+			continue
+		}
+
+		dst := d.hashMatch[:dstSize]
+		d.bulkHasher(toCheck, dst)
+		var newH uint32
+		for i, val := range dst {
+			di := i + index
+			newH = val
+			hh := &d.hashHead[newH&hashMask]
+			// Get previous value with the same hash.
+			// Our chain should point to the previous value.
+			d.hashPrev[di&windowMask] = *hh
+			// Set the head of the hash chain to us.
+			*hh = uint32(di + d.hashOffset)
+		}
+		d.hash = newH
+	}
+	// Update window information.
+	d.windowEnd = n
+	d.index = n
+}
+
+// Try to find a match starting at index whose length is greater than prevSize.
+// We only look at chainCount possibilities before giving up.
+func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
+	minMatchLook := maxMatchLength
+	if lookahead < minMatchLook {
+		minMatchLook = lookahead
+	}
+
+	win := d.window[0 : pos+minMatchLook]
+
+	// We quit when we get a match that's at least nice long
+	nice := len(win) - pos
+	if d.nice < nice {
+		nice = d.nice
+	}
+
+	// If we've got a match that's good enough, only look in 1/4 the chain.
+	tries := d.chain
+	length = prevLength
+	if length >= d.good {
+		tries >>= 2
+	}
+
+	wEnd := win[pos+length]
+	wPos := win[pos:]
+	minIndex := pos - windowSize
+
+	for i := prevHead; tries > 0; tries-- {
+		if wEnd == win[i+length] {
+			n := matchLen(win[i:], wPos, minMatchLook)
+
+			if n > length && (n > minMatchLength || pos-i <= 4096) {
+				length = n
+				offset = pos - i
+				ok = true
+				if n >= nice {
+					// The match is good enough that we don't try to find a better one.
+					break
+				}
+				wEnd = win[pos+n]
+			}
+		}
+		if i == minIndex {
+			// hashPrev[i & windowMask] has already been overwritten, so stop now.
+			break
+		}
+		i = int(d.hashPrev[i&windowMask]) - d.hashOffset
+		if i < minIndex || i < 0 {
+			break
+		}
+	}
+	return
+}
+
+func (d *compressor) writeStoredBlock(buf []byte) error {
+	if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
+		return d.w.err
+	}
+	d.w.writeBytes(buf)
+	return d.w.err
+}
+
+const hashmul = 0x1e35a7bd
+
+// hash4 returns a hash representation of the first 4 bytes
+// of the supplied slice.
+// The caller must ensure that len(b) >= 4.
+func hash4(b []byte) uint32 {
+	return ((uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24) * hashmul) >> (32 - hashBits)
+}
+
+// bulkHash4 will compute hashes using the same
+// algorithm as hash4
+func bulkHash4(b []byte, dst []uint32) {
+	if len(b) < minMatchLength {
+		return
+	}
+	hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
+	dst[0] = (hb * hashmul) >> (32 - hashBits)
+	end := len(b) - minMatchLength + 1
+	for i := 1; i < end; i++ {
+		hb = (hb << 8) | uint32(b[i+3])
+		dst[i] = (hb * hashmul) >> (32 - hashBits)
+	}
+}
+
+// matchLen returns the number of matching bytes in a and b
+// up to length 'max'. Both slices must be at least 'max'
+// bytes in size.
+func matchLen(a, b []byte, max int) int {
+	a = a[:max]
+	b = b[:len(a)]
+	for i, av := range a {
+		if b[i] != av {
+			return i
+		}
+	}
+	return max
+}
+
+// encSpeed will compress and store the currently added data,
+// if enough has been accumulated or we at the end of the stream.
+// Any error that occurred will be in d.err
+func (d *compressor) encSpeed() {
+	// We only compress if we have maxStoreBlockSize.
+	if d.windowEnd < maxStoreBlockSize {
+		if !d.sync {
+			return
+		}
+
+		// Handle small sizes.
+		if d.windowEnd < 128 {
+			switch {
+			case d.windowEnd == 0:
+				return
+			case d.windowEnd <= 16:
+				d.err = d.writeStoredBlock(d.window[:d.windowEnd])
+			default:
+				d.w.writeBlockHuff(false, d.window[:d.windowEnd])
+				d.err = d.w.err
+			}
+			d.windowEnd = 0
+			d.bestSpeed.reset()
+			return
+		}
+
+	}
+	// Encode the block.
+	d.tokens = d.bestSpeed.encode(d.tokens[:0], d.window[:d.windowEnd])
+
+	// If we removed less than 1/16th, Huffman compress the block.
+	if len(d.tokens) > d.windowEnd-(d.windowEnd>>4) {
+		d.w.writeBlockHuff(false, d.window[:d.windowEnd])
+	} else {
+		d.w.writeBlockDynamic(d.tokens, false, d.window[:d.windowEnd])
+	}
+	d.err = d.w.err
+	d.windowEnd = 0
+}
+
+func (d *compressor) initDeflate() {
+	d.window = make([]byte, 2*windowSize)
+	d.hashOffset = 1
+	d.tokens = make([]token, 0, maxFlateBlockTokens+1)
+	d.length = minMatchLength - 1
+	d.offset = 0
+	d.byteAvailable = false
+	d.index = 0
+	d.hash = 0
+	d.chainHead = -1
+	d.bulkHasher = bulkHash4
+}
+
+func (d *compressor) deflate() {
+	if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
+		return
+	}
+
+	d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
+	if d.index < d.maxInsertIndex {
+		d.hash = hash4(d.window[d.index : d.index+minMatchLength])
+	}
+
+Loop:
+	for {
+		if d.index > d.windowEnd {
+			panic("index > windowEnd")
+		}
+		lookahead := d.windowEnd - d.index
+		if lookahead < minMatchLength+maxMatchLength {
+			if !d.sync {
+				break Loop
+			}
+			if d.index > d.windowEnd {
+				panic("index > windowEnd")
+			}
+			if lookahead == 0 {
+				// Flush current output block if any.
+				if d.byteAvailable {
+					// There is still one pending token that needs to be flushed
+					d.tokens = append(d.tokens, literalToken(uint32(d.window[d.index-1])))
+					d.byteAvailable = false
+				}
+				if len(d.tokens) > 0 {
+					if d.err = d.writeBlock(d.tokens, d.index); d.err != nil {
+						return
+					}
+					d.tokens = d.tokens[:0]
+				}
+				break Loop
+			}
+		}
+		if d.index < d.maxInsertIndex {
+			// Update the hash
+			d.hash = hash4(d.window[d.index : d.index+minMatchLength])
+			hh := &d.hashHead[d.hash&hashMask]
+			d.chainHead = int(*hh)
+			d.hashPrev[d.index&windowMask] = uint32(d.chainHead)
+			*hh = uint32(d.index + d.hashOffset)
+		}
+		prevLength := d.length
+		prevOffset := d.offset
+		d.length = minMatchLength - 1
+		d.offset = 0
+		minIndex := d.index - windowSize
+		if minIndex < 0 {
+			minIndex = 0
+		}
+
+		if d.chainHead-d.hashOffset >= minIndex &&
+			(d.fastSkipHashing != skipNever && lookahead > minMatchLength-1 ||
+				d.fastSkipHashing == skipNever && lookahead > prevLength && prevLength < d.lazy) {
+			if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
+				d.length = newLength
+				d.offset = newOffset
+			}
+		}
+		if d.fastSkipHashing != skipNever && d.length >= minMatchLength ||
+			d.fastSkipHashing == skipNever && prevLength >= minMatchLength && d.length <= prevLength {
+			// There was a match at the previous step, and the current match is
+			// not better. Output the previous match.
+			if d.fastSkipHashing != skipNever {
+				d.tokens = append(d.tokens, matchToken(uint32(d.length-baseMatchLength), uint32(d.offset-baseMatchOffset)))
+			} else {
+				d.tokens = append(d.tokens, matchToken(uint32(prevLength-baseMatchLength), uint32(prevOffset-baseMatchOffset)))
+			}
+			// Insert in the hash table all strings up to the end of the match.
+			// index and index-1 are already inserted. If there is not enough
+			// lookahead, the last two strings are not inserted into the hash
+			// table.
+			if d.length <= d.fastSkipHashing {
+				var newIndex int
+				if d.fastSkipHashing != skipNever {
+					newIndex = d.index + d.length
+				} else {
+					newIndex = d.index + prevLength - 1
+				}
+				index := d.index
+				for index++; index < newIndex; index++ {
+					if index < d.maxInsertIndex {
+						d.hash = hash4(d.window[index : index+minMatchLength])
+						// Get previous value with the same hash.
+						// Our chain should point to the previous value.
+						hh := &d.hashHead[d.hash&hashMask]
+						d.hashPrev[index&windowMask] = *hh
+						// Set the head of the hash chain to us.
+						*hh = uint32(index + d.hashOffset)
+					}
+				}
+				d.index = index
+
+				if d.fastSkipHashing == skipNever {
+					d.byteAvailable = false
+					d.length = minMatchLength - 1
+				}
+			} else {
+				// For matches this long, we don't bother inserting each individual
+				// item into the table.
+				d.index += d.length
+				if d.index < d.maxInsertIndex {
+					d.hash = hash4(d.window[d.index : d.index+minMatchLength])
+				}
+			}
+			if len(d.tokens) == maxFlateBlockTokens {
+				// The block includes the current character
+				if d.err = d.writeBlock(d.tokens, d.index); d.err != nil {
+					return
+				}
+				d.tokens = d.tokens[:0]
+			}
+		} else {
+			if d.fastSkipHashing != skipNever || d.byteAvailable {
+				i := d.index - 1
+				if d.fastSkipHashing != skipNever {
+					i = d.index
+				}
+				d.tokens = append(d.tokens, literalToken(uint32(d.window[i])))
+				if len(d.tokens) == maxFlateBlockTokens {
+					if d.err = d.writeBlock(d.tokens, i+1); d.err != nil {
+						return
+					}
+					d.tokens = d.tokens[:0]
+				}
+			}
+			d.index++
+			if d.fastSkipHashing == skipNever {
+				d.byteAvailable = true
+			}
+		}
+	}
+}
+
+func (d *compressor) fillStore(b []byte) int {
+	n := copy(d.window[d.windowEnd:], b)
+	d.windowEnd += n
+	return n
+}
+
+func (d *compressor) store() {
+	if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) {
+		d.err = d.writeStoredBlock(d.window[:d.windowEnd])
+		d.windowEnd = 0
+	}
+}
+
+// storeHuff compresses and stores the currently added data
+// when the d.window is full or we are at the end of the stream.
+// Any error that occurred will be in d.err
+func (d *compressor) storeHuff() {
+	if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
+		return
+	}
+	d.w.writeBlockHuff(false, d.window[:d.windowEnd])
+	d.err = d.w.err
+	d.windowEnd = 0
+}
+
+func (d *compressor) write(b []byte) (n int, err error) {
+	if d.err != nil {
+		return 0, d.err
+	}
+	n = len(b)
+	for len(b) > 0 {
+		d.step(d)
+		b = b[d.fill(d, b):]
+		if d.err != nil {
+			return 0, d.err
+		}
+	}
+	return n, nil
+}
+
+func (d *compressor) syncFlush() error {
+	if d.err != nil {
+		return d.err
+	}
+	d.sync = true
+	d.step(d)
+	if d.err == nil {
+		d.w.writeStoredHeader(0, false)
+		d.w.flush()
+		d.err = d.w.err
+	}
+	d.sync = false
+	return d.err
+}
+
+func (d *compressor) init(w io.Writer, level int) (err error) {
+	d.w = newHuffmanBitWriter(w)
+
+	switch {
+	case level == NoCompression:
+		d.window = make([]byte, maxStoreBlockSize)
+		d.fill = (*compressor).fillStore
+		d.step = (*compressor).store
+	case level == HuffmanOnly:
+		d.window = make([]byte, maxStoreBlockSize)
+		d.fill = (*compressor).fillStore
+		d.step = (*compressor).storeHuff
+	case level == BestSpeed:
+		d.compressionLevel = levels[level]
+		d.window = make([]byte, maxStoreBlockSize)
+		d.fill = (*compressor).fillStore
+		d.step = (*compressor).encSpeed
+		d.bestSpeed = newDeflateFast()
+		d.tokens = make([]token, maxStoreBlockSize)
+	case level == DefaultCompression:
+		level = 6
+		fallthrough
+	case 2 <= level && level <= 9:
+		d.compressionLevel = levels[level]
+		d.initDeflate()
+		d.fill = (*compressor).fillDeflate
+		d.step = (*compressor).deflate
+	default:
+		return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
+	}
+	return nil
+}
+
+func (d *compressor) reset(w io.Writer) {
+	d.w.reset(w)
+	d.sync = false
+	d.err = nil
+	switch d.compressionLevel.level {
+	case NoCompression:
+		d.windowEnd = 0
+	case BestSpeed:
+		d.windowEnd = 0
+		d.tokens = d.tokens[:0]
+		d.bestSpeed.reset()
+	default:
+		d.chainHead = -1
+		for i := range d.hashHead {
+			d.hashHead[i] = 0
+		}
+		for i := range d.hashPrev {
+			d.hashPrev[i] = 0
+		}
+		d.hashOffset = 1
+		d.index, d.windowEnd = 0, 0
+		d.blockStart, d.byteAvailable = 0, false
+		d.tokens = d.tokens[:0]
+		d.length = minMatchLength - 1
+		d.offset = 0
+		d.hash = 0
+		d.maxInsertIndex = 0
+	}
+}
+
+func (d *compressor) close() error {
+	if d.err != nil {
+		return d.err
+	}
+	d.sync = true
+	d.step(d)
+	if d.err != nil {
+		return d.err
+	}
+	if d.w.writeStoredHeader(0, true); d.w.err != nil {
+		return d.w.err
+	}
+	d.w.flush()
+	return d.w.err
+}
+
+// NewWriter returns a new Writer compressing data at the given level.
+// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
+// higher levels typically run slower but compress more. Level 0
+// (NoCompression) does not attempt any compression; it only adds the
+// necessary DEFLATE framing.
+// Level -1 (DefaultCompression) uses the default compression level.
+// Level -2 (HuffmanOnly) will use Huffman compression only, giving
+// a very fast compression for all types of input, but sacrificing considerable
+// compression efficiency.
+//
+// If level is in the range [-2, 9] then the error returned will be nil.
+// Otherwise the error returned will be non-nil.
+func NewWriter(w io.Writer, level int) (*Writer, error) {
+	var dw Writer
+	if err := dw.d.init(w, level); err != nil {
+		return nil, err
+	}
+	return &dw, nil
+}
+
+// NewWriterDict is like NewWriter but initializes the new
+// Writer with a preset dictionary. The returned Writer behaves
+// as if the dictionary had been written to it without producing
+// any compressed output. The compressed data written to w
+// can only be decompressed by a Reader initialized with the
+// same dictionary.
+func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
+	dw := &dictWriter{w}
+	zw, err := NewWriter(dw, level)
+	if err != nil {
+		return nil, err
+	}
+	zw.d.fillWindow(dict)
+	zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
+	return zw, err
+}
+
+type dictWriter struct {
+	w io.Writer
+}
+
+func (w *dictWriter) Write(b []byte) (n int, err error) {
+	return w.w.Write(b)
+}
+
+// A Writer takes data written to it and writes the compressed
+// form of that data to an underlying writer (see NewWriter).
+type Writer struct {
+	d    compressor
+	dict []byte
+}
+
+// Write writes data to w, which will eventually write the
+// compressed form of data to its underlying writer.
+func (w *Writer) Write(data []byte) (n int, err error) {
+	return w.d.write(data)
+}
+
+// Flush flushes any pending data to the underlying writer.
+// It is useful mainly in compressed network protocols, to ensure that
+// a remote reader has enough data to reconstruct a packet.
+// Flush does not return until the data has been written.
+// Calling Flush when there is no pending data still causes the Writer
+// to emit a sync marker of at least 4 bytes.
+// If the underlying writer returns an error, Flush returns that error.
+//
+// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
+func (w *Writer) Flush() error {
+	// For more about flushing:
+	// https://www.bolet.org/~pornin/deflate-flush.html
+	return w.d.syncFlush()
+}
+
+// Close flushes and closes the writer.
+func (w *Writer) Close() error {
+	return w.d.close()
+}
+
+// Reset discards the writer's state and makes it equivalent to
+// the result of NewWriter or NewWriterDict called with dst
+// and w's level and dictionary.
+func (w *Writer) Reset(dst io.Writer) {
+	if dw, ok := w.d.w.writer.(*dictWriter); ok {
+		// w was created with NewWriterDict
+		dw.w = dst
+		w.d.reset(dw)
+		w.d.fillWindow(w.dict)
+	} else {
+		// w was created with NewWriter
+		w.d.reset(dst)
+	}
+}

image/internal/compress/flate/deflate_test.go

@@ -0,0 +1,984 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package flate
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"internal/testenv"
+	"io"
+	"math/rand"
+	"os"
+	"reflect"
+	"runtime/debug"
+	"sync"
+	"testing"
+)
+
+type deflateTest struct {
+	in    []byte
+	level int
+	out   []byte
+}
+
+type deflateInflateTest struct {
+	in []byte
+}
+
+type reverseBitsTest struct {
+	in       uint16
+	bitCount uint8
+	out      uint16
+}
+
+var deflateTests = []*deflateTest{
+	{[]byte{}, 0, []byte{1, 0, 0, 255, 255}},
+	{[]byte{0x11}, -1, []byte{18, 4, 4, 0, 0, 255, 255}},
+	{[]byte{0x11}, DefaultCompression, []byte{18, 4, 4, 0, 0, 255, 255}},
+	{[]byte{0x11}, 4, []byte{18, 4, 4, 0, 0, 255, 255}},
+
+	{[]byte{0x11}, 0, []byte{0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255}},
+	{[]byte{0x11, 0x12}, 0, []byte{0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255}},
+	{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 0,
+		[]byte{0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255},
+	},
+	{[]byte{}, 2, []byte{1, 0, 0, 255, 255}},
+	{[]byte{0x11}, 2, []byte{18, 4, 4, 0, 0, 255, 255}},
+	{[]byte{0x11, 0x12}, 2, []byte{18, 20, 2, 4, 0, 0, 255, 255}},
+	{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 2, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}},
+	{[]byte{}, 9, []byte{1, 0, 0, 255, 255}},
+	{[]byte{0x11}, 9, []byte{18, 4, 4, 0, 0, 255, 255}},
+	{[]byte{0x11, 0x12}, 9, []byte{18, 20, 2, 4, 0, 0, 255, 255}},
+	{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 9, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}},
+}
+
+var deflateInflateTests = []*deflateInflateTest{
+	{[]byte{}},
+	{[]byte{0x11}},
+	{[]byte{0x11, 0x12}},
+	{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}},
+	{[]byte{0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13}},
+	{largeDataChunk()},
+}
+
+var reverseBitsTests = []*reverseBitsTest{
+	{1, 1, 1},
+	{1, 2, 2},
+	{1, 3, 4},
+	{1, 4, 8},
+	{1, 5, 16},
+	{17, 5, 17},
+	{257, 9, 257},
+	{29, 5, 23},
+}
+
+func largeDataChunk() []byte {
+	result := make([]byte, 100000)
+	for i := range result {
+		result[i] = byte(i * i & 0xFF)
+	}
+	return result
+}
+
+func TestBulkHash4(t *testing.T) {
+	for _, x := range deflateTests {
+		y := x.out
+		if len(y) < minMatchLength {
+			continue
+		}
+		y = append(y, y...)
+		for j := 4; j < len(y); j++ {
+			y := y[:j]
+			dst := make([]uint32, len(y)-minMatchLength+1)
+			for i := range dst {
+				dst[i] = uint32(i + 100)
+			}
+			bulkHash4(y, dst)
+			for i, got := range dst {
+				want := hash4(y[i:])
+				if got != want && got == uint32(i)+100 {
+					t.Errorf("Len:%d Index:%d, want 0x%08x but not modified", len(y), i, want)
+				} else if got != want {
+					t.Errorf("Len:%d Index:%d, got 0x%08x want:0x%08x", len(y), i, got, want)
+				}
+			}
+		}
+	}
+}
+
+func TestDeflate(t *testing.T) {
+	for _, h := range deflateTests {
+		var buf bytes.Buffer
+		w, err := NewWriter(&buf, h.level)
+		if err != nil {
+			t.Errorf("NewWriter: %v", err)
+			continue
+		}
+		w.Write(h.in)
+		w.Close()
+		if !bytes.Equal(buf.Bytes(), h.out) {
+			t.Errorf("Deflate(%d, %x) = \n%#v, want \n%#v", h.level, h.in, buf.Bytes(), h.out)
+		}
+	}
+}
+
+// A sparseReader returns a stream consisting of 0s followed by 1<<16 1s.
+// This tests missing hash references in a very large input.
+type sparseReader struct {
+	l   int64
+	cur int64
+}
+
+func (r *sparseReader) Read(b []byte) (n int, err error) {
+	if r.cur >= r.l {
+		return 0, io.EOF
+	}
+	n = len(b)
+	cur := r.cur + int64(n)
+	if cur > r.l {
+		n -= int(cur - r.l)
+		cur = r.l
+	}
+	for i := range b[0:n] {
+		if r.cur+int64(i) >= r.l-1<<16 {
+			b[i] = 1
+		} else {
+			b[i] = 0
+		}
+	}
+	r.cur = cur
+	return
+}
+
+func TestVeryLongSparseChunk(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping sparse chunk during short test")
+	}
+	w, err := NewWriter(io.Discard, 1)
+	if err != nil {
+		t.Errorf("NewWriter: %v", err)
+		return
+	}
+	if _, err = io.Copy(w, &sparseReader{l: 23e8}); err != nil {
+		t.Errorf("Compress failed: %v", err)
+		return
+	}
+}
+
+type syncBuffer struct {
+	buf    bytes.Buffer
+	mu     sync.RWMutex
+	closed bool
+	ready  chan bool
+}
+
+func newSyncBuffer() *syncBuffer {
+	return &syncBuffer{ready: make(chan bool, 1)}
+}
+
+func (b *syncBuffer) Read(p []byte) (n int, err error) {
+	for {
+		b.mu.RLock()
+		n, err = b.buf.Read(p)
+		b.mu.RUnlock()
+		if n > 0 || b.closed {
+			return
+		}
+		<-b.ready
+	}
+}
+
+func (b *syncBuffer) signal() {
+	select {
+	case b.ready <- true:
+	default:
+	}
+}
+
+func (b *syncBuffer) Write(p []byte) (n int, err error) {
+	n, err = b.buf.Write(p)
+	b.signal()
+	return
+}
+
+func (b *syncBuffer) WriteMode() {
+	b.mu.Lock()
+}
+
+func (b *syncBuffer) ReadMode() {
+	b.mu.Unlock()
+	b.signal()
+}
+
+func (b *syncBuffer) Close() error {
+	b.closed = true
+	b.signal()
+	return nil
+}
+
+func testSync(t *testing.T, level int, input []byte, name string) {
+	if len(input) == 0 {
+		return
+	}
+
+	t.Logf("--testSync %d, %d, %s", level, len(input), name)
+	buf := newSyncBuffer()
+	buf1 := new(bytes.Buffer)
+	buf.WriteMode()
+	w, err := NewWriter(io.MultiWriter(buf, buf1), level)
+	if err != nil {
+		t.Errorf("NewWriter: %v", err)
+		return
+	}
+	r := NewReader(buf)
+
+	// Write half the input and read back.
+	for i := 0; i < 2; i++ {
+		var lo, hi int
+		if i == 0 {
+			lo, hi = 0, (len(input)+1)/2
+		} else {
+			lo, hi = (len(input)+1)/2, len(input)
+		}
+		t.Logf("#%d: write %d-%d", i, lo, hi)
+		if _, err := w.Write(input[lo:hi]); err != nil {
+			t.Errorf("testSync: write: %v", err)
+			return
+		}
+		if i == 0 {
+			if err := w.Flush(); err != nil {
+				t.Errorf("testSync: flush: %v", err)
+				return
+			}
+		} else {
+			if err := w.Close(); err != nil {
+				t.Errorf("testSync: close: %v", err)
+			}
+		}
+		buf.ReadMode()
+		out := make([]byte, hi-lo+1)
+		m, err := io.ReadAtLeast(r, out, hi-lo)
+		t.Logf("#%d: read %d", i, m)
+		if m != hi-lo || err != nil {
+			t.Errorf("testSync/%d (%d, %d, %s): read %d: %d, %v (%d left)", i, level, len(input), name, hi-lo, m, err, buf.buf.Len())
+			return
+		}
+		if !bytes.Equal(input[lo:hi], out[:hi-lo]) {
+			t.Errorf("testSync/%d: read wrong bytes: %x vs %x", i, input[lo:hi], out[:hi-lo])
+			return
+		}
+		// This test originally checked that after reading
+		// the first half of the input, there was nothing left
+		// in the read buffer (buf.buf.Len() != 0) but that is
+		// not necessarily the case: the write Flush may emit
+		// some extra framing bits that are not necessary
+		// to process to obtain the first half of the uncompressed
+		// data. The test ran correctly most of the time, because
+		// the background goroutine had usually read even
+		// those extra bits by now, but it's not a useful thing to
+		// check.
+		buf.WriteMode()
+	}
+	buf.ReadMode()
+	out := make([]byte, 10)
+	if n, err := r.Read(out); n > 0 || err != io.EOF {
+		t.Errorf("testSync (%d, %d, %s): final Read: %d, %v (hex: %x)", level, len(input), name, n, err, out[0:n])
+	}
+	if buf.buf.Len() != 0 {
+		t.Errorf("testSync (%d, %d, %s): extra data at end", level, len(input), name)
+	}
+	r.Close()
+
+	// stream should work for ordinary reader too
+	r = NewReader(buf1)
+	out, err = io.ReadAll(r)
+	if err != nil {
+		t.Errorf("testSync: read: %s", err)
+		return
+	}
+	r.Close()
+	if !bytes.Equal(input, out) {
+		t.Errorf("testSync: decompress(compress(data)) != data: level=%d input=%s", level, name)
+	}
+}
+
+func testToFromWithLevelAndLimit(t *testing.T, level int, input []byte, name string, limit int) {
+	var buffer bytes.Buffer
+	w, err := NewWriter(&buffer, level)
+	if err != nil {
+		t.Errorf("NewWriter: %v", err)
+		return
+	}
+	w.Write(input)
+	w.Close()
+	if limit > 0 && buffer.Len() > limit {
+		t.Errorf("level: %d, len(compress(data)) = %d > limit = %d", level, buffer.Len(), limit)
+		return
+	}
+	if limit > 0 {
+		t.Logf("level: %d, size:%.2f%%, %d b\n", level, float64(buffer.Len()*100)/float64(limit), buffer.Len())
+	}
+	r := NewReader(&buffer)
+	out, err := io.ReadAll(r)
+	if err != nil {
+		t.Errorf("read: %s", err)
+		return
+	}
+	r.Close()
+	if !bytes.Equal(input, out) {
+		t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name)
+		return
+	}
+	testSync(t, level, input, name)
+}
+
+func testToFromWithLimit(t *testing.T, input []byte, name string, limit [11]int) {
+	for i := 0; i < 10; i++ {
+		testToFromWithLevelAndLimit(t, i, input, name, limit[i])
+	}
+	// Test HuffmanCompression
+	testToFromWithLevelAndLimit(t, -2, input, name, limit[10])
+}
+
+func TestDeflateInflate(t *testing.T) {
+	t.Parallel()
+	for i, h := range deflateInflateTests {
+		if testing.Short() && len(h.in) > 10000 {
+			continue
+		}
+		testToFromWithLimit(t, h.in, fmt.Sprintf("#%d", i), [11]int{})
+	}
+}
+
+func TestReverseBits(t *testing.T) {
+	for _, h := range reverseBitsTests {
+		if v := reverseBits(h.in, h.bitCount); v != h.out {
+			t.Errorf("reverseBits(%v,%v) = %v, want %v",
+				h.in, h.bitCount, v, h.out)
+		}
+	}
+}
+
+type deflateInflateStringTest struct {
+	filename string
+	label    string
+	limit    [11]int
+}
+
+var deflateInflateStringTests = []deflateInflateStringTest{
+	{
+		"../testdata/e.txt",
+		"2.718281828...",
+		[...]int{100018, 50650, 50960, 51150, 50930, 50790, 50790, 50790, 50790, 50790, 43683},
+	},
+	{
+		"../../testdata/Isaac.Newton-Opticks.txt",
+		"Isaac.Newton-Opticks",
+		[...]int{567248, 218338, 198211, 193152, 181100, 175427, 175427, 173597, 173422, 173422, 325240},
+	},
+}
+
+func TestDeflateInflateString(t *testing.T) {
+	t.Parallel()
+	if testing.Short() && testenv.Builder() == "" {
+		t.Skip("skipping in short mode")
+	}
+	for _, test := range deflateInflateStringTests {
+		gold, err := os.ReadFile(test.filename)
+		if err != nil {
+			t.Error(err)
+		}
+		testToFromWithLimit(t, gold, test.label, test.limit)
+		if testing.Short() {
+			break
+		}
+	}
+}
+
+func TestReaderDict(t *testing.T) {
+	const (
+		dict = "hello world"
+		text = "hello again world"
+	)
+	var b bytes.Buffer
+	w, err := NewWriter(&b, 5)
+	if err != nil {
+		t.Fatalf("NewWriter: %v", err)
+	}
+	w.Write([]byte(dict))
+	w.Flush()
+	b.Reset()
+	w.Write([]byte(text))
+	w.Close()
+
+	r := NewReaderDict(&b, []byte(dict))
+	data, err := io.ReadAll(r)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(data) != "hello again world" {
+		t.Fatalf("read returned %q want %q", string(data), text)
+	}
+}
+
+func TestWriterDict(t *testing.T) {
+	const (
+		dict = "hello world"
+		text = "hello again world"
+	)
+	var b bytes.Buffer
+	w, err := NewWriter(&b, 5)
+	if err != nil {
+		t.Fatalf("NewWriter: %v", err)
+	}
+	w.Write([]byte(dict))
+	w.Flush()
+	b.Reset()
+	w.Write([]byte(text))
+	w.Close()
+
+	var b1 bytes.Buffer
+	w, _ = NewWriterDict(&b1, 5, []byte(dict))
+	w.Write([]byte(text))
+	w.Close()
+
+	if !bytes.Equal(b1.Bytes(), b.Bytes()) {
+		t.Fatalf("writer wrote %q want %q", b1.Bytes(), b.Bytes())
+	}
+}
+
+// See https://golang.org/issue/2508
+func TestRegression2508(t *testing.T) {
+	if testing.Short() {
+		t.Logf("test disabled with -short")
+		return
+	}
+	w, err := NewWriter(io.Discard, 1)
+	if err != nil {
+		t.Fatalf("NewWriter: %v", err)
+	}
+	buf := make([]byte, 1024)
+	for i := 0; i < 131072; i++ {
+		if _, err := w.Write(buf); err != nil {
+			t.Fatalf("writer failed: %v", err)
+		}
+	}
+	w.Close()
+}
+
+func TestWriterReset(t *testing.T) {
+	t.Parallel()
+	for level := 0; level <= 9; level++ {
+		if testing.Short() && level > 1 {
+			break
+		}
+		w, err := NewWriter(io.Discard, level)
+		if err != nil {
+			t.Fatalf("NewWriter: %v", err)
+		}
+		buf := []byte("hello world")
+		n := 1024
+		if testing.Short() {
+			n = 10
+		}
+		for i := 0; i < n; i++ {
+			w.Write(buf)
+		}
+		w.Reset(io.Discard)
+
+		wref, err := NewWriter(io.Discard, level)
+		if err != nil {
+			t.Fatalf("NewWriter: %v", err)
+		}
+
+		// DeepEqual doesn't compare functions.
+		w.d.fill, wref.d.fill = nil, nil
+		w.d.step, wref.d.step = nil, nil
+		w.d.bulkHasher, wref.d.bulkHasher = nil, nil
+		w.d.bestSpeed, wref.d.bestSpeed = nil, nil
+		// hashMatch is always overwritten when used.
+		copy(w.d.hashMatch[:], wref.d.hashMatch[:])
+		if len(w.d.tokens) != 0 {
+			t.Errorf("level %d Writer not reset after Reset. %d tokens were present", level, len(w.d.tokens))
+		}
+		// As long as the length is 0, we don't care about the content.
+		w.d.tokens = wref.d.tokens
+
+		// We don't care if there are values in the window, as long as it is at d.index is 0
+		w.d.window = wref.d.window
+		if !reflect.DeepEqual(w, wref) {
+			t.Errorf("level %d Writer not reset after Reset", level)
+		}
+	}
+
+	levels := []int{0, 1, 2, 5, 9}
+	for _, level := range levels {
+		t.Run(fmt.Sprint(level), func(t *testing.T) {
+			testResetOutput(t, level, nil)
+		})
+	}
+
+	t.Run("dict", func(t *testing.T) {
+		for _, level := range levels {
+			t.Run(fmt.Sprint(level), func(t *testing.T) {
+				testResetOutput(t, level, nil)
+			})
+		}
+	})
+}
+
+func testResetOutput(t *testing.T, level int, dict []byte) {
+	writeData := func(w *Writer) {
+		msg := []byte("now is the time for all good gophers")
+		w.Write(msg)
+		w.Flush()
+
+		hello := []byte("hello world")
+		for i := 0; i < 1024; i++ {
+			w.Write(hello)
+		}
+
+		fill := bytes.Repeat([]byte("x"), 65000)
+		w.Write(fill)
+	}
+
+	buf := new(bytes.Buffer)
+	var w *Writer
+	var err error
+	if dict == nil {
+		w, err = NewWriter(buf, level)
+	} else {
+		w, err = NewWriterDict(buf, level, dict)
+	}
+	if err != nil {
+		t.Fatalf("NewWriter: %v", err)
+	}
+
+	writeData(w)
+	w.Close()
+	out1 := buf.Bytes()
+
+	buf2 := new(bytes.Buffer)
+	w.Reset(buf2)
+	writeData(w)
+	w.Close()
+	out2 := buf2.Bytes()
+
+	if len(out1) != len(out2) {
+		t.Errorf("got %d, expected %d bytes", len(out2), len(out1))
+		return
+	}
+	if !bytes.Equal(out1, out2) {
+		mm := 0
+		for i, b := range out1[:len(out2)] {
+			if b != out2[i] {
+				t.Errorf("mismatch index %d: %#02x, expected %#02x", i, out2[i], b)
+			}
+			mm++
+			if mm == 10 {
+				t.Fatal("Stopping")
+			}
+		}
+	}
+	t.Logf("got %d bytes", len(out1))
+}
+
+// TestBestSpeed tests that round-tripping through deflate and then inflate
+// recovers the original input. The Write sizes are near the thresholds in the
+// compressor.encSpeed method (0, 16, 128), as well as near maxStoreBlockSize
+// (65535).
+func TestBestSpeed(t *testing.T) {
+	t.Parallel()
+	abc := make([]byte, 128)
+	for i := range abc {
+		abc[i] = byte(i)
+	}
+	abcabc := bytes.Repeat(abc, 131072/len(abc))
+	var want []byte
+
+	testCases := [][]int{
+		{65536, 0},
+		{65536, 1},
+		{65536, 1, 256},
+		{65536, 1, 65536},
+		{65536, 14},
+		{65536, 15},
+		{65536, 16},
+		{65536, 16, 256},
+		{65536, 16, 65536},
+		{65536, 127},
+		{65536, 128},
+		{65536, 128, 256},
+		{65536, 128, 65536},
+		{65536, 129},
+		{65536, 65536, 256},
+		{65536, 65536, 65536},
+	}
+
+	for i, tc := range testCases {
+		if i >= 3 && testing.Short() {
+			break
+		}
+		for _, firstN := range []int{1, 65534, 65535, 65536, 65537, 131072} {
+			tc[0] = firstN
+		outer:
+			for _, flush := range []bool{false, true} {
+				buf := new(bytes.Buffer)
+				want = want[:0]
+
+				w, err := NewWriter(buf, BestSpeed)
+				if err != nil {
+					t.Errorf("i=%d, firstN=%d, flush=%t: NewWriter: %v", i, firstN, flush, err)
+					continue
+				}
+				for _, n := range tc {
+					want = append(want, abcabc[:n]...)
+					if _, err := w.Write(abcabc[:n]); err != nil {
+						t.Errorf("i=%d, firstN=%d, flush=%t: Write: %v", i, firstN, flush, err)
+						continue outer
+					}
+					if !flush {
+						continue
+					}
+					if err := w.Flush(); err != nil {
+						t.Errorf("i=%d, firstN=%d, flush=%t: Flush: %v", i, firstN, flush, err)
+						continue outer
+					}
+				}
+				if err := w.Close(); err != nil {
+					t.Errorf("i=%d, firstN=%d, flush=%t: Close: %v", i, firstN, flush, err)
+					continue
+				}
+
+				r := NewReader(buf)
+				got, err := io.ReadAll(r)
+				if err != nil {
+					t.Errorf("i=%d, firstN=%d, flush=%t: ReadAll: %v", i, firstN, flush, err)
+					continue
+				}
+				r.Close()
+
+				if !bytes.Equal(got, want) {
+					t.Errorf("i=%d, firstN=%d, flush=%t: corruption during deflate-then-inflate", i, firstN, flush)
+					continue
+				}
+			}
+		}
+	}
+}
+
+var errIO = errors.New("IO error")
+
+// failWriter fails with errIO exactly at the nth call to Write.
+type failWriter struct{ n int }
+
+func (w *failWriter) Write(b []byte) (int, error) {
+	w.n--
+	if w.n == -1 {
+		return 0, errIO
+	}
+	return len(b), nil
+}
+
+func TestWriterPersistentError(t *testing.T) {
+	t.Parallel()
+	d, err := os.ReadFile("../../testdata/Isaac.Newton-Opticks.txt")
+	if err != nil {
+		t.Fatalf("ReadFile: %v", err)
+	}
+	d = d[:10000] // Keep this test short
+
+	zw, err := NewWriter(nil, DefaultCompression)
+	if err != nil {
+		t.Fatalf("NewWriter: %v", err)
+	}
+
+	// Sweep over the threshold at which an error is returned.
+	// The variable i makes it such that the ith call to failWriter.Write will
+	// return errIO. Since failWriter errors are not persistent, we must ensure
+	// that flate.Writer errors are persistent.
+	for i := 0; i < 1000; i++ {
+		fw := &failWriter{i}
+		zw.Reset(fw)
+
+		_, werr := zw.Write(d)
+		cerr := zw.Close()
+		if werr != errIO && werr != nil {
+			t.Errorf("test %d, mismatching Write error: got %v, want %v", i, werr, errIO)
+		}
+		if cerr != errIO && fw.n < 0 {
+			t.Errorf("test %d, mismatching Close error: got %v, want %v", i, cerr, errIO)
+		}
+		if fw.n >= 0 {
+			// At this point, the failure threshold was sufficiently high enough
+			// that we wrote the whole stream without any errors.
+			return
+		}
+	}
+}
+
+func TestBestSpeedMatch(t *testing.T) {
+	t.Parallel()
+	cases := []struct {
+		previous, current []byte
+		t, s, want        int32
+	}{{
+		previous: []byte{0, 0, 0, 1, 2},
+		current:  []byte{3, 4, 5, 0, 1, 2, 3, 4, 5},
+		t:        -3,
+		s:        3,
+		want:     6,
+	}, {
+		previous: []byte{0, 0, 0, 1, 2},
+		current:  []byte{2, 4, 5, 0, 1, 2, 3, 4, 5},
+		t:        -3,
+		s:        3,
+		want:     3,
+	}, {
+		previous: []byte{0, 0, 0, 1, 1},
+		current:  []byte{3, 4, 5, 0, 1, 2, 3, 4, 5},
+		t:        -3,
+		s:        3,
+		want:     2,
+	}, {
+		previous: []byte{0, 0, 0, 1, 2},
+		current:  []byte{2, 2, 2, 2, 1, 2, 3, 4, 5},
+		t:        -1,
+		s:        0,
+		want:     4,
+	}, {
+		previous: []byte{0, 0, 0, 1, 2, 3, 4, 5, 2, 2},
+		current:  []byte{2, 2, 2, 2, 1, 2, 3, 4, 5},
+		t:        -7,
+		s:        4,
+		want:     5,
+	}, {
+		previous: []byte{9, 9, 9, 9, 9},
+		current:  []byte{2, 2, 2, 2, 1, 2, 3, 4, 5},
+		t:        -1,
+		s:        0,
+		want:     0,
+	}, {
+		previous: []byte{9, 9, 9, 9, 9},
+		current:  []byte{9, 2, 2, 2, 1, 2, 3, 4, 5},
+		t:        0,
+		s:        1,
+		want:     0,
+	}, {
+		previous: []byte{},
+		current:  []byte{9, 2, 2, 2, 1, 2, 3, 4, 5},
+		t:        -5,
+		s:        1,
+		want:     0,
+	}, {
+		previous: []byte{},
+		current:  []byte{9, 2, 2, 2, 1, 2, 3, 4, 5},
+		t:        -1,
+		s:        1,
+		want:     0,
+	}, {
+		previous: []byte{},
+		current:  []byte{2, 2, 2, 2, 1, 2, 3, 4, 5},
+		t:        0,
+		s:        1,
+		want:     3,
+	}, {
+		previous: []byte{3, 4, 5},
+		current:  []byte{3, 4, 5},
+		t:        -3,
+		s:        0,
+		want:     3,
+	}, {
+		previous: make([]byte, 1000),
+		current:  make([]byte, 1000),
+		t:        -1000,
+		s:        0,
+		want:     maxMatchLength - 4,
+	}, {
+		previous: make([]byte, 200),
+		current:  make([]byte, 500),
+		t:        -200,
+		s:        0,
+		want:     maxMatchLength - 4,
+	}, {
+		previous: make([]byte, 200),
+		current:  make([]byte, 500),
+		t:        0,
+		s:        1,
+		want:     maxMatchLength - 4,
+	}, {
+		previous: make([]byte, maxMatchLength-4),
+		current:  make([]byte, 500),
+		t:        -(maxMatchLength - 4),
+		s:        0,
+		want:     maxMatchLength - 4,
+	}, {
+		previous: make([]byte, 200),
+		current:  make([]byte, 500),
+		t:        -200,
+		s:        400,
+		want:     100,
+	}, {
+		previous: make([]byte, 10),
+		current:  make([]byte, 500),
+		t:        200,
+		s:        400,
+		want:     100,
+	}}
+	for i, c := range cases {
+		e := deflateFast{prev: c.previous}
+		got := e.matchLen(c.s, c.t, c.current)
+		if got != c.want {
+			t.Errorf("Test %d: match length, want %d, got %d", i, c.want, got)
+		}
+	}
+}
+
+func TestBestSpeedMaxMatchOffset(t *testing.T) {
+	t.Parallel()
+	const abc, xyz = "abcdefgh", "stuvwxyz"
+	for _, matchBefore := range []bool{false, true} {
+		for _, extra := range []int{0, inputMargin - 1, inputMargin, inputMargin + 1, 2 * inputMargin} {
+			for offsetAdj := -5; offsetAdj <= +5; offsetAdj++ {
+				report := func(desc string, err error) {
+					t.Errorf("matchBefore=%t, extra=%d, offsetAdj=%d: %s%v",
+						matchBefore, extra, offsetAdj, desc, err)
+				}
+
+				offset := maxMatchOffset + offsetAdj
+
+				// Make src to be a []byte of the form
+				//	"%s%s%s%s%s" % (abc, zeros0, xyzMaybe, abc, zeros1)
+				// where:
+				//	zeros0 is approximately maxMatchOffset zeros.
+				//	xyzMaybe is either xyz or the empty string.
+				//	zeros1 is between 0 and 30 zeros.
+				// The difference between the two abc's will be offset, which
+				// is maxMatchOffset plus or minus a small adjustment.
+				src := make([]byte, offset+len(abc)+extra)
+				copy(src, abc)
+				if !matchBefore {
+					copy(src[offset-len(xyz):], xyz)
+				}
+				copy(src[offset:], abc)
+
+				buf := new(bytes.Buffer)
+				w, err := NewWriter(buf, BestSpeed)
+				if err != nil {
+					report("NewWriter: ", err)
+					continue
+				}
+				if _, err := w.Write(src); err != nil {
+					report("Write: ", err)
+					continue
+				}
+				if err := w.Close(); err != nil {
+					report("Writer.Close: ", err)
+					continue
+				}
+
+				r := NewReader(buf)
+				dst, err := io.ReadAll(r)
+				r.Close()
+				if err != nil {
+					report("ReadAll: ", err)
+					continue
+				}
+
+				if !bytes.Equal(dst, src) {
+					report("", fmt.Errorf("bytes differ after round-tripping"))
+					continue
+				}
+			}
+		}
+	}
+}
+
+func TestBestSpeedShiftOffsets(t *testing.T) {
+	// Test if shiftoffsets properly preserves matches and resets out-of-range matches
+	// seen in https://github.com/golang/go/issues/4142
+	enc := newDeflateFast()
+
+	// testData may not generate internal matches.
+	testData := make([]byte, 32)
+	rng := rand.New(rand.NewSource(0))
+	for i := range testData {
+		testData[i] = byte(rng.Uint32())
+	}
+
+	// Encode the testdata with clean state.
+	// Second part should pick up matches from the first block.
+	wantFirstTokens := len(enc.encode(nil, testData))
+	wantSecondTokens := len(enc.encode(nil, testData))
+
+	if wantFirstTokens <= wantSecondTokens {
+		t.Fatalf("test needs matches between inputs to be generated")
+	}
+	// Forward the current indicator to before wraparound.
+	enc.cur = bufferReset - int32(len(testData))
+
+	// Part 1 before wrap, should match clean state.
+	got := len(enc.encode(nil, testData))
+	if wantFirstTokens != got {
+		t.Errorf("got %d, want %d tokens", got, wantFirstTokens)
+	}
+
+	// Verify we are about to wrap.
+	if enc.cur != bufferReset {
+		t.Errorf("got %d, want e.cur to be at bufferReset (%d)", enc.cur, bufferReset)
+	}
+
+	// Part 2 should match clean state as well even if wrapped.
+	got = len(enc.encode(nil, testData))
+	if wantSecondTokens != got {
+		t.Errorf("got %d, want %d token", got, wantSecondTokens)
+	}
+
+	// Verify that we wrapped.
+	if enc.cur >= bufferReset {
+		t.Errorf("want e.cur to be < bufferReset (%d), got %d", bufferReset, enc.cur)
+	}
+
+	// Forward the current buffer, leaving the matches at the bottom.
+	enc.cur = bufferReset
+	enc.shiftOffsets()
+
+	// Ensure that no matches were picked up.
+	got = len(enc.encode(nil, testData))
+	if wantFirstTokens != got {
+		t.Errorf("got %d, want %d tokens", got, wantFirstTokens)
+	}
+}
+
+func TestMaxStackSize(t *testing.T) {
+	// This test must not run in parallel with other tests as debug.SetMaxStack
+	// affects all goroutines.
+	n := debug.SetMaxStack(1 << 16)
+	defer debug.SetMaxStack(n)
+
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	b := make([]byte, 1<<20)
+	for level := HuffmanOnly; level <= BestCompression; level++ {
+		// Run in separate goroutine to increase probability of stack regrowth.
+		wg.Add(1)
+		go func(level int) {
+			defer wg.Done()
+			zw, err := NewWriter(io.Discard, level)
+			if err != nil {
+				t.Errorf("level %d, NewWriter() = %v, want nil", level, err)
+			}
+			if n, err := zw.Write(b); n != len(b) || err != nil {
+				t.Errorf("level %d, Write() = (%d, %v), want (%d, nil)", level, n, err, len(b))
+			}
+			if err := zw.Close(); err != nil {
+				t.Errorf("level %d, Close() = %v, want nil", level, err)
+			}
+			zw.Reset(io.Discard)
+		}(level)
+	}
+}

image/internal/compress/flate/deflatefast.go

@@ -0,0 +1,309 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package flate
+
+import "math"
+
+// This encoding algorithm, which prioritizes speed over output size, is
+// based on Snappy's LZ77-style encoder: github.com/golang/snappy
+
+const (
+	tableBits  = 14             // Bits used in the table.
+	tableSize  = 1 << tableBits // Size of the table.
+	tableMask  = tableSize - 1  // Mask for table indices. Redundant, but can eliminate bounds checks.
+	tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
+
+	// Reset the buffer offset when reaching this.
+	// Offsets are stored between blocks as int32 values.
+	// Since the offset we are checking against is at the beginning
+	// of the buffer, we need to subtract the current and input
+	// buffer to not risk overflowing the int32.
+	bufferReset = math.MaxInt32 - maxStoreBlockSize*2
+)
+
+func load32(b []byte, i int32) uint32 {
+	b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
+	return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
+}
+
+func load64(b []byte, i int32) uint64 {
+	b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
+	return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
+		uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
+}
+
+func hash(u uint32) uint32 {
+	return (u * 0x1e35a7bd) >> tableShift
+}
+
+// These constants are defined by the Snappy implementation so that its
+// assembly implementation can fast-path some 16-bytes-at-a-time copies. They
+// aren't necessary in the pure Go implementation, as we don't use those same
+// optimizations, but using the same thresholds doesn't really hurt.
+const (
+	inputMargin            = 16 - 1
+	minNonLiteralBlockSize = 1 + 1 + inputMargin
+)
+
+type tableEntry struct {
+	val    uint32 // Value at destination
+	offset int32
+}
+
+// deflateFast maintains the table for matches,
+// and the previous byte block for cross block matching.
+type deflateFast struct {
+	table [tableSize]tableEntry
+	prev  []byte // Previous block, zero length if unknown.
+	cur   int32  // Current match offset.
+}
+
+func newDeflateFast() *deflateFast {
+	return &deflateFast{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}
+}
+
+// encode encodes a block given in src and appends tokens
+// to dst and returns the result.
+func (e *deflateFast) encode(dst []token, src []byte) []token {
+	// Ensure that e.cur doesn't wrap.
+	if e.cur >= bufferReset {
+		e.shiftOffsets()
+	}
+
+	// This check isn't in the Snappy implementation, but there, the caller
+	// instead of the callee handles this case.
+	if len(src) < minNonLiteralBlockSize {
+		e.cur += maxStoreBlockSize
+		e.prev = e.prev[:0]
+		return emitLiteral(dst, src)
+	}
+
+	// sLimit is when to stop looking for offset/length copies. The inputMargin
+	// lets us use a fast path for emitLiteral in the main loop, while we are
+	// looking for copies.
+	sLimit := int32(len(src) - inputMargin)
+
+	// nextEmit is where in src the next emitLiteral should start from.
+	nextEmit := int32(0)
+	s := int32(0)
+	cv := load32(src, s)
+	nextHash := hash(cv)
+
+	for {
+		// Copied from the C++ snappy implementation:
+		//
+		// Heuristic match skipping: If 32 bytes are scanned with no matches
+		// found, start looking only at every other byte. If 32 more bytes are
+		// scanned (or skipped), look at every third byte, etc.. When a match
+		// is found, immediately go back to looking at every byte. This is a
+		// small loss (~5% performance, ~0.1% density) for compressible data
+		// due to more bookkeeping, but for non-compressible data (such as
+		// JPEG) it's a huge win since the compressor quickly "realizes" the
+		// data is incompressible and doesn't bother looking for matches
+		// everywhere.
+		//
+		// The "skip" variable keeps track of how many bytes there are since
+		// the last match; dividing it by 32 (ie. right-shifting by five) gives
+		// the number of bytes to move ahead for each iteration.
+		skip := int32(32)
+
+		nextS := s
+		var candidate tableEntry
+		for {
+			s = nextS
+			bytesBetweenHashLookups := skip >> 5
+			nextS = s + bytesBetweenHashLookups
+			skip += bytesBetweenHashLookups
+			if nextS > sLimit {
+				goto emitRemainder
+			}
+			candidate = e.table[nextHash&tableMask]
+			now := load32(src, nextS)
+			e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv}
+			nextHash = hash(now)
+
+			offset := s - (candidate.offset - e.cur)
+			if offset > maxMatchOffset || cv != candidate.val {
+				// Out of range or not matched.
+				cv = now
+				continue
+			}
+			break
+		}
+
+		// A 4-byte match has been found. We'll later see if more than 4 bytes
+		// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
+		// them as literal bytes.
+		dst = emitLiteral(dst, src[nextEmit:s])
+
+		// Call emitCopy, and then see if another emitCopy could be our next
+		// move. Repeat until we find no match for the input immediately after
+		// what was consumed by the last emitCopy call.
+		//
+		// If we exit this loop normally then we need to call emitLiteral next,
+		// though we don't yet know how big the literal will be. We handle that
+		// by proceeding to the next iteration of the main loop. We also can
+		// exit this loop via goto if we get close to exhausting the input.
+		for {
+			// Invariant: we have a 4-byte match at s, and no need to emit any
+			// literal bytes prior to s.
+
+			// Extend the 4-byte match as long as possible.
+			//
+			s += 4
+			t := candidate.offset - e.cur + 4
+			l := e.matchLen(s, t, src)
+
+			// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
+			dst = append(dst, matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)))
+			s += l
+			nextEmit = s
+			if s >= sLimit {
+				goto emitRemainder
+			}
+
+			// We could immediately start working at s now, but to improve
+			// compression we first update the hash table at s-1 and at s. If
+			// another emitCopy is not our next move, also calculate nextHash
+			// at s+1. At least on GOARCH=amd64, these three hash calculations
+			// are faster as one load64 call (with some shifts) instead of
+			// three load32 calls.
+			x := load64(src, s-1)
+			prevHash := hash(uint32(x))
+			e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)}
+			x >>= 8
+			currHash := hash(uint32(x))
+			candidate = e.table[currHash&tableMask]
+			e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)}
+
+			offset := s - (candidate.offset - e.cur)
+			if offset > maxMatchOffset || uint32(x) != candidate.val {
+				cv = uint32(x >> 8)
+				nextHash = hash(cv)
+				s++
+				break
+			}
+		}
+	}
+
+emitRemainder:
+	if int(nextEmit) < len(src) {
+		dst = emitLiteral(dst, src[nextEmit:])
+	}
+	e.cur += int32(len(src))
+	e.prev = e.prev[:len(src)]
+	copy(e.prev, src)
+	return dst
+}
+
+func emitLiteral(dst []token, lit []byte) []token {
+	for _, v := range lit {
+		dst = append(dst, literalToken(uint32(v)))
+	}
+	return dst
+}
+
+// matchLen returns the match length between src[s:] and src[t:].
+// t can be negative to indicate the match is starting in e.prev.
+// We assume that src[s-4:s] and src[t-4:t] already match.
+func (e *deflateFast) matchLen(s, t int32, src []byte) int32 {
+	s1 := int(s) + maxMatchLength - 4
+	if s1 > len(src) {
+		s1 = len(src)
+	}
+
+	// If we are inside the current block
+	if t >= 0 {
+		b := src[t:]
+		a := src[s:s1]
+		b = b[:len(a)]
+		// Extend the match to be as long as possible.
+		for i := range a {
+			if a[i] != b[i] {
+				return int32(i)
+			}
+		}
+		return int32(len(a))
+	}
+
+	// We found a match in the previous block.
+	tp := int32(len(e.prev)) + t
+	if tp < 0 {
+		return 0
+	}
+
+	// Extend the match to be as long as possible.
+	a := src[s:s1]
+	b := e.prev[tp:]
+	if len(b) > len(a) {
+		b = b[:len(a)]
+	}
+	a = a[:len(b)]
+	for i := range b {
+		if a[i] != b[i] {
+			return int32(i)
+		}
+	}
+
+	// If we reached our limit, we matched everything we are
+	// allowed to in the previous block and we return.
+	n := int32(len(b))
+	if int(s+n) == s1 {
+		return n
+	}
+
+	// Continue looking for more matches in the current block.
+	a = src[s+n : s1]
+	b = src[:len(a)]
+	for i := range a {
+		if a[i] != b[i] {
+			return int32(i) + n
+		}
+	}
+	return int32(len(a)) + n
+}
+
+// Reset resets the encoding history.
+// This ensures that no matches are made to the previous block.
+func (e *deflateFast) reset() {
+	e.prev = e.prev[:0]
+	// Bump the offset, so all matches will fail distance check.
+	// Nothing should be >= e.cur in the table.
+	e.cur += maxMatchOffset
+
+	// Protect against e.cur wraparound.
+	if e.cur >= bufferReset {
+		e.shiftOffsets()
+	}
+}
+
+// shiftOffsets will shift down all match offset.
+// This is only called in rare situations to prevent integer overflow.
+//
+// See https://golang.org/issue/18636 and https://github.com/golang/go/issues/34121.
+func (e *deflateFast) shiftOffsets() {
+	if len(e.prev) == 0 {
+		// We have no history; just clear the table.
+		for i := range e.table[:] {
+			e.table[i] = tableEntry{}
+		}
+		e.cur = maxMatchOffset + 1
+		return
+	}
+
+	// Shift down everything in the table that isn't already too far away.
+	for i := range e.table[:] {
+		v := e.table[i].offset - e.cur + maxMatchOffset + 1
+		if v < 0 {
+			// We want to reset e.cur to maxMatchOffset + 1, so we need to shift
+			// all table entries down by (e.cur - (maxMatchOffset + 1)).
+			// Because we ignore matches > maxMatchOffset, we can cap
+			// any negative offsets at 0.
+			v = 0
+		}
+		e.table[i].offset = v
+	}
+	e.cur = maxMatchOffset + 1
+}

image/internal/compress/flate/dict_decoder.go

@@ -0,0 +1,186 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package flate
+
+// dictDecoder implements the LZ77 sliding dictionary as used in decompression.
+// LZ77 decompresses data through sequences of two forms of commands:
+//
+//	* Literal insertions: Runs of one or more symbols are inserted into the data
+//	stream as is. This is accomplished through the writeByte method for a
+//	single symbol, or combinations of writeSlice/writeMark for multiple symbols.
+//	Any valid stream must start with a literal insertion if no preset dictionary
+//	is used.
+//
+//	* Backward copies: Runs of one or more symbols are copied from previously
+//	emitted data. Backward copies come as the tuple (dist, length) where dist
+//	determines how far back in the stream to copy from and length determines how
+//	many bytes to copy. Note that it is valid for the length to be greater than
+//	the distance. Since LZ77 uses forward copies, that situation is used to
+//	perform a form of run-length encoding on repeated runs of symbols.
+//	The writeCopy and tryWriteCopy are used to implement this command.
+//
+// For performance reasons, this implementation performs little to no sanity
+// checks about the arguments. As such, the invariants documented for each
+// method call must be respected.
+type dictDecoder struct {
+	hist []byte // Sliding window history
+
+	// Invariant: 0 <= rdPos <= wrPos <= len(hist)
+	wrPos int  // Current output position in buffer
+	rdPos int  // Have emitted hist[:rdPos] already
+	full  bool // Has a full window length been written yet?
+}
+
+// To minimize the memory usage in TinyGo, it is defined as a fixed array
+// instead of a make().
+var ddHistBuf [1 << 15]byte
+
+// init initializes dictDecoder to have a sliding window dictionary of the given
+// size. If a preset dict is provided, it will initialize the dictionary with
+// the contents of dict.
+func (dd *dictDecoder) init(size int, dict []byte) {
+	*dd = dictDecoder{hist: dd.hist}
+
+	if cap(dd.hist) < size {
+		dd.hist = ddHistBuf[:size]
+	}
+	dd.hist = dd.hist[:size]
+
+	if len(dict) > len(dd.hist) {
+		dict = dict[len(dict)-len(dd.hist):]
+	}
+	dd.wrPos = copy(dd.hist, dict)
+	if dd.wrPos == len(dd.hist) {
+		dd.wrPos = 0
+		dd.full = true
+	}
+	dd.rdPos = dd.wrPos
+}
+
+// histSize reports the total amount of historical data in the dictionary.
+func (dd *dictDecoder) histSize() int {
+	if dd.full {
+		return len(dd.hist)
+	}
+	return dd.wrPos
+}
+
+// availRead reports the number of bytes that can be flushed by readFlush.
+func (dd *dictDecoder) availRead() int {
+	return dd.wrPos - dd.rdPos
+}
+
+// availWrite reports the available amount of output buffer space.
+func (dd *dictDecoder) availWrite() int {
+	return len(dd.hist) - dd.wrPos
+}
+
+// writeSlice returns a slice of the available buffer to write data to.
+//
+// This invariant will be kept: len(s) <= availWrite()
+func (dd *dictDecoder) writeSlice() []byte {
+	return dd.hist[dd.wrPos:]
+}
+
+// writeMark advances the writer pointer by cnt.
+//
+// This invariant must be kept: 0 <= cnt <= availWrite()
+func (dd *dictDecoder) writeMark(cnt int) {
+	dd.wrPos += cnt
+}
+
+// writeByte writes a single byte to the dictionary.
+//
+// This invariant must be kept: 0 < availWrite()
+func (dd *dictDecoder) writeByte(c byte) {
+	dd.hist[dd.wrPos] = c
+	dd.wrPos++
+}
+
+// writeCopy copies a string at a given (dist, length) to the output.
+// This returns the number of bytes copied and may be less than the requested
+// length if the available space in the output buffer is too small.
+//
+// This invariant must be kept: 0 < dist <= histSize()
+func (dd *dictDecoder) writeCopy(dist, length int) int {
+	dstBase := dd.wrPos
+	dstPos := dstBase
+	srcPos := dstPos - dist
+	endPos := dstPos + length
+	if endPos > len(dd.hist) {
+		endPos = len(dd.hist)
+	}
+
+	// Copy non-overlapping section after destination position.
+	//
+	// This section is non-overlapping in that the copy length for this section
+	// is always less than or equal to the backwards distance. This can occur
+	// if a distance refers to data that wraps-around in the buffer.
+	// Thus, a backwards copy is performed here; that is, the exact bytes in
+	// the source prior to the copy is placed in the destination.
+	if srcPos < 0 {
+		srcPos += len(dd.hist)
+		dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:])
+		srcPos = 0
+	}
+
+	// Copy possibly overlapping section before destination position.
+	//
+	// This section can overlap if the copy length for this section is larger
+	// than the backwards distance. This is allowed by LZ77 so that repeated
+	// strings can be succinctly represented using (dist, length) pairs.
+	// Thus, a forwards copy is performed here; that is, the bytes copied is
+	// possibly dependent on the resulting bytes in the destination as the copy
+	// progresses along. This is functionally equivalent to the following:
+	//
+	//	for i := 0; i < endPos-dstPos; i++ {
+	//		dd.hist[dstPos+i] = dd.hist[srcPos+i]
+	//	}
+	//	dstPos = endPos
+	//
+	for dstPos < endPos {
+		dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
+	}
+
+	dd.wrPos = dstPos
+	return dstPos - dstBase
+}
+
+// tryWriteCopy tries to copy a string at a given (distance, length) to the
+// output. This specialized version is optimized for short distances.
+//
+// This method is designed to be inlined for performance reasons.
+//
+// This invariant must be kept: 0 < dist <= histSize()
+func (dd *dictDecoder) tryWriteCopy(dist, length int) int {
+	dstPos := dd.wrPos
+	endPos := dstPos + length
+	if dstPos < dist || endPos > len(dd.hist) {
+		return 0
+	}
+	dstBase := dstPos
+	srcPos := dstPos - dist
+
+	// Copy possibly overlapping section before destination position.
+	for dstPos < endPos {
+		dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
+	}
+
+	dd.wrPos = dstPos
+	return dstPos - dstBase
+}
+
+// readFlush returns a slice of the historical buffer that is ready to be
+// emitted to the user. The data returned by readFlush must be fully consumed
+// before calling any other dictDecoder methods.
+func (dd *dictDecoder) readFlush() []byte {
+	toRead := dd.hist[dd.rdPos:dd.wrPos]
+	dd.rdPos = dd.wrPos
+	if dd.wrPos == len(dd.hist) {
+		dd.wrPos, dd.rdPos = 0, 0
+		dd.full = true
+	}
+	return toRead
+}

image/internal/compress/flate/dict_decoder_test.go

@@ -0,0 +1,139 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package flate
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+)
+
+func TestDictDecoder(t *testing.T) {
+	const (
+		abc  = "ABC\n"
+		fox  = "The quick brown fox jumped over the lazy dog!\n"
+		poem = "The Road Not Taken\nRobert Frost\n" +
+			"\n" +
+			"Two roads diverged in a yellow wood,\n" +
+			"And sorry I could not travel both\n" +
+			"And be one traveler, long I stood\n" +
+			"And looked down one as far as I could\n" +
+			"To where it bent in the undergrowth;\n" +
+			"\n" +
+			"Then took the other, as just as fair,\n" +
+			"And having perhaps the better claim,\n" +
+			"Because it was grassy and wanted wear;\n" +
+			"Though as for that the passing there\n" +
+			"Had worn them really about the same,\n" +
+			"\n" +
+			"And both that morning equally lay\n" +
+			"In leaves no step had trodden black.\n" +
+			"Oh, I kept the first for another day!\n" +
+			"Yet knowing how way leads on to way,\n" +
+			"I doubted if I should ever come back.\n" +
+			"\n" +
+			"I shall be telling this with a sigh\n" +
+			"Somewhere ages and ages hence:\n" +
+			"Two roads diverged in a wood, and I-\n" +
+			"I took the one less traveled by,\n" +
+			"And that has made all the difference.\n"
+	)
+
+	var poemRefs = []struct {
+		dist   int // Backward distance (0 if this is an insertion)
+		length int // Length of copy or insertion
+	}{
+		{0, 38}, {33, 3}, {0, 48}, {79, 3}, {0, 11}, {34, 5}, {0, 6}, {23, 7},
+		{0, 8}, {50, 3}, {0, 2}, {69, 3}, {34, 5}, {0, 4}, {97, 3}, {0, 4},
+		{43, 5}, {0, 6}, {7, 4}, {88, 7}, {0, 12}, {80, 3}, {0, 2}, {141, 4},
+		{0, 1}, {196, 3}, {0, 3}, {157, 3}, {0, 6}, {181, 3}, {0, 2}, {23, 3},
+		{77, 3}, {28, 5}, {128, 3}, {110, 4}, {70, 3}, {0, 4}, {85, 6}, {0, 2},
+		{182, 6}, {0, 4}, {133, 3}, {0, 7}, {47, 5}, {0, 20}, {112, 5}, {0, 1},
+		{58, 3}, {0, 8}, {59, 3}, {0, 4}, {173, 3}, {0, 5}, {114, 3}, {0, 4},
+		{92, 5}, {0, 2}, {71, 3}, {0, 2}, {76, 5}, {0, 1}, {46, 3}, {96, 4},
+		{130, 4}, {0, 3}, {360, 3}, {0, 3}, {178, 5}, {0, 7}, {75, 3}, {0, 3},
+		{45, 6}, {0, 6}, {299, 6}, {180, 3}, {70, 6}, {0, 1}, {48, 3}, {66, 4},
+		{0, 3}, {47, 5}, {0, 9}, {325, 3}, {0, 1}, {359, 3}, {318, 3}, {0, 2},
+		{199, 3}, {0, 1}, {344, 3}, {0, 3}, {248, 3}, {0, 10}, {310, 3}, {0, 3},
+		{93, 6}, {0, 3}, {252, 3}, {157, 4}, {0, 2}, {273, 5}, {0, 14}, {99, 4},
+		{0, 1}, {464, 4}, {0, 2}, {92, 4}, {495, 3}, {0, 1}, {322, 4}, {16, 4},
+		{0, 3}, {402, 3}, {0, 2}, {237, 4}, {0, 2}, {432, 4}, {0, 1}, {483, 5},
+		{0, 2}, {294, 4}, {0, 2}, {306, 3}, {113, 5}, {0, 1}, {26, 4}, {164, 3},
+		{488, 4}, {0, 1}, {542, 3}, {248, 6}, {0, 5}, {205, 3}, {0, 8}, {48, 3},
+		{449, 6}, {0, 2}, {192, 3}, {328, 4}, {9, 5}, {433, 3}, {0, 3}, {622, 25},
+		{615, 5}, {46, 5}, {0, 2}, {104, 3}, {475, 10}, {549, 3}, {0, 4}, {597, 8},
+		{314, 3}, {0, 1}, {473, 6}, {317, 5}, {0, 1}, {400, 3}, {0, 3}, {109, 3},
+		{151, 3}, {48, 4}, {0, 4}, {125, 3}, {108, 3}, {0, 2},
+	}
+
+	var got, want bytes.Buffer
+	var dd dictDecoder
+	dd.init(1<<11, nil)
+
+	var writeCopy = func(dist, length int) {
+		for length > 0 {
+			cnt := dd.tryWriteCopy(dist, length)
+			if cnt == 0 {
+				cnt = dd.writeCopy(dist, length)
+			}
+
+			length -= cnt
+			if dd.availWrite() == 0 {
+				got.Write(dd.readFlush())
+			}
+		}
+	}
+	var writeString = func(str string) {
+		for len(str) > 0 {
+			cnt := copy(dd.writeSlice(), str)
+			str = str[cnt:]
+			dd.writeMark(cnt)
+			if dd.availWrite() == 0 {
+				got.Write(dd.readFlush())
+			}
+		}
+	}
+
+	writeString(".")
+	want.WriteByte('.')
+
+	str := poem
+	for _, ref := range poemRefs {
+		if ref.dist == 0 {
+			writeString(str[:ref.length])
+		} else {
+			writeCopy(ref.dist, ref.length)
+		}
+		str = str[ref.length:]
+	}
+	want.WriteString(poem)
+
+	writeCopy(dd.histSize(), 33)
+	want.Write(want.Bytes()[:33])
+
+	writeString(abc)
+	writeCopy(len(abc), 59*len(abc))
+	want.WriteString(strings.Repeat(abc, 60))
+
+	writeString(fox)
+	writeCopy(len(fox), 9*len(fox))
+	want.WriteString(strings.Repeat(fox, 10))
+
+	writeString(".")
+	writeCopy(1, 9)
+	want.WriteString(strings.Repeat(".", 10))
+
+	writeString(strings.ToUpper(poem))
+	writeCopy(len(poem), 7*len(poem))
+	want.WriteString(strings.Repeat(strings.ToUpper(poem), 8))
+
+	writeCopy(dd.histSize(), 10)
+	want.Write(want.Bytes()[want.Len()-dd.histSize():][:10])
+
+	got.Write(dd.readFlush())
+	if got.String() != want.String() {
+		t.Errorf("final string mismatch:\ngot  %q\nwant %q", got.String(), want.String())
+	}
+}

image/internal/compress/flate/example_test.go

@@ -0,0 +1,243 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package flate_test
+
+import (
+	"bytes"
+	"compress/flate"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+	"sync"
+)
+
+// In performance critical applications, Reset can be used to discard the
+// current compressor or decompressor state and reinitialize them quickly
+// by taking advantage of previously allocated memory.
+func Example_reset() {
+	proverbs := []string{
+		"Don't communicate by sharing memory, share memory by communicating.\n",
+		"Concurrency is not parallelism.\n",
+		"The bigger the interface, the weaker the abstraction.\n",
+		"Documentation is for users.\n",
+	}
+
+	var r strings.Reader
+	var b bytes.Buffer
+	buf := make([]byte, 32<<10)
+
+	zw, err := flate.NewWriter(nil, flate.DefaultCompression)
+	if err != nil {
+		log.Fatal(err)
+	}
+	zr := flate.NewReader(nil)
+
+	for _, s := range proverbs {
+		r.Reset(s)
+		b.Reset()
+
+		// Reset the compressor and encode from some input stream.
+		zw.Reset(&b)
+		if _, err := io.CopyBuffer(zw, &r, buf); err != nil {
+			log.Fatal(err)
+		}
+		if err := zw.Close(); err != nil {
+			log.Fatal(err)
+		}
+
+		// Reset the decompressor and decode to some output stream.
+		if err := zr.(flate.Resetter).Reset(&b, nil); err != nil {
+			log.Fatal(err)
+		}
+		if _, err := io.CopyBuffer(os.Stdout, zr, buf); err != nil {
+			log.Fatal(err)
+		}
+		if err := zr.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	// Output:
+	// Don't communicate by sharing memory, share memory by communicating.
+	// Concurrency is not parallelism.
+	// The bigger the interface, the weaker the abstraction.
+	// Documentation is for users.
+}
+
+// A preset dictionary can be used to improve the compression ratio.
+// The downside to using a dictionary is that the compressor and decompressor
+// must agree in advance what dictionary to use.
+func Example_dictionary() {
+	// The dictionary is a string of bytes. When compressing some input data,
+	// the compressor will attempt to substitute substrings with matches found
+	// in the dictionary. As such, the dictionary should only contain substrings
+	// that are expected to be found in the actual data stream.
+	const dict = `<?xml version="1.0"?>` + `<book>` + `<data>` + `<meta name="` + `" content="`
+
+	// The data to compress should (but is not required to) contain frequent
+	// substrings that match those in the dictionary.
+	const data = `<?xml version="1.0"?>
+<book>
+	<meta name="title" content="The Go Programming Language"/>
+	<meta name="authors" content="Alan Donovan and Brian Kernighan"/>
+	<meta name="published" content="2015-10-26"/>
+	<meta name="isbn" content="978-0134190440"/>
+	<data>...</data>
+</book>
+`
+
+	var b bytes.Buffer
+
+	// Compress the data using the specially crafted dictionary.
+	zw, err := flate.NewWriterDict(&b, flate.DefaultCompression, []byte(dict))
+	if err != nil {
+		log.Fatal(err)
+	}
+	if _, err := io.Copy(zw, strings.NewReader(data)); err != nil {
+		log.Fatal(err)
+	}
+	if err := zw.Close(); err != nil {
+		log.Fatal(err)
+	}
+
+	// The decompressor must use the same dictionary as the compressor.
+	// Otherwise, the input may appear as corrupted.
+	fmt.Println("Decompressed output using the dictionary:")
+	zr := flate.NewReaderDict(bytes.NewReader(b.Bytes()), []byte(dict))
+	if _, err := io.Copy(os.Stdout, zr); err != nil {
+		log.Fatal(err)
+	}
+	if err := zr.Close(); err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println()
+
+	// Substitute all of the bytes in the dictionary with a '#' to visually
+	// demonstrate the approximate effectiveness of using a preset dictionary.
+	fmt.Println("Substrings matched by the dictionary are marked with #:")
+	hashDict := []byte(dict)
+	for i := range hashDict {
+		hashDict[i] = '#'
+	}
+	zr = flate.NewReaderDict(&b, hashDict)
+	if _, err := io.Copy(os.Stdout, zr); err != nil {
+		log.Fatal(err)
+	}
+	if err := zr.Close(); err != nil {
+		log.Fatal(err)
+	}
+
+	// Output:
+	// Decompressed output using the dictionary:
+	// <?xml version="1.0"?>
+	// <book>
+	// 	<meta name="title" content="The Go Programming Language"/>
+	// 	<meta name="authors" content="Alan Donovan and Brian Kernighan"/>
+	// 	<meta name="published" content="2015-10-26"/>
+	// 	<meta name="isbn" content="978-0134190440"/>
+	// 	<data>...</data>
+	// </book>
+	//
+	// Substrings matched by the dictionary are marked with #:
+	// #####################
+	// ######
+	// 	############title###########The Go Programming Language"/#
+	// 	############authors###########Alan Donovan and Brian Kernighan"/#
+	// 	############published###########2015-10-26"/#
+	// 	############isbn###########978-0134190440"/#
+	// 	######...</#####
+	// </#####
+}
+
+// DEFLATE is suitable for transmitting compressed data across the network.
+func Example_synchronization() {
+	var wg sync.WaitGroup
+	defer wg.Wait()
+
+	// Use io.Pipe to simulate a network connection.
+	// A real network application should take care to properly close the
+	// underlying connection.
+	rp, wp := io.Pipe()
+
+	// Start a goroutine to act as the transmitter.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		zw, err := flate.NewWriter(wp, flate.BestSpeed)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		b := make([]byte, 256)
+		for _, m := range strings.Fields("A long time ago in a galaxy far, far away...") {
+			// We use a simple framing format where the first byte is the
+			// message length, followed the message itself.
+			b[0] = uint8(copy(b[1:], m))
+
+			if _, err := zw.Write(b[:1+len(m)]); err != nil {
+				log.Fatal(err)
+			}
+
+			// Flush ensures that the receiver can read all data sent so far.
+			if err := zw.Flush(); err != nil {
+				log.Fatal(err)
+			}
+		}
+
+		if err := zw.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}()
+
+	// Start a goroutine to act as the receiver.
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		zr := flate.NewReader(rp)
+
+		b := make([]byte, 256)
+		for {
+			// Read the message length.
+			// This is guaranteed to return for every corresponding
+			// Flush and Close on the transmitter side.
+			if _, err := io.ReadFull(zr, b[:1]); err != nil {
+				if err == io.EOF {
+					break // The transmitter closed the stream
+				}
+				log.Fatal(err)
+			}
+
+			// Read the message content.
+			n := int(b[0])
+			if _, err := io.ReadFull(zr, b[:n]); err != nil {
+				log.Fatal(err)
+			}
+
+			fmt.Printf("Received %d bytes: %s\n", n, b[:n])
+		}
+		fmt.Println()
+
+		if err := zr.Close(); err != nil {
+			log.Fatal(err)
+		}
+	}()
+
+	// Output:
+	// Received 1 bytes: A
+	// Received 4 bytes: long
+	// Received 4 bytes: time
+	// Received 3 bytes: ago
+	// Received 2 bytes: in
+	// Received 1 bytes: a
+	// Received 6 bytes: galaxy
+	// Received 4 bytes: far,
+	// Received 3 bytes: far
+	// Received 7 bytes: away...
+}

image/internal/compress/flate/flate_test.go

@@ -0,0 +1,352 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This test tests some internals of the flate package.
+// The tests in package compress/gzip serve as the
+// end-to-end test of the decompressor.
+
+package flate
+
+import (
+	"bytes"
+	"encoding/hex"
+	"io"
+	"strings"
+	"testing"
+)
+
+// The following test should not panic.
+func TestIssue5915(t *testing.T) {
+	bits := []int{4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0, 5, 5, 6,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 6, 0, 11, 0, 8, 0, 6, 6, 10, 8}
+	var h huffmanDecoder
+	if h.init(bits) {
+		t.Fatalf("Given sequence of bits is bad, and should not succeed.")
+	}
+}
+
+// The following test should not panic.
+func TestIssue5962(t *testing.T) {
+	bits := []int{4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0,
+		5, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11}
+	var h huffmanDecoder
+	if h.init(bits) {
+		t.Fatalf("Given sequence of bits is bad, and should not succeed.")
+	}
+}
+
+// The following test should not panic.
+func TestIssue6255(t *testing.T) {
+	bits1 := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 11}
+	bits2 := []int{11, 13}
+	var h huffmanDecoder
+	if !h.init(bits1) {
+		t.Fatalf("Given sequence of bits is good and should succeed.")
+	}
+	if h.init(bits2) {
+		t.Fatalf("Given sequence of bits is bad and should not succeed.")
+	}
+}
+
+func TestInvalidEncoding(t *testing.T) {
+	// Initialize Huffman decoder to recognize "0".
+	var h huffmanDecoder
+	if !h.init([]int{1}) {
+		t.Fatal("Failed to initialize Huffman decoder")
+	}
+
+	// Initialize decompressor with invalid Huffman coding.
+	var f decompressor
+	f.r = bytes.NewReader([]byte{0xff})
+
+	_, err := f.huffSym(&h)
+	if err == nil {
+		t.Fatal("Should have rejected invalid bit sequence")
+	}
+}
+
+func TestInvalidBits(t *testing.T) {
+	oversubscribed := []int{1, 2, 3, 4, 4, 5}
+	incomplete := []int{1, 2, 4, 4}
+	var h huffmanDecoder
+	if h.init(oversubscribed) {
+		t.Fatal("Should reject oversubscribed bit-length set")
+	}
+	if h.init(incomplete) {
+		t.Fatal("Should reject incomplete bit-length set")
+	}
+}
+
+func TestStreams(t *testing.T) {
+	// To verify any of these hexstrings as valid or invalid flate streams
+	// according to the C zlib library, you can use the Python wrapper library:
+	// >>> hex_string = "010100feff11"
+	// >>> import zlib
+	// >>> zlib.decompress(hex_string.decode("hex"), -15) # Negative means raw DEFLATE
+	// '\x11'
+
+	testCases := []struct {
+		desc   string // Description of the stream
+		stream string // Hexstring of the input DEFLATE stream
+		want   string // Expected result. Use "fail" to expect failure
+	}{{
+		"degenerate HCLenTree",
+		"05e0010000000000100000000000000000000000000000000000000000000000" +
+			"00000000000000000004",
+		"fail",
+	}, {
+		"complete HCLenTree, empty HLitTree, empty HDistTree",
+		"05e0010400000000000000000000000000000000000000000000000000000000" +
+			"00000000000000000010",
+		"fail",
+	}, {
+		"empty HCLenTree",
+		"05e0010000000000000000000000000000000000000000000000000000000000" +
+			"00000000000000000010",
+		"fail",
+	}, {
+		"complete HCLenTree, complete HLitTree, empty HDistTree, use missing HDist symbol",
+		"000100feff000de0010400000000100000000000000000000000000000000000" +
+			"0000000000000000000000000000002c",
+		"fail",
+	}, {
+		"complete HCLenTree, complete HLitTree, degenerate HDistTree, use missing HDist symbol",
+		"000100feff000de0010000000000000000000000000000000000000000000000" +
+			"00000000000000000610000000004070",
+		"fail",
+	}, {
+		"complete HCLenTree, empty HLitTree, empty HDistTree",
+		"05e0010400000000100400000000000000000000000000000000000000000000" +
+			"0000000000000000000000000008",
+		"fail",
+	}, {
+		"complete HCLenTree, empty HLitTree, degenerate HDistTree",
+		"05e0010400000000100400000000000000000000000000000000000000000000" +
+			"0000000000000000000800000008",
+		"fail",
+	}, {
+		"complete HCLenTree, degenerate HLitTree, degenerate HDistTree, use missing HLit symbol",
+		"05e0010400000000100000000000000000000000000000000000000000000000" +
+			"0000000000000000001c",
+		"fail",
+	}, {
+		"complete HCLenTree, complete HLitTree, too large HDistTree",
+		"edff870500000000200400000000000000000000000000000000000000000000" +
+			"000000000000000000080000000000000004",
+		"fail",
+	}, {
+		"complete HCLenTree, complete HLitTree, empty HDistTree, excessive repeater code",
+		"edfd870500000000200400000000000000000000000000000000000000000000" +
+			"000000000000000000e8b100",
+		"fail",
+	}, {
+		"complete HCLenTree, complete HLitTree, empty HDistTree of normal length 30",
+		"05fd01240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffff07000000fe01",
+		"",
+	}, {
+		"complete HCLenTree, complete HLitTree, empty HDistTree of excessive length 31",
+		"05fe01240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffff07000000fc03",
+		"fail",
+	}, {
+		"complete HCLenTree, over-subscribed HLitTree, empty HDistTree",
+		"05e001240000000000fcffffffffffffffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffff07f00f",
+		"fail",
+	}, {
+		"complete HCLenTree, under-subscribed HLitTree, empty HDistTree",
+		"05e001240000000000fcffffffffffffffffffffffffffffffffffffffffffff" +
+			"fffffffffcffffffff07f00f",
+		"fail",
+	}, {
+		"complete HCLenTree, complete HLitTree with single code, empty HDistTree",
+		"05e001240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffff07f00f",
+		"01",
+	}, {
+		"complete HCLenTree, complete HLitTree with multiple codes, empty HDistTree",
+		"05e301240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" +
+			"ffffffffffffffffff07807f",
+		"01",
+	}, {
+		"complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HDist symbol",
+		"000100feff000de0010400000000100000000000000000000000000000000000" +
+			"0000000000000000000000000000003c",
+		"00000000",
+	}, {
+		"complete HCLenTree, degenerate HLitTree, degenerate HDistTree",
+		"05e0010400000000100000000000000000000000000000000000000000000000" +
+			"0000000000000000000c",
+		"",
+	}, {
+		"complete HCLenTree, degenerate HLitTree, empty HDistTree",
+		"05e0010400000000100000000000000000000000000000000000000000000000" +
+			"00000000000000000004",
+		"",
+	}, {
+		"complete HCLenTree, complete HLitTree, empty HDistTree, spanning repeater code",
+		"edfd870500000000200400000000000000000000000000000000000000000000" +
+			"000000000000000000e8b000",
+		"",
+	}, {
+		"complete HCLenTree with length codes, complete HLitTree, empty HDistTree",
+		"ede0010400000000100000000000000000000000000000000000000000000000" +
+			"0000000000000000000400004000",
+		"",
+	}, {
+		"complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HLit symbol 284 with count 31",
+		"000100feff00ede0010400000000100000000000000000000000000000000000" +
+			"000000000000000000000000000000040000407f00",
+		"0000000000000000000000000000000000000000000000000000000000000000" +
+			"0000000000000000000000000000000000000000000000000000000000000000" +
+			"0000000000000000000000000000000000000000000000000000000000000000" +
+			"0000000000000000000000000000000000000000000000000000000000000000" +
+			"0000000000000000000000000000000000000000000000000000000000000000" +
+			"0000000000000000000000000000000000000000000000000000000000000000" +
+			"0000000000000000000000000000000000000000000000000000000000000000" +
+			"0000000000000000000000000000000000000000000000000000000000000000" +
+			"000000",
+	}, {
+		"complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HLit and HDist symbols",
+		"0cc2010d00000082b0ac4aff0eb07d27060000ffff",
+		"616263616263",
+	}, {
+		"fixed block, use reserved symbol 287",
+		"33180700",
+		"fail",
+	}, {
+		"raw block",
+		"010100feff11",
+		"11",
+	}, {
+		"issue 10426 - over-subscribed HCLenTree causes a hang",
+		"344c4a4e494d4b070000ff2e2eff2e2e2e2e2eff",
+		"fail",
+	}, {
+		"issue 11030 - empty HDistTree unexpectedly leads to error",
+		"05c0070600000080400fff37a0ca",
+		"",
+	}, {
+		"issue 11033 - empty HDistTree unexpectedly leads to error",
+		"050fb109c020cca5d017dcbca044881ee1034ec149c8980bbc413c2ab35be9dc" +
+			"b1473449922449922411202306ee97b0383a521b4ffdcf3217f9f7d3adb701",
+		"3130303634342068652e706870005d05355f7ed957ff084a90925d19e3ebc6d0" +
+			"c6d7",
+	}}
+
+	for i, tc := range testCases {
+		data, err := hex.DecodeString(tc.stream)
+		if err != nil {
+			t.Fatal(err)
+		}
+		data, err = io.ReadAll(NewReader(bytes.NewReader(data)))
+		if tc.want == "fail" {
+			if err == nil {
+				t.Errorf("#%d (%s): got nil error, want non-nil", i, tc.desc)
+			}
+		} else {
+			if err != nil {
+				t.Errorf("#%d (%s): %v", i, tc.desc, err)
+				continue
+			}
+			if got := hex.EncodeToString(data); got != tc.want {
+				t.Errorf("#%d (%s):\ngot  %q\nwant %q", i, tc.desc, got, tc.want)
+			}
+
+		}
+	}
+}
+
+func TestTruncatedStreams(t *testing.T) {
+	const data = "\x00\f\x00\xf3\xffhello, world\x01\x00\x00\xff\xff"
+
+	for i := 0; i < len(data)-1; i++ {
+		r := NewReader(strings.NewReader(data[:i]))
+		_, err := io.Copy(io.Discard, r)
+		if err != io.ErrUnexpectedEOF {
+			t.Errorf("io.Copy(%d) on truncated stream: got %v, want %v", i, err, io.ErrUnexpectedEOF)
+		}
+	}
+}
+
+// Verify that flate.Reader.Read returns (n, io.EOF) instead
+// of (n, nil) + (0, io.EOF) when possible.
+//
+// This helps net/http.Transport reuse HTTP/1 connections more
+// aggressively.
+//
+// See https://github.com/google/go-github/pull/317 for background.
+func TestReaderEarlyEOF(t *testing.T) {
+	t.Parallel()
+	testSizes := []int{
+		1, 2, 3, 4, 5, 6, 7, 8,
+		100, 1000, 10000, 100000,
+		128, 1024, 16384, 131072,
+
+		// Testing multiples of windowSize triggers the case
+		// where Read will fail to return an early io.EOF.
+		windowSize * 1, windowSize * 2, windowSize * 3,
+	}
+
+	var maxSize int
+	for _, n := range testSizes {
+		if maxSize < n {
+			maxSize = n
+		}
+	}
+
+	readBuf := make([]byte, 40)
+	data := make([]byte, maxSize)
+	for i := range data {
+		data[i] = byte(i)
+	}
+
+	for _, sz := range testSizes {
+		if testing.Short() && sz > windowSize {
+			continue
+		}
+		for _, flush := range []bool{true, false} {
+			earlyEOF := true // Do we expect early io.EOF?
+
+			var buf bytes.Buffer
+			w, _ := NewWriter(&buf, 5)
+			w.Write(data[:sz])
+			if flush {
+				// If a Flush occurs after all the actual data, the flushing
+				// semantics dictate that we will observe a (0, io.EOF) since
+				// Read must return data before it knows that the stream ended.
+				w.Flush()
+				earlyEOF = false
+			}
+			w.Close()
+
+			r := NewReader(&buf)
+			for {
+				n, err := r.Read(readBuf)
+				if err == io.EOF {
+					// If the availWrite == windowSize, then that means that the
+					// previous Read returned because the write buffer was full
+					// and it just so happened that the stream had no more data.
+					// This situation is rare, but unavoidable.
+					if r.(*decompressor).dict.availWrite() == windowSize {
+						earlyEOF = false
+					}
+
+					if n == 0 && earlyEOF {
+						t.Errorf("On size:%d flush:%v, Read() = (0, io.EOF), want (n, io.EOF)", sz, flush)
+					}
+					if n != 0 && !earlyEOF {
+						t.Errorf("On size:%d flush:%v, Read() = (%d, io.EOF), want (0, io.EOF)", sz, flush, n)
+					}
+					break
+				}
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+	}
+}

image/internal/compress/flate/huffman_bit_writer.go

@@ -0,0 +1,704 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package flate
+
+import (
+	"io"
+)
+
+const (
+	// The largest offset code.
+	offsetCodeCount = 30
+
+	// The special code used to mark the end of a block.
+	endBlockMarker = 256
+
+	// The first length code.
+	lengthCodesStart = 257
+
+	// The number of codegen codes.
+	codegenCodeCount = 19
+	badCode          = 255
+
+	// bufferFlushSize indicates the buffer size
+	// after which bytes are flushed to the writer.
+	// Should preferably be a multiple of 6, since
+	// we accumulate 6 bytes between writes to the buffer.
+	bufferFlushSize = 240
+
+	// bufferSize is the actual output byte buffer size.
+	// It must have additional headroom for a flush
+	// which can contain up to 8 bytes.
+	bufferSize = bufferFlushSize + 8
+)
+
+// The number of extra bits needed by length code X - LENGTH_CODES_START.
+var lengthExtraBits = []int8{
+	/* 257 */ 0, 0, 0,
+	/* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2,
+	/* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,
+	/* 280 */ 4, 5, 5, 5, 5, 0,
+}
+
+// The length indicated by length code X - LENGTH_CODES_START.
+var lengthBase = []uint32{
+	0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
+	12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
+	64, 80, 96, 112, 128, 160, 192, 224, 255,
+}
+
+// offset code word extra bits.
+var offsetExtraBits = []int8{
+	0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
+	4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
+	9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
+}
+
+var offsetBase = []uint32{
+	0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
+	0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
+	0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
+	0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
+	0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
+	0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
+}
+
+// The odd order in which the codegen code sizes are written.
+var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
+
+type huffmanBitWriter struct {
+	// writer is the underlying writer.
+	// Do not use it directly; use the write method, which ensures
+	// that Write errors are sticky.
+	writer io.Writer
+
+	// Data waiting to be written is bytes[0:nbytes]
+	// and then the low nbits of bits.  Data is always written
+	// sequentially into the bytes array.
+	bits            uint64
+	nbits           uint
+	bytes           [bufferSize]byte
+	codegenFreq     [codegenCodeCount]int32
+	nbytes          int
+	literalFreq     []int32
+	offsetFreq      []int32
+	codegen         []uint8
+	literalEncoding *huffmanEncoder
+	offsetEncoding  *huffmanEncoder
+	codegenEncoding *huffmanEncoder
+	err             error
+}
+
+func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter {
+	return &huffmanBitWriter{
+		writer:          w,
+		literalFreq:     make([]int32, maxNumLit),
+		offsetFreq:      make([]int32, offsetCodeCount),
+		codegen:         make([]uint8, maxNumLit+offsetCodeCount+1),
+		literalEncoding: newHuffmanEncoder(maxNumLit),
+		codegenEncoding: newHuffmanEncoder(codegenCodeCount),
+		offsetEncoding:  newHuffmanEncoder(offsetCodeCount),
+	}
+}
+
+func (w *huffmanBitWriter) reset(writer io.Writer) {
+	w.writer = writer
+	w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil
+}
+
+func (w *huffmanBitWriter) flush() {
+	if w.err != nil {
+		w.nbits = 0
+		return
+	}
+	n := w.nbytes
+	for w.nbits != 0 {
+		w.bytes[n] = byte(w.bits)
+		w.bits >>= 8
+		if w.nbits > 8 { // Avoid underflow
+			w.nbits -= 8
+		} else {
+			w.nbits = 0
+		}
+		n++
+	}
+	w.bits = 0
+	w.write(w.bytes[:n])
+	w.nbytes = 0
+}
+
+func (w *huffmanBitWriter) write(b []byte) {
+	if w.err != nil {
+		return
+	}
+	_, w.err = w.writer.Write(b)
+}
+
+func (w *huffmanBitWriter) writeBits(b int32, nb uint) {
+	if w.err != nil {
+		return
+	}
+	w.bits |= uint64(b) << w.nbits
+	w.nbits += nb
+	if w.nbits >= 48 {
+		bits := w.bits
+		w.bits >>= 48
+		w.nbits -= 48
+		n := w.nbytes
+		bytes := w.bytes[n : n+6]
+		bytes[0] = byte(bits)
+		bytes[1] = byte(bits >> 8)
+		bytes[2] = byte(bits >> 16)
+		bytes[3] = byte(bits >> 24)
+		bytes[4] = byte(bits >> 32)
+		bytes[5] = byte(bits >> 40)
+		n += 6
+		if n >= bufferFlushSize {
+			w.write(w.bytes[:n])
+			n = 0
+		}
+		w.nbytes = n
+	}
+}
+
+func (w *huffmanBitWriter) writeBytes(bytes []byte) {
+	if w.err != nil {
+		return
+	}
+	n := w.nbytes
+	if w.nbits&7 != 0 {
+		w.err = InternalError("writeBytes with unfinished bits")
+		return
+	}
+	for w.nbits != 0 {
+		w.bytes[n] = byte(w.bits)
+		w.bits >>= 8
+		w.nbits -= 8
+		n++
+	}
+	if n != 0 {
+		w.write(w.bytes[:n])
+	}
+	w.nbytes = 0
+	w.write(bytes)
+}
+
+// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
+// the literal and offset lengths arrays (which are concatenated into a single
+// array).  This method generates that run-length encoding.
+//
+// The result is written into the codegen array, and the frequencies
+// of each code is written into the codegenFreq array.
+// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
+// information. Code badCode is an end marker
+//
+//  numLiterals      The number of literals in literalEncoding
+//  numOffsets       The number of offsets in offsetEncoding
+//  litenc, offenc   The literal and offset encoder to use
+func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) {
+	for i := range w.codegenFreq {
+		w.codegenFreq[i] = 0
+	}
+	// Note that we are using codegen both as a temporary variable for holding
+	// a copy of the frequencies, and as the place where we put the result.
+	// This is fine because the output is always shorter than the input used
+	// so far.
+	codegen := w.codegen // cache
+	// Copy the concatenated code sizes to codegen. Put a marker at the end.
+	cgnl := codegen[:numLiterals]
+	for i := range cgnl {
+		cgnl[i] = uint8(litEnc.codes[i].len)
+	}
+
+	cgnl = codegen[numLiterals : numLiterals+numOffsets]
+	for i := range cgnl {
+		cgnl[i] = uint8(offEnc.codes[i].len)
+	}
+	codegen[numLiterals+numOffsets] = badCode
+
+	size := codegen[0]
+	count := 1
+	outIndex := 0
+	for inIndex := 1; size != badCode; inIndex++ {
+		// INVARIANT: We have seen "count" copies of size that have not yet
+		// had output generated for them.
+		nextSize := codegen[inIndex]
+		if nextSize == size {
+			count++
+			continue
+		}
+		// We need to generate codegen indicating "count" of size.
+		if size != 0 {
+			codegen[outIndex] = size
+			outIndex++
+			w.codegenFreq[size]++
+			count--
+			for count >= 3 {
+				n := 6
+				if n > count {
+					n = count
+				}
+				codegen[outIndex] = 16
+				outIndex++
+				codegen[outIndex] = uint8(n - 3)
+				outIndex++
+				w.codegenFreq[16]++
+				count -= n
+			}
+		} else {
+			for count >= 11 {
+				n := 138
+				if n > count {
+					n = count
+				}
+				codegen[outIndex] = 18
+				outIndex++
+				codegen[outIndex] = uint8(n - 11)
+				outIndex++
+				w.codegenFreq[18]++
+				count -= n
+			}
+			if count >= 3 {
+				// count >= 3 && count <= 10
+				codegen[outIndex] = 17
+				outIndex++
+				codegen[outIndex] = uint8(count - 3)
+				outIndex++
+				w.codegenFreq[17]++
+				count = 0
+			}
+		}
+		count--
+		for ; count >= 0; count-- {
+			codegen[outIndex] = size
+			outIndex++
+			w.codegenFreq[size]++
+		}
+		// Set up invariant for next time through the loop.
+		size = nextSize
+		count = 1
+	}
+	// Marker indicating the end of the codegen.
+	codegen[outIndex] = badCode
+}
+
+// dynamicSize returns the size of dynamically encoded data in bits.
+func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) {
+	numCodegens = len(w.codegenFreq)
+	for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
+		numCodegens--
+	}
+	header := 3 + 5 + 5 + 4 + (3 * numCodegens) +
+		w.codegenEncoding.bitLength(w.codegenFreq[:]) +
+		int(w.codegenFreq[16])*2 +
+		int(w.codegenFreq[17])*3 +
+		int(w.codegenFreq[18])*7
+	size = header +
+		litEnc.bitLength(w.literalFreq) +
+		offEnc.bitLength(w.offsetFreq) +
+		extraBits
+
+	return size, numCodegens
+}
+
+// fixedSize returns the size of dynamically encoded data in bits.
+func (w *huffmanBitWriter) fixedSize(extraBits int) int {
+	return 3 +
+		fixedLiteralEncoding.bitLength(w.literalFreq) +
+		fixedOffsetEncoding.bitLength(w.offsetFreq) +
+		extraBits
+}
+
+// storedSize calculates the stored size, including header.
+// The function returns the size in bits and whether the block
+// fits inside a single block.
+func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) {
+	if in == nil {
+		return 0, false
+	}
+	if len(in) <= maxStoreBlockSize {
+		return (len(in) + 5) * 8, true
+	}
+	return 0, false
+}
+
+func (w *huffmanBitWriter) writeCode(c hcode) {
+	if w.err != nil {
+		return
+	}
+	w.bits |= uint64(c.code) << w.nbits
+	w.nbits += uint(c.len)
+	if w.nbits >= 48 {
+		bits := w.bits
+		w.bits >>= 48
+		w.nbits -= 48
+		n := w.nbytes
+		bytes := w.bytes[n : n+6]
+		bytes[0] = byte(bits)
+		bytes[1] = byte(bits >> 8)
+		bytes[2] = byte(bits >> 16)
+		bytes[3] = byte(bits >> 24)
+		bytes[4] = byte(bits >> 32)
+		bytes[5] = byte(bits >> 40)
+		n += 6
+		if n >= bufferFlushSize {
+			w.write(w.bytes[:n])
+			n = 0
+		}
+		w.nbytes = n
+	}
+}
+
+// Write the header of a dynamic Huffman block to the output stream.
+//
+//  numLiterals  The number of literals specified in codegen
+//  numOffsets   The number of offsets specified in codegen
+//  numCodegens  The number of codegens used in codegen
+func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) {
+	if w.err != nil {
+		return
+	}
+	var firstBits int32 = 4
+	if isEof {
+		firstBits = 5
+	}
+	w.writeBits(firstBits, 3)
+	w.writeBits(int32(numLiterals-257), 5)
+	w.writeBits(int32(numOffsets-1), 5)
+	w.writeBits(int32(numCodegens-4), 4)
+
+	for i := 0; i < numCodegens; i++ {
+		value := uint(w.codegenEncoding.codes[codegenOrder[i]].len)
+		w.writeBits(int32(value), 3)
+	}
+
+	i := 0
+	for {
+		var codeWord int = int(w.codegen[i])
+		i++
+		if codeWord == badCode {
+			break
+		}
+		w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
+
+		switch codeWord {
+		case 16:
+			w.writeBits(int32(w.codegen[i]), 2)
+			i++
+			break
+		case 17:
+			w.writeBits(int32(w.codegen[i]), 3)
+			i++
+			break
+		case 18:
+			w.writeBits(int32(w.codegen[i]), 7)
+			i++
+			break
+		}
+	}
+}
+
+func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) {
+	if w.err != nil {
+		return
+	}
+	var flag int32
+	if isEof {
+		flag = 1
+	}
+	w.writeBits(flag, 3)
+	w.flush()
+	w.writeBits(int32(length), 16)
+	w.writeBits(int32(^uint16(length)), 16)
+}
+
+func (w *huffmanBitWriter) writeFixedHeader(isEof bool) {
+	if w.err != nil {
+		return
+	}
+	// Indicate that we are a fixed Huffman block
+	var value int32 = 2
+	if isEof {
+		value = 3
+	}
+	w.writeBits(value, 3)
+}
+
+// writeBlock will write a block of tokens with the smallest encoding.
+// The original input can be supplied, and if the huffman encoded data
+// is larger than the original bytes, the data will be written as a
+// stored block.
+// If the input is nil, the tokens will always be Huffman encoded.
+func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
+	if w.err != nil {
+		return
+	}
+
+	tokens = append(tokens, endBlockMarker)
+	numLiterals, numOffsets := w.indexTokens(tokens)
+
+	var extraBits int
+	storedSize, storable := w.storedSize(input)
+	if storable {
+		// We only bother calculating the costs of the extra bits required by
+		// the length of offset fields (which will be the same for both fixed
+		// and dynamic encoding), if we need to compare those two encodings
+		// against stored encoding.
+		for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ {
+			// First eight length codes have extra size = 0.
+			extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart])
+		}
+		for offsetCode := 4; offsetCode < numOffsets; offsetCode++ {
+			// First four offset codes have extra size = 0.
+			extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode])
+		}
+	}
+
+	// Figure out smallest code.
+	// Fixed Huffman baseline.
+	var literalEncoding = fixedLiteralEncoding
+	var offsetEncoding = fixedOffsetEncoding
+	var size = w.fixedSize(extraBits)
+
+	// Dynamic Huffman?
+	var numCodegens int
+
+	// Generate codegen and codegenFrequencies, which indicates how to encode
+	// the literalEncoding and the offsetEncoding.
+	w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
+	w.codegenEncoding.generate(w.codegenFreq[:], 7)
+	dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)
+
+	if dynamicSize < size {
+		size = dynamicSize
+		literalEncoding = w.literalEncoding
+		offsetEncoding = w.offsetEncoding
+	}
+
+	// Stored bytes?
+	if storable && storedSize < size {
+		w.writeStoredHeader(len(input), eof)
+		w.writeBytes(input)
+		return
+	}
+
+	// Huffman.
+	if literalEncoding == fixedLiteralEncoding {
+		w.writeFixedHeader(eof)
+	} else {
+		w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
+	}
+
+	// Write the tokens.
+	w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes)
+}
+
+// writeBlockDynamic encodes a block using a dynamic Huffman table.
+// This should be used if the symbols used have a disproportionate
+// histogram distribution.
+// If input is supplied and the compression savings are below 1/16th of the
+// input size the block is stored.
+func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) {
+	if w.err != nil {
+		return
+	}
+
+	tokens = append(tokens, endBlockMarker)
+	numLiterals, numOffsets := w.indexTokens(tokens)
+
+	// Generate codegen and codegenFrequencies, which indicates how to encode
+	// the literalEncoding and the offsetEncoding.
+	w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
+	w.codegenEncoding.generate(w.codegenFreq[:], 7)
+	size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0)
+
+	// Store bytes, if we don't get a reasonable improvement.
+	if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
+		w.writeStoredHeader(len(input), eof)
+		w.writeBytes(input)
+		return
+	}
+
+	// Write Huffman table.
+	w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
+
+	// Write the tokens.
+	w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes)
+}
+
+// indexTokens indexes a slice of tokens, and updates
+// literalFreq and offsetFreq, and generates literalEncoding
+// and offsetEncoding.
+// The number of literal and offset tokens is returned.
+func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) {
+	for i := range w.literalFreq {
+		w.literalFreq[i] = 0
+	}
+	for i := range w.offsetFreq {
+		w.offsetFreq[i] = 0
+	}
+
+	for _, t := range tokens {
+		if t < matchType {
+			w.literalFreq[t.literal()]++
+			continue
+		}
+		length := t.length()
+		offset := t.offset()
+		w.literalFreq[lengthCodesStart+lengthCode(length)]++
+		w.offsetFreq[offsetCode(offset)]++
+	}
+
+	// get the number of literals
+	numLiterals = len(w.literalFreq)
+	for w.literalFreq[numLiterals-1] == 0 {
+		numLiterals--
+	}
+	// get the number of offsets
+	numOffsets = len(w.offsetFreq)
+	for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 {
+		numOffsets--
+	}
+	if numOffsets == 0 {
+		// We haven't found a single match. If we want to go with the dynamic encoding,
+		// we should count at least one offset to be sure that the offset huffman tree could be encoded.
+		w.offsetFreq[0] = 1
+		numOffsets = 1
+	}
+	w.literalEncoding.generate(w.literalFreq, 15)
+	w.offsetEncoding.generate(w.offsetFreq, 15)
+	return
+}
+
+// writeTokens writes a slice of tokens to the output.
+// codes for literal and offset encoding must be supplied.
+func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) {
+	if w.err != nil {
+		return
+	}
+	for _, t := range tokens {
+		if t < matchType {
+			w.writeCode(leCodes[t.literal()])
+			continue
+		}
+		// Write the length
+		length := t.length()
+		lengthCode := lengthCode(length)
+		w.writeCode(leCodes[lengthCode+lengthCodesStart])
+		extraLengthBits := uint(lengthExtraBits[lengthCode])
+		if extraLengthBits > 0 {
+			extraLength := int32(length - lengthBase[lengthCode])
+			w.writeBits(extraLength, extraLengthBits)
+		}
+		// Write the offset
+		offset := t.offset()
+		offsetCode := offsetCode(offset)
+		w.writeCode(oeCodes[offsetCode])
+		extraOffsetBits := uint(offsetExtraBits[offsetCode])
+		if extraOffsetBits > 0 {
+			extraOffset := int32(offset - offsetBase[offsetCode])
+			w.writeBits(extraOffset, extraOffsetBits)
+		}
+	}
+}
+
+// huffOffset is a static offset encoder used for huffman only encoding.
+// It can be reused since we will not be encoding offset values.
+var huffOffset *huffmanEncoder
+
+func init() {
+	offsetFreq := make([]int32, offsetCodeCount)
+	offsetFreq[0] = 1
+	huffOffset = newHuffmanEncoder(offsetCodeCount)
+	huffOffset.generate(offsetFreq, 15)
+}
+
+// writeBlockHuff encodes a block of bytes as either
+// Huffman encoded literals or uncompressed bytes if the
+// results only gains very little from compression.
+func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) {
+	if w.err != nil {
+		return
+	}
+
+	// Clear histogram
+	for i := range w.literalFreq {
+		w.literalFreq[i] = 0
+	}
+
+	// Add everything as literals
+	histogram(input, w.literalFreq)
+
+	w.literalFreq[endBlockMarker] = 1
+
+	const numLiterals = endBlockMarker + 1
+	w.offsetFreq[0] = 1
+	const numOffsets = 1
+
+	w.literalEncoding.generate(w.literalFreq, 15)
+
+	// Figure out smallest code.
+	// Always use dynamic Huffman or Store
+	var numCodegens int
+
+	// Generate codegen and codegenFrequencies, which indicates how to encode
+	// the literalEncoding and the offsetEncoding.
+	w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset)
+	w.codegenEncoding.generate(w.codegenFreq[:], 7)
+	size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0)
+
+	// Store bytes, if we don't get a reasonable improvement.
+	if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
+		w.writeStoredHeader(len(input), eof)
+		w.writeBytes(input)
+		return
+	}
+
+	// Huffman.
+	w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
+	encoding := w.literalEncoding.codes[:257]
+	n := w.nbytes
+	for _, t := range input {
+		// Bitwriting inlined, ~30% speedup
+		c := encoding[t]
+		w.bits |= uint64(c.code) << w.nbits
+		w.nbits += uint(c.len)
+		if w.nbits < 48 {
+			continue
+		}
+		// Store 6 bytes
+		bits := w.bits
+		w.bits >>= 48
+		w.nbits -= 48
+		bytes := w.bytes[n : n+6]
+		bytes[0] = byte(bits)
+		bytes[1] = byte(bits >> 8)
+		bytes[2] = byte(bits >> 16)
+		bytes[3] = byte(bits >> 24)
+		bytes[4] = byte(bits >> 32)
+		bytes[5] = byte(bits >> 40)
+		n += 6
+		if n < bufferFlushSize {
+			continue
+		}
+		w.write(w.bytes[:n])
+		if w.err != nil {
+			return // Return early in the event of write failures
+		}
+		n = 0
+	}
+	w.nbytes = n
+	w.writeCode(encoding[endBlockMarker])
+}
+
+// histogram accumulates a histogram of b in h.
+//
+// len(h) must be >= 256, and h's elements must be all zeroes.
+func histogram(b []byte, h []int32) {
+	h = h[:256]
+	for _, t := range b {
+		h[t]++
+	}
+}