tinygo-org/drivers
1
0
Fork 0
mirror of https://github.com/tinygo-org/drivers.git synced 2026-08-17 13:23:26 +00:00
Code Issues Packages Projects Releases 10 Wiki Activity

image: add support for image/jpeg and image/png (#303)

Browse Source 
* Copy from go1.17 image package
* Remove unnecessary files
* Reduce memory usage
* Add examples/ili9341/slideshow
* image: add ./image/README.md
* image: change convert2bin to . /cmd
* Makefile: add ./cmd to NOTEST
This commit is contained in:
sago35
2021-09-18 19:18:02 +09:00
committed by GitHub
parent 259593e608
commit f2e5278965
52 changed files with 19324 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
image/internal/compress/flate/deflate.go
+748
View File
@@ -0,0 +1,748 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"fmt"
"io"
"math"
)
const (
NoCompression = 0
BestSpeed = 1
BestCompression = 9
DefaultCompression = -1
// HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
// entropy encoding. This mode is useful in compressing data that has
// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
// that lacks an entropy encoder. Compression gains are achieved when
// certain bytes in the input stream occur more frequently than others.
//
// Note that HuffmanOnly produces a compressed output that is
// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
// continue to be able to decompress this output.
HuffmanOnly = -2
)
const (
logWindowSize = 15
windowSize = 1 << logWindowSize
windowMask = windowSize - 1
// The LZ77 step produces a sequence of literal tokens and <length, offset>
// pair tokens. The offset is also known as distance. The underlying wire
// format limits the range of lengths and offsets. For example, there are
// 256 legitimate lengths: those in the range [3, 258]. This package's
// compressor uses a higher minimum match length, enabling optimizations
// such as finding matches via 32-bit loads and compares.
baseMatchLength = 3 // The smallest match length per the RFC section 3.2.5
minMatchLength = 4 // The smallest match length that the compressor actually emits
maxMatchLength = 258 // The largest match length
baseMatchOffset = 1 // The smallest match offset
maxMatchOffset = 1 << 15 // The largest match offset
// The maximum number of tokens we put into a single flate block, just to
// stop things from getting too large.
maxFlateBlockTokens = 1 << 14
maxStoreBlockSize = 65535
hashBits = 17 // After 17 performance degrades
hashSize = 1 << hashBits
hashMask = (1 << hashBits) - 1
maxHashOffset = 1 << 24
skipNever = math.MaxInt32
)
type compressionLevel struct {
level, good, lazy, nice, chain, fastSkipHashing int
}
var levels = []compressionLevel{
{0, 0, 0, 0, 0, 0}, // NoCompression.
{1, 0, 0, 0, 0, 0}, // BestSpeed uses a custom algorithm; see deflatefast.go.
// For levels 2-3 we don't bother trying with lazy matches.
{2, 4, 0, 16, 8, 5},
{3, 4, 0, 32, 32, 6},
// Levels 4-9 use increasingly more lazy matching
// and increasingly stringent conditions for "good enough".
{4, 4, 4, 16, 16, skipNever},
{5, 8, 16, 32, 32, skipNever},
{6, 8, 16, 128, 128, skipNever},
{7, 8, 32, 128, 256, skipNever},
{8, 32, 128, 258, 1024, skipNever},
{9, 32, 258, 258, 4096, skipNever},
}
type compressor struct {
compressionLevel
w *huffmanBitWriter
bulkHasher func([]byte, []uint32)
// compression algorithm
fill func(*compressor, []byte) int // copy data to window
step func(*compressor) // process window
sync bool // requesting flush
bestSpeed *deflateFast // Encoder for BestSpeed
// Input hash chains
// hashHead[hashValue] contains the largest inputIndex with the specified hash value
// If hashHead[hashValue] is within the current window, then
// hashPrev[hashHead[hashValue] & windowMask] contains the previous index
// with the same hash value.
chainHead int
hashHead [hashSize]uint32
hashPrev [windowSize]uint32
hashOffset int
// input window: unprocessed data is window[index:windowEnd]
index int
window []byte
windowEnd int
blockStart int // window index where current tokens start
byteAvailable bool // if true, still need to process window[index-1].
// queued output tokens
tokens []token
// deflate state
length int
offset int
hash uint32
maxInsertIndex int
err error
// hashMatch must be able to contain hashes for the maximum match length.
hashMatch [maxMatchLength - 1]uint32
}
func (d *compressor) fillDeflate(b []byte) int {
if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
// shift the window by windowSize
copy(d.window, d.window[windowSize:2*windowSize])
d.index -= windowSize
d.windowEnd -= windowSize
if d.blockStart >= windowSize {
d.blockStart -= windowSize
} else {
d.blockStart = math.MaxInt32
}
d.hashOffset += windowSize
if d.hashOffset > maxHashOffset {
delta := d.hashOffset - 1
d.hashOffset -= delta
d.chainHead -= delta
// Iterate over slices instead of arrays to avoid copying
// the entire table onto the stack (Issue #18625).
for i, v := range d.hashPrev[:] {
if int(v) > delta {
d.hashPrev[i] = uint32(int(v) - delta)
} else {
d.hashPrev[i] = 0
}
}
for i, v := range d.hashHead[:] {
if int(v) > delta {
d.hashHead[i] = uint32(int(v) - delta)
} else {
d.hashHead[i] = 0
}
}
}
}
n := copy(d.window[d.windowEnd:], b)
d.windowEnd += n
return n
}
func (d *compressor) writeBlock(tokens []token, index int) error {
if index > 0 {
var window []byte
if d.blockStart <= index {
window = d.window[d.blockStart:index]
}
d.blockStart = index
d.w.writeBlock(tokens, false, window)
return d.w.err
}
return nil
}
// fillWindow will fill the current window with the supplied
// dictionary and calculate all hashes.
// This is much faster than doing a full encode.
// Should only be used after a reset.
func (d *compressor) fillWindow(b []byte) {
// Do not fill window if we are in store-only mode.
if d.compressionLevel.level < 2 {
return
}
if d.index != 0 || d.windowEnd != 0 {
panic("internal error: fillWindow called with stale data")
}
// If we are given too much, cut it.
if len(b) > windowSize {
b = b[len(b)-windowSize:]
}
// Add all to window.
n := copy(d.window, b)
// Calculate 256 hashes at the time (more L1 cache hits)
loops := (n + 256 - minMatchLength) / 256
for j := 0; j < loops; j++ {
index := j * 256
end := index + 256 + minMatchLength - 1
if end > n {
end = n
}
toCheck := d.window[index:end]
dstSize := len(toCheck) - minMatchLength + 1
if dstSize <= 0 {
continue
}
dst := d.hashMatch[:dstSize]
d.bulkHasher(toCheck, dst)
var newH uint32
for i, val := range dst {
di := i + index
newH = val
hh := &d.hashHead[newH&hashMask]
// Get previous value with the same hash.
// Our chain should point to the previous value.
d.hashPrev[di&windowMask] = *hh
// Set the head of the hash chain to us.
*hh = uint32(di + d.hashOffset)
}
d.hash = newH
}
// Update window information.
d.windowEnd = n
d.index = n
}
// Try to find a match starting at index whose length is greater than prevSize.
// We only look at chainCount possibilities before giving up.
func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
minMatchLook := maxMatchLength
if lookahead < minMatchLook {
minMatchLook = lookahead
}
win := d.window[0 : pos+minMatchLook]
// We quit when we get a match that's at least nice long
nice := len(win) - pos
if d.nice < nice {
nice = d.nice
}
// If we've got a match that's good enough, only look in 1/4 the chain.
tries := d.chain
length = prevLength
if length >= d.good {
tries >>= 2
}
wEnd := win[pos+length]
wPos := win[pos:]
minIndex := pos - windowSize
for i := prevHead; tries > 0; tries-- {
if wEnd == win[i+length] {
n := matchLen(win[i:], wPos, minMatchLook)
if n > length && (n > minMatchLength || pos-i <= 4096) {
length = n
offset = pos - i
ok = true
if n >= nice {
// The match is good enough that we don't try to find a better one.
break
}
wEnd = win[pos+n]
}
}
if i == minIndex {
// hashPrev[i & windowMask] has already been overwritten, so stop now.
break
}
i = int(d.hashPrev[i&windowMask]) - d.hashOffset
if i < minIndex || i < 0 {
break
}
}
return
}
func (d *compressor) writeStoredBlock(buf []byte) error {
if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
return d.w.err
}
d.w.writeBytes(buf)
return d.w.err
}
const hashmul = 0x1e35a7bd
// hash4 returns a hash representation of the first 4 bytes
// of the supplied slice.
// The caller must ensure that len(b) >= 4.
func hash4(b []byte) uint32 {
return ((uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24) * hashmul) >> (32 - hashBits)
}
// bulkHash4 will compute hashes using the same
// algorithm as hash4
func bulkHash4(b []byte, dst []uint32) {
if len(b) < minMatchLength {
return
}
hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
dst[0] = (hb * hashmul) >> (32 - hashBits)
end := len(b) - minMatchLength + 1
for i := 1; i < end; i++ {
hb = (hb << 8) | uint32(b[i+3])
dst[i] = (hb * hashmul) >> (32 - hashBits)
}
}
// matchLen returns the number of matching bytes in a and b
// up to length 'max'. Both slices must be at least 'max'
// bytes in size.
func matchLen(a, b []byte, max int) int {
a = a[:max]
b = b[:len(a)]
for i, av := range a {
if b[i] != av {
return i
}
}
return max
}
// encSpeed will compress and store the currently added data,
// if enough has been accumulated or we at the end of the stream.
// Any error that occurred will be in d.err
func (d *compressor) encSpeed() {
// We only compress if we have maxStoreBlockSize.
if d.windowEnd < maxStoreBlockSize {
if !d.sync {
return
}
// Handle small sizes.
if d.windowEnd < 128 {
switch {
case d.windowEnd == 0:
return
case d.windowEnd <= 16:
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
default:
d.w.writeBlockHuff(false, d.window[:d.windowEnd])
d.err = d.w.err
}
d.windowEnd = 0
d.bestSpeed.reset()
return
}
}
// Encode the block.
d.tokens = d.bestSpeed.encode(d.tokens[:0], d.window[:d.windowEnd])
// If we removed less than 1/16th, Huffman compress the block.
if len(d.tokens) > d.windowEnd-(d.windowEnd>>4) {
d.w.writeBlockHuff(false, d.window[:d.windowEnd])
} else {
d.w.writeBlockDynamic(d.tokens, false, d.window[:d.windowEnd])
}
d.err = d.w.err
d.windowEnd = 0
}
func (d *compressor) initDeflate() {
d.window = make([]byte, 2*windowSize)
d.hashOffset = 1
d.tokens = make([]token, 0, maxFlateBlockTokens+1)
d.length = minMatchLength - 1
d.offset = 0
d.byteAvailable = false
d.index = 0
d.hash = 0
d.chainHead = -1
d.bulkHasher = bulkHash4
}
func (d *compressor) deflate() {
if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync {
return
}
d.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
if d.index < d.maxInsertIndex {
d.hash = hash4(d.window[d.index : d.index+minMatchLength])
}
Loop:
for {
if d.index > d.windowEnd {
panic("index > windowEnd")
}
lookahead := d.windowEnd - d.index
if lookahead < minMatchLength+maxMatchLength {
if !d.sync {
break Loop
}
if d.index > d.windowEnd {
panic("index > windowEnd")
}
if lookahead == 0 {
// Flush current output block if any.
if d.byteAvailable {
// There is still one pending token that needs to be flushed
d.tokens = append(d.tokens, literalToken(uint32(d.window[d.index-1])))
d.byteAvailable = false
}
if len(d.tokens) > 0 {
if d.err = d.writeBlock(d.tokens, d.index); d.err != nil {
return
}
d.tokens = d.tokens[:0]
}
break Loop
}
}
if d.index < d.maxInsertIndex {
// Update the hash
d.hash = hash4(d.window[d.index : d.index+minMatchLength])
hh := &d.hashHead[d.hash&hashMask]
d.chainHead = int(*hh)
d.hashPrev[d.index&windowMask] = uint32(d.chainHead)
*hh = uint32(d.index + d.hashOffset)
}
prevLength := d.length
prevOffset := d.offset
d.length = minMatchLength - 1
d.offset = 0
minIndex := d.index - windowSize
if minIndex < 0 {
minIndex = 0
}
if d.chainHead-d.hashOffset >= minIndex &&
(d.fastSkipHashing != skipNever && lookahead > minMatchLength-1 ||
d.fastSkipHashing == skipNever && lookahead > prevLength && prevLength < d.lazy) {
if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok {
d.length = newLength
d.offset = newOffset
}
}
if d.fastSkipHashing != skipNever && d.length >= minMatchLength ||
d.fastSkipHashing == skipNever && prevLength >= minMatchLength && d.length <= prevLength {
// There was a match at the previous step, and the current match is
// not better. Output the previous match.
if d.fastSkipHashing != skipNever {
d.tokens = append(d.tokens, matchToken(uint32(d.length-baseMatchLength), uint32(d.offset-baseMatchOffset)))
} else {
d.tokens = append(d.tokens, matchToken(uint32(prevLength-baseMatchLength), uint32(prevOffset-baseMatchOffset)))
}
// Insert in the hash table all strings up to the end of the match.
// index and index-1 are already inserted. If there is not enough
// lookahead, the last two strings are not inserted into the hash
// table.
if d.length <= d.fastSkipHashing {
var newIndex int
if d.fastSkipHashing != skipNever {
newIndex = d.index + d.length
} else {
newIndex = d.index + prevLength - 1
}
index := d.index
for index++; index < newIndex; index++ {
if index < d.maxInsertIndex {
d.hash = hash4(d.window[index : index+minMatchLength])
// Get previous value with the same hash.
// Our chain should point to the previous value.
hh := &d.hashHead[d.hash&hashMask]
d.hashPrev[index&windowMask] = *hh
// Set the head of the hash chain to us.
*hh = uint32(index + d.hashOffset)
}
}
d.index = index
if d.fastSkipHashing == skipNever {
d.byteAvailable = false
d.length = minMatchLength - 1
}
} else {
// For matches this long, we don't bother inserting each individual
// item into the table.
d.index += d.length
if d.index < d.maxInsertIndex {
d.hash = hash4(d.window[d.index : d.index+minMatchLength])
}
}
if len(d.tokens) == maxFlateBlockTokens {
// The block includes the current character
if d.err = d.writeBlock(d.tokens, d.index); d.err != nil {
return
}
d.tokens = d.tokens[:0]
}
} else {
if d.fastSkipHashing != skipNever || d.byteAvailable {
i := d.index - 1
if d.fastSkipHashing != skipNever {
i = d.index
}
d.tokens = append(d.tokens, literalToken(uint32(d.window[i])))
if len(d.tokens) == maxFlateBlockTokens {
if d.err = d.writeBlock(d.tokens, i+1); d.err != nil {
return
}
d.tokens = d.tokens[:0]
}
}
d.index++
if d.fastSkipHashing == skipNever {
d.byteAvailable = true
}
}
}
}
func (d *compressor) fillStore(b []byte) int {
n := copy(d.window[d.windowEnd:], b)
d.windowEnd += n
return n
}
func (d *compressor) store() {
if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) {
d.err = d.writeStoredBlock(d.window[:d.windowEnd])
d.windowEnd = 0
}
}
// storeHuff compresses and stores the currently added data
// when the d.window is full or we are at the end of the stream.
// Any error that occurred will be in d.err
func (d *compressor) storeHuff() {
if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
return
}
d.w.writeBlockHuff(false, d.window[:d.windowEnd])
d.err = d.w.err
d.windowEnd = 0
}
func (d *compressor) write(b []byte) (n int, err error) {
if d.err != nil {
return 0, d.err
}
n = len(b)
for len(b) > 0 {
d.step(d)
b = b[d.fill(d, b):]
if d.err != nil {
return 0, d.err
}
}
return n, nil
}
func (d *compressor) syncFlush() error {
if d.err != nil {
return d.err
}
d.sync = true
d.step(d)
if d.err == nil {
d.w.writeStoredHeader(0, false)
d.w.flush()
d.err = d.w.err
}
d.sync = false
return d.err
}
func (d *compressor) init(w io.Writer, level int) (err error) {
d.w = newHuffmanBitWriter(w)
switch {
case level == NoCompression:
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillStore
d.step = (*compressor).store
case level == HuffmanOnly:
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillStore
d.step = (*compressor).storeHuff
case level == BestSpeed:
d.compressionLevel = levels[level]
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillStore
d.step = (*compressor).encSpeed
d.bestSpeed = newDeflateFast()
d.tokens = make([]token, maxStoreBlockSize)
case level == DefaultCompression:
level = 6
fallthrough
case 2 <= level && level <= 9:
d.compressionLevel = levels[level]
d.initDeflate()
d.fill = (*compressor).fillDeflate
d.step = (*compressor).deflate
default:
return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
}
return nil
}
func (d *compressor) reset(w io.Writer) {
d.w.reset(w)
d.sync = false
d.err = nil
switch d.compressionLevel.level {
case NoCompression:
d.windowEnd = 0
case BestSpeed:
d.windowEnd = 0
d.tokens = d.tokens[:0]
d.bestSpeed.reset()
default:
d.chainHead = -1
for i := range d.hashHead {
d.hashHead[i] = 0
}
for i := range d.hashPrev {
d.hashPrev[i] = 0
}
d.hashOffset = 1
d.index, d.windowEnd = 0, 0
d.blockStart, d.byteAvailable = 0, false
d.tokens = d.tokens[:0]
d.length = minMatchLength - 1
d.offset = 0
d.hash = 0
d.maxInsertIndex = 0
}
}
func (d *compressor) close() error {
if d.err != nil {
return d.err
}
d.sync = true
d.step(d)
if d.err != nil {
return d.err
}
if d.w.writeStoredHeader(0, true); d.w.err != nil {
return d.w.err
}
d.w.flush()
return d.w.err
}
// NewWriter returns a new Writer compressing data at the given level.
// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
// higher levels typically run slower but compress more. Level 0
// (NoCompression) does not attempt any compression; it only adds the
// necessary DEFLATE framing.
// Level -1 (DefaultCompression) uses the default compression level.
// Level -2 (HuffmanOnly) will use Huffman compression only, giving
// a very fast compression for all types of input, but sacrificing considerable
// compression efficiency.
//
// If level is in the range [-2, 9] then the error returned will be nil.
// Otherwise the error returned will be non-nil.
func NewWriter(w io.Writer, level int) (*Writer, error) {
var dw Writer
if err := dw.d.init(w, level); err != nil {
return nil, err
}
return &dw, nil
}
// NewWriterDict is like NewWriter but initializes the new
// Writer with a preset dictionary. The returned Writer behaves
// as if the dictionary had been written to it without producing
// any compressed output. The compressed data written to w
// can only be decompressed by a Reader initialized with the
// same dictionary.
func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
dw := &dictWriter{w}
zw, err := NewWriter(dw, level)
if err != nil {
return nil, err
}
zw.d.fillWindow(dict)
zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
return zw, err
}
type dictWriter struct {
w io.Writer
}
func (w *dictWriter) Write(b []byte) (n int, err error) {
return w.w.Write(b)
}
// A Writer takes data written to it and writes the compressed
// form of that data to an underlying writer (see NewWriter).
type Writer struct {
d compressor
dict []byte
}
// Write writes data to w, which will eventually write the
// compressed form of data to its underlying writer.
func (w *Writer) Write(data []byte) (n int, err error) {
return w.d.write(data)
}
// Flush flushes any pending data to the underlying writer.
// It is useful mainly in compressed network protocols, to ensure that
// a remote reader has enough data to reconstruct a packet.
// Flush does not return until the data has been written.
// Calling Flush when there is no pending data still causes the Writer
// to emit a sync marker of at least 4 bytes.
// If the underlying writer returns an error, Flush returns that error.
//
// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
func (w *Writer) Flush() error {
// For more about flushing:
// https://www.bolet.org/~pornin/deflate-flush.html
return w.d.syncFlush()
}
// Close flushes and closes the writer.
func (w *Writer) Close() error {
return w.d.close()
}
// Reset discards the writer's state and makes it equivalent to
// the result of NewWriter or NewWriterDict called with dst
// and w's level and dictionary.
func (w *Writer) Reset(dst io.Writer) {
if dw, ok := w.d.w.writer.(*dictWriter); ok {
// w was created with NewWriterDict
dw.w = dst
w.d.reset(dw)
w.d.fillWindow(w.dict)
} else {
// w was created with NewWriter
w.d.reset(dst)
}
}
image/internal/compress/flate/deflate_test.go
+984
View File
@@ -0,0 +1,984 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"bytes"
"errors"
"fmt"
"internal/testenv"
"io"
"math/rand"
"os"
"reflect"
"runtime/debug"
"sync"
"testing"
)
type deflateTest struct {
in []byte
level int
out []byte
}
type deflateInflateTest struct {
in []byte
}
type reverseBitsTest struct {
in uint16
bitCount uint8
out uint16
}
var deflateTests = []*deflateTest{
{[]byte{}, 0, []byte{1, 0, 0, 255, 255}},
{[]byte{0x11}, -1, []byte{18, 4, 4, 0, 0, 255, 255}},
{[]byte{0x11}, DefaultCompression, []byte{18, 4, 4, 0, 0, 255, 255}},
{[]byte{0x11}, 4, []byte{18, 4, 4, 0, 0, 255, 255}},
{[]byte{0x11}, 0, []byte{0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255}},
{[]byte{0x11, 0x12}, 0, []byte{0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255}},
{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 0,
[]byte{0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255},
},
{[]byte{}, 2, []byte{1, 0, 0, 255, 255}},
{[]byte{0x11}, 2, []byte{18, 4, 4, 0, 0, 255, 255}},
{[]byte{0x11, 0x12}, 2, []byte{18, 20, 2, 4, 0, 0, 255, 255}},
{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 2, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}},
{[]byte{}, 9, []byte{1, 0, 0, 255, 255}},
{[]byte{0x11}, 9, []byte{18, 4, 4, 0, 0, 255, 255}},
{[]byte{0x11, 0x12}, 9, []byte{18, 20, 2, 4, 0, 0, 255, 255}},
{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}, 9, []byte{18, 132, 2, 64, 0, 0, 0, 255, 255}},
}
var deflateInflateTests = []*deflateInflateTest{
{[]byte{}},
{[]byte{0x11}},
{[]byte{0x11, 0x12}},
{[]byte{0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}},
{[]byte{0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13}},
{largeDataChunk()},
}
var reverseBitsTests = []*reverseBitsTest{
{1, 1, 1},
{1, 2, 2},
{1, 3, 4},
{1, 4, 8},
{1, 5, 16},
{17, 5, 17},
{257, 9, 257},
{29, 5, 23},
}
func largeDataChunk() []byte {
result := make([]byte, 100000)
for i := range result {
result[i] = byte(i * i & 0xFF)
}
return result
}
func TestBulkHash4(t *testing.T) {
for _, x := range deflateTests {
y := x.out
if len(y) < minMatchLength {
continue
}
y = append(y, y...)
for j := 4; j < len(y); j++ {
y := y[:j]
dst := make([]uint32, len(y)-minMatchLength+1)
for i := range dst {
dst[i] = uint32(i + 100)
}
bulkHash4(y, dst)
for i, got := range dst {
want := hash4(y[i:])
if got != want && got == uint32(i)+100 {
t.Errorf("Len:%d Index:%d, want 0x%08x but not modified", len(y), i, want)
} else if got != want {
t.Errorf("Len:%d Index:%d, got 0x%08x want:0x%08x", len(y), i, got, want)
}
}
}
}
}
func TestDeflate(t *testing.T) {
for _, h := range deflateTests {
var buf bytes.Buffer
w, err := NewWriter(&buf, h.level)
if err != nil {
t.Errorf("NewWriter: %v", err)
continue
}
w.Write(h.in)
w.Close()
if !bytes.Equal(buf.Bytes(), h.out) {
t.Errorf("Deflate(%d, %x) = \n%#v, want \n%#v", h.level, h.in, buf.Bytes(), h.out)
}
}
}
// A sparseReader returns a stream consisting of 0s followed by 1<<16 1s.
// This tests missing hash references in a very large input.
type sparseReader struct {
l int64
cur int64
}
func (r *sparseReader) Read(b []byte) (n int, err error) {
if r.cur >= r.l {
return 0, io.EOF
}
n = len(b)
cur := r.cur + int64(n)
if cur > r.l {
n -= int(cur - r.l)
cur = r.l
}
for i := range b[0:n] {
if r.cur+int64(i) >= r.l-1<<16 {
b[i] = 1
} else {
b[i] = 0
}
}
r.cur = cur
return
}
func TestVeryLongSparseChunk(t *testing.T) {
if testing.Short() {
t.Skip("skipping sparse chunk during short test")
}
w, err := NewWriter(io.Discard, 1)
if err != nil {
t.Errorf("NewWriter: %v", err)
return
}
if _, err = io.Copy(w, &sparseReader{l: 23e8}); err != nil {
t.Errorf("Compress failed: %v", err)
return
}
}
type syncBuffer struct {
buf bytes.Buffer
mu sync.RWMutex
closed bool
ready chan bool
}
func newSyncBuffer() *syncBuffer {
return &syncBuffer{ready: make(chan bool, 1)}
}
func (b *syncBuffer) Read(p []byte) (n int, err error) {
for {
b.mu.RLock()
n, err = b.buf.Read(p)
b.mu.RUnlock()
if n > 0 || b.closed {
return
}
<-b.ready
}
}
func (b *syncBuffer) signal() {
select {
case b.ready <- true:
default:
}
}
func (b *syncBuffer) Write(p []byte) (n int, err error) {
n, err = b.buf.Write(p)
b.signal()
return
}
func (b *syncBuffer) WriteMode() {
b.mu.Lock()
}
func (b *syncBuffer) ReadMode() {
b.mu.Unlock()
b.signal()
}
func (b *syncBuffer) Close() error {
b.closed = true
b.signal()
return nil
}
func testSync(t *testing.T, level int, input []byte, name string) {
if len(input) == 0 {
return
}
t.Logf("--testSync %d, %d, %s", level, len(input), name)
buf := newSyncBuffer()
buf1 := new(bytes.Buffer)
buf.WriteMode()
w, err := NewWriter(io.MultiWriter(buf, buf1), level)
if err != nil {
t.Errorf("NewWriter: %v", err)
return
}
r := NewReader(buf)
// Write half the input and read back.
for i := 0; i < 2; i++ {
var lo, hi int
if i == 0 {
lo, hi = 0, (len(input)+1)/2
} else {
lo, hi = (len(input)+1)/2, len(input)
}
t.Logf("#%d: write %d-%d", i, lo, hi)
if _, err := w.Write(input[lo:hi]); err != nil {
t.Errorf("testSync: write: %v", err)
return
}
if i == 0 {
if err := w.Flush(); err != nil {
t.Errorf("testSync: flush: %v", err)
return
}
} else {
if err := w.Close(); err != nil {
t.Errorf("testSync: close: %v", err)
}
}
buf.ReadMode()
out := make([]byte, hi-lo+1)
m, err := io.ReadAtLeast(r, out, hi-lo)
t.Logf("#%d: read %d", i, m)
if m != hi-lo || err != nil {
t.Errorf("testSync/%d (%d, %d, %s): read %d: %d, %v (%d left)", i, level, len(input), name, hi-lo, m, err, buf.buf.Len())
return
}
if !bytes.Equal(input[lo:hi], out[:hi-lo]) {
t.Errorf("testSync/%d: read wrong bytes: %x vs %x", i, input[lo:hi], out[:hi-lo])
return
}
// This test originally checked that after reading
// the first half of the input, there was nothing left
// in the read buffer (buf.buf.Len() != 0) but that is
// not necessarily the case: the write Flush may emit
// some extra framing bits that are not necessary
// to process to obtain the first half of the uncompressed
// data. The test ran correctly most of the time, because
// the background goroutine had usually read even
// those extra bits by now, but it's not a useful thing to
// check.
buf.WriteMode()
}
buf.ReadMode()
out := make([]byte, 10)
if n, err := r.Read(out); n > 0 || err != io.EOF {
t.Errorf("testSync (%d, %d, %s): final Read: %d, %v (hex: %x)", level, len(input), name, n, err, out[0:n])
}
if buf.buf.Len() != 0 {
t.Errorf("testSync (%d, %d, %s): extra data at end", level, len(input), name)
}
r.Close()
// stream should work for ordinary reader too
r = NewReader(buf1)
out, err = io.ReadAll(r)
if err != nil {
t.Errorf("testSync: read: %s", err)
return
}
r.Close()
if !bytes.Equal(input, out) {
t.Errorf("testSync: decompress(compress(data)) != data: level=%d input=%s", level, name)
}
}
func testToFromWithLevelAndLimit(t *testing.T, level int, input []byte, name string, limit int) {
var buffer bytes.Buffer
w, err := NewWriter(&buffer, level)
if err != nil {
t.Errorf("NewWriter: %v", err)
return
}
w.Write(input)
w.Close()
if limit > 0 && buffer.Len() > limit {
t.Errorf("level: %d, len(compress(data)) = %d > limit = %d", level, buffer.Len(), limit)
return
}
if limit > 0 {
t.Logf("level: %d, size:%.2f%%, %d b\n", level, float64(buffer.Len()*100)/float64(limit), buffer.Len())
}
r := NewReader(&buffer)
out, err := io.ReadAll(r)
if err != nil {
t.Errorf("read: %s", err)
return
}
r.Close()
if !bytes.Equal(input, out) {
t.Errorf("decompress(compress(data)) != data: level=%d input=%s", level, name)
return
}
testSync(t, level, input, name)
}
func testToFromWithLimit(t *testing.T, input []byte, name string, limit [11]int) {
for i := 0; i < 10; i++ {
testToFromWithLevelAndLimit(t, i, input, name, limit[i])
}
// Test HuffmanCompression
testToFromWithLevelAndLimit(t, -2, input, name, limit[10])
}
func TestDeflateInflate(t *testing.T) {
t.Parallel()
for i, h := range deflateInflateTests {
if testing.Short() && len(h.in) > 10000 {
continue
}
testToFromWithLimit(t, h.in, fmt.Sprintf("#%d", i), [11]int{})
}
}
func TestReverseBits(t *testing.T) {
for _, h := range reverseBitsTests {
if v := reverseBits(h.in, h.bitCount); v != h.out {
t.Errorf("reverseBits(%v,%v) = %v, want %v",
h.in, h.bitCount, v, h.out)
}
}
}
type deflateInflateStringTest struct {
filename string
label string
limit [11]int
}
var deflateInflateStringTests = []deflateInflateStringTest{
{
"../testdata/e.txt",
"2.718281828...",
[...]int{100018, 50650, 50960, 51150, 50930, 50790, 50790, 50790, 50790, 50790, 43683},
},
{
"../../testdata/Isaac.Newton-Opticks.txt",
"Isaac.Newton-Opticks",
[...]int{567248, 218338, 198211, 193152, 181100, 175427, 175427, 173597, 173422, 173422, 325240},
},
}
func TestDeflateInflateString(t *testing.T) {
t.Parallel()
if testing.Short() && testenv.Builder() == "" {
t.Skip("skipping in short mode")
}
for _, test := range deflateInflateStringTests {
gold, err := os.ReadFile(test.filename)
if err != nil {
t.Error(err)
}
testToFromWithLimit(t, gold, test.label, test.limit)
if testing.Short() {
break
}
}
}
func TestReaderDict(t *testing.T) {
const (
dict = "hello world"
text = "hello again world"
)
var b bytes.Buffer
w, err := NewWriter(&b, 5)
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
w.Write([]byte(dict))
w.Flush()
b.Reset()
w.Write([]byte(text))
w.Close()
r := NewReaderDict(&b, []byte(dict))
data, err := io.ReadAll(r)
if err != nil {
t.Fatal(err)
}
if string(data) != "hello again world" {
t.Fatalf("read returned %q want %q", string(data), text)
}
}
func TestWriterDict(t *testing.T) {
const (
dict = "hello world"
text = "hello again world"
)
var b bytes.Buffer
w, err := NewWriter(&b, 5)
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
w.Write([]byte(dict))
w.Flush()
b.Reset()
w.Write([]byte(text))
w.Close()
var b1 bytes.Buffer
w, _ = NewWriterDict(&b1, 5, []byte(dict))
w.Write([]byte(text))
w.Close()
if !bytes.Equal(b1.Bytes(), b.Bytes()) {
t.Fatalf("writer wrote %q want %q", b1.Bytes(), b.Bytes())
}
}
// See https://golang.org/issue/2508
func TestRegression2508(t *testing.T) {
if testing.Short() {
t.Logf("test disabled with -short")
return
}
w, err := NewWriter(io.Discard, 1)
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
buf := make([]byte, 1024)
for i := 0; i < 131072; i++ {
if _, err := w.Write(buf); err != nil {
t.Fatalf("writer failed: %v", err)
}
}
w.Close()
}
func TestWriterReset(t *testing.T) {
t.Parallel()
for level := 0; level <= 9; level++ {
if testing.Short() && level > 1 {
break
}
w, err := NewWriter(io.Discard, level)
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
buf := []byte("hello world")
n := 1024
if testing.Short() {
n = 10
}
for i := 0; i < n; i++ {
w.Write(buf)
}
w.Reset(io.Discard)
wref, err := NewWriter(io.Discard, level)
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
// DeepEqual doesn't compare functions.
w.d.fill, wref.d.fill = nil, nil
w.d.step, wref.d.step = nil, nil
w.d.bulkHasher, wref.d.bulkHasher = nil, nil
w.d.bestSpeed, wref.d.bestSpeed = nil, nil
// hashMatch is always overwritten when used.
copy(w.d.hashMatch[:], wref.d.hashMatch[:])
if len(w.d.tokens) != 0 {
t.Errorf("level %d Writer not reset after Reset. %d tokens were present", level, len(w.d.tokens))
}
// As long as the length is 0, we don't care about the content.
w.d.tokens = wref.d.tokens
// We don't care if there are values in the window, as long as it is at d.index is 0
w.d.window = wref.d.window
if !reflect.DeepEqual(w, wref) {
t.Errorf("level %d Writer not reset after Reset", level)
}
}
levels := []int{0, 1, 2, 5, 9}
for _, level := range levels {
t.Run(fmt.Sprint(level), func(t *testing.T) {
testResetOutput(t, level, nil)
})
}
t.Run("dict", func(t *testing.T) {
for _, level := range levels {
t.Run(fmt.Sprint(level), func(t *testing.T) {
testResetOutput(t, level, nil)
})
}
})
}
func testResetOutput(t *testing.T, level int, dict []byte) {
writeData := func(w *Writer) {
msg := []byte("now is the time for all good gophers")
w.Write(msg)
w.Flush()
hello := []byte("hello world")
for i := 0; i < 1024; i++ {
w.Write(hello)
}
fill := bytes.Repeat([]byte("x"), 65000)
w.Write(fill)
}
buf := new(bytes.Buffer)
var w *Writer
var err error
if dict == nil {
w, err = NewWriter(buf, level)
} else {
w, err = NewWriterDict(buf, level, dict)
}
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
writeData(w)
w.Close()
out1 := buf.Bytes()
buf2 := new(bytes.Buffer)
w.Reset(buf2)
writeData(w)
w.Close()
out2 := buf2.Bytes()
if len(out1) != len(out2) {
t.Errorf("got %d, expected %d bytes", len(out2), len(out1))
return
}
if !bytes.Equal(out1, out2) {
mm := 0
for i, b := range out1[:len(out2)] {
if b != out2[i] {
t.Errorf("mismatch index %d: %#02x, expected %#02x", i, out2[i], b)
}
mm++
if mm == 10 {
t.Fatal("Stopping")
}
}
}
t.Logf("got %d bytes", len(out1))
}
// TestBestSpeed tests that round-tripping through deflate and then inflate
// recovers the original input. The Write sizes are near the thresholds in the
// compressor.encSpeed method (0, 16, 128), as well as near maxStoreBlockSize
// (65535).
func TestBestSpeed(t *testing.T) {
t.Parallel()
abc := make([]byte, 128)
for i := range abc {
abc[i] = byte(i)
}
abcabc := bytes.Repeat(abc, 131072/len(abc))
var want []byte
testCases := [][]int{
{65536, 0},
{65536, 1},
{65536, 1, 256},
{65536, 1, 65536},
{65536, 14},
{65536, 15},
{65536, 16},
{65536, 16, 256},
{65536, 16, 65536},
{65536, 127},
{65536, 128},
{65536, 128, 256},
{65536, 128, 65536},
{65536, 129},
{65536, 65536, 256},
{65536, 65536, 65536},
}
for i, tc := range testCases {
if i >= 3 && testing.Short() {
break
}
for _, firstN := range []int{1, 65534, 65535, 65536, 65537, 131072} {
tc[0] = firstN
outer:
for _, flush := range []bool{false, true} {
buf := new(bytes.Buffer)
want = want[:0]
w, err := NewWriter(buf, BestSpeed)
if err != nil {
t.Errorf("i=%d, firstN=%d, flush=%t: NewWriter: %v", i, firstN, flush, err)
continue
}
for _, n := range tc {
want = append(want, abcabc[:n]...)
if _, err := w.Write(abcabc[:n]); err != nil {
t.Errorf("i=%d, firstN=%d, flush=%t: Write: %v", i, firstN, flush, err)
continue outer
}
if !flush {
continue
}
if err := w.Flush(); err != nil {
t.Errorf("i=%d, firstN=%d, flush=%t: Flush: %v", i, firstN, flush, err)
continue outer
}
}
if err := w.Close(); err != nil {
t.Errorf("i=%d, firstN=%d, flush=%t: Close: %v", i, firstN, flush, err)
continue
}
r := NewReader(buf)
got, err := io.ReadAll(r)
if err != nil {
t.Errorf("i=%d, firstN=%d, flush=%t: ReadAll: %v", i, firstN, flush, err)
continue
}
r.Close()
if !bytes.Equal(got, want) {
t.Errorf("i=%d, firstN=%d, flush=%t: corruption during deflate-then-inflate", i, firstN, flush)
continue
}
}
}
}
}
var errIO = errors.New("IO error")
// failWriter fails with errIO exactly at the nth call to Write.
type failWriter struct{ n int }
func (w *failWriter) Write(b []byte) (int, error) {
w.n--
if w.n == -1 {
return 0, errIO
}
return len(b), nil
}
func TestWriterPersistentError(t *testing.T) {
t.Parallel()
d, err := os.ReadFile("../../testdata/Isaac.Newton-Opticks.txt")
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
d = d[:10000] // Keep this test short
zw, err := NewWriter(nil, DefaultCompression)
if err != nil {
t.Fatalf("NewWriter: %v", err)
}
// Sweep over the threshold at which an error is returned.
// The variable i makes it such that the ith call to failWriter.Write will
// return errIO. Since failWriter errors are not persistent, we must ensure
// that flate.Writer errors are persistent.
for i := 0; i < 1000; i++ {
fw := &failWriter{i}
zw.Reset(fw)
_, werr := zw.Write(d)
cerr := zw.Close()
if werr != errIO && werr != nil {
t.Errorf("test %d, mismatching Write error: got %v, want %v", i, werr, errIO)
}
if cerr != errIO && fw.n < 0 {
t.Errorf("test %d, mismatching Close error: got %v, want %v", i, cerr, errIO)
}
if fw.n >= 0 {
// At this point, the failure threshold was sufficiently high enough
// that we wrote the whole stream without any errors.
return
}
}
}
func TestBestSpeedMatch(t *testing.T) {
t.Parallel()
cases := []struct {
previous, current []byte
t, s, want int32
}{{
previous: []byte{0, 0, 0, 1, 2},
current: []byte{3, 4, 5, 0, 1, 2, 3, 4, 5},
t: -3,
s: 3,
want: 6,
}, {
previous: []byte{0, 0, 0, 1, 2},
current: []byte{2, 4, 5, 0, 1, 2, 3, 4, 5},
t: -3,
s: 3,
want: 3,
}, {
previous: []byte{0, 0, 0, 1, 1},
current: []byte{3, 4, 5, 0, 1, 2, 3, 4, 5},
t: -3,
s: 3,
want: 2,
}, {
previous: []byte{0, 0, 0, 1, 2},
current: []byte{2, 2, 2, 2, 1, 2, 3, 4, 5},
t: -1,
s: 0,
want: 4,
}, {
previous: []byte{0, 0, 0, 1, 2, 3, 4, 5, 2, 2},
current: []byte{2, 2, 2, 2, 1, 2, 3, 4, 5},
t: -7,
s: 4,
want: 5,
}, {
previous: []byte{9, 9, 9, 9, 9},
current: []byte{2, 2, 2, 2, 1, 2, 3, 4, 5},
t: -1,
s: 0,
want: 0,
}, {
previous: []byte{9, 9, 9, 9, 9},
current: []byte{9, 2, 2, 2, 1, 2, 3, 4, 5},
t: 0,
s: 1,
want: 0,
}, {
previous: []byte{},
current: []byte{9, 2, 2, 2, 1, 2, 3, 4, 5},
t: -5,
s: 1,
want: 0,
}, {
previous: []byte{},
current: []byte{9, 2, 2, 2, 1, 2, 3, 4, 5},
t: -1,
s: 1,
want: 0,
}, {
previous: []byte{},
current: []byte{2, 2, 2, 2, 1, 2, 3, 4, 5},
t: 0,
s: 1,
want: 3,
}, {
previous: []byte{3, 4, 5},
current: []byte{3, 4, 5},
t: -3,
s: 0,
want: 3,
}, {
previous: make([]byte, 1000),
current: make([]byte, 1000),
t: -1000,
s: 0,
want: maxMatchLength - 4,
}, {
previous: make([]byte, 200),
current: make([]byte, 500),
t: -200,
s: 0,
want: maxMatchLength - 4,
}, {
previous: make([]byte, 200),
current: make([]byte, 500),
t: 0,
s: 1,
want: maxMatchLength - 4,
}, {
previous: make([]byte, maxMatchLength-4),
current: make([]byte, 500),
t: -(maxMatchLength - 4),
s: 0,
want: maxMatchLength - 4,
}, {
previous: make([]byte, 200),
current: make([]byte, 500),
t: -200,
s: 400,
want: 100,
}, {
previous: make([]byte, 10),
current: make([]byte, 500),
t: 200,
s: 400,
want: 100,
}}
for i, c := range cases {
e := deflateFast{prev: c.previous}
got := e.matchLen(c.s, c.t, c.current)
if got != c.want {
t.Errorf("Test %d: match length, want %d, got %d", i, c.want, got)
}
}
}
func TestBestSpeedMaxMatchOffset(t *testing.T) {
t.Parallel()
const abc, xyz = "abcdefgh", "stuvwxyz"
for _, matchBefore := range []bool{false, true} {
for _, extra := range []int{0, inputMargin - 1, inputMargin, inputMargin + 1, 2 * inputMargin} {
for offsetAdj := -5; offsetAdj <= +5; offsetAdj++ {
report := func(desc string, err error) {
t.Errorf("matchBefore=%t, extra=%d, offsetAdj=%d: %s%v",
matchBefore, extra, offsetAdj, desc, err)
}
offset := maxMatchOffset + offsetAdj
// Make src to be a []byte of the form
// "%s%s%s%s%s" % (abc, zeros0, xyzMaybe, abc, zeros1)
// where:
// zeros0 is approximately maxMatchOffset zeros.
// xyzMaybe is either xyz or the empty string.
// zeros1 is between 0 and 30 zeros.
// The difference between the two abc's will be offset, which
// is maxMatchOffset plus or minus a small adjustment.
src := make([]byte, offset+len(abc)+extra)
copy(src, abc)
if !matchBefore {
copy(src[offset-len(xyz):], xyz)
}
copy(src[offset:], abc)
buf := new(bytes.Buffer)
w, err := NewWriter(buf, BestSpeed)
if err != nil {
report("NewWriter: ", err)
continue
}
if _, err := w.Write(src); err != nil {
report("Write: ", err)
continue
}
if err := w.Close(); err != nil {
report("Writer.Close: ", err)
continue
}
r := NewReader(buf)
dst, err := io.ReadAll(r)
r.Close()
if err != nil {
report("ReadAll: ", err)
continue
}
if !bytes.Equal(dst, src) {
report("", fmt.Errorf("bytes differ after round-tripping"))
continue
}
}
}
}
}
func TestBestSpeedShiftOffsets(t *testing.T) {
// Test if shiftoffsets properly preserves matches and resets out-of-range matches
// seen in https://github.com/golang/go/issues/4142
enc := newDeflateFast()
// testData may not generate internal matches.
testData := make([]byte, 32)
rng := rand.New(rand.NewSource(0))
for i := range testData {
testData[i] = byte(rng.Uint32())
}
// Encode the testdata with clean state.
// Second part should pick up matches from the first block.
wantFirstTokens := len(enc.encode(nil, testData))
wantSecondTokens := len(enc.encode(nil, testData))
if wantFirstTokens <= wantSecondTokens {
t.Fatalf("test needs matches between inputs to be generated")
}
// Forward the current indicator to before wraparound.
enc.cur = bufferReset - int32(len(testData))
// Part 1 before wrap, should match clean state.
got := len(enc.encode(nil, testData))
if wantFirstTokens != got {
t.Errorf("got %d, want %d tokens", got, wantFirstTokens)
}
// Verify we are about to wrap.
if enc.cur != bufferReset {
t.Errorf("got %d, want e.cur to be at bufferReset (%d)", enc.cur, bufferReset)
}
// Part 2 should match clean state as well even if wrapped.
got = len(enc.encode(nil, testData))
if wantSecondTokens != got {
t.Errorf("got %d, want %d token", got, wantSecondTokens)
}
// Verify that we wrapped.
if enc.cur >= bufferReset {
t.Errorf("want e.cur to be < bufferReset (%d), got %d", bufferReset, enc.cur)
}
// Forward the current buffer, leaving the matches at the bottom.
enc.cur = bufferReset
enc.shiftOffsets()
// Ensure that no matches were picked up.
got = len(enc.encode(nil, testData))
if wantFirstTokens != got {
t.Errorf("got %d, want %d tokens", got, wantFirstTokens)
}
}
func TestMaxStackSize(t *testing.T) {
// This test must not run in parallel with other tests as debug.SetMaxStack
// affects all goroutines.
n := debug.SetMaxStack(1 << 16)
defer debug.SetMaxStack(n)
var wg sync.WaitGroup
defer wg.Wait()
b := make([]byte, 1<<20)
for level := HuffmanOnly; level <= BestCompression; level++ {
// Run in separate goroutine to increase probability of stack regrowth.
wg.Add(1)
go func(level int) {
defer wg.Done()
zw, err := NewWriter(io.Discard, level)
if err != nil {
t.Errorf("level %d, NewWriter() = %v, want nil", level, err)
}
if n, err := zw.Write(b); n != len(b) || err != nil {
t.Errorf("level %d, Write() = (%d, %v), want (%d, nil)", level, n, err, len(b))
}
if err := zw.Close(); err != nil {
t.Errorf("level %d, Close() = %v, want nil", level, err)
}
zw.Reset(io.Discard)
}(level)
}
}
image/internal/compress/flate/deflatefast.go
+309
View File
@@ -0,0 +1,309 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import "math"
// This encoding algorithm, which prioritizes speed over output size, is
// based on Snappy's LZ77-style encoder: github.com/golang/snappy
const (
tableBits = 14 // Bits used in the table.
tableSize = 1 << tableBits // Size of the table.
tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
tableShift = 32 - tableBits // Right-shift to get the tableBits most significant bits of a uint32.
// Reset the buffer offset when reaching this.
// Offsets are stored between blocks as int32 values.
// Since the offset we are checking against is at the beginning
// of the buffer, we need to subtract the current and input
// buffer to not risk overflowing the int32.
bufferReset = math.MaxInt32 - maxStoreBlockSize*2
)
func load32(b []byte, i int32) uint32 {
b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
}
func load64(b []byte, i int32) uint64 {
b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line.
return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
}
func hash(u uint32) uint32 {
return (u * 0x1e35a7bd) >> tableShift
}
// These constants are defined by the Snappy implementation so that its
// assembly implementation can fast-path some 16-bytes-at-a-time copies. They
// aren't necessary in the pure Go implementation, as we don't use those same
// optimizations, but using the same thresholds doesn't really hurt.
const (
inputMargin = 16 - 1
minNonLiteralBlockSize = 1 + 1 + inputMargin
)
type tableEntry struct {
val uint32 // Value at destination
offset int32
}
// deflateFast maintains the table for matches,
// and the previous byte block for cross block matching.
type deflateFast struct {
table [tableSize]tableEntry
prev []byte // Previous block, zero length if unknown.
cur int32 // Current match offset.
}
func newDeflateFast() *deflateFast {
return &deflateFast{cur: maxStoreBlockSize, prev: make([]byte, 0, maxStoreBlockSize)}
}
// encode encodes a block given in src and appends tokens
// to dst and returns the result.
func (e *deflateFast) encode(dst []token, src []byte) []token {
// Ensure that e.cur doesn't wrap.
if e.cur >= bufferReset {
e.shiftOffsets()
}
// This check isn't in the Snappy implementation, but there, the caller
// instead of the callee handles this case.
if len(src) < minNonLiteralBlockSize {
e.cur += maxStoreBlockSize
e.prev = e.prev[:0]
return emitLiteral(dst, src)
}
// sLimit is when to stop looking for offset/length copies. The inputMargin
// lets us use a fast path for emitLiteral in the main loop, while we are
// looking for copies.
sLimit := int32(len(src) - inputMargin)
// nextEmit is where in src the next emitLiteral should start from.
nextEmit := int32(0)
s := int32(0)
cv := load32(src, s)
nextHash := hash(cv)
for {
// Copied from the C++ snappy implementation:
//
// Heuristic match skipping: If 32 bytes are scanned with no matches
// found, start looking only at every other byte. If 32 more bytes are
// scanned (or skipped), look at every third byte, etc.. When a match
// is found, immediately go back to looking at every byte. This is a
// small loss (~5% performance, ~0.1% density) for compressible data
// due to more bookkeeping, but for non-compressible data (such as
// JPEG) it's a huge win since the compressor quickly "realizes" the
// data is incompressible and doesn't bother looking for matches
// everywhere.
//
// The "skip" variable keeps track of how many bytes there are since
// the last match; dividing it by 32 (ie. right-shifting by five) gives
// the number of bytes to move ahead for each iteration.
skip := int32(32)
nextS := s
var candidate tableEntry
for {
s = nextS
bytesBetweenHashLookups := skip >> 5
nextS = s + bytesBetweenHashLookups
skip += bytesBetweenHashLookups
if nextS > sLimit {
goto emitRemainder
}
candidate = e.table[nextHash&tableMask]
now := load32(src, nextS)
e.table[nextHash&tableMask] = tableEntry{offset: s + e.cur, val: cv}
nextHash = hash(now)
offset := s - (candidate.offset - e.cur)
if offset > maxMatchOffset || cv != candidate.val {
// Out of range or not matched.
cv = now
continue
}
break
}
// A 4-byte match has been found. We'll later see if more than 4 bytes
// match. But, prior to the match, src[nextEmit:s] are unmatched. Emit
// them as literal bytes.
dst = emitLiteral(dst, src[nextEmit:s])
// Call emitCopy, and then see if another emitCopy could be our next
// move. Repeat until we find no match for the input immediately after
// what was consumed by the last emitCopy call.
//
// If we exit this loop normally then we need to call emitLiteral next,
// though we don't yet know how big the literal will be. We handle that
// by proceeding to the next iteration of the main loop. We also can
// exit this loop via goto if we get close to exhausting the input.
for {
// Invariant: we have a 4-byte match at s, and no need to emit any
// literal bytes prior to s.
// Extend the 4-byte match as long as possible.
//
s += 4
t := candidate.offset - e.cur + 4
l := e.matchLen(s, t, src)
// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
dst = append(dst, matchToken(uint32(l+4-baseMatchLength), uint32(s-t-baseMatchOffset)))
s += l
nextEmit = s
if s >= sLimit {
goto emitRemainder
}
// We could immediately start working at s now, but to improve
// compression we first update the hash table at s-1 and at s. If
// another emitCopy is not our next move, also calculate nextHash
// at s+1. At least on GOARCH=amd64, these three hash calculations
// are faster as one load64 call (with some shifts) instead of
// three load32 calls.
x := load64(src, s-1)
prevHash := hash(uint32(x))
e.table[prevHash&tableMask] = tableEntry{offset: e.cur + s - 1, val: uint32(x)}
x >>= 8
currHash := hash(uint32(x))
candidate = e.table[currHash&tableMask]
e.table[currHash&tableMask] = tableEntry{offset: e.cur + s, val: uint32(x)}
offset := s - (candidate.offset - e.cur)
if offset > maxMatchOffset || uint32(x) != candidate.val {
cv = uint32(x >> 8)
nextHash = hash(cv)
s++
break
}
}
}
emitRemainder:
if int(nextEmit) < len(src) {
dst = emitLiteral(dst, src[nextEmit:])
}
e.cur += int32(len(src))
e.prev = e.prev[:len(src)]
copy(e.prev, src)
return dst
}
func emitLiteral(dst []token, lit []byte) []token {
for _, v := range lit {
dst = append(dst, literalToken(uint32(v)))
}
return dst
}
// matchLen returns the match length between src[s:] and src[t:].
// t can be negative to indicate the match is starting in e.prev.
// We assume that src[s-4:s] and src[t-4:t] already match.
func (e *deflateFast) matchLen(s, t int32, src []byte) int32 {
s1 := int(s) + maxMatchLength - 4
if s1 > len(src) {
s1 = len(src)
}
// If we are inside the current block
if t >= 0 {
b := src[t:]
a := src[s:s1]
b = b[:len(a)]
// Extend the match to be as long as possible.
for i := range a {
if a[i] != b[i] {
return int32(i)
}
}
return int32(len(a))
}
// We found a match in the previous block.
tp := int32(len(e.prev)) + t
if tp < 0 {
return 0
}
// Extend the match to be as long as possible.
a := src[s:s1]
b := e.prev[tp:]
if len(b) > len(a) {
b = b[:len(a)]
}
a = a[:len(b)]
for i := range b {
if a[i] != b[i] {
return int32(i)
}
}
// If we reached our limit, we matched everything we are
// allowed to in the previous block and we return.
n := int32(len(b))
if int(s+n) == s1 {
return n
}
// Continue looking for more matches in the current block.
a = src[s+n : s1]
b = src[:len(a)]
for i := range a {
if a[i] != b[i] {
return int32(i) + n
}
}
return int32(len(a)) + n
}
// Reset resets the encoding history.
// This ensures that no matches are made to the previous block.
func (e *deflateFast) reset() {
e.prev = e.prev[:0]
// Bump the offset, so all matches will fail distance check.
// Nothing should be >= e.cur in the table.
e.cur += maxMatchOffset
// Protect against e.cur wraparound.
if e.cur >= bufferReset {
e.shiftOffsets()
}
}
// shiftOffsets will shift down all match offset.
// This is only called in rare situations to prevent integer overflow.
//
// See https://golang.org/issue/18636 and https://github.com/golang/go/issues/34121.
func (e *deflateFast) shiftOffsets() {
if len(e.prev) == 0 {
// We have no history; just clear the table.
for i := range e.table[:] {
e.table[i] = tableEntry{}
}
e.cur = maxMatchOffset + 1
return
}
// Shift down everything in the table that isn't already too far away.
for i := range e.table[:] {
v := e.table[i].offset - e.cur + maxMatchOffset + 1
if v < 0 {
// We want to reset e.cur to maxMatchOffset + 1, so we need to shift
// all table entries down by (e.cur - (maxMatchOffset + 1)).
// Because we ignore matches > maxMatchOffset, we can cap
// any negative offsets at 0.
v = 0
}
e.table[i].offset = v
}
e.cur = maxMatchOffset + 1
}
image/internal/compress/flate/dict_decoder.go
+186
View File
@@ -0,0 +1,186 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
// dictDecoder implements the LZ77 sliding dictionary as used in decompression.
// LZ77 decompresses data through sequences of two forms of commands:
//
// * Literal insertions: Runs of one or more symbols are inserted into the data
// stream as is. This is accomplished through the writeByte method for a
// single symbol, or combinations of writeSlice/writeMark for multiple symbols.
// Any valid stream must start with a literal insertion if no preset dictionary
// is used.
//
// * Backward copies: Runs of one or more symbols are copied from previously
// emitted data. Backward copies come as the tuple (dist, length) where dist
// determines how far back in the stream to copy from and length determines how
// many bytes to copy. Note that it is valid for the length to be greater than
// the distance. Since LZ77 uses forward copies, that situation is used to
// perform a form of run-length encoding on repeated runs of symbols.
// The writeCopy and tryWriteCopy are used to implement this command.
//
// For performance reasons, this implementation performs little to no sanity
// checks about the arguments. As such, the invariants documented for each
// method call must be respected.
type dictDecoder struct {
hist []byte // Sliding window history
// Invariant: 0 <= rdPos <= wrPos <= len(hist)
wrPos int // Current output position in buffer
rdPos int // Have emitted hist[:rdPos] already
full bool // Has a full window length been written yet?
}
// To minimize the memory usage in TinyGo, it is defined as a fixed array
// instead of a make().
var ddHistBuf [1 << 15]byte
// init initializes dictDecoder to have a sliding window dictionary of the given
// size. If a preset dict is provided, it will initialize the dictionary with
// the contents of dict.
func (dd *dictDecoder) init(size int, dict []byte) {
*dd = dictDecoder{hist: dd.hist}
if cap(dd.hist) < size {
dd.hist = ddHistBuf[:size]
}
dd.hist = dd.hist[:size]
if len(dict) > len(dd.hist) {
dict = dict[len(dict)-len(dd.hist):]
}
dd.wrPos = copy(dd.hist, dict)
if dd.wrPos == len(dd.hist) {
dd.wrPos = 0
dd.full = true
}
dd.rdPos = dd.wrPos
}
// histSize reports the total amount of historical data in the dictionary.
func (dd *dictDecoder) histSize() int {
if dd.full {
return len(dd.hist)
}
return dd.wrPos
}
// availRead reports the number of bytes that can be flushed by readFlush.
func (dd *dictDecoder) availRead() int {
return dd.wrPos - dd.rdPos
}
// availWrite reports the available amount of output buffer space.
func (dd *dictDecoder) availWrite() int {
return len(dd.hist) - dd.wrPos
}
// writeSlice returns a slice of the available buffer to write data to.
//
// This invariant will be kept: len(s) <= availWrite()
func (dd *dictDecoder) writeSlice() []byte {
return dd.hist[dd.wrPos:]
}
// writeMark advances the writer pointer by cnt.
//
// This invariant must be kept: 0 <= cnt <= availWrite()
func (dd *dictDecoder) writeMark(cnt int) {
dd.wrPos += cnt
}
// writeByte writes a single byte to the dictionary.
//
// This invariant must be kept: 0 < availWrite()
func (dd *dictDecoder) writeByte(c byte) {
dd.hist[dd.wrPos] = c
dd.wrPos++
}
// writeCopy copies a string at a given (dist, length) to the output.
// This returns the number of bytes copied and may be less than the requested
// length if the available space in the output buffer is too small.
//
// This invariant must be kept: 0 < dist <= histSize()
func (dd *dictDecoder) writeCopy(dist, length int) int {
dstBase := dd.wrPos
dstPos := dstBase
srcPos := dstPos - dist
endPos := dstPos + length
if endPos > len(dd.hist) {
endPos = len(dd.hist)
}
// Copy non-overlapping section after destination position.
//
// This section is non-overlapping in that the copy length for this section
// is always less than or equal to the backwards distance. This can occur
// if a distance refers to data that wraps-around in the buffer.
// Thus, a backwards copy is performed here; that is, the exact bytes in
// the source prior to the copy is placed in the destination.
if srcPos < 0 {
srcPos += len(dd.hist)
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:])
srcPos = 0
}
// Copy possibly overlapping section before destination position.
//
// This section can overlap if the copy length for this section is larger
// than the backwards distance. This is allowed by LZ77 so that repeated
// strings can be succinctly represented using (dist, length) pairs.
// Thus, a forwards copy is performed here; that is, the bytes copied is
// possibly dependent on the resulting bytes in the destination as the copy
// progresses along. This is functionally equivalent to the following:
//
// for i := 0; i < endPos-dstPos; i++ {
// dd.hist[dstPos+i] = dd.hist[srcPos+i]
// }
// dstPos = endPos
//
for dstPos < endPos {
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
}
dd.wrPos = dstPos
return dstPos - dstBase
}
// tryWriteCopy tries to copy a string at a given (distance, length) to the
// output. This specialized version is optimized for short distances.
//
// This method is designed to be inlined for performance reasons.
//
// This invariant must be kept: 0 < dist <= histSize()
func (dd *dictDecoder) tryWriteCopy(dist, length int) int {
dstPos := dd.wrPos
endPos := dstPos + length
if dstPos < dist || endPos > len(dd.hist) {
return 0
}
dstBase := dstPos
srcPos := dstPos - dist
// Copy possibly overlapping section before destination position.
for dstPos < endPos {
dstPos += copy(dd.hist[dstPos:endPos], dd.hist[srcPos:dstPos])
}
dd.wrPos = dstPos
return dstPos - dstBase
}
// readFlush returns a slice of the historical buffer that is ready to be
// emitted to the user. The data returned by readFlush must be fully consumed
// before calling any other dictDecoder methods.
func (dd *dictDecoder) readFlush() []byte {
toRead := dd.hist[dd.rdPos:dd.wrPos]
dd.rdPos = dd.wrPos
if dd.wrPos == len(dd.hist) {
dd.wrPos, dd.rdPos = 0, 0
dd.full = true
}
return toRead
}
image/internal/compress/flate/dict_decoder_test.go
+139
View File
@@ -0,0 +1,139 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"bytes"
"strings"
"testing"
)
func TestDictDecoder(t *testing.T) {
const (
abc = "ABC\n"
fox = "The quick brown fox jumped over the lazy dog!\n"
poem = "The Road Not Taken\nRobert Frost\n" +
"\n" +
"Two roads diverged in a yellow wood,\n" +
"And sorry I could not travel both\n" +
"And be one traveler, long I stood\n" +
"And looked down one as far as I could\n" +
"To where it bent in the undergrowth;\n" +
"\n" +
"Then took the other, as just as fair,\n" +
"And having perhaps the better claim,\n" +
"Because it was grassy and wanted wear;\n" +
"Though as for that the passing there\n" +
"Had worn them really about the same,\n" +
"\n" +
"And both that morning equally lay\n" +
"In leaves no step had trodden black.\n" +
"Oh, I kept the first for another day!\n" +
"Yet knowing how way leads on to way,\n" +
"I doubted if I should ever come back.\n" +
"\n" +
"I shall be telling this with a sigh\n" +
"Somewhere ages and ages hence:\n" +
"Two roads diverged in a wood, and I-\n" +
"I took the one less traveled by,\n" +
"And that has made all the difference.\n"
)
var poemRefs = []struct {
dist int // Backward distance (0 if this is an insertion)
length int // Length of copy or insertion
}{
{0, 38}, {33, 3}, {0, 48}, {79, 3}, {0, 11}, {34, 5}, {0, 6}, {23, 7},
{0, 8}, {50, 3}, {0, 2}, {69, 3}, {34, 5}, {0, 4}, {97, 3}, {0, 4},
{43, 5}, {0, 6}, {7, 4}, {88, 7}, {0, 12}, {80, 3}, {0, 2}, {141, 4},
{0, 1}, {196, 3}, {0, 3}, {157, 3}, {0, 6}, {181, 3}, {0, 2}, {23, 3},
{77, 3}, {28, 5}, {128, 3}, {110, 4}, {70, 3}, {0, 4}, {85, 6}, {0, 2},
{182, 6}, {0, 4}, {133, 3}, {0, 7}, {47, 5}, {0, 20}, {112, 5}, {0, 1},
{58, 3}, {0, 8}, {59, 3}, {0, 4}, {173, 3}, {0, 5}, {114, 3}, {0, 4},
{92, 5}, {0, 2}, {71, 3}, {0, 2}, {76, 5}, {0, 1}, {46, 3}, {96, 4},
{130, 4}, {0, 3}, {360, 3}, {0, 3}, {178, 5}, {0, 7}, {75, 3}, {0, 3},
{45, 6}, {0, 6}, {299, 6}, {180, 3}, {70, 6}, {0, 1}, {48, 3}, {66, 4},
{0, 3}, {47, 5}, {0, 9}, {325, 3}, {0, 1}, {359, 3}, {318, 3}, {0, 2},
{199, 3}, {0, 1}, {344, 3}, {0, 3}, {248, 3}, {0, 10}, {310, 3}, {0, 3},
{93, 6}, {0, 3}, {252, 3}, {157, 4}, {0, 2}, {273, 5}, {0, 14}, {99, 4},
{0, 1}, {464, 4}, {0, 2}, {92, 4}, {495, 3}, {0, 1}, {322, 4}, {16, 4},
{0, 3}, {402, 3}, {0, 2}, {237, 4}, {0, 2}, {432, 4}, {0, 1}, {483, 5},
{0, 2}, {294, 4}, {0, 2}, {306, 3}, {113, 5}, {0, 1}, {26, 4}, {164, 3},
{488, 4}, {0, 1}, {542, 3}, {248, 6}, {0, 5}, {205, 3}, {0, 8}, {48, 3},
{449, 6}, {0, 2}, {192, 3}, {328, 4}, {9, 5}, {433, 3}, {0, 3}, {622, 25},
{615, 5}, {46, 5}, {0, 2}, {104, 3}, {475, 10}, {549, 3}, {0, 4}, {597, 8},
{314, 3}, {0, 1}, {473, 6}, {317, 5}, {0, 1}, {400, 3}, {0, 3}, {109, 3},
{151, 3}, {48, 4}, {0, 4}, {125, 3}, {108, 3}, {0, 2},
}
var got, want bytes.Buffer
var dd dictDecoder
dd.init(1<<11, nil)
var writeCopy = func(dist, length int) {
for length > 0 {
cnt := dd.tryWriteCopy(dist, length)
if cnt == 0 {
cnt = dd.writeCopy(dist, length)
}
length -= cnt
if dd.availWrite() == 0 {
got.Write(dd.readFlush())
}
}
}
var writeString = func(str string) {
for len(str) > 0 {
cnt := copy(dd.writeSlice(), str)
str = str[cnt:]
dd.writeMark(cnt)
if dd.availWrite() == 0 {
got.Write(dd.readFlush())
}
}
}
writeString(".")
want.WriteByte('.')
str := poem
for _, ref := range poemRefs {
if ref.dist == 0 {
writeString(str[:ref.length])
} else {
writeCopy(ref.dist, ref.length)
}
str = str[ref.length:]
}
want.WriteString(poem)
writeCopy(dd.histSize(), 33)
want.Write(want.Bytes()[:33])
writeString(abc)
writeCopy(len(abc), 59*len(abc))
want.WriteString(strings.Repeat(abc, 60))
writeString(fox)
writeCopy(len(fox), 9*len(fox))
want.WriteString(strings.Repeat(fox, 10))
writeString(".")
writeCopy(1, 9)
want.WriteString(strings.Repeat(".", 10))
writeString(strings.ToUpper(poem))
writeCopy(len(poem), 7*len(poem))
want.WriteString(strings.Repeat(strings.ToUpper(poem), 8))
writeCopy(dd.histSize(), 10)
want.Write(want.Bytes()[want.Len()-dd.histSize():][:10])
got.Write(dd.readFlush())
if got.String() != want.String() {
t.Errorf("final string mismatch:\ngot %q\nwant %q", got.String(), want.String())
}
}
image/internal/compress/flate/example_test.go
+243
View File
@@ -0,0 +1,243 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate_test
import (
"bytes"
"compress/flate"
"fmt"
"io"
"log"
"os"
"strings"
"sync"
)
// In performance critical applications, Reset can be used to discard the
// current compressor or decompressor state and reinitialize them quickly
// by taking advantage of previously allocated memory.
func Example_reset() {
proverbs := []string{
"Don't communicate by sharing memory, share memory by communicating.\n",
"Concurrency is not parallelism.\n",
"The bigger the interface, the weaker the abstraction.\n",
"Documentation is for users.\n",
}
var r strings.Reader
var b bytes.Buffer
buf := make([]byte, 32<<10)
zw, err := flate.NewWriter(nil, flate.DefaultCompression)
if err != nil {
log.Fatal(err)
}
zr := flate.NewReader(nil)
for _, s := range proverbs {
r.Reset(s)
b.Reset()
// Reset the compressor and encode from some input stream.
zw.Reset(&b)
if _, err := io.CopyBuffer(zw, &r, buf); err != nil {
log.Fatal(err)
}
if err := zw.Close(); err != nil {
log.Fatal(err)
}
// Reset the decompressor and decode to some output stream.
if err := zr.(flate.Resetter).Reset(&b, nil); err != nil {
log.Fatal(err)
}
if _, err := io.CopyBuffer(os.Stdout, zr, buf); err != nil {
log.Fatal(err)
}
if err := zr.Close(); err != nil {
log.Fatal(err)
}
}
// Output:
// Don't communicate by sharing memory, share memory by communicating.
// Concurrency is not parallelism.
// The bigger the interface, the weaker the abstraction.
// Documentation is for users.
}
// A preset dictionary can be used to improve the compression ratio.
// The downside to using a dictionary is that the compressor and decompressor
// must agree in advance what dictionary to use.
func Example_dictionary() {
// The dictionary is a string of bytes. When compressing some input data,
// the compressor will attempt to substitute substrings with matches found
// in the dictionary. As such, the dictionary should only contain substrings
// that are expected to be found in the actual data stream.
const dict = `<?xml version="1.0"?>` + `<book>` + `<data>` + `<meta name="` + `" content="`
// The data to compress should (but is not required to) contain frequent
// substrings that match those in the dictionary.
const data = `<?xml version="1.0"?>
<book>
<meta name="title" content="The Go Programming Language"/>
<meta name="authors" content="Alan Donovan and Brian Kernighan"/>
<meta name="published" content="2015-10-26"/>
<meta name="isbn" content="978-0134190440"/>
<data>...</data>
</book>
`
var b bytes.Buffer
// Compress the data using the specially crafted dictionary.
zw, err := flate.NewWriterDict(&b, flate.DefaultCompression, []byte(dict))
if err != nil {
log.Fatal(err)
}
if _, err := io.Copy(zw, strings.NewReader(data)); err != nil {
log.Fatal(err)
}
if err := zw.Close(); err != nil {
log.Fatal(err)
}
// The decompressor must use the same dictionary as the compressor.
// Otherwise, the input may appear as corrupted.
fmt.Println("Decompressed output using the dictionary:")
zr := flate.NewReaderDict(bytes.NewReader(b.Bytes()), []byte(dict))
if _, err := io.Copy(os.Stdout, zr); err != nil {
log.Fatal(err)
}
if err := zr.Close(); err != nil {
log.Fatal(err)
}
fmt.Println()
// Substitute all of the bytes in the dictionary with a '#' to visually
// demonstrate the approximate effectiveness of using a preset dictionary.
fmt.Println("Substrings matched by the dictionary are marked with #:")
hashDict := []byte(dict)
for i := range hashDict {
hashDict[i] = '#'
}
zr = flate.NewReaderDict(&b, hashDict)
if _, err := io.Copy(os.Stdout, zr); err != nil {
log.Fatal(err)
}
if err := zr.Close(); err != nil {
log.Fatal(err)
}
// Output:
// Decompressed output using the dictionary:
// <?xml version="1.0"?>
// <book>
// <meta name="title" content="The Go Programming Language"/>
// <meta name="authors" content="Alan Donovan and Brian Kernighan"/>
// <meta name="published" content="2015-10-26"/>
// <meta name="isbn" content="978-0134190440"/>
// <data>...</data>
// </book>
//
// Substrings matched by the dictionary are marked with #:
// #####################
// ######
// ############title###########The Go Programming Language"/#
// ############authors###########Alan Donovan and Brian Kernighan"/#
// ############published###########2015-10-26"/#
// ############isbn###########978-0134190440"/#
// ######...</#####
// </#####
}
// DEFLATE is suitable for transmitting compressed data across the network.
func Example_synchronization() {
var wg sync.WaitGroup
defer wg.Wait()
// Use io.Pipe to simulate a network connection.
// A real network application should take care to properly close the
// underlying connection.
rp, wp := io.Pipe()
// Start a goroutine to act as the transmitter.
wg.Add(1)
go func() {
defer wg.Done()
zw, err := flate.NewWriter(wp, flate.BestSpeed)
if err != nil {
log.Fatal(err)
}
b := make([]byte, 256)
for _, m := range strings.Fields("A long time ago in a galaxy far, far away...") {
// We use a simple framing format where the first byte is the
// message length, followed the message itself.
b[0] = uint8(copy(b[1:], m))
if _, err := zw.Write(b[:1+len(m)]); err != nil {
log.Fatal(err)
}
// Flush ensures that the receiver can read all data sent so far.
if err := zw.Flush(); err != nil {
log.Fatal(err)
}
}
if err := zw.Close(); err != nil {
log.Fatal(err)
}
}()
// Start a goroutine to act as the receiver.
wg.Add(1)
go func() {
defer wg.Done()
zr := flate.NewReader(rp)
b := make([]byte, 256)
for {
// Read the message length.
// This is guaranteed to return for every corresponding
// Flush and Close on the transmitter side.
if _, err := io.ReadFull(zr, b[:1]); err != nil {
if err == io.EOF {
break // The transmitter closed the stream
}
log.Fatal(err)
}
// Read the message content.
n := int(b[0])
if _, err := io.ReadFull(zr, b[:n]); err != nil {
log.Fatal(err)
}
fmt.Printf("Received %d bytes: %s\n", n, b[:n])
}
fmt.Println()
if err := zr.Close(); err != nil {
log.Fatal(err)
}
}()
// Output:
// Received 1 bytes: A
// Received 4 bytes: long
// Received 4 bytes: time
// Received 3 bytes: ago
// Received 2 bytes: in
// Received 1 bytes: a
// Received 6 bytes: galaxy
// Received 4 bytes: far,
// Received 3 bytes: far
// Received 7 bytes: away...
}
image/internal/compress/flate/flate_test.go
+352
View File
@@ -0,0 +1,352 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This test tests some internals of the flate package.
// The tests in package compress/gzip serve as the
// end-to-end test of the decompressor.
package flate
import (
"bytes"
"encoding/hex"
"io"
"strings"
"testing"
)
// The following test should not panic.
func TestIssue5915(t *testing.T) {
bits := []int{4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0, 5, 5, 6,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8, 6, 0, 11, 0, 8, 0, 6, 6, 10, 8}
var h huffmanDecoder
if h.init(bits) {
t.Fatalf("Given sequence of bits is bad, and should not succeed.")
}
}
// The following test should not panic.
func TestIssue5962(t *testing.T) {
bits := []int{4, 0, 0, 6, 4, 3, 2, 3, 3, 4, 4, 5, 0, 0, 0, 0,
5, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11}
var h huffmanDecoder
if h.init(bits) {
t.Fatalf("Given sequence of bits is bad, and should not succeed.")
}
}
// The following test should not panic.
func TestIssue6255(t *testing.T) {
bits1 := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 11}
bits2 := []int{11, 13}
var h huffmanDecoder
if !h.init(bits1) {
t.Fatalf("Given sequence of bits is good and should succeed.")
}
if h.init(bits2) {
t.Fatalf("Given sequence of bits is bad and should not succeed.")
}
}
func TestInvalidEncoding(t *testing.T) {
// Initialize Huffman decoder to recognize "0".
var h huffmanDecoder
if !h.init([]int{1}) {
t.Fatal("Failed to initialize Huffman decoder")
}
// Initialize decompressor with invalid Huffman coding.
var f decompressor
f.r = bytes.NewReader([]byte{0xff})
_, err := f.huffSym(&h)
if err == nil {
t.Fatal("Should have rejected invalid bit sequence")
}
}
func TestInvalidBits(t *testing.T) {
oversubscribed := []int{1, 2, 3, 4, 4, 5}
incomplete := []int{1, 2, 4, 4}
var h huffmanDecoder
if h.init(oversubscribed) {
t.Fatal("Should reject oversubscribed bit-length set")
}
if h.init(incomplete) {
t.Fatal("Should reject incomplete bit-length set")
}
}
func TestStreams(t *testing.T) {
// To verify any of these hexstrings as valid or invalid flate streams
// according to the C zlib library, you can use the Python wrapper library:
// >>> hex_string = "010100feff11"
// >>> import zlib
// >>> zlib.decompress(hex_string.decode("hex"), -15) # Negative means raw DEFLATE
// '\x11'
testCases := []struct {
desc string // Description of the stream
stream string // Hexstring of the input DEFLATE stream
want string // Expected result. Use "fail" to expect failure
}{{
"degenerate HCLenTree",
"05e0010000000000100000000000000000000000000000000000000000000000" +
"00000000000000000004",
"fail",
}, {
"complete HCLenTree, empty HLitTree, empty HDistTree",
"05e0010400000000000000000000000000000000000000000000000000000000" +
"00000000000000000010",
"fail",
}, {
"empty HCLenTree",
"05e0010000000000000000000000000000000000000000000000000000000000" +
"00000000000000000010",
"fail",
}, {
"complete HCLenTree, complete HLitTree, empty HDistTree, use missing HDist symbol",
"000100feff000de0010400000000100000000000000000000000000000000000" +
"0000000000000000000000000000002c",
"fail",
}, {
"complete HCLenTree, complete HLitTree, degenerate HDistTree, use missing HDist symbol",
"000100feff000de0010000000000000000000000000000000000000000000000" +
"00000000000000000610000000004070",
"fail",
}, {
"complete HCLenTree, empty HLitTree, empty HDistTree",
"05e0010400000000100400000000000000000000000000000000000000000000" +
"0000000000000000000000000008",
"fail",
}, {
"complete HCLenTree, empty HLitTree, degenerate HDistTree",
"05e0010400000000100400000000000000000000000000000000000000000000" +
"0000000000000000000800000008",
"fail",
}, {
"complete HCLenTree, degenerate HLitTree, degenerate HDistTree, use missing HLit symbol",
"05e0010400000000100000000000000000000000000000000000000000000000" +
"0000000000000000001c",
"fail",
}, {
"complete HCLenTree, complete HLitTree, too large HDistTree",
"edff870500000000200400000000000000000000000000000000000000000000" +
"000000000000000000080000000000000004",
"fail",
}, {
"complete HCLenTree, complete HLitTree, empty HDistTree, excessive repeater code",
"edfd870500000000200400000000000000000000000000000000000000000000" +
"000000000000000000e8b100",
"fail",
}, {
"complete HCLenTree, complete HLitTree, empty HDistTree of normal length 30",
"05fd01240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" +
"ffffffffffffffffff07000000fe01",
"",
}, {
"complete HCLenTree, complete HLitTree, empty HDistTree of excessive length 31",
"05fe01240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" +
"ffffffffffffffffff07000000fc03",
"fail",
}, {
"complete HCLenTree, over-subscribed HLitTree, empty HDistTree",
"05e001240000000000fcffffffffffffffffffffffffffffffffffffffffffff" +
"ffffffffffffffffff07f00f",
"fail",
}, {
"complete HCLenTree, under-subscribed HLitTree, empty HDistTree",
"05e001240000000000fcffffffffffffffffffffffffffffffffffffffffffff" +
"fffffffffcffffffff07f00f",
"fail",
}, {
"complete HCLenTree, complete HLitTree with single code, empty HDistTree",
"05e001240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" +
"ffffffffffffffffff07f00f",
"01",
}, {
"complete HCLenTree, complete HLitTree with multiple codes, empty HDistTree",
"05e301240000000000f8ffffffffffffffffffffffffffffffffffffffffffff" +
"ffffffffffffffffff07807f",
"01",
}, {
"complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HDist symbol",
"000100feff000de0010400000000100000000000000000000000000000000000" +
"0000000000000000000000000000003c",
"00000000",
}, {
"complete HCLenTree, degenerate HLitTree, degenerate HDistTree",
"05e0010400000000100000000000000000000000000000000000000000000000" +
"0000000000000000000c",
"",
}, {
"complete HCLenTree, degenerate HLitTree, empty HDistTree",
"05e0010400000000100000000000000000000000000000000000000000000000" +
"00000000000000000004",
"",
}, {
"complete HCLenTree, complete HLitTree, empty HDistTree, spanning repeater code",
"edfd870500000000200400000000000000000000000000000000000000000000" +
"000000000000000000e8b000",
"",
}, {
"complete HCLenTree with length codes, complete HLitTree, empty HDistTree",
"ede0010400000000100000000000000000000000000000000000000000000000" +
"0000000000000000000400004000",
"",
}, {
"complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HLit symbol 284 with count 31",
"000100feff00ede0010400000000100000000000000000000000000000000000" +
"000000000000000000000000000000040000407f00",
"0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"0000000000000000000000000000000000000000000000000000000000000000" +
"000000",
}, {
"complete HCLenTree, complete HLitTree, degenerate HDistTree, use valid HLit and HDist symbols",
"0cc2010d00000082b0ac4aff0eb07d27060000ffff",
"616263616263",
}, {
"fixed block, use reserved symbol 287",
"33180700",
"fail",
}, {
"raw block",
"010100feff11",
"11",
}, {
"issue 10426 - over-subscribed HCLenTree causes a hang",
"344c4a4e494d4b070000ff2e2eff2e2e2e2e2eff",
"fail",
}, {
"issue 11030 - empty HDistTree unexpectedly leads to error",
"05c0070600000080400fff37a0ca",
"",
}, {
"issue 11033 - empty HDistTree unexpectedly leads to error",
"050fb109c020cca5d017dcbca044881ee1034ec149c8980bbc413c2ab35be9dc" +
"b1473449922449922411202306ee97b0383a521b4ffdcf3217f9f7d3adb701",
"3130303634342068652e706870005d05355f7ed957ff084a90925d19e3ebc6d0" +
"c6d7",
}}
for i, tc := range testCases {
data, err := hex.DecodeString(tc.stream)
if err != nil {
t.Fatal(err)
}
data, err = io.ReadAll(NewReader(bytes.NewReader(data)))
if tc.want == "fail" {
if err == nil {
t.Errorf("#%d (%s): got nil error, want non-nil", i, tc.desc)
}
} else {
if err != nil {
t.Errorf("#%d (%s): %v", i, tc.desc, err)
continue
}
if got := hex.EncodeToString(data); got != tc.want {
t.Errorf("#%d (%s):\ngot %q\nwant %q", i, tc.desc, got, tc.want)
}
}
}
}
func TestTruncatedStreams(t *testing.T) {
const data = "\x00\f\x00\xf3\xffhello, world\x01\x00\x00\xff\xff"
for i := 0; i < len(data)-1; i++ {
r := NewReader(strings.NewReader(data[:i]))
_, err := io.Copy(io.Discard, r)
if err != io.ErrUnexpectedEOF {
t.Errorf("io.Copy(%d) on truncated stream: got %v, want %v", i, err, io.ErrUnexpectedEOF)
}
}
}
// Verify that flate.Reader.Read returns (n, io.EOF) instead
// of (n, nil) + (0, io.EOF) when possible.
//
// This helps net/http.Transport reuse HTTP/1 connections more
// aggressively.
//
// See https://github.com/google/go-github/pull/317 for background.
func TestReaderEarlyEOF(t *testing.T) {
t.Parallel()
testSizes := []int{
1, 2, 3, 4, 5, 6, 7, 8,
100, 1000, 10000, 100000,
128, 1024, 16384, 131072,
// Testing multiples of windowSize triggers the case
// where Read will fail to return an early io.EOF.
windowSize * 1, windowSize * 2, windowSize * 3,
}
var maxSize int
for _, n := range testSizes {
if maxSize < n {
maxSize = n
}
}
readBuf := make([]byte, 40)
data := make([]byte, maxSize)
for i := range data {
data[i] = byte(i)
}
for _, sz := range testSizes {
if testing.Short() && sz > windowSize {
continue
}
for _, flush := range []bool{true, false} {
earlyEOF := true // Do we expect early io.EOF?
var buf bytes.Buffer
w, _ := NewWriter(&buf, 5)
w.Write(data[:sz])
if flush {
// If a Flush occurs after all the actual data, the flushing
// semantics dictate that we will observe a (0, io.EOF) since
// Read must return data before it knows that the stream ended.
w.Flush()
earlyEOF = false
}
w.Close()
r := NewReader(&buf)
for {
n, err := r.Read(readBuf)
if err == io.EOF {
// If the availWrite == windowSize, then that means that the
// previous Read returned because the write buffer was full
// and it just so happened that the stream had no more data.
// This situation is rare, but unavoidable.
if r.(*decompressor).dict.availWrite() == windowSize {
earlyEOF = false
}
if n == 0 && earlyEOF {
t.Errorf("On size:%d flush:%v, Read() = (0, io.EOF), want (n, io.EOF)", sz, flush)
}
if n != 0 && !earlyEOF {
t.Errorf("On size:%d flush:%v, Read() = (%d, io.EOF), want (0, io.EOF)", sz, flush, n)
}
break
}
if err != nil {
t.Fatal(err)
}
}
}
}
}
image/internal/compress/flate/huffman_bit_writer.go
+704
View File
@@ -0,0 +1,704 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"io"
)
const (
// The largest offset code.
offsetCodeCount = 30
// The special code used to mark the end of a block.
endBlockMarker = 256
// The first length code.
lengthCodesStart = 257
// The number of codegen codes.
codegenCodeCount = 19
badCode = 255
// bufferFlushSize indicates the buffer size
// after which bytes are flushed to the writer.
// Should preferably be a multiple of 6, since
// we accumulate 6 bytes between writes to the buffer.
bufferFlushSize = 240
// bufferSize is the actual output byte buffer size.
// It must have additional headroom for a flush
// which can contain up to 8 bytes.
bufferSize = bufferFlushSize + 8
)
// The number of extra bits needed by length code X - LENGTH_CODES_START.
var lengthExtraBits = []int8{
/* 257 */ 0, 0, 0,
/* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2,
/* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4,
/* 280 */ 4, 5, 5, 5, 5, 0,
}
// The length indicated by length code X - LENGTH_CODES_START.
var lengthBase = []uint32{
0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
64, 80, 96, 112, 128, 160, 192, 224, 255,
}
// offset code word extra bits.
var offsetExtraBits = []int8{
0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
}
var offsetBase = []uint32{
0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
}
// The odd order in which the codegen code sizes are written.
var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
type huffmanBitWriter struct {
// writer is the underlying writer.
// Do not use it directly; use the write method, which ensures
// that Write errors are sticky.
writer io.Writer
// Data waiting to be written is bytes[0:nbytes]
// and then the low nbits of bits. Data is always written
// sequentially into the bytes array.
bits uint64
nbits uint
bytes [bufferSize]byte
codegenFreq [codegenCodeCount]int32
nbytes int
literalFreq []int32
offsetFreq []int32
codegen []uint8
literalEncoding *huffmanEncoder
offsetEncoding *huffmanEncoder
codegenEncoding *huffmanEncoder
err error
}
func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter {
return &huffmanBitWriter{
writer: w,
literalFreq: make([]int32, maxNumLit),
offsetFreq: make([]int32, offsetCodeCount),
codegen: make([]uint8, maxNumLit+offsetCodeCount+1),
literalEncoding: newHuffmanEncoder(maxNumLit),
codegenEncoding: newHuffmanEncoder(codegenCodeCount),
offsetEncoding: newHuffmanEncoder(offsetCodeCount),
}
}
func (w *huffmanBitWriter) reset(writer io.Writer) {
w.writer = writer
w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil
}
func (w *huffmanBitWriter) flush() {
if w.err != nil {
w.nbits = 0
return
}
n := w.nbytes
for w.nbits != 0 {
w.bytes[n] = byte(w.bits)
w.bits >>= 8
if w.nbits > 8 { // Avoid underflow
w.nbits -= 8
} else {
w.nbits = 0
}
n++
}
w.bits = 0
w.write(w.bytes[:n])
w.nbytes = 0
}
func (w *huffmanBitWriter) write(b []byte) {
if w.err != nil {
return
}
_, w.err = w.writer.Write(b)
}
func (w *huffmanBitWriter) writeBits(b int32, nb uint) {
if w.err != nil {
return
}
w.bits |= uint64(b) << w.nbits
w.nbits += nb
if w.nbits >= 48 {
bits := w.bits
w.bits >>= 48
w.nbits -= 48
n := w.nbytes
bytes := w.bytes[n : n+6]
bytes[0] = byte(bits)
bytes[1] = byte(bits >> 8)
bytes[2] = byte(bits >> 16)
bytes[3] = byte(bits >> 24)
bytes[4] = byte(bits >> 32)
bytes[5] = byte(bits >> 40)
n += 6
if n >= bufferFlushSize {
w.write(w.bytes[:n])
n = 0
}
w.nbytes = n
}
}
func (w *huffmanBitWriter) writeBytes(bytes []byte) {
if w.err != nil {
return
}
n := w.nbytes
if w.nbits&7 != 0 {
w.err = InternalError("writeBytes with unfinished bits")
return
}
for w.nbits != 0 {
w.bytes[n] = byte(w.bits)
w.bits >>= 8
w.nbits -= 8
n++
}
if n != 0 {
w.write(w.bytes[:n])
}
w.nbytes = 0
w.write(bytes)
}
// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
// the literal and offset lengths arrays (which are concatenated into a single
// array). This method generates that run-length encoding.
//
// The result is written into the codegen array, and the frequencies
// of each code is written into the codegenFreq array.
// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
// information. Code badCode is an end marker
//
// numLiterals The number of literals in literalEncoding
// numOffsets The number of offsets in offsetEncoding
// litenc, offenc The literal and offset encoder to use
func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, litEnc, offEnc *huffmanEncoder) {
for i := range w.codegenFreq {
w.codegenFreq[i] = 0
}
// Note that we are using codegen both as a temporary variable for holding
// a copy of the frequencies, and as the place where we put the result.
// This is fine because the output is always shorter than the input used
// so far.
codegen := w.codegen // cache
// Copy the concatenated code sizes to codegen. Put a marker at the end.
cgnl := codegen[:numLiterals]
for i := range cgnl {
cgnl[i] = uint8(litEnc.codes[i].len)
}
cgnl = codegen[numLiterals : numLiterals+numOffsets]
for i := range cgnl {
cgnl[i] = uint8(offEnc.codes[i].len)
}
codegen[numLiterals+numOffsets] = badCode
size := codegen[0]
count := 1
outIndex := 0
for inIndex := 1; size != badCode; inIndex++ {
// INVARIANT: We have seen "count" copies of size that have not yet
// had output generated for them.
nextSize := codegen[inIndex]
if nextSize == size {
count++
continue
}
// We need to generate codegen indicating "count" of size.
if size != 0 {
codegen[outIndex] = size
outIndex++
w.codegenFreq[size]++
count--
for count >= 3 {
n := 6
if n > count {
n = count
}
codegen[outIndex] = 16
outIndex++
codegen[outIndex] = uint8(n - 3)
outIndex++
w.codegenFreq[16]++
count -= n
}
} else {
for count >= 11 {
n := 138
if n > count {
n = count
}
codegen[outIndex] = 18
outIndex++
codegen[outIndex] = uint8(n - 11)
outIndex++
w.codegenFreq[18]++
count -= n
}
if count >= 3 {
// count >= 3 && count <= 10
codegen[outIndex] = 17
outIndex++
codegen[outIndex] = uint8(count - 3)
outIndex++
w.codegenFreq[17]++
count = 0
}
}
count--
for ; count >= 0; count-- {
codegen[outIndex] = size
outIndex++
w.codegenFreq[size]++
}
// Set up invariant for next time through the loop.
size = nextSize
count = 1
}
// Marker indicating the end of the codegen.
codegen[outIndex] = badCode
}
// dynamicSize returns the size of dynamically encoded data in bits.
func (w *huffmanBitWriter) dynamicSize(litEnc, offEnc *huffmanEncoder, extraBits int) (size, numCodegens int) {
numCodegens = len(w.codegenFreq)
for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 {
numCodegens--
}
header := 3 + 5 + 5 + 4 + (3 * numCodegens) +
w.codegenEncoding.bitLength(w.codegenFreq[:]) +
int(w.codegenFreq[16])*2 +
int(w.codegenFreq[17])*3 +
int(w.codegenFreq[18])*7
size = header +
litEnc.bitLength(w.literalFreq) +
offEnc.bitLength(w.offsetFreq) +
extraBits
return size, numCodegens
}
// fixedSize returns the size of dynamically encoded data in bits.
func (w *huffmanBitWriter) fixedSize(extraBits int) int {
return 3 +
fixedLiteralEncoding.bitLength(w.literalFreq) +
fixedOffsetEncoding.bitLength(w.offsetFreq) +
extraBits
}
// storedSize calculates the stored size, including header.
// The function returns the size in bits and whether the block
// fits inside a single block.
func (w *huffmanBitWriter) storedSize(in []byte) (int, bool) {
if in == nil {
return 0, false
}
if len(in) <= maxStoreBlockSize {
return (len(in) + 5) * 8, true
}
return 0, false
}
func (w *huffmanBitWriter) writeCode(c hcode) {
if w.err != nil {
return
}
w.bits |= uint64(c.code) << w.nbits
w.nbits += uint(c.len)
if w.nbits >= 48 {
bits := w.bits
w.bits >>= 48
w.nbits -= 48
n := w.nbytes
bytes := w.bytes[n : n+6]
bytes[0] = byte(bits)
bytes[1] = byte(bits >> 8)
bytes[2] = byte(bits >> 16)
bytes[3] = byte(bits >> 24)
bytes[4] = byte(bits >> 32)
bytes[5] = byte(bits >> 40)
n += 6
if n >= bufferFlushSize {
w.write(w.bytes[:n])
n = 0
}
w.nbytes = n
}
}
// Write the header of a dynamic Huffman block to the output stream.
//
// numLiterals The number of literals specified in codegen
// numOffsets The number of offsets specified in codegen
// numCodegens The number of codegens used in codegen
func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) {
if w.err != nil {
return
}
var firstBits int32 = 4
if isEof {
firstBits = 5
}
w.writeBits(firstBits, 3)
w.writeBits(int32(numLiterals-257), 5)
w.writeBits(int32(numOffsets-1), 5)
w.writeBits(int32(numCodegens-4), 4)
for i := 0; i < numCodegens; i++ {
value := uint(w.codegenEncoding.codes[codegenOrder[i]].len)
w.writeBits(int32(value), 3)
}
i := 0
for {
var codeWord int = int(w.codegen[i])
i++
if codeWord == badCode {
break
}
w.writeCode(w.codegenEncoding.codes[uint32(codeWord)])
switch codeWord {
case 16:
w.writeBits(int32(w.codegen[i]), 2)
i++
break
case 17:
w.writeBits(int32(w.codegen[i]), 3)
i++
break
case 18:
w.writeBits(int32(w.codegen[i]), 7)
i++
break
}
}
}
func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) {
if w.err != nil {
return
}
var flag int32
if isEof {
flag = 1
}
w.writeBits(flag, 3)
w.flush()
w.writeBits(int32(length), 16)
w.writeBits(int32(^uint16(length)), 16)
}
func (w *huffmanBitWriter) writeFixedHeader(isEof bool) {
if w.err != nil {
return
}
// Indicate that we are a fixed Huffman block
var value int32 = 2
if isEof {
value = 3
}
w.writeBits(value, 3)
}
// writeBlock will write a block of tokens with the smallest encoding.
// The original input can be supplied, and if the huffman encoded data
// is larger than the original bytes, the data will be written as a
// stored block.
// If the input is nil, the tokens will always be Huffman encoded.
func (w *huffmanBitWriter) writeBlock(tokens []token, eof bool, input []byte) {
if w.err != nil {
return
}
tokens = append(tokens, endBlockMarker)
numLiterals, numOffsets := w.indexTokens(tokens)
var extraBits int
storedSize, storable := w.storedSize(input)
if storable {
// We only bother calculating the costs of the extra bits required by
// the length of offset fields (which will be the same for both fixed
// and dynamic encoding), if we need to compare those two encodings
// against stored encoding.
for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ {
// First eight length codes have extra size = 0.
extraBits += int(w.literalFreq[lengthCode]) * int(lengthExtraBits[lengthCode-lengthCodesStart])
}
for offsetCode := 4; offsetCode < numOffsets; offsetCode++ {
// First four offset codes have extra size = 0.
extraBits += int(w.offsetFreq[offsetCode]) * int(offsetExtraBits[offsetCode])
}
}
// Figure out smallest code.
// Fixed Huffman baseline.
var literalEncoding = fixedLiteralEncoding
var offsetEncoding = fixedOffsetEncoding
var size = w.fixedSize(extraBits)
// Dynamic Huffman?
var numCodegens int
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
dynamicSize, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, extraBits)
if dynamicSize < size {
size = dynamicSize
literalEncoding = w.literalEncoding
offsetEncoding = w.offsetEncoding
}
// Stored bytes?
if storable && storedSize < size {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
// Huffman.
if literalEncoding == fixedLiteralEncoding {
w.writeFixedHeader(eof)
} else {
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
}
// Write the tokens.
w.writeTokens(tokens, literalEncoding.codes, offsetEncoding.codes)
}
// writeBlockDynamic encodes a block using a dynamic Huffman table.
// This should be used if the symbols used have a disproportionate
// histogram distribution.
// If input is supplied and the compression savings are below 1/16th of the
// input size the block is stored.
func (w *huffmanBitWriter) writeBlockDynamic(tokens []token, eof bool, input []byte) {
if w.err != nil {
return
}
tokens = append(tokens, endBlockMarker)
numLiterals, numOffsets := w.indexTokens(tokens)
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, w.offsetEncoding)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
size, numCodegens := w.dynamicSize(w.literalEncoding, w.offsetEncoding, 0)
// Store bytes, if we don't get a reasonable improvement.
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
// Write Huffman table.
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
// Write the tokens.
w.writeTokens(tokens, w.literalEncoding.codes, w.offsetEncoding.codes)
}
// indexTokens indexes a slice of tokens, and updates
// literalFreq and offsetFreq, and generates literalEncoding
// and offsetEncoding.
// The number of literal and offset tokens is returned.
func (w *huffmanBitWriter) indexTokens(tokens []token) (numLiterals, numOffsets int) {
for i := range w.literalFreq {
w.literalFreq[i] = 0
}
for i := range w.offsetFreq {
w.offsetFreq[i] = 0
}
for _, t := range tokens {
if t < matchType {
w.literalFreq[t.literal()]++
continue
}
length := t.length()
offset := t.offset()
w.literalFreq[lengthCodesStart+lengthCode(length)]++
w.offsetFreq[offsetCode(offset)]++
}
// get the number of literals
numLiterals = len(w.literalFreq)
for w.literalFreq[numLiterals-1] == 0 {
numLiterals--
}
// get the number of offsets
numOffsets = len(w.offsetFreq)
for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 {
numOffsets--
}
if numOffsets == 0 {
// We haven't found a single match. If we want to go with the dynamic encoding,
// we should count at least one offset to be sure that the offset huffman tree could be encoded.
w.offsetFreq[0] = 1
numOffsets = 1
}
w.literalEncoding.generate(w.literalFreq, 15)
w.offsetEncoding.generate(w.offsetFreq, 15)
return
}
// writeTokens writes a slice of tokens to the output.
// codes for literal and offset encoding must be supplied.
func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) {
if w.err != nil {
return
}
for _, t := range tokens {
if t < matchType {
w.writeCode(leCodes[t.literal()])
continue
}
// Write the length
length := t.length()
lengthCode := lengthCode(length)
w.writeCode(leCodes[lengthCode+lengthCodesStart])
extraLengthBits := uint(lengthExtraBits[lengthCode])
if extraLengthBits > 0 {
extraLength := int32(length - lengthBase[lengthCode])
w.writeBits(extraLength, extraLengthBits)
}
// Write the offset
offset := t.offset()
offsetCode := offsetCode(offset)
w.writeCode(oeCodes[offsetCode])
extraOffsetBits := uint(offsetExtraBits[offsetCode])
if extraOffsetBits > 0 {
extraOffset := int32(offset - offsetBase[offsetCode])
w.writeBits(extraOffset, extraOffsetBits)
}
}
}
// huffOffset is a static offset encoder used for huffman only encoding.
// It can be reused since we will not be encoding offset values.
var huffOffset *huffmanEncoder
func init() {
offsetFreq := make([]int32, offsetCodeCount)
offsetFreq[0] = 1
huffOffset = newHuffmanEncoder(offsetCodeCount)
huffOffset.generate(offsetFreq, 15)
}
// writeBlockHuff encodes a block of bytes as either
// Huffman encoded literals or uncompressed bytes if the
// results only gains very little from compression.
func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) {
if w.err != nil {
return
}
// Clear histogram
for i := range w.literalFreq {
w.literalFreq[i] = 0
}
// Add everything as literals
histogram(input, w.literalFreq)
w.literalFreq[endBlockMarker] = 1
const numLiterals = endBlockMarker + 1
w.offsetFreq[0] = 1
const numOffsets = 1
w.literalEncoding.generate(w.literalFreq, 15)
// Figure out smallest code.
// Always use dynamic Huffman or Store
var numCodegens int
// Generate codegen and codegenFrequencies, which indicates how to encode
// the literalEncoding and the offsetEncoding.
w.generateCodegen(numLiterals, numOffsets, w.literalEncoding, huffOffset)
w.codegenEncoding.generate(w.codegenFreq[:], 7)
size, numCodegens := w.dynamicSize(w.literalEncoding, huffOffset, 0)
// Store bytes, if we don't get a reasonable improvement.
if ssize, storable := w.storedSize(input); storable && ssize < (size+size>>4) {
w.writeStoredHeader(len(input), eof)
w.writeBytes(input)
return
}
// Huffman.
w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof)
encoding := w.literalEncoding.codes[:257]
n := w.nbytes
for _, t := range input {
// Bitwriting inlined, ~30% speedup
c := encoding[t]
w.bits |= uint64(c.code) << w.nbits
w.nbits += uint(c.len)
if w.nbits < 48 {
continue
}
// Store 6 bytes
bits := w.bits
w.bits >>= 48
w.nbits -= 48
bytes := w.bytes[n : n+6]
bytes[0] = byte(bits)
bytes[1] = byte(bits >> 8)
bytes[2] = byte(bits >> 16)
bytes[3] = byte(bits >> 24)
bytes[4] = byte(bits >> 32)
bytes[5] = byte(bits >> 40)
n += 6
if n < bufferFlushSize {
continue
}
w.write(w.bytes[:n])
if w.err != nil {
return // Return early in the event of write failures
}
n = 0
}
w.nbytes = n
w.writeCode(encoding[endBlockMarker])
}
// histogram accumulates a histogram of b in h.
//
// len(h) must be >= 256, and h's elements must be all zeroes.
func histogram(b []byte, h []int32) {
h = h[:256]
for _, t := range b {
h[t]++
}
}
image/internal/compress/flate/huffman_bit_writer_test.go
+365
View File
File diff suppressed because one or more lines are too long
image/internal/compress/flate/huffman_code.go
+349
View File
@@ -0,0 +1,349 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"math"
"math/bits"
"sort"
)
// hcode is a huffman code with a bit code and bit length.
type hcode struct {
code, len uint16
}
type huffmanEncoder struct {
codes []hcode
freqcache []literalNode
bitCount [17]int32
lns byLiteral // stored to avoid repeated allocation in generate
lfs byFreq // stored to avoid repeated allocation in generate
}
type literalNode struct {
literal uint16
freq int32
}
// A levelInfo describes the state of the constructed tree for a given depth.
type levelInfo struct {
// Our level. for better printing
level int32
// The frequency of the last node at this level
lastFreq int32
// The frequency of the next character to add to this level
nextCharFreq int32
// The frequency of the next pair (from level below) to add to this level.
// Only valid if the "needed" value of the next lower level is 0.
nextPairFreq int32
// The number of chains remaining to generate for this level before moving
// up to the next level
needed int32
}
// set sets the code and length of an hcode.
func (h *hcode) set(code uint16, length uint16) {
h.len = length
h.code = code
}
func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} }
func newHuffmanEncoder(size int) *huffmanEncoder {
return &huffmanEncoder{codes: make([]hcode, size)}
}
// Generates a HuffmanCode corresponding to the fixed literal table
func generateFixedLiteralEncoding() *huffmanEncoder {
h := newHuffmanEncoder(maxNumLit)
codes := h.codes
var ch uint16
for ch = 0; ch < maxNumLit; ch++ {
var bits uint16
var size uint16
switch {
case ch < 144:
// size 8, 000110000 .. 10111111
bits = ch + 48
size = 8
break
case ch < 256:
// size 9, 110010000 .. 111111111
bits = ch + 400 - 144
size = 9
break
case ch < 280:
// size 7, 0000000 .. 0010111
bits = ch - 256
size = 7
break
default:
// size 8, 11000000 .. 11000111
bits = ch + 192 - 280
size = 8
}
codes[ch] = hcode{code: reverseBits(bits, byte(size)), len: size}
}
return h
}
func generateFixedOffsetEncoding() *huffmanEncoder {
h := newHuffmanEncoder(30)
codes := h.codes
for ch := range codes {
codes[ch] = hcode{code: reverseBits(uint16(ch), 5), len: 5}
}
return h
}
var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding()
var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding()
func (h *huffmanEncoder) bitLength(freq []int32) int {
var total int
for i, f := range freq {
if f != 0 {
total += int(f) * int(h.codes[i].len)
}
}
return total
}
const maxBitsLimit = 16
// Return the number of literals assigned to each bit size in the Huffman encoding
//
// This method is only called when list.length >= 3
// The cases of 0, 1, and 2 literals are handled by special case code.
//
// list An array of the literals with non-zero frequencies
// and their associated frequencies. The array is in order of increasing
// frequency, and has as its last element a special element with frequency
// MaxInt32
// maxBits The maximum number of bits that should be used to encode any literal.
// Must be less than 16.
// return An integer array in which array[i] indicates the number of literals
// that should be encoded in i bits.
func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 {
if maxBits >= maxBitsLimit {
panic("flate: maxBits too large")
}
n := int32(len(list))
list = list[0 : n+1]
list[n] = maxNode()
// The tree can't have greater depth than n - 1, no matter what. This
// saves a little bit of work in some small cases
if maxBits > n-1 {
maxBits = n - 1
}
// Create information about each of the levels.
// A bogus "Level 0" whose sole purpose is so that
// level1.prev.needed==0. This makes level1.nextPairFreq
// be a legitimate value that never gets chosen.
var levels [maxBitsLimit]levelInfo
// leafCounts[i] counts the number of literals at the left
// of ancestors of the rightmost node at level i.
// leafCounts[i][j] is the number of literals at the left
// of the level j ancestor.
var leafCounts [maxBitsLimit][maxBitsLimit]int32
for level := int32(1); level <= maxBits; level++ {
// For every level, the first two items are the first two characters.
// We initialize the levels as if we had already figured this out.
levels[level] = levelInfo{
level: level,
lastFreq: list[1].freq,
nextCharFreq: list[2].freq,
nextPairFreq: list[0].freq + list[1].freq,
}
leafCounts[level][level] = 2
if level == 1 {
levels[level].nextPairFreq = math.MaxInt32
}
}
// We need a total of 2*n - 2 items at top level and have already generated 2.
levels[maxBits].needed = 2*n - 4
level := maxBits
for {
l := &levels[level]
if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 {
// We've run out of both leafs and pairs.
// End all calculations for this level.
// To make sure we never come back to this level or any lower level,
// set nextPairFreq impossibly large.
l.needed = 0
levels[level+1].nextPairFreq = math.MaxInt32
level++
continue
}
prevFreq := l.lastFreq
if l.nextCharFreq < l.nextPairFreq {
// The next item on this row is a leaf node.
n := leafCounts[level][level] + 1
l.lastFreq = l.nextCharFreq
// Lower leafCounts are the same of the previous node.
leafCounts[level][level] = n
l.nextCharFreq = list[n].freq
} else {
// The next item on this row is a pair from the previous row.
// nextPairFreq isn't valid until we generate two
// more values in the level below
l.lastFreq = l.nextPairFreq
// Take leaf counts from the lower level, except counts[level] remains the same.
copy(leafCounts[level][:level], leafCounts[level-1][:level])
levels[l.level-1].needed = 2
}
if l.needed--; l.needed == 0 {
// We've done everything we need to do for this level.
// Continue calculating one level up. Fill in nextPairFreq
// of that level with the sum of the two nodes we've just calculated on
// this level.
if l.level == maxBits {
// All done!
break
}
levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq
level++
} else {
// If we stole from below, move down temporarily to replenish it.
for levels[level-1].needed > 0 {
level--
}
}
}
// Somethings is wrong if at the end, the top level is null or hasn't used
// all of the leaves.
if leafCounts[maxBits][maxBits] != n {
panic("leafCounts[maxBits][maxBits] != n")
}
bitCount := h.bitCount[:maxBits+1]
bits := 1
counts := &leafCounts[maxBits]
for level := maxBits; level > 0; level-- {
// chain.leafCount gives the number of literals requiring at least "bits"
// bits to encode.
bitCount[bits] = counts[level] - counts[level-1]
bits++
}
return bitCount
}
// Look at the leaves and assign them a bit count and an encoding as specified
// in RFC 1951 3.2.2
func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) {
code := uint16(0)
for n, bits := range bitCount {
code <<= 1
if n == 0 || bits == 0 {
continue
}
// The literals list[len(list)-bits] .. list[len(list)-bits]
// are encoded using "bits" bits, and get the values
// code, code + 1, .... The code values are
// assigned in literal order (not frequency order).
chunk := list[len(list)-int(bits):]
h.lns.sort(chunk)
for _, node := range chunk {
h.codes[node.literal] = hcode{code: reverseBits(code, uint8(n)), len: uint16(n)}
code++
}
list = list[0 : len(list)-int(bits)]
}
}
// Update this Huffman Code object to be the minimum code for the specified frequency count.
//
// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
// maxBits The maximum number of bits to use for any literal.
func (h *huffmanEncoder) generate(freq []int32, maxBits int32) {
if h.freqcache == nil {
// Allocate a reusable buffer with the longest possible frequency table.
// Possible lengths are codegenCodeCount, offsetCodeCount and maxNumLit.
// The largest of these is maxNumLit, so we allocate for that case.
h.freqcache = make([]literalNode, maxNumLit+1)
}
list := h.freqcache[:len(freq)+1]
// Number of non-zero literals
count := 0
// Set list to be the set of all non-zero literals and their frequencies
for i, f := range freq {
if f != 0 {
list[count] = literalNode{uint16(i), f}
count++
} else {
list[count] = literalNode{}
h.codes[i].len = 0
}
}
list[len(freq)] = literalNode{}
list = list[:count]
if count <= 2 {
// Handle the small cases here, because they are awkward for the general case code. With
// two or fewer literals, everything has bit length 1.
for i, node := range list {
// "list" is in order of increasing literal value.
h.codes[node.literal].set(uint16(i), 1)
}
return
}
h.lfs.sort(list)
// Get the number of literals for each bit count
bitCount := h.bitCounts(list, maxBits)
// And do the assignment
h.assignEncodingAndSize(bitCount, list)
}
type byLiteral []literalNode
func (s *byLiteral) sort(a []literalNode) {
*s = byLiteral(a)
sort.Sort(s)
}
func (s byLiteral) Len() int { return len(s) }
func (s byLiteral) Less(i, j int) bool {
return s[i].literal < s[j].literal
}
func (s byLiteral) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
type byFreq []literalNode
func (s *byFreq) sort(a []literalNode) {
*s = byFreq(a)
sort.Sort(s)
}
func (s byFreq) Len() int { return len(s) }
func (s byFreq) Less(i, j int) bool {
if s[i].freq == s[j].freq {
return s[i].literal < s[j].literal
}
return s[i].freq < s[j].freq
}
func (s byFreq) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func reverseBits(number uint16, bitLength byte) uint16 {
return bits.Reverse16(number << (16 - bitLength))
}
image/internal/compress/flate/inflate.go
+825
View File
@@ -0,0 +1,825 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package flate implements the DEFLATE compressed data format, described in
// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file
// formats.
package flate
import (
"bufio"
"io"
"math/bits"
"strconv"
"sync"
)
const (
maxCodeLen = 16 // max length of Huffman code
// The next three numbers come from the RFC section 3.2.7, with the
// additional proviso in section 3.2.5 which implies that distance codes
// 30 and 31 should never occur in compressed data.
maxNumLit = 286
maxNumDist = 30
numCodes = 19 // number of codes in Huffman meta-code
)
// Initialize the fixedHuffmanDecoder only once upon first use.
var fixedOnce sync.Once
var fixedHuffmanDecoder huffmanDecoder
// A CorruptInputError reports the presence of corrupt input at a given offset.
type CorruptInputError int64
func (e CorruptInputError) Error() string {
return "flate: corrupt input before offset " + strconv.FormatInt(int64(e), 10)
}
// An InternalError reports an error in the flate code itself.
type InternalError string
func (e InternalError) Error() string { return "flate: internal error: " + string(e) }
// A ReadError reports an error encountered while reading input.
//
// Deprecated: No longer returned.
type ReadError struct {
Offset int64 // byte offset where error occurred
Err error // error returned by underlying Read
}
func (e *ReadError) Error() string {
return "flate: read error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error()
}
// A WriteError reports an error encountered while writing output.
//
// Deprecated: No longer returned.
type WriteError struct {
Offset int64 // byte offset where error occurred
Err error // error returned by underlying Write
}
func (e *WriteError) Error() string {
return "flate: write error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error()
}
// Resetter resets a ReadCloser returned by NewReader or NewReaderDict
// to switch to a new underlying Reader. This permits reusing a ReadCloser
// instead of allocating a new one.
type Resetter interface {
// Reset discards any buffered data and resets the Resetter as if it was
// newly initialized with the given reader.
Reset(r io.Reader, dict []byte) error
}
// The data structure for decoding Huffman tables is based on that of
// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits),
// For codes smaller than the table width, there are multiple entries
// (each combination of trailing bits has the same value). For codes
// larger than the table width, the table contains a link to an overflow
// table. The width of each entry in the link table is the maximum code
// size minus the chunk width.
//
// Note that you can do a lookup in the table even without all bits
// filled. Since the extra bits are zero, and the DEFLATE Huffman codes
// have the property that shorter codes come before longer ones, the
// bit length estimate in the result is a lower bound on the actual
// number of bits.
//
// See the following:
// https://github.com/madler/zlib/raw/master/doc/algorithm.txt
// chunk & 15 is number of bits
// chunk >> 4 is value, including table link
const (
huffmanChunkBits = 9
huffmanNumChunks = 1 << huffmanChunkBits
huffmanCountMask = 15
huffmanValueShift = 4
)
type huffmanDecoder struct {
min int // the minimum code length
chunks [huffmanNumChunks]uint32 // chunks as described above
links [][]uint32 // overflow links
linkMask uint32 // mask the width of the link table
}
// Initialize Huffman decoding tables from array of code lengths.
// Following this function, h is guaranteed to be initialized into a complete
// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
// degenerate case where the tree has only a single symbol with length 1. Empty
// trees are permitted.
func (h *huffmanDecoder) init(lengths []int) bool {
// Sanity enables additional runtime tests during Huffman
// table construction. It's intended to be used during
// development to supplement the currently ad-hoc unit tests.
const sanity = false
if h.min != 0 {
*h = huffmanDecoder{}
}
// Count number of codes of each length,
// compute min and max length.
var count [maxCodeLen]int
var min, max int
for _, n := range lengths {
if n == 0 {
continue
}
if min == 0 || n < min {
min = n
}
if n > max {
max = n
}
count[n]++
}
// Empty tree. The decompressor.huffSym function will fail later if the tree
// is used. Technically, an empty tree is only valid for the HDIST tree and
// not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
// is guaranteed to fail since it will attempt to use the tree to decode the
// codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
// guaranteed to fail later since the compressed data section must be
// composed of at least one symbol (the end-of-block marker).
if max == 0 {
return true
}
code := 0
var nextcode [maxCodeLen]int
for i := min; i <= max; i++ {
code <<= 1
nextcode[i] = code
code += count[i]
}
// Check that the coding is complete (i.e., that we've
// assigned all 2-to-the-max possible bit sequences).
// Exception: To be compatible with zlib, we also need to
// accept degenerate single-code codings. See also
// TestDegenerateHuffmanCoding.
if code != 1<<uint(max) && !(code == 1 && max == 1) {
return false
}
h.min = min
if max > huffmanChunkBits {
numLinks := 1 << (uint(max) - huffmanChunkBits)
h.linkMask = uint32(numLinks - 1)
// create link tables
link := nextcode[huffmanChunkBits+1] >> 1
h.links = make([][]uint32, huffmanNumChunks-link)
for j := uint(link); j < huffmanNumChunks; j++ {
reverse := int(bits.Reverse16(uint16(j)))
reverse >>= uint(16 - huffmanChunkBits)
off := j - uint(link)
if sanity && h.chunks[reverse] != 0 {
panic("impossible: overwriting existing chunk")
}
h.chunks[reverse] = uint32(off<<huffmanValueShift | (huffmanChunkBits + 1))
h.links[off] = make([]uint32, numLinks)
}
}
for i, n := range lengths {
if n == 0 {
continue
}
code := nextcode[n]
nextcode[n]++
chunk := uint32(i<<huffmanValueShift | n)
reverse := int(bits.Reverse16(uint16(code)))
reverse >>= uint(16 - n)
if n <= huffmanChunkBits {
for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
// We should never need to overwrite
// an existing chunk. Also, 0 is
// never a valid chunk, because the
// lower 4 "count" bits should be
// between 1 and 15.
if sanity && h.chunks[off] != 0 {
panic("impossible: overwriting existing chunk")
}
h.chunks[off] = chunk
}
} else {
j := reverse & (huffmanNumChunks - 1)
if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 {
// Longer codes should have been
// associated with a link table above.
panic("impossible: not an indirect chunk")
}
value := h.chunks[j] >> huffmanValueShift
linktab := h.links[value]
reverse >>= huffmanChunkBits
for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) {
if sanity && linktab[off] != 0 {
panic("impossible: overwriting existing chunk")
}
linktab[off] = chunk
}
}
}
if sanity {
// Above we've sanity checked that we never overwrote
// an existing entry. Here we additionally check that
// we filled the tables completely.
for i, chunk := range h.chunks {
if chunk == 0 {
// As an exception, in the degenerate
// single-code case, we allow odd
// chunks to be missing.
if code == 1 && i%2 == 1 {
continue
}
panic("impossible: missing chunk")
}
}
for _, linktab := range h.links {
for _, chunk := range linktab {
if chunk == 0 {
panic("impossible: missing chunk")
}
}
}
}
return true
}
// The actual read interface needed by NewReader.
// If the passed in io.Reader does not also have ReadByte,
// the NewReader will introduce its own buffering.
type Reader interface {
io.Reader
io.ByteReader
}
// Decompress state.
type decompressor struct {
// Input source.
r Reader
roffset int64
// Input bits, in top of b.
b uint32
nb uint
// Huffman decoders for literal/length, distance.
h1, h2 huffmanDecoder
// Length arrays used to define Huffman codes.
bits *[maxNumLit + maxNumDist]int
codebits *[numCodes]int
// Output history, buffer.
dict dictDecoder
// Temporary buffer (avoids repeated allocation).
buf [4]byte
// Next step in the decompression,
// and decompression state.
step func(*decompressor)
stepState int
final bool
err error
toRead []byte
hl, hd *huffmanDecoder
copyLen int
copyDist int
}
func (f *decompressor) nextBlock() {
for f.nb < 1+2 {
if f.err = f.moreBits(); f.err != nil {
return
}
}
f.final = f.b&1 == 1
f.b >>= 1
typ := f.b & 3
f.b >>= 2
f.nb -= 1 + 2
switch typ {
case 0:
f.dataBlock()
case 1:
// compressed, fixed Huffman tables
f.hl = &fixedHuffmanDecoder
f.hd = nil
f.huffmanBlock()
case 2:
// compressed, dynamic Huffman tables
if f.err = f.readHuffman(); f.err != nil {
break
}
f.hl = &f.h1
f.hd = &f.h2
f.huffmanBlock()
default:
// 3 is reserved.
f.err = CorruptInputError(f.roffset)
}
}
func (f *decompressor) Read(b []byte) (int, error) {
for {
if len(f.toRead) > 0 {
n := copy(b, f.toRead)
f.toRead = f.toRead[n:]
if len(f.toRead) == 0 {
return n, f.err
}
return n, nil
}
if f.err != nil {
return 0, f.err
}
f.step(f)
if f.err != nil && len(f.toRead) == 0 {
f.toRead = f.dict.readFlush() // Flush what's left in case of error
}
}
}
func (f *decompressor) Close() error {
if f.err == io.EOF {
return nil
}
return f.err
}
// RFC 1951 section 3.2.7.
// Compression with dynamic Huffman codes
var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}
func (f *decompressor) readHuffman() error {
// HLIT[5], HDIST[5], HCLEN[4].
for f.nb < 5+5+4 {
if err := f.moreBits(); err != nil {
return err
}
}
nlit := int(f.b&0x1F) + 257
if nlit > maxNumLit {
return CorruptInputError(f.roffset)
}
f.b >>= 5
ndist := int(f.b&0x1F) + 1
if ndist > maxNumDist {
return CorruptInputError(f.roffset)
}
f.b >>= 5
nclen := int(f.b&0xF) + 4
// numCodes is 19, so nclen is always valid.
f.b >>= 4
f.nb -= 5 + 5 + 4
// (HCLEN+4)*3 bits: code lengths in the magic codeOrder order.
for i := 0; i < nclen; i++ {
for f.nb < 3 {
if err := f.moreBits(); err != nil {
return err
}
}
f.codebits[codeOrder[i]] = int(f.b & 0x7)
f.b >>= 3
f.nb -= 3
}
for i := nclen; i < len(codeOrder); i++ {
f.codebits[codeOrder[i]] = 0
}
if !f.h1.init(f.codebits[0:]) {
return CorruptInputError(f.roffset)
}
// HLIT + 257 code lengths, HDIST + 1 code lengths,
// using the code length Huffman code.
for i, n := 0, nlit+ndist; i < n; {
x, err := f.huffSym(&f.h1)
if err != nil {
return err
}
if x < 16 {
// Actual length.
f.bits[i] = x
i++
continue
}
// Repeat previous length or zero.
var rep int
var nb uint
var b int
switch x {
default:
return InternalError("unexpected length code")
case 16:
rep = 3
nb = 2
if i == 0 {
return CorruptInputError(f.roffset)
}
b = f.bits[i-1]
case 17:
rep = 3
nb = 3
b = 0
case 18:
rep = 11
nb = 7
b = 0
}
for f.nb < nb {
if err := f.moreBits(); err != nil {
return err
}
}
rep += int(f.b & uint32(1<<nb-1))
f.b >>= nb
f.nb -= nb
if i+rep > n {
return CorruptInputError(f.roffset)
}
for j := 0; j < rep; j++ {
f.bits[i] = b
i++
}
}
if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) {
return CorruptInputError(f.roffset)
}
// As an optimization, we can initialize the min bits to read at a time
// for the HLIT tree to the length of the EOB marker since we know that
// every block must terminate with one. This preserves the property that
// we never read any extra bytes after the end of the DEFLATE stream.
if f.h1.min < f.bits[endBlockMarker] {
f.h1.min = f.bits[endBlockMarker]
}
return nil
}
// Decode a single Huffman block from f.
// hl and hd are the Huffman states for the lit/length values
// and the distance values, respectively. If hd == nil, using the
// fixed distance encoding associated with fixed Huffman blocks.
func (f *decompressor) huffmanBlock() {
const (
stateInit = iota // Zero value must be stateInit
stateDict
)
switch f.stepState {
case stateInit:
goto readLiteral
case stateDict:
goto copyHistory
}
readLiteral:
// Read literal and/or (length, distance) according to RFC section 3.2.3.
{
v, err := f.huffSym(f.hl)
if err != nil {
f.err = err
return
}
var n uint // number of bits extra
var length int
switch {
case v < 256:
f.dict.writeByte(byte(v))
if f.dict.availWrite() == 0 {
f.toRead = f.dict.readFlush()
f.step = (*decompressor).huffmanBlock
f.stepState = stateInit
return
}
goto readLiteral
case v == 256:
f.finishBlock()
return
// otherwise, reference to older data
case v < 265:
length = v - (257 - 3)
n = 0
case v < 269:
length = v*2 - (265*2 - 11)
n = 1
case v < 273:
length = v*4 - (269*4 - 19)
n = 2
case v < 277:
length = v*8 - (273*8 - 35)
n = 3
case v < 281:
length = v*16 - (277*16 - 67)
n = 4
case v < 285:
length = v*32 - (281*32 - 131)
n = 5
case v < maxNumLit:
length = 258
n = 0
default:
f.err = CorruptInputError(f.roffset)
return
}
if n > 0 {
for f.nb < n {
if err = f.moreBits(); err != nil {
f.err = err
return
}
}
length += int(f.b & uint32(1<<n-1))
f.b >>= n
f.nb -= n
}
var dist int
if f.hd == nil {
for f.nb < 5 {
if err = f.moreBits(); err != nil {
f.err = err
return
}
}
dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3)))
f.b >>= 5
f.nb -= 5
} else {
if dist, err = f.huffSym(f.hd); err != nil {
f.err = err
return
}
}
switch {
case dist < 4:
dist++
case dist < maxNumDist:
nb := uint(dist-2) >> 1
// have 1 bit in bottom of dist, need nb more.
extra := (dist & 1) << nb
for f.nb < nb {
if err = f.moreBits(); err != nil {
f.err = err
return
}
}
extra |= int(f.b & uint32(1<<nb-1))
f.b >>= nb
f.nb -= nb
dist = 1<<(nb+1) + 1 + extra
default:
f.err = CorruptInputError(f.roffset)
return
}
// No check on length; encoding can be prescient.
if dist > f.dict.histSize() {
f.err = CorruptInputError(f.roffset)
return
}
f.copyLen, f.copyDist = length, dist
goto copyHistory
}
copyHistory:
// Perform a backwards copy according to RFC section 3.2.3.
{
cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen)
if cnt == 0 {
cnt = f.dict.writeCopy(f.copyDist, f.copyLen)
}
f.copyLen -= cnt
if f.dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = f.dict.readFlush()
f.step = (*decompressor).huffmanBlock // We need to continue this work
f.stepState = stateDict
return
}
goto readLiteral
}
}
// Copy a single uncompressed data block from input to output.
func (f *decompressor) dataBlock() {
// Uncompressed.
// Discard current half-byte.
f.nb = 0
f.b = 0
// Length then ones-complement of length.
nr, err := io.ReadFull(f.r, f.buf[0:4])
f.roffset += int64(nr)
if err != nil {
f.err = noEOF(err)
return
}
n := int(f.buf[0]) | int(f.buf[1])<<8
nn := int(f.buf[2]) | int(f.buf[3])<<8
if uint16(nn) != uint16(^n) {
f.err = CorruptInputError(f.roffset)
return
}
if n == 0 {
f.toRead = f.dict.readFlush()
f.finishBlock()
return
}
f.copyLen = n
f.copyData()
}
// copyData copies f.copyLen bytes from the underlying reader into f.hist.
// It pauses for reads when f.hist is full.
func (f *decompressor) copyData() {
buf := f.dict.writeSlice()
if len(buf) > f.copyLen {
buf = buf[:f.copyLen]
}
cnt, err := io.ReadFull(f.r, buf)
f.roffset += int64(cnt)
f.copyLen -= cnt
f.dict.writeMark(cnt)
if err != nil {
f.err = noEOF(err)
return
}
if f.dict.availWrite() == 0 || f.copyLen > 0 {
f.toRead = f.dict.readFlush()
f.step = (*decompressor).copyData
return
}
f.finishBlock()
}
func (f *decompressor) finishBlock() {
if f.final {
if f.dict.availRead() > 0 {
f.toRead = f.dict.readFlush()
}
f.err = io.EOF
}
f.step = (*decompressor).nextBlock
}
// noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF.
func noEOF(e error) error {
if e == io.EOF {
return io.ErrUnexpectedEOF
}
return e
}
func (f *decompressor) moreBits() error {
c, err := f.r.ReadByte()
if err != nil {
return noEOF(err)
}
f.roffset++
f.b |= uint32(c) << f.nb
f.nb += 8
return nil
}
// Read the next Huffman-encoded symbol from f according to h.
func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) {
// Since a huffmanDecoder can be empty or be composed of a degenerate tree
// with single element, huffSym must error on these two edge cases. In both
// cases, the chunks slice will be 0 for the invalid sequence, leading it
// satisfy the n == 0 check below.
n := uint(h.min)
// Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers,
// but is smart enough to keep local variables in registers, so use nb and b,
// inline call to moreBits and reassign b,nb back to f on return.
nb, b := f.nb, f.b
for {
for nb < n {
c, err := f.r.ReadByte()
if err != nil {
f.b = b
f.nb = nb
return 0, noEOF(err)
}
f.roffset++
b |= uint32(c) << (nb & 31)
nb += 8
}
chunk := h.chunks[b&(huffmanNumChunks-1)]
n = uint(chunk & huffmanCountMask)
if n > huffmanChunkBits {
chunk = h.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&h.linkMask]
n = uint(chunk & huffmanCountMask)
}
if n <= nb {
if n == 0 {
f.b = b
f.nb = nb
f.err = CorruptInputError(f.roffset)
return 0, f.err
}
f.b = b >> (n & 31)
f.nb = nb - n
return int(chunk >> huffmanValueShift), nil
}
}
}
func makeReader(r io.Reader) Reader {
if rr, ok := r.(Reader); ok {
return rr
}
return bufio.NewReader(r)
}
func fixedHuffmanDecoderInit() {
fixedOnce.Do(func() {
// These come from the RFC section 3.2.6.
var bits [288]int
for i := 0; i < 144; i++ {
bits[i] = 8
}
for i := 144; i < 256; i++ {
bits[i] = 9
}
for i := 256; i < 280; i++ {
bits[i] = 7
}
for i := 280; i < 288; i++ {
bits[i] = 8
}
fixedHuffmanDecoder.init(bits[:])
})
}
func (f *decompressor) Reset(r io.Reader, dict []byte) error {
*f = decompressor{
r: makeReader(r),
bits: f.bits,
codebits: f.codebits,
dict: f.dict,
step: (*decompressor).nextBlock,
}
f.dict.init(maxMatchOffset, dict)
return nil
}
// NewReader returns a new ReadCloser that can be used
// to read the uncompressed version of r.
// If r does not also implement io.ByteReader,
// the decompressor may read more data than necessary from r.
// It is the caller's responsibility to call Close on the ReadCloser
// when finished reading.
//
// The ReadCloser returned by NewReader also implements Resetter.
func NewReader(r io.Reader) io.ReadCloser {
fixedHuffmanDecoderInit()
var f decompressor
f.r = makeReader(r)
f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int)
f.step = (*decompressor).nextBlock
f.dict.init(maxMatchOffset, nil)
return &f
}
// NewReaderDict is like NewReader but initializes the reader
// with a preset dictionary. The returned Reader behaves as if
// the uncompressed data stream started with the given dictionary,
// which has already been read. NewReaderDict is typically used
// to read data compressed by NewWriterDict.
//
// The ReadCloser returned by NewReader also implements Resetter.
func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser {
fixedHuffmanDecoderInit()
var f decompressor
f.r = makeReader(r)
f.bits = new([maxNumLit + maxNumDist]int)
f.codebits = new([numCodes]int)
f.step = (*decompressor).nextBlock
f.dict.init(maxMatchOffset, dict)
return &f
}
image/internal/compress/flate/inflate_test.go
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"bytes"
"io"
"strings"
"testing"
)
func TestReset(t *testing.T) {
ss := []string{
"lorem ipsum izzle fo rizzle",
"the quick brown fox jumped over",
}
deflated := make([]bytes.Buffer, 2)
for i, s := range ss {
w, _ := NewWriter(&deflated[i], 1)
w.Write([]byte(s))
w.Close()
}
inflated := make([]bytes.Buffer, 2)
f := NewReader(&deflated[0])
io.Copy(&inflated[0], f)
f.(Resetter).Reset(&deflated[1], nil)
io.Copy(&inflated[1], f)
f.Close()
for i, s := range ss {
if s != inflated[i].String() {
t.Errorf("inflated[%d]:\ngot %q\nwant %q", i, inflated[i], s)
}
}
}
func TestReaderTruncated(t *testing.T) {
vectors := []struct{ input, output string }{
{"\x00", ""},
{"\x00\f", ""},
{"\x00\f\x00", ""},
{"\x00\f\x00\xf3\xff", ""},
{"\x00\f\x00\xf3\xffhello", "hello"},
{"\x00\f\x00\xf3\xffhello, world", "hello, world"},
{"\x02", ""},
{"\xf2H\xcd", "He"},
{"\xf2H͙0a\u0084\t", "Hel\x90\x90\x90\x90\x90"},
{"\xf2H͙0a\u0084\t\x00", "Hel\x90\x90\x90\x90\x90"},
}
for i, v := range vectors {
r := strings.NewReader(v.input)
zr := NewReader(r)
b, err := io.ReadAll(zr)
if err != io.ErrUnexpectedEOF {
t.Errorf("test %d, error mismatch: got %v, want io.ErrUnexpectedEOF", i, err)
}
if string(b) != v.output {
t.Errorf("test %d, output mismatch: got %q, want %q", i, b, v.output)
}
}
}
func TestResetDict(t *testing.T) {
dict := []byte("the lorem fox")
ss := []string{
"lorem ipsum izzle fo rizzle",
"the quick brown fox jumped over",
}
deflated := make([]bytes.Buffer, len(ss))
for i, s := range ss {
w, _ := NewWriterDict(&deflated[i], DefaultCompression, dict)
w.Write([]byte(s))
w.Close()
}
inflated := make([]bytes.Buffer, len(ss))
f := NewReader(nil)
for i := range inflated {
f.(Resetter).Reset(&deflated[i], dict)
io.Copy(&inflated[i], f)
}
f.Close()
for i, s := range ss {
if s != inflated[i].String() {
t.Errorf("inflated[%d]:\ngot %q\nwant %q", i, inflated[i], s)
}
}
}
image/internal/compress/flate/reader_test.go
+98
View File
@@ -0,0 +1,98 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"bytes"
"io"
"os"
"runtime"
"strings"
"testing"
)
func TestNlitOutOfRange(t *testing.T) {
// Trying to decode this bogus flate data, which has a Huffman table
// with nlit=288, should not panic.
io.Copy(io.Discard, NewReader(strings.NewReader(
"\xfc\xfe\x36\xe7\x5e\x1c\xef\xb3\x55\x58\x77\xb6\x56\xb5\x43\xf4"+
"\x6f\xf2\xd2\xe6\x3d\x99\xa0\x85\x8c\x48\xeb\xf8\xda\x83\x04\x2a"+
"\x75\xc4\xf8\x0f\x12\x11\xb9\xb4\x4b\x09\xa0\xbe\x8b\x91\x4c")))
}
var suites = []struct{ name, file string }{
// Digits is the digits of the irrational number e. Its decimal representation
// does not repeat, but there are only 10 possible digits, so it should be
// reasonably compressible.
{"Digits", "../testdata/e.txt"},
// Newton is Isaac Newtons's educational text on Opticks.
{"Newton", "../../testdata/Isaac.Newton-Opticks.txt"},
}
func BenchmarkDecode(b *testing.B) {
doBench(b, func(b *testing.B, buf0 []byte, level, n int) {
b.ReportAllocs()
b.StopTimer()
b.SetBytes(int64(n))
compressed := new(bytes.Buffer)
w, err := NewWriter(compressed, level)
if err != nil {
b.Fatal(err)
}
for i := 0; i < n; i += len(buf0) {
if len(buf0) > n-i {
buf0 = buf0[:n-i]
}
io.Copy(w, bytes.NewReader(buf0))
}
w.Close()
buf1 := compressed.Bytes()
buf0, compressed, w = nil, nil, nil
runtime.GC()
b.StartTimer()
for i := 0; i < b.N; i++ {
io.Copy(io.Discard, NewReader(bytes.NewReader(buf1)))
}
})
}
var levelTests = []struct {
name string
level int
}{
{"Huffman", HuffmanOnly},
{"Speed", BestSpeed},
{"Default", DefaultCompression},
{"Compression", BestCompression},
}
var sizes = []struct {
name string
n int
}{
{"1e4", 1e4},
{"1e5", 1e5},
{"1e6", 1e6},
}
func doBench(b *testing.B, f func(b *testing.B, buf []byte, level, n int)) {
for _, suite := range suites {
buf, err := os.ReadFile(suite.file)
if err != nil {
b.Fatal(err)
}
if len(buf) == 0 {
b.Fatalf("test file %q has no data", suite.file)
}
for _, l := range levelTests {
for _, s := range sizes {
b.Run(suite.name+"/"+l.name+"/"+s.name, func(b *testing.B) {
f(b, buf, l.level, s.n)
})
}
}
}
}
image/internal/compress/flate/token.go
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
const (
// 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused
// 8 bits: xlength = length - MIN_MATCH_LENGTH
// 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal
lengthShift = 22
offsetMask = 1<<lengthShift - 1
typeMask = 3 << 30
literalType = 0 << 30
matchType = 1 << 30
)
// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
// is lengthCodes[length - MIN_MATCH_LENGTH]
var lengthCodes = [...]uint32{
0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
27, 27, 27, 27, 27, 28,
}
var offsetCodes = [...]uint32{
0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
}
type token uint32
// Convert a literal into a literal token.
func literalToken(literal uint32) token { return token(literalType + literal) }
// Convert a < xlength, xoffset > pair into a match token.
func matchToken(xlength uint32, xoffset uint32) token {
return token(matchType + xlength<<lengthShift + xoffset)
}
// Returns the literal of a literal token
func (t token) literal() uint32 { return uint32(t - literalType) }
// Returns the extra offset of a match token
func (t token) offset() uint32 { return uint32(t) & offsetMask }
func (t token) length() uint32 { return uint32((t - matchType) >> lengthShift) }
func lengthCode(len uint32) uint32 { return lengthCodes[len] }
// Returns the offset code corresponding to a specific offset
func offsetCode(off uint32) uint32 {
if off < uint32(len(offsetCodes)) {
return offsetCodes[off]
}
if off>>7 < uint32(len(offsetCodes)) {
return offsetCodes[off>>7] + 14
}
return offsetCodes[off>>14] + 28
}
image/internal/compress/flate/writer_test.go
+237
View File
@@ -0,0 +1,237 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package flate
import (
"bytes"
"fmt"
"io"
"math/rand"
"runtime"
"testing"
)
func BenchmarkEncode(b *testing.B) {
doBench(b, func(b *testing.B, buf0 []byte, level, n int) {
b.StopTimer()
b.SetBytes(int64(n))
buf1 := make([]byte, n)
for i := 0; i < n; i += len(buf0) {
if len(buf0) > n-i {
buf0 = buf0[:n-i]
}
copy(buf1[i:], buf0)
}
buf0 = nil
w, err := NewWriter(io.Discard, level)
if err != nil {
b.Fatal(err)
}
runtime.GC()
b.StartTimer()
for i := 0; i < b.N; i++ {
w.Reset(io.Discard)
w.Write(buf1)
w.Close()
}
})
}
// errorWriter is a writer that fails after N writes.
type errorWriter struct {
N int
}
func (e *errorWriter) Write(b []byte) (int, error) {
if e.N <= 0 {
return 0, io.ErrClosedPipe
}
e.N--
return len(b), nil
}
// Test if errors from the underlying writer is passed upwards.
func TestWriteError(t *testing.T) {
t.Parallel()
buf := new(bytes.Buffer)
n := 65536
if !testing.Short() {
n *= 4
}
for i := 0; i < n; i++ {
fmt.Fprintf(buf, "asdasfasf%d%dfghfgujyut%dyutyu\n", i, i, i)
}
in := buf.Bytes()
// We create our own buffer to control number of writes.
copyBuffer := make([]byte, 128)
for l := 0; l < 10; l++ {
for fail := 1; fail <= 256; fail *= 2 {
// Fail after 'fail' writes
ew := &errorWriter{N: fail}
w, err := NewWriter(ew, l)
if err != nil {
t.Fatalf("NewWriter: level %d: %v", l, err)
}
n, err := io.CopyBuffer(w, struct{ io.Reader }{bytes.NewBuffer(in)}, copyBuffer)
if err == nil {
t.Fatalf("Level %d: Expected an error, writer was %#v", l, ew)
}
n2, err := w.Write([]byte{1, 2, 2, 3, 4, 5})
if n2 != 0 {
t.Fatal("Level", l, "Expected 0 length write, got", n)
}
if err == nil {
t.Fatal("Level", l, "Expected an error")
}
err = w.Flush()
if err == nil {
t.Fatal("Level", l, "Expected an error on flush")
}
err = w.Close()
if err == nil {
t.Fatal("Level", l, "Expected an error on close")
}
w.Reset(io.Discard)
n2, err = w.Write([]byte{1, 2, 3, 4, 5, 6})
if err != nil {
t.Fatal("Level", l, "Got unexpected error after reset:", err)
}
if n2 == 0 {
t.Fatal("Level", l, "Got 0 length write, expected > 0")
}
if testing.Short() {
return
}
}
}
}
// Test if two runs produce identical results
// even when writing different sizes to the Writer.
func TestDeterministic(t *testing.T) {
t.Parallel()
for i := 0; i <= 9; i++ {
t.Run(fmt.Sprint("L", i), func(t *testing.T) { testDeterministic(i, t) })
}
t.Run("LM2", func(t *testing.T) { testDeterministic(-2, t) })
}
func testDeterministic(i int, t *testing.T) {
t.Parallel()
// Test so much we cross a good number of block boundaries.
var length = maxStoreBlockSize*30 + 500
if testing.Short() {
length /= 10
}
// Create a random, but compressible stream.
rng := rand.New(rand.NewSource(1))
t1 := make([]byte, length)
for i := range t1 {
t1[i] = byte(rng.Int63() & 7)
}
// Do our first encode.
var b1 bytes.Buffer
br := bytes.NewBuffer(t1)
w, err := NewWriter(&b1, i)
if err != nil {
t.Fatal(err)
}
// Use a very small prime sized buffer.
cbuf := make([]byte, 787)
_, err = io.CopyBuffer(w, struct{ io.Reader }{br}, cbuf)
if err != nil {
t.Fatal(err)
}
w.Close()
// We choose a different buffer size,
// bigger than a maximum block, and also a prime.
var b2 bytes.Buffer
cbuf = make([]byte, 81761)
br2 := bytes.NewBuffer(t1)
w2, err := NewWriter(&b2, i)
if err != nil {
t.Fatal(err)
}
_, err = io.CopyBuffer(w2, struct{ io.Reader }{br2}, cbuf)
if err != nil {
t.Fatal(err)
}
w2.Close()
b1b := b1.Bytes()
b2b := b2.Bytes()
if !bytes.Equal(b1b, b2b) {
t.Errorf("level %d did not produce deterministic result, result mismatch, len(a) = %d, len(b) = %d", i, len(b1b), len(b2b))
}
}
// TestDeflateFast_Reset will test that encoding is consistent
// across a warparound of the table offset.
// See https://github.com/golang/go/issues/34121
func TestDeflateFast_Reset(t *testing.T) {
buf := new(bytes.Buffer)
n := 65536
for i := 0; i < n; i++ {
fmt.Fprintf(buf, "asdfasdfasdfasdf%d%dfghfgujyut%dyutyu\n", i, i, i)
}
// This is specific to level 1.
const level = 1
in := buf.Bytes()
offset := 1
if testing.Short() {
offset = 256
}
// We do an encode with a clean buffer to compare.
var want bytes.Buffer
w, err := NewWriter(&want, level)
if err != nil {
t.Fatalf("NewWriter: level %d: %v", level, err)
}
// Output written 3 times.
w.Write(in)
w.Write(in)
w.Write(in)
w.Close()
for ; offset <= 256; offset *= 2 {
w, err := NewWriter(io.Discard, level)
if err != nil {
t.Fatalf("NewWriter: level %d: %v", level, err)
}
// Reset until we are right before the wraparound.
// Each reset adds maxMatchOffset to the offset.
for i := 0; i < (bufferReset-len(in)-offset-maxMatchOffset)/maxMatchOffset; i++ {
// skip ahead to where we are close to wrap around...
w.d.reset(nil)
}
var got bytes.Buffer
w.Reset(&got)
// Write 3 times, close.
for i := 0; i < 3; i++ {
_, err = w.Write(in)
if err != nil {
t.Fatal(err)
}
}
err = w.Close()
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got.Bytes(), want.Bytes()) {
t.Fatalf("output did not match at wraparound, len(want) = %d, len(got) = %d", want.Len(), got.Len())
}
}
}
image/internal/compress/zlib/example_test.go
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package zlib_test
import (
"bytes"
"compress/zlib"
"fmt"
"io"
"os"
)
func ExampleNewWriter() {
var b bytes.Buffer
w := zlib.NewWriter(&b)
w.Write([]byte("hello, world\n"))
w.Close()
fmt.Println(b.Bytes())
// Output: [120 156 202 72 205 201 201 215 81 40 207 47 202 73 225 2 4 0 0 255 255 33 231 4 147]
}
func ExampleNewReader() {
buff := []byte{120, 156, 202, 72, 205, 201, 201, 215, 81, 40, 207,
47, 202, 73, 225, 2, 4, 0, 0, 255, 255, 33, 231, 4, 147}
b := bytes.NewReader(buff)
r, err := zlib.NewReader(b)
if err != nil {
panic(err)
}
io.Copy(os.Stdout, r)
// Output: hello, world
r.Close()
}
image/internal/compress/zlib/reader.go
+178
View File
@@ -0,0 +1,178 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
Package zlib implements reading and writing of zlib format compressed data,
as specified in RFC 1950.
The implementation provides filters that uncompress during reading
and compress during writing. For example, to write compressed data
to a buffer:
var b bytes.Buffer
w := zlib.NewWriter(&b)
w.Write([]byte("hello, world\n"))
w.Close()
and to read that data back:
r, err := zlib.NewReader(&b)
io.Copy(os.Stdout, r)
r.Close()
*/
package zlib
import (
"bufio"
"errors"
"hash"
"hash/adler32"
"io"
"tinygo.org/x/drivers/image/internal/compress/flate"
)
const zlibDeflate = 8
var (
// ErrChecksum is returned when reading ZLIB data that has an invalid checksum.
ErrChecksum = errors.New("zlib: invalid checksum")
// ErrDictionary is returned when reading ZLIB data that has an invalid dictionary.
ErrDictionary = errors.New("zlib: invalid dictionary")
// ErrHeader is returned when reading ZLIB data that has an invalid header.
ErrHeader = errors.New("zlib: invalid header")
)
type reader struct {
r flate.Reader
decompressor io.ReadCloser
digest hash.Hash32
err error
scratch [4]byte
}
// Resetter resets a ReadCloser returned by NewReader or NewReaderDict
// to switch to a new underlying Reader. This permits reusing a ReadCloser
// instead of allocating a new one.
type Resetter interface {
// Reset discards any buffered data and resets the Resetter as if it was
// newly initialized with the given reader.
Reset(r io.Reader, dict []byte) error
}
// NewReader creates a new ReadCloser.
// Reads from the returned ReadCloser read and decompress data from r.
// If r does not implement io.ByteReader, the decompressor may read more
// data than necessary from r.
// It is the caller's responsibility to call Close on the ReadCloser when done.
//
// The ReadCloser returned by NewReader also implements Resetter.
func NewReader(r io.Reader) (io.ReadCloser, error) {
return NewReaderDict(r, nil)
}
// NewReaderDict is like NewReader but uses a preset dictionary.
// NewReaderDict ignores the dictionary if the compressed data does not refer to it.
// If the compressed data refers to a different dictionary, NewReaderDict returns ErrDictionary.
//
// The ReadCloser returned by NewReaderDict also implements Resetter.
func NewReaderDict(r io.Reader, dict []byte) (io.ReadCloser, error) {
z := new(reader)
err := z.Reset(r, dict)
if err != nil {
return nil, err
}
return z, nil
}
func (z *reader) Read(p []byte) (int, error) {
if z.err != nil {
return 0, z.err
}
var n int
n, z.err = z.decompressor.Read(p)
z.digest.Write(p[0:n])
if z.err != io.EOF {
// In the normal case we return here.
return n, z.err
}
// Finished file; check checksum.
if _, err := io.ReadFull(z.r, z.scratch[0:4]); err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
z.err = err
return n, z.err
}
// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])
if checksum != z.digest.Sum32() {
z.err = ErrChecksum
return n, z.err
}
return n, io.EOF
}
// Calling Close does not close the wrapped io.Reader originally passed to NewReader.
// In order for the ZLIB checksum to be verified, the reader must be
// fully consumed until the io.EOF.
func (z *reader) Close() error {
if z.err != nil && z.err != io.EOF {
return z.err
}
z.err = z.decompressor.Close()
return z.err
}
func (z *reader) Reset(r io.Reader, dict []byte) error {
*z = reader{decompressor: z.decompressor}
if fr, ok := r.(flate.Reader); ok {
z.r = fr
} else {
z.r = bufio.NewReader(r)
}
// Read the header (RFC 1950 section 2.2.).
_, z.err = io.ReadFull(z.r, z.scratch[0:2])
if z.err != nil {
if z.err == io.EOF {
z.err = io.ErrUnexpectedEOF
}
return z.err
}
h := uint(z.scratch[0])<<8 | uint(z.scratch[1])
if (z.scratch[0]&0x0f != zlibDeflate) || (h%31 != 0) {
z.err = ErrHeader
return z.err
}
haveDict := z.scratch[1]&0x20 != 0
if haveDict {
_, z.err = io.ReadFull(z.r, z.scratch[0:4])
if z.err != nil {
if z.err == io.EOF {
z.err = io.ErrUnexpectedEOF
}
return z.err
}
checksum := uint32(z.scratch[0])<<24 | uint32(z.scratch[1])<<16 | uint32(z.scratch[2])<<8 | uint32(z.scratch[3])
if checksum != adler32.Checksum(dict) {
z.err = ErrDictionary
return z.err
}
}
if z.decompressor == nil {
if haveDict {
z.decompressor = flate.NewReaderDict(z.r, dict)
} else {
z.decompressor = flate.NewReader(z.r)
}
} else {
z.decompressor.(flate.Resetter).Reset(z.r, dict)
}
z.digest = adler32.New()
return nil
}
image/internal/compress/zlib/reader_test.go
+179
View File
@@ -0,0 +1,179 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package zlib
import (
"bytes"
"io"
"testing"
)
type zlibTest struct {
desc string
raw string
compressed []byte
dict []byte
err error
}
// Compare-to-golden test data was generated by the ZLIB example program at
// https://www.zlib.net/zpipe.c
var zlibTests = []zlibTest{
{
"truncated empty",
"",
[]byte{},
nil,
io.ErrUnexpectedEOF,
},
{
"truncated dict",
"",
[]byte{0x78, 0xbb},
[]byte{0x00},
io.ErrUnexpectedEOF,
},
{
"truncated checksum",
"",
[]byte{0x78, 0xbb, 0x00, 0x01, 0x00, 0x01, 0xca, 0x48,
0xcd, 0xc9, 0xc9, 0xd7, 0x51, 0x28, 0xcf, 0x2f,
0xca, 0x49, 0x01, 0x04, 0x00, 0x00, 0xff, 0xff,
},
[]byte{0x00},
io.ErrUnexpectedEOF,
},
{
"empty",
"",
[]byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01},
nil,
nil,
},
{
"goodbye",
"goodbye, world",
[]byte{
0x78, 0x9c, 0x4b, 0xcf, 0xcf, 0x4f, 0x49, 0xaa,
0x4c, 0xd5, 0x51, 0x28, 0xcf, 0x2f, 0xca, 0x49,
0x01, 0x00, 0x28, 0xa5, 0x05, 0x5e,
},
nil,
nil,
},
{
"bad header",
"",
[]byte{0x78, 0x9f, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01},
nil,
ErrHeader,
},
{
"bad checksum",
"",
[]byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0xff},
nil,
ErrChecksum,
},
{
"not enough data",
"",
[]byte{0x78, 0x9c, 0x03, 0x00, 0x00, 0x00},
nil,
io.ErrUnexpectedEOF,
},
{
"excess data is silently ignored",
"",
[]byte{
0x78, 0x9c, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
0x78, 0x9c, 0xff,
},
nil,
nil,
},
{
"dictionary",
"Hello, World!\n",
[]byte{
0x78, 0xbb, 0x1c, 0x32, 0x04, 0x27, 0xf3, 0x00,
0xb1, 0x75, 0x20, 0x1c, 0x45, 0x2e, 0x00, 0x24,
0x12, 0x04, 0x74,
},
[]byte{
0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64, 0x0a,
},
nil,
},
{
"wrong dictionary",
"",
[]byte{
0x78, 0xbb, 0x1c, 0x32, 0x04, 0x27, 0xf3, 0x00,
0xb1, 0x75, 0x20, 0x1c, 0x45, 0x2e, 0x00, 0x24,
0x12, 0x04, 0x74,
},
[]byte{
0x48, 0x65, 0x6c, 0x6c,
},
ErrDictionary,
},
{
"truncated zlib stream amid raw-block",
"hello",
[]byte{
0x78, 0x9c, 0x00, 0x0c, 0x00, 0xf3, 0xff, 0x68, 0x65, 0x6c, 0x6c, 0x6f,
},
nil,
io.ErrUnexpectedEOF,
},
{
"truncated zlib stream amid fixed-block",
"He",
[]byte{
0x78, 0x9c, 0xf2, 0x48, 0xcd,
},
nil,
io.ErrUnexpectedEOF,
},
}
func TestDecompressor(t *testing.T) {
b := new(bytes.Buffer)
for _, tt := range zlibTests {
in := bytes.NewReader(tt.compressed)
zr, err := NewReaderDict(in, tt.dict)
if err != nil {
if err != tt.err {
t.Errorf("%s: NewReader: %s", tt.desc, err)
}
continue
}
defer zr.Close()
// Read and verify correctness of data.
b.Reset()
n, err := io.Copy(b, zr)
if err != nil {
if err != tt.err {
t.Errorf("%s: io.Copy: %v want %v", tt.desc, err, tt.err)
}
continue
}
s := b.String()
if s != tt.raw {
t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.desc, n, s, len(tt.raw), tt.raw)
}
// Check for sticky errors.
if n, err := zr.Read([]byte{0}); n != 0 || err != io.EOF {
t.Errorf("%s: Read() = (%d, %v), want (0, io.EOF)", tt.desc, n, err)
}
if err := zr.Close(); err != nil {
t.Errorf("%s: Close() = %v, want nil", tt.desc, err)
}
}
}
image/internal/compress/zlib/writer.go
+193
View File
@@ -0,0 +1,193 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package zlib
import (
"compress/flate"
"encoding/binary"
"fmt"
"hash"
"hash/adler32"
"io"
)
// These constants are copied from the flate package, so that code that imports
// "compress/zlib" does not also have to import "compress/flate".
const (
NoCompression = flate.NoCompression
BestSpeed = flate.BestSpeed
BestCompression = flate.BestCompression
DefaultCompression = flate.DefaultCompression
HuffmanOnly = flate.HuffmanOnly
)
// A Writer takes data written to it and writes the compressed
// form of that data to an underlying writer (see NewWriter).
type Writer struct {
w io.Writer
level int
dict []byte
compressor *flate.Writer
digest hash.Hash32
err error
scratch [4]byte
wroteHeader bool
}
// NewWriter creates a new Writer.
// Writes to the returned Writer are compressed and written to w.
//
// It is the caller's responsibility to call Close on the Writer when done.
// Writes may be buffered and not flushed until Close.
func NewWriter(w io.Writer) *Writer {
z, _ := NewWriterLevelDict(w, DefaultCompression, nil)
return z
}
// NewWriterLevel is like NewWriter but specifies the compression level instead
// of assuming DefaultCompression.
//
// The compression level can be DefaultCompression, NoCompression, HuffmanOnly
// or any integer value between BestSpeed and BestCompression inclusive.
// The error returned will be nil if the level is valid.
func NewWriterLevel(w io.Writer, level int) (*Writer, error) {
return NewWriterLevelDict(w, level, nil)
}
// NewWriterLevelDict is like NewWriterLevel but specifies a dictionary to
// compress with.
//
// The dictionary may be nil. If not, its contents should not be modified until
// the Writer is closed.
func NewWriterLevelDict(w io.Writer, level int, dict []byte) (*Writer, error) {
if level < HuffmanOnly || level > BestCompression {
return nil, fmt.Errorf("zlib: invalid compression level: %d", level)
}
return &Writer{
w: w,
level: level,
dict: dict,
}, nil
}
// Reset clears the state of the Writer z such that it is equivalent to its
// initial state from NewWriterLevel or NewWriterLevelDict, but instead writing
// to w.
func (z *Writer) Reset(w io.Writer) {
z.w = w
// z.level and z.dict left unchanged.
if z.compressor != nil {
z.compressor.Reset(w)
}
if z.digest != nil {
z.digest.Reset()
}
z.err = nil
z.scratch = [4]byte{}
z.wroteHeader = false
}
// writeHeader writes the ZLIB header.
func (z *Writer) writeHeader() (err error) {
z.wroteHeader = true
// ZLIB has a two-byte header (as documented in RFC 1950).
// The first four bits is the CINFO (compression info), which is 7 for the default deflate window size.
// The next four bits is the CM (compression method), which is 8 for deflate.
z.scratch[0] = 0x78
// The next two bits is the FLEVEL (compression level). The four values are:
// 0=fastest, 1=fast, 2=default, 3=best.
// The next bit, FDICT, is set if a dictionary is given.
// The final five FCHECK bits form a mod-31 checksum.
switch z.level {
case -2, 0, 1:
z.scratch[1] = 0 << 6
case 2, 3, 4, 5:
z.scratch[1] = 1 << 6
case 6, -1:
z.scratch[1] = 2 << 6
case 7, 8, 9:
z.scratch[1] = 3 << 6
default:
panic("unreachable")
}
if z.dict != nil {
z.scratch[1] |= 1 << 5
}
z.scratch[1] += uint8(31 - (uint16(z.scratch[0])<<8+uint16(z.scratch[1]))%31)
if _, err = z.w.Write(z.scratch[0:2]); err != nil {
return err
}
if z.dict != nil {
// The next four bytes are the Adler-32 checksum of the dictionary.
binary.BigEndian.PutUint32(z.scratch[:], adler32.Checksum(z.dict))
if _, err = z.w.Write(z.scratch[0:4]); err != nil {
return err
}
}
if z.compressor == nil {
// Initialize deflater unless the Writer is being reused
// after a Reset call.
z.compressor, err = flate.NewWriterDict(z.w, z.level, z.dict)
if err != nil {
return err
}
z.digest = adler32.New()
}
return nil
}
// Write writes a compressed form of p to the underlying io.Writer. The
// compressed bytes are not necessarily flushed until the Writer is closed or
// explicitly flushed.
func (z *Writer) Write(p []byte) (n int, err error) {
if !z.wroteHeader {
z.err = z.writeHeader()
}
if z.err != nil {
return 0, z.err
}
if len(p) == 0 {
return 0, nil
}
n, err = z.compressor.Write(p)
if err != nil {
z.err = err
return
}
z.digest.Write(p)
return
}
// Flush flushes the Writer to its underlying io.Writer.
func (z *Writer) Flush() error {
if !z.wroteHeader {
z.err = z.writeHeader()
}
if z.err != nil {
return z.err
}
z.err = z.compressor.Flush()
return z.err
}
// Close closes the Writer, flushing any unwritten data to the underlying
// io.Writer, but does not close the underlying io.Writer.
func (z *Writer) Close() error {
if !z.wroteHeader {
z.err = z.writeHeader()
}
if z.err != nil {
return z.err
}
z.err = z.compressor.Close()
if z.err != nil {
return z.err
}
checksum := z.digest.Sum32()
// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
binary.BigEndian.PutUint32(z.scratch[:], checksum)
_, z.err = z.w.Write(z.scratch[0:4])
return z.err
}
image/internal/compress/zlib/writer_test.go
+224
View File
@@ -0,0 +1,224 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package zlib
import (
"bytes"
"fmt"
"internal/testenv"
"io"
"os"
"testing"
)
var filenames = []string{
"../testdata/gettysburg.txt",
"../testdata/e.txt",
"../testdata/pi.txt",
}
var data = []string{
"test a reasonable sized string that can be compressed",
}
// Tests that compressing and then decompressing the given file at the given compression level and dictionary
// yields equivalent bytes to the original file.
func testFileLevelDict(t *testing.T, fn string, level int, d string) {
// Read the file, as golden output.
golden, err := os.Open(fn)
if err != nil {
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err)
return
}
defer golden.Close()
b0, err0 := io.ReadAll(golden)
if err0 != nil {
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err0)
return
}
testLevelDict(t, fn, b0, level, d)
}
func testLevelDict(t *testing.T, fn string, b0 []byte, level int, d string) {
// Make dictionary, if given.
var dict []byte
if d != "" {
dict = []byte(d)
}
// Push data through a pipe that compresses at the write end, and decompresses at the read end.
piper, pipew := io.Pipe()
defer piper.Close()
go func() {
defer pipew.Close()
zlibw, err := NewWriterLevelDict(pipew, level, dict)
if err != nil {
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err)
return
}
defer zlibw.Close()
_, err = zlibw.Write(b0)
if err != nil {
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err)
return
}
}()
zlibr, err := NewReaderDict(piper, dict)
if err != nil {
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err)
return
}
defer zlibr.Close()
// Compare the decompressed data.
b1, err1 := io.ReadAll(zlibr)
if err1 != nil {
t.Errorf("%s (level=%d, dict=%q): %v", fn, level, d, err1)
return
}
if len(b0) != len(b1) {
t.Errorf("%s (level=%d, dict=%q): length mismatch %d versus %d", fn, level, d, len(b0), len(b1))
return
}
for i := 0; i < len(b0); i++ {
if b0[i] != b1[i] {
t.Errorf("%s (level=%d, dict=%q): mismatch at %d, 0x%02x versus 0x%02x\n", fn, level, d, i, b0[i], b1[i])
return
}
}
}
func testFileLevelDictReset(t *testing.T, fn string, level int, dict []byte) {
var b0 []byte
var err error
if fn != "" {
b0, err = os.ReadFile(fn)
if err != nil {
t.Errorf("%s (level=%d): %v", fn, level, err)
return
}
}
// Compress once.
buf := new(bytes.Buffer)
var zlibw *Writer
if dict == nil {
zlibw, err = NewWriterLevel(buf, level)
} else {
zlibw, err = NewWriterLevelDict(buf, level, dict)
}
if err == nil {
_, err = zlibw.Write(b0)
}
if err == nil {
err = zlibw.Close()
}
if err != nil {
t.Errorf("%s (level=%d): %v", fn, level, err)
return
}
out := buf.String()
// Reset and compress again.
buf2 := new(bytes.Buffer)
zlibw.Reset(buf2)
_, err = zlibw.Write(b0)
if err == nil {
err = zlibw.Close()
}
if err != nil {
t.Errorf("%s (level=%d): %v", fn, level, err)
return
}
out2 := buf2.String()
if out2 != out {
t.Errorf("%s (level=%d): different output after reset (got %d bytes, expected %d",
fn, level, len(out2), len(out))
}
}
func TestWriter(t *testing.T) {
for i, s := range data {
b := []byte(s)
tag := fmt.Sprintf("#%d", i)
testLevelDict(t, tag, b, DefaultCompression, "")
testLevelDict(t, tag, b, NoCompression, "")
testLevelDict(t, tag, b, HuffmanOnly, "")
for level := BestSpeed; level <= BestCompression; level++ {
testLevelDict(t, tag, b, level, "")
}
}
}
func TestWriterBig(t *testing.T) {
for i, fn := range filenames {
testFileLevelDict(t, fn, DefaultCompression, "")
testFileLevelDict(t, fn, NoCompression, "")
testFileLevelDict(t, fn, HuffmanOnly, "")
for level := BestSpeed; level <= BestCompression; level++ {
testFileLevelDict(t, fn, level, "")
if level >= 1 && testing.Short() && testenv.Builder() == "" {
break
}
}
if i == 0 && testing.Short() && testenv.Builder() == "" {
break
}
}
}
func TestWriterDict(t *testing.T) {
const dictionary = "0123456789."
for i, fn := range filenames {
testFileLevelDict(t, fn, DefaultCompression, dictionary)
testFileLevelDict(t, fn, NoCompression, dictionary)
testFileLevelDict(t, fn, HuffmanOnly, dictionary)
for level := BestSpeed; level <= BestCompression; level++ {
testFileLevelDict(t, fn, level, dictionary)
if level >= 1 && testing.Short() && testenv.Builder() == "" {
break
}
}
if i == 0 && testing.Short() && testenv.Builder() == "" {
break
}
}
}
func TestWriterReset(t *testing.T) {
const dictionary = "0123456789."
for _, fn := range filenames {
testFileLevelDictReset(t, fn, NoCompression, nil)
testFileLevelDictReset(t, fn, DefaultCompression, nil)
testFileLevelDictReset(t, fn, HuffmanOnly, nil)
testFileLevelDictReset(t, fn, NoCompression, []byte(dictionary))
testFileLevelDictReset(t, fn, DefaultCompression, []byte(dictionary))
testFileLevelDictReset(t, fn, HuffmanOnly, []byte(dictionary))
if testing.Short() {
break
}
for level := BestSpeed; level <= BestCompression; level++ {
testFileLevelDictReset(t, fn, level, nil)
}
}
}
func TestWriterDictIsUsed(t *testing.T) {
var input = []byte("Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.")
var buf bytes.Buffer
compressor, err := NewWriterLevelDict(&buf, BestCompression, input)
if err != nil {
t.Errorf("error in NewWriterLevelDict: %s", err)
return
}
compressor.Write(input)
compressor.Close()
const expectedMaxSize = 25
output := buf.Bytes()
if len(output) > expectedMaxSize {
t.Errorf("result too large (got %d, want <= %d bytes). Is the dictionary being used?", len(output), expectedMaxSize)
}
}