Skip to content

Commit 17e5901

Browse files
andybalholmandybalholm
committed
Make my matchfinder work more accessible.
1 parent cf812c0 commit 17e5901

File tree

4 files changed

+118
-41
lines changed

4 files changed

+118
-41
lines changed

‎README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,13 @@ This package is a brotli compressor and decompressor implemented in Go.
22
It was translated from the reference implementation (https://github.com/google/brotli)
33
with the `c2go` tool at https://github.com/andybalholm/c2go.
44

5+
I have been working on new compression algorithms (not translated from C)
6+
in the matchfinder package.
7+
You can use them with the NewWriterV2 function.
8+
Currently they give better results than the old implementation
9+
(at least for compressing my test file, Newton’s *Opticks*)
10+
on levels 2 to 6.
11+
512
I am using it in production with https://github.com/andybalholm/redwood.
613

714
API documentation is found at https://pkg.go.dev/github.com/andybalholm/brotli?tab=doc.

‎brotli_test.go‎

Lines changed: 67 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -69,46 +69,6 @@ func TestEncoderEmptyWrite(t *testing.T) {
6969
t.Errorf("Close()=%v, want nil", err)
7070
}
7171
}
72-
73-
func TestWriter(t *testing.T) {
74-
for level := BestSpeed; level <= BestCompression; level++ {
75-
// Test basic encoder usage.
76-
input := []byte("<html><body><H1>Hello world</H1></body></html>")
77-
out := bytes.Buffer{}
78-
e := NewWriterOptions(&out, WriterOptions{Quality: level})
79-
in := bytes.NewReader([]byte(input))
80-
n, err := io.Copy(e, in)
81-
if err != nil {
82-
t.Errorf("Copy Error: %v", err)
83-
}
84-
if int(n) != len(input) {
85-
t.Errorf("Copy() n=%v, want %v", n, len(input))
86-
}
87-
if err := e.Close(); err != nil {
88-
t.Errorf("Close Error after copied %d bytes: %v", n, err)
89-
}
90-
if err := checkCompressedData(out.Bytes(), input); err != nil {
91-
t.Error(err)
92-
}
93-
94-
out2 := bytes.Buffer{}
95-
e.Reset(&out2)
96-
n2, err := e.Write(input)
97-
if err != nil {
98-
t.Errorf("Write error after Reset: %v", err)
99-
}
100-
if n2 != len(input) {
101-
t.Errorf("Write() after Reset n=%d, want %d", n2, len(input))
102-
}
103-
if err := e.Close(); err != nil {
104-
t.Errorf("Close error after Reset (copied %d) bytes: %v", n2, err)
105-
}
106-
if !bytes.Equal(out.Bytes(), out2.Bytes()) {
107-
t.Error("Compressed data after Reset doesn't equal first time")
108-
}
109-
}
110-
}
111-
11272
func TestIssue22(t *testing.T) {
11373
f, err := os.Open("testdata/issue22.gz")
11474
if err != nil {
@@ -149,6 +109,45 @@ func TestIssue22(t *testing.T) {
149109
}
150110
}
151111

112+
func TestWriterV2(t *testing.T) {
113+
for level := BestSpeed; level <= BestCompression; level++ {
114+
// Test basic encoder usage.
115+
input := []byte("<html><body><H1>Hello world</H1></body></html>")
116+
out := bytes.Buffer{}
117+
e := NewWriterV2(&out, level)
118+
in := bytes.NewReader([]byte(input))
119+
n, err := io.Copy(e, in)
120+
if err != nil {
121+
t.Errorf("Copy Error: %v", err)
122+
}
123+
if int(n) != len(input) {
124+
t.Errorf("Copy() n=%v, want %v", n, len(input))
125+
}
126+
if err := e.Close(); err != nil {
127+
t.Errorf("Close Error after copied %d bytes: %v", n, err)
128+
}
129+
if err := checkCompressedData(out.Bytes(), input); err != nil {
130+
t.Error(err)
131+
}
132+
133+
out2 := bytes.Buffer{}
134+
e.Reset(&out2)
135+
n2, err := e.Write(input)
136+
if err != nil {
137+
t.Errorf("Write error after Reset: %v", err)
138+
}
139+
if n2 != len(input) {
140+
t.Errorf("Write() after Reset n=%d, want %d", n2, len(input))
141+
}
142+
if err := e.Close(); err != nil {
143+
t.Errorf("Close error after Reset (copied %d) bytes: %v", n2, err)
144+
}
145+
if !bytes.Equal(out.Bytes(), out2.Bytes()) {
146+
t.Error("Compressed data after Reset doesn't equal first time")
147+
}
148+
}
149+
}
150+
152151
func TestEncoderStreams(t *testing.T) {
153152
// Test that output is streamed.
154153
// Adjust window size to ensure the encoder outputs at least enough bytes
@@ -576,6 +575,30 @@ func BenchmarkEncodeLevelsReset(b *testing.B) {
576575
}
577576
}
578577

578+
func BenchmarkEncodeLevelsResetV2(b *testing.B) {
579+
opticks, err := ioutil.ReadFile("testdata/Isaac.Newton-Opticks.txt")
580+
if err != nil {
581+
b.Fatal(err)
582+
}
583+
584+
for level := BestSpeed; level <= 7; level++ {
585+
buf := new(bytes.Buffer)
586+
w := NewWriterV2(buf, level)
587+
w.Write(opticks)
588+
w.Close()
589+
b.Run(fmt.Sprintf("%d", level), func(b *testing.B) {
590+
b.ReportAllocs()
591+
b.ReportMetric(float64(len(opticks))/float64(buf.Len()), "ratio")
592+
b.SetBytes(int64(len(opticks)))
593+
for i := 0; i < b.N; i++ {
594+
w.Reset(ioutil.Discard)
595+
w.Write(opticks)
596+
w.Close()
597+
}
598+
})
599+
}
600+
}
601+
579602
func BenchmarkDecodeLevels(b *testing.B) {
580603
opticks, err := ioutil.ReadFile("testdata/Isaac.Newton-Opticks.txt")
581604
if err != nil {
@@ -694,6 +717,10 @@ func BenchmarkEncodeM4Chain128(b *testing.B) {
694717
benchmark(b, "testdata/Isaac.Newton-Opticks.txt", &matchfinder.M4{MaxDistance: 1 << 20, ChainLength: 128, HashLen: 5, DistanceBitCost: 57}, 1<<16)
695718
}
696719

720+
func BenchmarkEncodeM4Chain256(b *testing.B) {
721+
benchmark(b, "testdata/Isaac.Newton-Opticks.txt", &matchfinder.M4{MaxDistance: 1 << 20, ChainLength: 256, HashLen: 5, DistanceBitCost: 57}, 1<<16)
722+
}
723+
697724
func TestEncodeM0(t *testing.T) {
698725
test(t, "testdata/Isaac.Newton-Opticks.txt", matchfinder.M0{}, 1<<16)
699726
}

‎http.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ func HTTPCompressor(w http.ResponseWriter, r *http.Request) io.WriteCloser {
2020
switch encoding {
2121
case "br":
2222
w.Header().Set("Content-Encoding", "br")
23-
return NewWriter(w)
23+
return NewWriterV2(w, DefaultCompression)
2424
case "gzip":
2525
w.Header().Set("Content-Encoding", "gzip")
2626
return gzip.NewWriter(w)

‎writer.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package brotli
33
import (
44
"errors"
55
"io"
6+
7+
"github.com/andybalholm/brotli/matchfinder"
68
)
79

810
const (
@@ -117,3 +119,44 @@ type nopCloser struct {
117119
}
118120

119121
func (nopCloser) Close() error { return nil }
122+
123+
// NewWriterV2 is like NewWriterLevel, but it uses the new implementation
124+
// based on the matchfinder package. It currently supports up to level 7;
125+
// if a higher level is specified, level 7 will be used.
126+
func NewWriterV2(dst io.Writer, level int) *matchfinder.Writer {
127+
var mf matchfinder.MatchFinder
128+
if level < 2 {
129+
mf = matchfinder.M0{Lazy: level == 1}
130+
} else {
131+
hashLen := 6
132+
if level >= 6 {
133+
hashLen = 5
134+
}
135+
chainLen := 64
136+
switch level {
137+
case 2:
138+
chainLen = 0
139+
case 3:
140+
chainLen = 1
141+
case 4:
142+
chainLen = 2
143+
case 5:
144+
chainLen = 4
145+
case 6:
146+
chainLen = 8
147+
}
148+
mf = &matchfinder.M4{
149+
MaxDistance: 1 << 20,
150+
ChainLength: chainLen,
151+
HashLen: hashLen,
152+
DistanceBitCost: 57,
153+
}
154+
}
155+
156+
return &matchfinder.Writer{
157+
Dest: dst,
158+
MatchFinder: mf,
159+
Encoder: &Encoder{},
160+
BlockSize: 1 << 16,
161+
}
162+
}

0 commit comments

Comments
 (0)