oss/forgejo
oss/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2025-10-20 19:52:04 +00:00
Code Wiki Activity

blob: GetBlobContent: reduce allocations (#8223)

Browse source 
See #8222 for context.

## git.Blob.NewTruncatedReader

This introduce a new `NewTruncatedReader` method to return a blob-reader which silently truncates when the limit is reached (io.EOF will be returned).
Since the actual size is also returned `GetBlobContent` can pre-allocate a `[]byte` of the full-size (min of the asked size and the actual size) and call `io.ReadFull(rc, buf)` (instead of `util.ReadWithLimit(dataRc, int(limit))` which is convoluted and not used anywhere else).

### Tests

- I added test coverage for Go changes...
  - [x] in their respective `*_test.go` for unit tests.

### Documentation

- [x] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [x] I do not want this change to show in the release notes.

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/8223
Reviewed-by: Lucas <sclu1034@noreply.codeberg.org>
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: oliverpool <git@olivier.pfad.fr>
Co-committed-by: oliverpool <git@olivier.pfad.fr>
This commit is contained in:
oliverpool 2025-06-19 18:36:12 +02:00 • committed by Earl Warren
parent 913eaffb8a
commit 31ad7c9353
4 changed files with 95 additions and 135 deletions
Download patch file Download diff file Expand all files Collapse all files

37
modules/util/io.go
View file

@ -4,7 +4,6 @@
package util
import (
"bytes"
"errors"
"io"
)
@ -20,42 +19,6 @@ func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
return n, err
}
// ReadWithLimit reads at most "limit" bytes from r into buf.
// If EOF or ErrUnexpectedEOF occurs while reading, err will be nil.
func ReadWithLimit(r io.Reader, n int) (buf []byte, err error) {
return readWithLimit(r, 1024, n)
}
func readWithLimit(r io.Reader, batch, limit int) ([]byte, error) {
if limit <= batch {
buf := make([]byte, limit)
n, err := ReadAtMost(r, buf)
if err != nil {
return nil, err
}
return buf[:n], nil
}
res := bytes.NewBuffer(make([]byte, 0, batch))
bufFix := make([]byte, batch)
eof := false
for res.Len() < limit && !eof {
bufTmp := bufFix
if res.Len()+batch > limit {
bufTmp = bufFix[:limit-res.Len()]
}
n, err := io.ReadFull(r, bufTmp)
if err == io.EOF || err == io.ErrUnexpectedEOF {
eof = true
} else if err != nil {
return nil, err
}
if _, err = res.Write(bufTmp[:n]); err != nil {
return nil, err
}
}
return res.Bytes(), nil
}
// ErrNotEmpty is an error reported when there is a non-empty reader
var ErrNotEmpty = errors.New("not-empty")

67
modules/util/io_test.go
View file

@ -1,67 +0,0 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package util
import (
"bytes"
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type readerWithError struct {
buf *bytes.Buffer
}
func (r *readerWithError) Read(p []byte) (n int, err error) {
if r.buf.Len() < 2 {
return 0, errors.New("test error")
}
return r.buf.Read(p)
}
func TestReadWithLimit(t *testing.T) {
bs := []byte("0123456789abcdef")
// normal test
buf, err := readWithLimit(bytes.NewBuffer(bs), 5, 2)
require.NoError(t, err)
assert.Equal(t, []byte("01"), buf)
buf, err = readWithLimit(bytes.NewBuffer(bs), 5, 5)
require.NoError(t, err)
assert.Equal(t, []byte("01234"), buf)
buf, err = readWithLimit(bytes.NewBuffer(bs), 5, 6)
require.NoError(t, err)
assert.Equal(t, []byte("012345"), buf)
buf, err = readWithLimit(bytes.NewBuffer(bs), 5, len(bs))
require.NoError(t, err)
assert.Equal(t, []byte("0123456789abcdef"), buf)
buf, err = readWithLimit(bytes.NewBuffer(bs), 5, 100)
require.NoError(t, err)
assert.Equal(t, []byte("0123456789abcdef"), buf)
// test with error
buf, err = readWithLimit(&readerWithError{bytes.NewBuffer(bs)}, 5, 10)
require.NoError(t, err)
assert.Equal(t, []byte("0123456789"), buf)
buf, err = readWithLimit(&readerWithError{bytes.NewBuffer(bs)}, 5, 100)
require.ErrorContains(t, err, "test error")
assert.Empty(t, buf)
// test public function
buf, err = ReadWithLimit(bytes.NewBuffer(bs), 2)
require.NoError(t, err)
assert.Equal(t, []byte("01"), buf)
buf, err = ReadWithLimit(bytes.NewBuffer(bs), 9999999)
require.NoError(t, err)
assert.Equal(t, []byte("0123456789abcdef"), buf)
}