[v11.0/forgejo] storage test: reader should not be closed on save (#9031)

Backport of the fixes for #8529 (v11 is currently not affected because minio-client is an older version. Porting the tests should help in preventing future breakages).

- #8541: storage test (reader should not get closed)
- #8527 #8816: defer uploader.Close (most robust way)
- #8166: enable storage tests

The test can be run locally:

```
docker run --rm -e MINIO_DOMAIN=minio -e MINIO_ROOT_USER=123456 -e MINIO_ROOT_PASSWORD=12345678 -p 9000:9000  data.forgejo.org/oci/bitnami/minio:2024.8.17
```

```
TEST_MINIO_ENDPOINT=localhost:9000  go test -v -run ^TestMinioStorageIterator$ ./modules/storage
```

### Release notes

- [x] I do not want this change to show in the release notes.
- [ ] I want the title to show in the release notes with a link to this pull request.
- [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title.

Co-authored-by: Earl Warren <contact@earl-warren.org>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/9031
Reviewed-by: Michael Kriese <michael.kriese@gmx.de>
Co-authored-by: oliverpool <git@olivier.pfad.fr>
Co-committed-by: oliverpool <git@olivier.pfad.fr>

.forgejo/workflows/testing.yml

@@ -91,6 +91,7 @@ jobs:
           RACE_ENABLED: 'true'
           TAGS: bindata
           TEST_ELASTICSEARCH_URL: http://elasticsearch:9200
+          TEST_MINIO_ENDPOINT: minio:9000
   test-e2e:
     if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing'
     runs-on: docker

modules/indexer/issues/elasticsearch/elasticsearch_test.go

@@ -14,16 +14,10 @@ import (
 )
 
 func TestElasticsearchIndexer(t *testing.T) {
-	// The elasticsearch instance started by testing.yml > test-unit > services > elasticsearch
+	url := os.Getenv("TEST_ELASTICSEARCH_URL")
-	url := "http://elastic:changeme@elasticsearch:9200"
+	if url == "" {
-
+		t.Skip("TEST_ELASTICSEARCH_URL not set")
-	if os.Getenv("CI") == "" {
+		return
-		// Make it possible to run tests against a local elasticsearch instance
-		url = os.Getenv("TEST_ELASTICSEARCH_URL")
-		if url == "" {
-			t.Skip("TEST_ELASTICSEARCH_URL not set and not running in CI")
-			return
-		}
 	}
 
 	ok := false

modules/storage/minio_test.go

@@ -18,13 +18,14 @@ import (
 )
 
 func TestMinioStorageIterator(t *testing.T) {
-	if os.Getenv("CI") == "" {
+	endpoint := os.Getenv("TEST_MINIO_ENDPOINT")
-		t.Skip("minioStorage not present outside of CI")
+	if endpoint == "" {
+		t.Skip("TEST_MINIO_ENDPOINT not set")
 		return
 	}
 	testStorageIterator(t, setting.MinioStorageType, &setting.Storage{
 		MinioConfig: setting.MinioStorageConfig{
-			Endpoint:        "minio:9000",
+			Endpoint:        endpoint,
 			AccessKeyID:     "123456",
 			SecretAccessKey: "12345678",
 			Bucket:          "gitea",
@@ -34,13 +35,14 @@ func TestMinioStorageIterator(t *testing.T) {
 }
 
 func TestVirtualHostMinioStorage(t *testing.T) {
-	if os.Getenv("CI") == "" {
+	endpoint := os.Getenv("TEST_MINIO_ENDPOINT")
-		t.Skip("minioStorage not present outside of CI")
+	if endpoint == "" {
+		t.Skip("TEST_MINIO_ENDPOINT not set")
 		return
 	}
 	testStorageIterator(t, setting.MinioStorageType, &setting.Storage{
 		MinioConfig: setting.MinioStorageConfig{
-			Endpoint:        "minio:9000",
+			Endpoint:        endpoint,
 			AccessKeyID:     "123456",
 			SecretAccessKey: "12345678",
 			Bucket:          "gitea",
@@ -85,13 +87,14 @@ func TestMinioStoragePath(t *testing.T) {
 }
 
 func TestS3StorageBadRequest(t *testing.T) {
-	if os.Getenv("CI") == "" {
+	endpoint := os.Getenv("TEST_MINIO_ENDPOINT")
-		t.Skip("S3Storage not present outside of CI")
+	if endpoint == "" {
+		t.Skip("TEST_MINIO_ENDPOINT not set")
 		return
 	}
 	cfg := &setting.Storage{
 		MinioConfig: setting.MinioStorageConfig{
-			Endpoint:        "minio:9000",
+			Endpoint:        endpoint,
 			AccessKeyID:     "123456",
 			SecretAccessKey: "12345678",
 			Bucket:          "bucket",

modules/storage/storage_test.go

@@ -5,6 +5,7 @@ package storage
 
 import (
 	"bytes"
+	"io"
 	"testing"
 
 	"forgejo.org/modules/setting"
@@ -13,22 +14,39 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+type spyCloser struct {
+	io.Reader
+	closed int
+}
+
+func (s *spyCloser) Close() error {
+	s.closed++
+	return nil
+}
+
+var _ io.ReadCloser = &spyCloser{}
+
 func testStorageIterator(t *testing.T, typStr Type, cfg *setting.Storage) {
 	l, err := NewStorage(typStr, cfg)
 	require.NoError(t, err)
 
-	testFiles := [][]string{
+	testFiles := []struct {
-		{"a/1.txt", "a1"},
+		path, content string
-		{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
+		size          int64
-		{"ab/1.txt", "ab1"},
+	}{
-		{"b/1.txt", "b1"},
+		{"a/1.txt", "a1", -1},
-		{"b/2.txt", "b2"},
+		{"/a/1.txt", "aa1", -1}, // same as above, but with leading slash that will be trim
-		{"b/3.txt", "b3"},
+		{"ab/1.txt", "ab1", 3},
-		{"b/x 4.txt", "bx4"},
+		{"b/1.txt", "b1", 2}, // minio closes when the size is set
+		{"b/2.txt", "b2", -1},
+		{"b/3.txt", "b3", -1},
+		{"b/x 4.txt", "bx4", -1},
 	}
 	for _, f := range testFiles {
-		_, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1)
+		sc := &spyCloser{bytes.NewBufferString(f.content), 0}
+		_, err = l.Save(f.path, sc, f.size)
 		require.NoError(t, err)
+		assert.Equal(t, 0, sc.closed)
 	}
 
 	expectedList := map[string][]string{

routers/api/packages/container/container.go

@@ -389,12 +389,7 @@ func EndUploadBlob(ctx *context.Context) {
 		}
 		return
 	}
-	doClose := true
+	defer uploader.Close()
-	defer func() {
-		if doClose {
-			uploader.Close()
-		}
-	}()
 
 	if ctx.Req.Body != nil {
 		if err := uploader.Append(ctx, ctx.Req.Body); err != nil {
@@ -427,12 +422,6 @@ func EndUploadBlob(ctx *context.Context) {
 		return
 	}
 
-	if err := uploader.Close(); err != nil {
-		apiError(ctx, http.StatusInternalServerError, err)
-		return
-	}
-	doClose = false
-
 	if err := container_service.RemoveBlobUploadByID(ctx, uploader.ID); err != nil {
 		apiError(ctx, http.StatusInternalServerError, err)
 		return