mirror of
https://code.forgejo.org/forgejo/runner.git
synced 2025-09-15 18:57:01 +00:00
add unit tests for all changes in artifactcache
This commit is contained in:
Mathieu Fenniak
Earl Warren
parent
da7ef7c2a1
commit
6c35ea4fd9
1 changed files with 227 additions and 9 deletions
|
|
@ -12,6 +12,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
"code.forgejo.org/forgejo/runner/v9/testutils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/timshannon/bolthold"
|
||||
|
|
@ -28,7 +29,8 @@ const (
|
|||
var handlerExternalURL string
|
||||
|
||||
type AuthHeaderTransport struct {
|
||||
T http.RoundTripper
|
||||
T http.RoundTripper
|
||||
WriteIsolationKey string
|
||||
}
|
||||
|
||||
func (t *AuthHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
|
|
@ -37,11 +39,14 @@ func (t *AuthHeaderTransport) RoundTrip(req *http.Request) (*http.Response, erro
|
|||
req.Header.Set("Forgejo-Cache-Timestamp", cacheTimestamp)
|
||||
req.Header.Set("Forgejo-Cache-MAC", cacheMac)
|
||||
req.Header.Set("Forgejo-Cache-Host", handlerExternalURL)
|
||||
if t.WriteIsolationKey != "" {
|
||||
req.Header.Set("Forgejo-Cache-WriteIsolationKey", t.WriteIsolationKey)
|
||||
}
|
||||
return t.T.RoundTrip(req)
|
||||
}
|
||||
|
||||
var (
|
||||
httpClientTransport = AuthHeaderTransport{http.DefaultTransport}
|
||||
httpClientTransport = AuthHeaderTransport{T: http.DefaultTransport}
|
||||
httpClient = http.Client{Transport: &httpClientTransport}
|
||||
)
|
||||
|
||||
|
|
@ -88,7 +93,7 @@ func TestHandler(t *testing.T) {
|
|||
content := make([]byte, 100)
|
||||
_, err := rand.Read(content)
|
||||
require.NoError(t, err)
|
||||
uploadCacheNormally(t, base, key, version, content)
|
||||
uploadCacheNormally(t, base, key, version, "", content)
|
||||
})
|
||||
|
||||
t.Run("clean", func(t *testing.T) {
|
||||
|
|
@ -380,7 +385,7 @@ func TestHandler(t *testing.T) {
|
|||
_, err := rand.Read(content)
|
||||
require.NoError(t, err)
|
||||
|
||||
uploadCacheNormally(t, base, key, version, content)
|
||||
uploadCacheNormally(t, base, key, version, "", content)
|
||||
|
||||
// Perform the request with the custom `httpClient` which will send correct MAC data
|
||||
resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version))
|
||||
|
|
@ -416,7 +421,7 @@ func TestHandler(t *testing.T) {
|
|||
for i := range contents {
|
||||
_, err := rand.Read(contents[i])
|
||||
require.NoError(t, err)
|
||||
uploadCacheNormally(t, base, keys[i], version, contents[i])
|
||||
uploadCacheNormally(t, base, keys[i], version, "", contents[i])
|
||||
time.Sleep(time.Second) // ensure CreatedAt of caches are different
|
||||
}
|
||||
|
||||
|
|
@ -454,13 +459,91 @@ func TestHandler(t *testing.T) {
|
|||
assert.Equal(t, contents[except], content)
|
||||
})
|
||||
|
||||
t.Run("find can't match without WriteIsolationKey match", func(t *testing.T) {
|
||||
defer func() { httpClientTransport.WriteIsolationKey = "" }()
|
||||
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
||||
key := strings.ToLower(t.Name())
|
||||
|
||||
uploadCacheNormally(t, base, key, version, "TestWriteKey", make([]byte, 64))
|
||||
|
||||
httpClientTransport.WriteIsolationKey = "AnotherTestWriteKey"
|
||||
resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 204, resp.StatusCode)
|
||||
|
||||
httpClientTransport.WriteIsolationKey = ""
|
||||
resp, err = httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 204, resp.StatusCode)
|
||||
|
||||
httpClientTransport.WriteIsolationKey = "TestWriteKey"
|
||||
resp, err = httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("find prefers WriteIsolationKey match", func(t *testing.T) {
|
||||
defer func() { httpClientTransport.WriteIsolationKey = "" }()
|
||||
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d21"
|
||||
key := strings.ToLower(t.Name())
|
||||
|
||||
// Between two values with the same `key`...
|
||||
uploadCacheNormally(t, base, key, version, "TestWriteKey", make([]byte, 64))
|
||||
uploadCacheNormally(t, base, key, version, "", make([]byte, 128))
|
||||
|
||||
// We should read the value with the matching WriteIsolationKey from the cache...
|
||||
httpClientTransport.WriteIsolationKey = "TestWriteKey"
|
||||
resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
|
||||
got := struct {
|
||||
ArchiveLocation string `json:"archiveLocation"`
|
||||
}{}
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
||||
contentResp, err := httpClient.Get(got.ArchiveLocation)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, contentResp.StatusCode)
|
||||
content, err := io.ReadAll(contentResp.Body)
|
||||
require.NoError(t, err)
|
||||
// Which we finally check matches the correct WriteIsolationKey's content here.
|
||||
assert.Equal(t, make([]byte, 64), content)
|
||||
})
|
||||
|
||||
t.Run("find falls back if matching WriteIsolationKey not available", func(t *testing.T) {
|
||||
defer func() { httpClientTransport.WriteIsolationKey = "" }()
|
||||
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d21"
|
||||
key := strings.ToLower(t.Name())
|
||||
|
||||
uploadCacheNormally(t, base, key, version, "", make([]byte, 128))
|
||||
|
||||
httpClientTransport.WriteIsolationKey = "TestWriteKey"
|
||||
resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
|
||||
got := struct {
|
||||
ArchiveLocation string `json:"archiveLocation"`
|
||||
}{}
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
||||
contentResp, err := httpClient.Get(got.ArchiveLocation)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, contentResp.StatusCode)
|
||||
content, err := io.ReadAll(contentResp.Body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, make([]byte, 128), content)
|
||||
})
|
||||
|
||||
t.Run("case insensitive", func(t *testing.T) {
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
||||
key := strings.ToLower(t.Name())
|
||||
content := make([]byte, 100)
|
||||
_, err := rand.Read(content)
|
||||
require.NoError(t, err)
|
||||
uploadCacheNormally(t, base, key+"_ABC", version, content)
|
||||
uploadCacheNormally(t, base, key+"_ABC", version, "", content)
|
||||
|
||||
{
|
||||
reqKey := key + "_aBc"
|
||||
|
|
@ -494,7 +577,7 @@ func TestHandler(t *testing.T) {
|
|||
for i := range contents {
|
||||
_, err := rand.Read(contents[i])
|
||||
require.NoError(t, err)
|
||||
uploadCacheNormally(t, base, keys[i], version, contents[i])
|
||||
uploadCacheNormally(t, base, keys[i], version, "", contents[i])
|
||||
time.Sleep(time.Second) // ensure CreatedAt of caches are different
|
||||
}
|
||||
|
||||
|
|
@ -545,7 +628,7 @@ func TestHandler(t *testing.T) {
|
|||
for i := range contents {
|
||||
_, err := rand.Read(contents[i])
|
||||
require.NoError(t, err)
|
||||
uploadCacheNormally(t, base, keys[i], version, contents[i])
|
||||
uploadCacheNormally(t, base, keys[i], version, "", contents[i])
|
||||
time.Sleep(time.Second) // ensure CreatedAt of caches are different
|
||||
}
|
||||
|
||||
|
|
@ -581,9 +664,144 @@ func TestHandler(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
assert.Equal(t, contents[expect], content)
|
||||
})
|
||||
|
||||
t.Run("upload across WriteIsolationKey", func(t *testing.T) {
|
||||
defer testutils.MockVariable(&httpClientTransport.WriteIsolationKey, "CorrectKey")()
|
||||
|
||||
key := strings.ToLower(t.Name())
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
||||
content := make([]byte, 256)
|
||||
|
||||
var id uint64
|
||||
// reserve
|
||||
{
|
||||
body, err := json.Marshal(&Request{
|
||||
Key: key,
|
||||
Version: version,
|
||||
Size: int64(len(content)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
resp, err := httpClient.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 200, resp.StatusCode)
|
||||
|
||||
got := struct {
|
||||
CacheID uint64 `json:"cacheId"`
|
||||
}{}
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
||||
id = got.CacheID
|
||||
}
|
||||
// upload, but with the incorrect write isolation key relative to the cache obj created
|
||||
{
|
||||
httpClientTransport.WriteIsolationKey = "WrongKey"
|
||||
req, err := http.NewRequest(http.MethodPatch,
|
||||
fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
req.Header.Set("Content-Range", "bytes 0-99/*")
|
||||
resp, err := httpClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 403, resp.StatusCode)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("commit across WriteIsolationKey", func(t *testing.T) {
|
||||
defer testutils.MockVariable(&httpClientTransport.WriteIsolationKey, "CorrectKey")()
|
||||
|
||||
key := strings.ToLower(t.Name())
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
||||
content := make([]byte, 256)
|
||||
|
||||
var id uint64
|
||||
// reserve
|
||||
{
|
||||
body, err := json.Marshal(&Request{
|
||||
Key: key,
|
||||
Version: version,
|
||||
Size: int64(len(content)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
resp, err := httpClient.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 200, resp.StatusCode)
|
||||
|
||||
got := struct {
|
||||
CacheID uint64 `json:"cacheId"`
|
||||
}{}
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
||||
id = got.CacheID
|
||||
}
|
||||
// upload
|
||||
{
|
||||
req, err := http.NewRequest(http.MethodPatch,
|
||||
fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content))
|
||||
require.NoError(t, err)
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
req.Header.Set("Content-Range", "bytes 0-99/*")
|
||||
resp, err := httpClient.Do(req)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 200, resp.StatusCode)
|
||||
}
|
||||
// commit, but with the incorrect write isolation key relative to the cache obj created
|
||||
{
|
||||
httpClientTransport.WriteIsolationKey = "WrongKey"
|
||||
resp, err := httpClient.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 403, resp.StatusCode)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("get across WriteIsolationKey", func(t *testing.T) {
|
||||
defer func() { httpClientTransport.WriteIsolationKey = "" }()
|
||||
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d21"
|
||||
key := strings.ToLower(t.Name())
|
||||
uploadCacheNormally(t, base, key, version, "", make([]byte, 128))
|
||||
keyIsolated := strings.ToLower(t.Name()) + "_isolated"
|
||||
uploadCacheNormally(t, base, keyIsolated, version, "CorrectKey", make([]byte, 128))
|
||||
|
||||
// Perform the 'get' without the right WriteIsolationKey for the cache entry... should be OK for `key` since it
|
||||
// was written with WriteIsolationKey "" meaning it is available for non-isolated access
|
||||
{
|
||||
httpClientTransport.WriteIsolationKey = "WhoopsWrongKey"
|
||||
|
||||
resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
got := struct {
|
||||
ArchiveLocation string `json:"archiveLocation"`
|
||||
}{}
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
||||
|
||||
contentResp, err := httpClient.Get(got.ArchiveLocation)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, contentResp.StatusCode)
|
||||
httpClientTransport.WriteIsolationKey = "CorrectKey" // reset for next find
|
||||
}
|
||||
|
||||
// Perform the 'get' without the right WriteIsolationKey for the cache entry... should be 403 for `keyIsolated`
|
||||
// because it was written with a different WriteIsolationKey.
|
||||
{
|
||||
httpClientTransport.WriteIsolationKey = "CorrectKey" // for test purposes make the `find` successful...
|
||||
resp, err := httpClient.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, keyIsolated, version))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 200, resp.StatusCode)
|
||||
got := struct {
|
||||
ArchiveLocation string `json:"archiveLocation"`
|
||||
}{}
|
||||
require.NoError(t, json.NewDecoder(resp.Body).Decode(&got))
|
||||
|
||||
httpClientTransport.WriteIsolationKey = "WhoopsWrongKey" // but then access w/ the wrong key for `get`
|
||||
contentResp, err := httpClient.Get(got.ArchiveLocation)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 403, contentResp.StatusCode)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func uploadCacheNormally(t *testing.T, base, key, version string, content []byte) {
|
||||
func uploadCacheNormally(t *testing.T, base, key, version, writeIsolationKey string, content []byte) {
|
||||
defer testutils.MockVariable(&httpClientTransport.WriteIsolationKey, writeIsolationKey)()
|
||||
|
||||
var id uint64
|
||||
{
|
||||
body, err := json.Marshal(&Request{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue