oss/forgejo-runner
oss/forgejo-runner
1
0
Fork 0
mirror of https://code.forgejo.org/forgejo/runner.git synced 2025-08-06 17:40:58 +00:00
Code Wiki Activity

fix: partial secure cache

Browse source
This commit is contained in:
Michael Kriese 2024-11-21 22:49:12 +01:00 • committed by Kwonunn
parent ea79e3de41
commit 1082b31367
6 changed files with 143 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

123
act/artifactcache/handler.go
View file

@ -34,6 +34,7 @@ type Handler struct {
listener net.Listener
server *http.Server
logger logrus.FieldLogger
secret string
gcing atomic.Bool
gcAt time.Time
@ -41,8 +42,10 @@ type Handler struct {
outboundIP string
}
func StartHandler(dir, outboundIP string, port uint16, logger logrus.FieldLogger) (*Handler, error) {
h := &Handler{}
func StartHandler(dir, outboundIP string, port uint16, secret string, logger logrus.FieldLogger) (*Handler, error) {
h := &Handler{
secret: secret,
}
if logger == nil {
discard := logrus.New()
@ -80,12 +83,12 @@ func StartHandler(dir, outboundIP string, port uint16, logger logrus.FieldLogger
}
router := httprouter.New()
router.GET(urlBase+"/cache", h.middleware(h.find))
router.POST(urlBase+"/caches", h.middleware(h.reserve))
router.PATCH(urlBase+"/caches/:id", h.middleware(h.upload))
router.POST(urlBase+"/caches/:id", h.middleware(h.commit))
router.GET(urlBase+"/artifacts/:id", h.middleware(h.get))
router.POST(urlBase+"/clean", h.middleware(h.clean))
router.GET(cachePrefixPath+urlBase+"/cache", h.middleware(h.find))
router.POST(cachePrefixPath+urlBase+"/caches", h.middleware(h.reserve))
router.PATCH(cachePrefixPath+urlBase+"/caches/:id", h.middleware(h.upload))
router.POST(cachePrefixPath+urlBase+"/caches/:id", h.middleware(h.commit))
router.GET(cachePrefixPath+urlBase+"/artifacts/:id", h.middleware(h.get))
router.POST(cachePrefixPath+urlBase+"/clean", h.middleware(h.clean))
h.router = router
@ -155,7 +158,13 @@ func (h *Handler) openDB() (*bolthold.Store, error) {
}
// GET /_apis/artifactcache/cache
func (h *Handler) find(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
func (h *Handler) find(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
repo, err := h.validateMac(params)
if err != nil {
h.responseJSON(w, r, 500, err)
return
}
keys := strings.Split(r.URL.Query().Get("keys"), ",")
// cache keys are case insensitive
for i, key := range keys {
@ -170,7 +179,7 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, _ httprouter.Para
}
defer db.Close()
cache, err := findCache(db, keys, version)
cache, err := findCache(db, repo, keys, version)
if err != nil {
h.responseJSON(w, r, 500, err)
return
@ -196,7 +205,13 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, _ httprouter.Para
}
// POST /_apis/artifactcache/caches
func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
repo, err := h.validateMac(params)
if err != nil {
h.responseJSON(w, r, 500, err)
return
}
api := &Request{}
if err := json.NewDecoder(r.Body).Decode(api); err != nil {
h.responseJSON(w, r, 400, err)
@ -216,6 +231,7 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, _ httprouter.P
now := time.Now().Unix()
cache.CreatedAt = now
cache.UsedAt = now
cache.Repo = repo
if err := insertCache(db, cache); err != nil {
h.responseJSON(w, r, 500, err)
return
@ -227,6 +243,12 @@ func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, _ httprouter.P
// PATCH /_apis/artifactcache/caches/:id
func (h *Handler) upload(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
repo, err := h.validateMac(params)
if err != nil {
h.responseJSON(w, r, 500, err)
return
}
id, err := strconv.ParseUint(params.ByName("id"), 10, 64)
if err != nil {
h.responseJSON(w, r, 400, err)
@ -249,11 +271,17 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request, params httprout
return
}
// Should not happen
if cache.Repo != repo {
h.responseJSON(w, r, 500, ErrValidation)
return
}
if cache.Complete {
h.responseJSON(w, r, 400, fmt.Errorf("cache %v %q: already complete", cache.ID, cache.Key))
return
}
db.Close()
defer db.Close()
start, _, err := parseContentRange(r.Header.Get("Content-Range"))
if err != nil {
h.responseJSON(w, r, 400, err)
@ -262,12 +290,18 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request, params httprout
if err := h.storage.Write(cache.ID, start, r.Body); err != nil {
h.responseJSON(w, r, 500, err)
}
h.useCache(id)
h.useCache(db, cache)
h.responseJSON(w, r, 200)
}
// POST /_apis/artifactcache/caches/:id
func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
repo, err := h.validateMac(params)
if err != nil {
h.responseJSON(w, r, 500, err)
return
}
id, err := strconv.ParseUint(params.ByName("id"), 10, 64)
if err != nil {
h.responseJSON(w, r, 400, err)
@ -290,6 +324,12 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprout
return
}
// Should not happen
if cache.Repo != repo {
h.responseJSON(w, r, 500, ErrValidation)
return
}
if cache.Complete {
h.responseJSON(w, r, 400, fmt.Errorf("cache %v %q: already complete", cache.ID, cache.Key))
return
@ -323,17 +363,51 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprout
// GET /_apis/artifactcache/artifacts/:id
func (h *Handler) get(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
repo, err := h.validateMac(params)
if err != nil {
h.responseJSON(w, r, 500, err)
return
}
id, err := strconv.ParseUint(params.ByName("id"), 10, 64)
if err != nil {
h.responseJSON(w, r, 400, err)
return
}
h.useCache(id)
cache := &Cache{}
db, err := h.openDB()
if err != nil {
h.responseJSON(w, r, 500, err)
return
}
defer db.Close()
if err := db.Get(id, cache); err != nil {
if errors.Is(err, bolthold.ErrNotFound) {
h.responseJSON(w, r, 400, fmt.Errorf("cache %d: not reserved", id))
return
}
h.responseJSON(w, r, 500, err)
return
}
// Should not happen
if cache.Repo != repo {
h.responseJSON(w, r, 500, ErrValidation)
return
}
h.useCache(db, cache)
h.storage.Serve(w, r, id)
}
// POST /_apis/artifactcache/clean
func (h *Handler) clean(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
func (h *Handler) clean(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
_, err := h.validateMac(params)
if err != nil {
h.responseJSON(w, r, 500, err)
return
}
// TODO: don't support force deleting cache entries
// see: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
@ -349,12 +423,13 @@ func (h *Handler) middleware(handler httprouter.Handle) httprouter.Handle {
}
// if not found, return (nil, nil) instead of an error.
func findCache(db *bolthold.Store, keys []string, version string) (*Cache, error) {
func findCache(db *bolthold.Store, repo string, keys []string, version string) (*Cache, error) {
cache := &Cache{}
for _, prefix := range keys {
// if a key in the list matches exactly, don't return partial matches
if err := db.FindOne(cache,
bolthold.Where("Key").Eq(prefix).
bolthold.Where("Repo").Eq(repo).
And("Key").Eq(prefix).
And("Version").Eq(version).
And("Complete").Eq(true).
SortBy("CreatedAt").Reverse()); err == nil || !errors.Is(err, bolthold.ErrNotFound) {
@ -369,7 +444,8 @@ func findCache(db *bolthold.Store, keys []string, version string) (*Cache, error
continue
}
if err := db.FindOne(cache,
bolthold.Where("Key").RegExp(re).
bolthold.Where("Repo").Eq(repo).
And("Key").RegExp(re).
And("Version").Eq(version).
And("Complete").Eq(true).
SortBy("CreatedAt").Reverse()); err != nil {
@ -394,16 +470,7 @@ func insertCache(db *bolthold.Store, cache *Cache) error {
return nil
}
func (h *Handler) useCache(id uint64) {
db, err := h.openDB()
if err != nil {
return
}
defer db.Close()
cache := &Cache{}
if err := db.Get(id, cache); err != nil {
return
}
func (h *Handler) useCache(db *bolthold.Store, cache *Cache) {
cache.UsedAt = time.Now().Unix()
_ = db.Update(cache.ID, cache)
}

6
act/artifactcache/handler_test.go
View file

@ -20,9 +20,11 @@ import (
func TestHandler(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(dir, "", 0, nil)
handler, err := StartHandler(dir, "", 0, "secret", nil)
require.NoError(t, err)
t.Skip("TODO: handle secret")
base := fmt.Sprintf("%s%s", handler.ExternalURL(), urlBase)
defer func() {
@ -589,7 +591,7 @@ func uploadCacheNormally(t *testing.T, base, key, version string, content []byte
func TestHandler_gcCache(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(dir, "", 0, nil)
handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err)
defer func() {

41
act/artifactcache/mac.go Normal file
View file

@ -0,0 +1,41 @@
// Copyright 2024 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package artifactcache
import (
"crypto/hmac"
"crypto/sha256"
"errors"
"hash"
"github.com/julienschmidt/httprouter"
)
var (
ErrValidation = errors.New("repo validation error")
cachePrefixPath = "repo:/run:/time:/mac:/"
)
func (h *Handler) validateMac(params httprouter.Params) (string, error) {
repo := params.ByName("repo")
run := params.ByName("run")
time := params.ByName("time")
messageMAC := params.ByName("mac")
mac := computeMac(h.secret, repo, run, time)
expectedMAC := mac.Sum(nil)
if hmac.Equal([]byte(messageMAC), expectedMAC) {
return repo, nil
}
return repo, ErrValidation
}
func computeMac(key, repo, run, time string) hash.Hash {
mac := hmac.New(sha256.New, []byte(key))
mac.Write([]byte(repo))
mac.Write([]byte(run))
mac.Write([]byte(time))
return mac
}

1
act/artifactcache/model.go
View file

@ -25,6 +25,7 @@ func (c *Request) ToCache() *Cache {
type Cache struct {
ID uint64 `json:"id" boltholdKey:"ID"`
Repo string `json:"repo" boltholdIndex:"Repo"`
Key string `json:"key" boltholdIndex:"Key"`
Version string `json:"version" boltholdIndex:"Version"`
Size int64 `json:"cacheSize"`

1
cmd/input.go
View file

@ -59,6 +59,7 @@ type Input struct {
logPrefixJobID bool
networkName string
useNewActionCache bool
secret string
}
func (i *Input) resolve(path string) string {

2
cmd/root.go
View file

@ -632,7 +632,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str
var cacheHandler *artifactcache.Handler
if !input.noCacheServer && envs[cacheURLKey] == "" {
var err error
cacheHandler, err = artifactcache.StartHandler(input.cacheServerPath, input.cacheServerAddr, input.cacheServerPort, common.Logger(ctx))
cacheHandler, err = artifactcache.StartHandler(input.cacheServerPath, input.cacheServerAddr, input.cacheServerPort, input.secret, common.Logger(ctx))
if err != nil {
return err
}