From f3daf48fef659a474c3af0f44eab90c0cee90c37 Mon Sep 17 00:00:00 2001 From: Shubh Bapna <38372682+shubhbapna@users.noreply.github.com> Date: Wed, 3 May 2023 11:46:28 -0400 Subject: [PATCH 1/9] fix: remove hardcoded reference to github.com when using reusable remote workflows and remote actions (#1784) * fix filename for remote reusable workflow and remove hardcoded reference to github.com * remove hardcoded reference to github.com for remote action --- act/runner/reusable_workflow.go | 26 ++++++++++++++++---------- act/runner/step_action_remote.go | 11 +++++------ 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/act/runner/reusable_workflow.go b/act/runner/reusable_workflow.go index 1ffa22b7..67e04037 100644 --- a/act/runner/reusable_workflow.go +++ b/act/runner/reusable_workflow.go @@ -26,9 +26,12 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor { if remoteReusableWorkflow == nil { return common.NewErrorExecutor(fmt.Errorf("expected format {owner}/{repo}/.github/workflows/{filename}@{ref}. Actual '%s' Input string was not in a correct format", uses)) } - remoteReusableWorkflow.URL = rc.Config.GitHubInstance - workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(uses)) + // uses with safe filename makes the target directory look something like this {owner}-{repo}-.github-workflows-{filename}@{ref} + // instead we will just use {owner}-{repo}@{ref} as our target directory. This should also improve performance when we are using + // multiple reusable workflows from the same repository and ref since for each workflow we won't have to clone it again + filename := fmt.Sprintf("%s/%s@%s", remoteReusableWorkflow.Org, remoteReusableWorkflow.Repo, remoteReusableWorkflow.Ref) + workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(filename)) return common.NewPipelineExecutor( newMutexExecutor(cloneIfRequired(rc, *remoteReusableWorkflow, workflowDir)), @@ -56,12 +59,15 @@ func cloneIfRequired(rc *RunContext, remoteReusableWorkflow remoteReusableWorkfl notExists := errors.Is(err, fs.ErrNotExist) return notExists }, - git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{ - URL: remoteReusableWorkflow.CloneURL(), - Ref: remoteReusableWorkflow.Ref, - Dir: targetDirectory, - Token: rc.Config.Token, - }), + func(ctx context.Context) error { + remoteReusableWorkflow.URL = rc.getGithubContext(ctx).ServerURL + return git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{ + URL: remoteReusableWorkflow.CloneURL(), + Ref: remoteReusableWorkflow.Ref, + Dir: targetDirectory, + Token: rc.Config.Token, + })(ctx) + }, nil, ) } @@ -108,7 +114,7 @@ type remoteReusableWorkflow struct { } func (r *remoteReusableWorkflow) CloneURL() string { - return fmt.Sprintf("https://%s/%s/%s", r.URL, r.Org, r.Repo) + return fmt.Sprintf("%s/%s/%s", r.URL, r.Org, r.Repo) } func newRemoteReusableWorkflow(uses string) *remoteReusableWorkflow { @@ -124,6 +130,6 @@ func newRemoteReusableWorkflow(uses string) *remoteReusableWorkflow { Repo: matches[2], Filename: matches[3], Ref: matches[4], - URL: "github.com", + URL: "https://github.com", } } diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index 029ed5c4..e23dcf99 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -46,18 +46,17 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { return fmt.Errorf("Expected format {org}/{repo}[/path]@ref. Actual '%s' Input string was not in a correct format", sar.Step.Uses) } - sar.remoteAction.URL = sar.RunContext.Config.GitHubInstance - github := sar.getGithubContext(ctx) + sar.remoteAction.URL = github.ServerURL + if sar.remoteAction.IsCheckout() && isLocalCheckout(github, sar.Step) && !sar.RunContext.Config.NoSkipCheckout { common.Logger(ctx).Debugf("Skipping local actions/checkout because workdir was already copied") return nil } - sar.remoteAction.URL = sar.RunContext.Config.GitHubInstance for _, action := range sar.RunContext.Config.ReplaceGheActionWithGithubCom { if strings.EqualFold(fmt.Sprintf("%s/%s", sar.remoteAction.Org, sar.remoteAction.Repo), action) { - sar.remoteAction.URL = "github.com" + sar.remoteAction.URL = "https://github.com" github.Token = sar.RunContext.Config.ReplaceGheActionTokenWithGithubCom } } @@ -214,7 +213,7 @@ type remoteAction struct { } func (ra *remoteAction) CloneURL() string { - return fmt.Sprintf("https://%s/%s/%s", ra.URL, ra.Org, ra.Repo) + return fmt.Sprintf("%s/%s/%s", ra.URL, ra.Org, ra.Repo) } func (ra *remoteAction) IsCheckout() bool { @@ -240,7 +239,7 @@ func newRemoteAction(action string) *remoteAction { Repo: matches[2], Path: matches[4], Ref: matches[6], - URL: "github.com", + URL: "https://github.com", } } From e879afcd308dd14e54194d56b7fd65b9a3cfdbeb Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 3 May 2023 18:44:26 +0200 Subject: [PATCH 2/9] fix: fallback to unauthenticated pull (#1774) * fix: fallback to unauthenticated pull * move logger def * fixup * add import * . --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/docker_pull.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/act/container/docker_pull.go b/act/container/docker_pull.go index 75bfed16..6fb29613 100644 --- a/act/container/docker_pull.go +++ b/act/container/docker_pull.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "strings" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types" @@ -59,6 +60,13 @@ func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor { _ = logDockerResponse(logger, reader, err != nil) if err != nil { + if imagePullOptions.RegistryAuth != "" && strings.Contains(err.Error(), "unauthorized") { + logger.Errorf("pulling image '%v' (%s) failed with credentials %s retrying without them, please check for stale docker config files", imageRef, input.Platform, err.Error()) + imagePullOptions.RegistryAuth = "" + reader, err = cli.ImagePull(ctx, imageRef, imagePullOptions) + + _ = logDockerResponse(logger, reader, err != nil) + } return err } return nil @@ -69,9 +77,9 @@ func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput) imagePullOptions := types.ImagePullOptions{ Platform: input.Platform, } + logger := common.Logger(ctx) if input.Username != "" && input.Password != "" { - logger := common.Logger(ctx) logger.Debugf("using authentication for docker pull") authConfig := types.AuthConfig{ @@ -93,6 +101,7 @@ func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput) if authConfig.Username == "" && authConfig.Password == "" { return imagePullOptions, nil } + logger.Info("using DockerAuthConfig authentication for docker pull") encodedJSON, err := json.Marshal(authConfig) if err != nil { From 9ae1554d79342bdfe79cee00361506d326e2d80c Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 3 May 2023 19:26:28 +0200 Subject: [PATCH 3/9] Refactor evaluate yaml node do not alter nested nodes (#1761) * refactor: EvaluateYamlNode do not alter nested nodes * fix build error * fix op * fix lint * ... * fixup --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/expression.go | 121 +++++++++++++++++++++++++++++---------- 1 file changed, 91 insertions(+), 30 deletions(-) diff --git a/act/runner/expression.go b/act/runner/expression.go index ca40e95c..111274b0 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -158,67 +158,117 @@ func (ee expressionEvaluator) evaluate(ctx context.Context, in string, defaultSt return evaluated, err } -func (ee expressionEvaluator) evaluateScalarYamlNode(ctx context.Context, node *yaml.Node) error { +func (ee expressionEvaluator) evaluateScalarYamlNode(ctx context.Context, node *yaml.Node) (*yaml.Node, error) { var in string if err := node.Decode(&in); err != nil { - return err + return nil, err } if !strings.Contains(in, "${{") || !strings.Contains(in, "}}") { - return nil + return nil, nil } expr, _ := rewriteSubExpression(ctx, in, false) res, err := ee.evaluate(ctx, expr, exprparser.DefaultStatusCheckNone) if err != nil { - return err + return nil, err } - return node.Encode(res) + ret := &yaml.Node{} + if err := ret.Encode(res); err != nil { + return nil, err + } + return ret, err } -func (ee expressionEvaluator) evaluateMappingYamlNode(ctx context.Context, node *yaml.Node) error { +func (ee expressionEvaluator) evaluateMappingYamlNode(ctx context.Context, node *yaml.Node) (*yaml.Node, error) { + var ret *yaml.Node = nil // GitHub has this undocumented feature to merge maps, called insert directive insertDirective := regexp.MustCompile(`\${{\s*insert\s*}}`) - for i := 0; i < len(node.Content)/2; { + for i := 0; i < len(node.Content)/2; i++ { + changed := func() error { + if ret == nil { + ret = &yaml.Node{} + if err := ret.Encode(node); err != nil { + return err + } + ret.Content = ret.Content[:i*2] + } + return nil + } k := node.Content[i*2] v := node.Content[i*2+1] - if err := ee.EvaluateYamlNode(ctx, v); err != nil { - return err + ev, err := ee.evaluateYamlNodeInternal(ctx, v) + if err != nil { + return nil, err + } + if ev != nil { + if err := changed(); err != nil { + return nil, err + } + } else { + ev = v } var sk string // Merge the nested map of the insert directive if k.Decode(&sk) == nil && insertDirective.MatchString(sk) { - node.Content = append(append(node.Content[:i*2], v.Content...), node.Content[(i+1)*2:]...) - i += len(v.Content) / 2 - } else { - if err := ee.EvaluateYamlNode(ctx, k); err != nil { - return err + if ev.Kind != yaml.MappingNode { + return nil, fmt.Errorf("failed to insert node %v into mapping %v unexpected type %v expected MappingNode", ev, node, ev.Kind) + } + if err := changed(); err != nil { + return nil, err + } + ret.Content = append(ret.Content, ev.Content...) + } else { + ek, err := ee.evaluateYamlNodeInternal(ctx, k) + if err != nil { + return nil, err + } + if ek != nil { + if err := changed(); err != nil { + return nil, err + } + } else { + ek = k + } + if ret != nil { + ret.Content = append(ret.Content, ek, ev) } - i++ } } - return nil + return ret, nil } -func (ee expressionEvaluator) evaluateSequenceYamlNode(ctx context.Context, node *yaml.Node) error { - for i := 0; i < len(node.Content); { +func (ee expressionEvaluator) evaluateSequenceYamlNode(ctx context.Context, node *yaml.Node) (*yaml.Node, error) { + var ret *yaml.Node = nil + for i := 0; i < len(node.Content); i++ { v := node.Content[i] // Preserve nested sequences wasseq := v.Kind == yaml.SequenceNode - if err := ee.EvaluateYamlNode(ctx, v); err != nil { - return err + ev, err := ee.evaluateYamlNodeInternal(ctx, v) + if err != nil { + return nil, err } - // GitHub has this undocumented feature to merge sequences / arrays - // We have a nested sequence via evaluation, merge the arrays - if v.Kind == yaml.SequenceNode && !wasseq { - node.Content = append(append(node.Content[:i], v.Content...), node.Content[i+1:]...) - i += len(v.Content) - } else { - i++ + if ev != nil { + if ret == nil { + ret = &yaml.Node{} + if err := ret.Encode(node); err != nil { + return nil, err + } + ret.Content = ret.Content[:i] + } + // GitHub has this undocumented feature to merge sequences / arrays + // We have a nested sequence via evaluation, merge the arrays + if ev.Kind == yaml.SequenceNode && !wasseq { + ret.Content = append(ret.Content, ev.Content...) + } else { + ret.Content = append(ret.Content, ev) + } + } else if ret != nil { + ret.Content = append(ret.Content, v) } } - return nil + return ret, nil } -func (ee expressionEvaluator) EvaluateYamlNode(ctx context.Context, node *yaml.Node) error { +func (ee expressionEvaluator) evaluateYamlNodeInternal(ctx context.Context, node *yaml.Node) (*yaml.Node, error) { switch node.Kind { case yaml.ScalarNode: return ee.evaluateScalarYamlNode(ctx, node) @@ -227,10 +277,21 @@ func (ee expressionEvaluator) EvaluateYamlNode(ctx context.Context, node *yaml.N case yaml.SequenceNode: return ee.evaluateSequenceYamlNode(ctx, node) default: - return nil + return nil, nil } } +func (ee expressionEvaluator) EvaluateYamlNode(ctx context.Context, node *yaml.Node) error { + ret, err := ee.evaluateYamlNodeInternal(ctx, node) + if err != nil { + return err + } + if ret != nil { + return ret.Decode(node) + } + return nil +} + func (ee expressionEvaluator) Interpolate(ctx context.Context, in string) string { if !strings.Contains(in, "${{") || !strings.Contains(in, "}}") { return in From fc2883b75464136e62ed5c63e2c04d2050591689 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 3 May 2023 19:49:17 +0200 Subject: [PATCH 4/9] fix: don't allow `-self-hosted` mode as container image (#1783) * fix: don't allow `-self-hosted` mode as container image * fix: jobcontainer in hostmode platform * Update run_context.go --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index e6f5dfc7..acffb76d 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -420,8 +420,9 @@ func (rc *RunContext) startContainer() common.Executor { } func (rc *RunContext) IsHostEnv(ctx context.Context) bool { - image := rc.platformImage(ctx) - return strings.EqualFold(image, "-self-hosted") + platform := rc.runsOnImage(ctx) + image := rc.containerImage(ctx) + return image == "" && strings.EqualFold(platform, "-self-hosted") } func (rc *RunContext) stopContainer() common.Executor { @@ -474,7 +475,7 @@ func (rc *RunContext) Executor() common.Executor { } } -func (rc *RunContext) platformImage(ctx context.Context) string { +func (rc *RunContext) containerImage(ctx context.Context) string { job := rc.Run.Job() c := job.Container() @@ -482,6 +483,12 @@ func (rc *RunContext) platformImage(ctx context.Context) string { return rc.ExprEval.Interpolate(ctx, c.Image) } + return "" +} + +func (rc *RunContext) runsOnImage(ctx context.Context) string { + job := rc.Run.Job() + if job.RunsOn() == nil { common.Logger(ctx).Errorf("'runs-on' key not defined in %s", rc.String()) } @@ -497,6 +504,14 @@ func (rc *RunContext) platformImage(ctx context.Context) string { return "" } +func (rc *RunContext) platformImage(ctx context.Context) string { + if containerImage := rc.containerImage(ctx); containerImage != "" { + return containerImage + } + + return rc.runsOnImage(ctx) +} + func (rc *RunContext) options(ctx context.Context) string { job := rc.Run.Job() c := job.Container() From d6d05d4e562df89fa2e54d9f8247d8f774ed2216 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 3 May 2023 20:08:11 +0200 Subject: [PATCH 5/9] feat: implement steps.timeout-minutes (#1776) * feat: implement steps.timeout-minutes * Add imports * refactor code --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/step.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/act/runner/step.go b/act/runner/step.go index cff48b1b..9cc6aea4 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -4,7 +4,9 @@ import ( "context" "fmt" "path" + "strconv" "strings" + "time" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" @@ -134,7 +136,9 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo Mode: 0o666, })(ctx) - err = executor(ctx) + timeoutctx, cancelTimeOut := evaluateStepTimeout(ctx, rc.ExprEval, stepModel) + defer cancelTimeOut() + err = executor(timeoutctx) if err == nil { logger.WithField("stepResult", stepResult.Outcome).Infof(" \u2705 Success - %s %s", stage, stepString) @@ -182,6 +186,16 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo } } +func evaluateStepTimeout(ctx context.Context, exprEval ExpressionEvaluator, stepModel *model.Step) (context.Context, context.CancelFunc) { + timeout := exprEval.Interpolate(ctx, stepModel.TimeoutMinutes) + if timeout != "" { + if timeOutMinutes, err := strconv.ParseInt(timeout, 10, 64); err == nil { + return context.WithTimeout(ctx, time.Duration(timeOutMinutes)*time.Minute) + } + } + return ctx, func() {} +} + func setupEnv(ctx context.Context, step step) error { rc := step.getRunContext() From 9f9d26fb61e49b40703320437d8065e8d656e7bb Mon Sep 17 00:00:00 2001 From: benbaker76 Date: Wed, 3 May 2023 12:12:36 -0700 Subject: [PATCH 6/9] Replace backslash in GetActPath() for Windows (#1777) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/host_environment.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/act/container/host_environment.go b/act/container/host_environment.go index abc1115a..e0a18c7c 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -362,7 +362,11 @@ func (e *HostEnvironment) ToContainerPath(path string) string { } func (e *HostEnvironment) GetActPath() string { - return e.ActPath + actPath := e.ActPath + if runtime.GOOS == "windows" { + actPath = strings.ReplaceAll(actPath, "\\", "/") + } + return actPath } func (*HostEnvironment) GetPathVariableName() string { From 32e94ab212dccf76aa70eaaae79eced66651e945 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 03:12:14 +0000 Subject: [PATCH 7/9] build(deps): bump codecov/codecov-action from 3.1.3 to 3.1.4 (#1813) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3.1.3 to 3.1.4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3.1.3...v3.1.4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5d7cf8a6..bc0d765c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -61,7 +61,7 @@ jobs: - name: Run act from cli run: go run main.go -P ubuntu-latest=node:16-buster-slim -C ./pkg/runner/testdata/ -W ./basic/push.yml - name: Upload Codecov report - uses: codecov/codecov-action@v3.1.3 + uses: codecov/codecov-action@v3.1.4 with: files: coverage.txt fail_ci_if_error: true # optional (default = false) From 2aea0f766afea9da47f375be0c12d8d81288e2bc Mon Sep 17 00:00:00 2001 From: ab-pkandhari <132302443+ab-pkandhari@users.noreply.github.com> Date: Tue, 23 May 2023 07:26:47 -0500 Subject: [PATCH 8/9] fix: Update ARCH environment variable used in runners/actions (#1818) --- act/container/docker_run.go | 4 ++-- act/container/host_environment.go | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index df296e0d..65a373ff 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -240,8 +240,8 @@ func RunnerArch(ctx context.Context) string { archMapper := map[string]string{ "x86_64": "X64", - "386": "x86", - "aarch64": "arm64", + "386": "X86", + "aarch64": "ARM64", } if arch, ok := archMapper[info.Architecture]; ok { return arch diff --git a/act/container/host_environment.go b/act/container/host_environment.go index e0a18c7c..c6a22288 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -387,11 +387,13 @@ func (*HostEnvironment) JoinPathVariable(paths ...string) string { return strings.Join(paths, string(filepath.ListSeparator)) } +// Reference for Arch values for runner.arch +// https://docs.github.com/en/actions/learn-github-actions/contexts#runner-context func goArchToActionArch(arch string) string { archMapper := map[string]string{ "x86_64": "X64", - "386": "x86", - "aarch64": "arm64", + "386": "X86", + "aarch64": "ARM64", } if arch, ok := archMapper[arch]; ok { return arch From 414bb1b24399e9366623257b465526d8b539c05d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 May 2023 03:12:45 +0000 Subject: [PATCH 9/9] build(deps): bump megalinter/megalinter from 6.22.2 to 7.0.2 (#1827) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.22.2 to 7.0.2. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.22.2...v7.0.2) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index bc0d765c..b364c6f4 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -26,7 +26,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.4.0 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.22.2 + - uses: megalinter/megalinter/flavors/go@v7.0.2 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}