From 7a4d3e467bb6d00c888347fec1170bc14e9c72b5 Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Tue, 1 Nov 2022 16:58:07 +0100 Subject: [PATCH 01/12] feat: parse types of reusable workflows (#1414) This change does parse the different types of workflow jobs. It is not much by itself but the start to implement reusable workflows. Relates to #826 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/model/workflow.go | 50 ++++++++++++++++++++++++++++++++++++++ act/model/workflow_test.go | 25 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/act/model/workflow.go b/act/model/workflow.go index 214927e7..4e1d3496 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -380,6 +380,42 @@ func commonKeysMatch2(a map[string]interface{}, b map[string]interface{}, m map[ return true } +// JobType describes what type of job we are about to run +type JobType int + +const ( + // StepTypeRun is all steps that have a `run` attribute + JobTypeDefault JobType = iota + + // StepTypeReusableWorkflowLocal is all steps that have a `uses` that is a local workflow in the .github/workflows directory + JobTypeReusableWorkflowLocal + + // JobTypeReusableWorkflowRemote is all steps that have a `uses` that references a workflow file in a github repo + JobTypeReusableWorkflowRemote +) + +func (j JobType) String() string { + switch j { + case JobTypeDefault: + return "default" + case JobTypeReusableWorkflowLocal: + return "local-reusable-workflow" + case JobTypeReusableWorkflowRemote: + return "remote-reusable-workflow" + } + return "unknown" +} + +// Type returns the type of the job +func (j *Job) Type() JobType { + if strings.HasPrefix(j.Uses, "./.github/workflows") && (strings.HasSuffix(j.Uses, ".yml") || strings.HasSuffix(j.Uses, ".yaml")) { + return JobTypeReusableWorkflowLocal + } else if !strings.HasPrefix(j.Uses, "./") && strings.Contains(j.Uses, ".github/workflows") && (strings.Contains(j.Uses, ".yml@") || strings.Contains(j.Uses, ".yaml@")) { + return JobTypeReusableWorkflowRemote + } + return JobTypeDefault +} + // ContainerSpec is the specification of the container to use for the job type ContainerSpec struct { Image string `yaml:"image"` @@ -486,6 +522,12 @@ const ( // StepTypeUsesActionRemote is all steps that have a `uses` that is a reference to a github repo StepTypeUsesActionRemote + // StepTypeReusableWorkflowLocal is all steps that have a `uses` that is a local workflow in the .github/workflows directory + StepTypeReusableWorkflowLocal + + // StepTypeReusableWorkflowRemote is all steps that have a `uses` that references a workflow file in a github repo + StepTypeReusableWorkflowRemote + // StepTypeInvalid is for steps that have invalid step action StepTypeInvalid ) @@ -502,6 +544,10 @@ func (s StepType) String() string { return "remote-action" case StepTypeUsesDockerURL: return "docker" + case StepTypeReusableWorkflowLocal: + return "local-reusable-workflow" + case StepTypeReusableWorkflowRemote: + return "remote-reusable-workflow" } return "unknown" } @@ -519,6 +565,10 @@ func (s *Step) Type() StepType { return StepTypeRun } else if strings.HasPrefix(s.Uses, "docker://") { return StepTypeUsesDockerURL + } else if strings.HasPrefix(s.Uses, "./.github/workflows") && (strings.HasSuffix(s.Uses, ".yml") || strings.HasSuffix(s.Uses, ".yaml")) { + return StepTypeReusableWorkflowLocal + } else if !strings.HasPrefix(s.Uses, "./") && strings.Contains(s.Uses, ".github/workflows") && (strings.Contains(s.Uses, ".yml@") || strings.Contains(s.Uses, ".yaml@")) { + return StepTypeReusableWorkflowRemote } else if strings.HasPrefix(s.Uses, "./") { return StepTypeUsesActionLocal } diff --git a/act/model/workflow_test.go b/act/model/workflow_test.go index 6d3b307f..d978f163 100644 --- a/act/model/workflow_test.go +++ b/act/model/workflow_test.go @@ -138,6 +138,31 @@ jobs: }) } +func TestReadWorkflow_JobTypes(t *testing.T) { + yaml := ` +name: invalid job definition + +jobs: + default-job: + runs-on: ubuntu-latest + steps: + - run: echo + remote-reusable-workflow: + runs-on: ubuntu-latest + uses: remote/repo/.github/workflows/workflow.yml@main + local-reusable-workflow: + runs-on: ubuntu-latest + uses: ./.github/workflows/workflow.yml +` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + assert.Len(t, workflow.Jobs, 3) + assert.Equal(t, workflow.Jobs["default-job"].Type(), JobTypeDefault) + assert.Equal(t, workflow.Jobs["remote-reusable-workflow"].Type(), JobTypeReusableWorkflowRemote) + assert.Equal(t, workflow.Jobs["local-reusable-workflow"].Type(), JobTypeReusableWorkflowLocal) +} + func TestReadWorkflow_StepsTypes(t *testing.T) { yaml := ` name: invalid step definition From 3676e990390cf8150bc47e4428b0836eb9c4e259 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Nov 2022 02:25:24 +0000 Subject: [PATCH 02/12] build(deps): bump megalinter/megalinter from 6.13.0 to 6.14.0 (#1426) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.13.0 to 6.14.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.13.0...v6.14.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 80d04e93..33fa8ba3 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.3.0 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.13.0 + - uses: megalinter/megalinter/flavors/go@v6.14.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d72ce8baa19c5530d88f4e2f1a988d021cdd8b71 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Thu, 10 Nov 2022 21:16:00 +0100 Subject: [PATCH 03/12] fix: nil pointer access ( workflow_dispatch ) --- act/runner/expression.go | 20 ++++++++++--------- act/runner/runner_test.go | 3 +++ .../workflow_dispatch.yml | 17 ++++++++++++++++ .../workflow_dispatch.yml | 9 +++++++++ .../workflow_dispatch.yml | 10 ++++++++++ 5 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 act/runner/testdata/workflow_dispatch-scalar-composite-action/workflow_dispatch.yml create mode 100644 act/runner/testdata/workflow_dispatch-scalar/workflow_dispatch.yml create mode 100644 act/runner/testdata/workflow_dispatch_no_inputs_mapping/workflow_dispatch.yml diff --git a/act/runner/expression.go b/act/runner/expression.go index adba5699..43699800 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -331,15 +331,17 @@ func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *mod if ghc.EventName == "workflow_dispatch" { config := rc.Run.Workflow.WorkflowDispatchConfig() - for k, v := range config.Inputs { - value := nestedMapLookup(ghc.Event, "inputs", k) - if value == nil { - value = v.Default - } - if v.Type == "boolean" { - inputs[k] = value == "true" - } else { - inputs[k] = value + if config != nil && config.Inputs != nil { + for k, v := range config.Inputs { + value := nestedMapLookup(ghc.Event, "inputs", k) + if value == nil { + value = v.Default + } + if v.Type == "boolean" { + inputs[k] = value == "true" + } else { + inputs[k] = value + } } } } diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index dbf8a007..232fc163 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -183,6 +183,9 @@ func TestRunEvent(t *testing.T) { {workdir, "evalenv", "push", "", platforms}, {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms}, {workdir, "workflow_dispatch", "workflow_dispatch", "", platforms}, + {workdir, "workflow_dispatch_no_inputs_mapping", "workflow_dispatch", "", platforms}, + {workdir, "workflow_dispatch-scalar", "workflow_dispatch", "", platforms}, + {workdir, "workflow_dispatch-scalar-composite-action", "workflow_dispatch", "", platforms}, {"../model/testdata", "strategy", "push", "", platforms}, // TODO: move all testdata into pkg so we can validate it with planner and runner // {"testdata", "issue-228", "push", "", platforms, }, // TODO [igni]: Remove this once everything passes {"../model/testdata", "container-volumes", "push", "", platforms}, diff --git a/act/runner/testdata/workflow_dispatch-scalar-composite-action/workflow_dispatch.yml b/act/runner/testdata/workflow_dispatch-scalar-composite-action/workflow_dispatch.yml new file mode 100644 index 00000000..f8447b45 --- /dev/null +++ b/act/runner/testdata/workflow_dispatch-scalar-composite-action/workflow_dispatch.yml @@ -0,0 +1,17 @@ +name: workflow_dispatch + +on: workflow_dispatch + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: | + runs: + using: composite + steps: + - run: | + exit 0 + shell: bash + shell: cp {0} action.yml + - uses: ./ diff --git a/act/runner/testdata/workflow_dispatch-scalar/workflow_dispatch.yml b/act/runner/testdata/workflow_dispatch-scalar/workflow_dispatch.yml new file mode 100644 index 00000000..9c900e87 --- /dev/null +++ b/act/runner/testdata/workflow_dispatch-scalar/workflow_dispatch.yml @@ -0,0 +1,9 @@ +name: workflow_dispatch + +on: workflow_dispatch + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: exit 0 diff --git a/act/runner/testdata/workflow_dispatch_no_inputs_mapping/workflow_dispatch.yml b/act/runner/testdata/workflow_dispatch_no_inputs_mapping/workflow_dispatch.yml new file mode 100644 index 00000000..9fc6b097 --- /dev/null +++ b/act/runner/testdata/workflow_dispatch_no_inputs_mapping/workflow_dispatch.yml @@ -0,0 +1,10 @@ +name: workflow_dispatch + +on: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: exit 0 From 280420f3fa2dabf33bb0b6f05374ae71cc9b599b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Nov 2022 02:14:53 +0000 Subject: [PATCH 04/12] build(deps): bump golangci/golangci-lint-action from 3.3.0 to 3.3.1 (#1436) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.3.0 to 3.3.1. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.3.0...v3.3.1) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 33fa8ba3..d69a32d9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,7 +19,7 @@ jobs: with: go-version: ${{ env.GO_VERSION }} check-latest: true - - uses: golangci/golangci-lint-action@v3.3.0 + - uses: golangci/golangci-lint-action@v3.3.1 with: version: v1.47.2 - uses: megalinter/megalinter/flavors/go@v6.14.0 From 865c4556e04a4a7e0468185bfe1b66f5dc6efd8b Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Wed, 16 Nov 2022 18:00:49 +0100 Subject: [PATCH 05/12] fix: keep path to event json file in composite actions (#1428) * fix: keep path to event json file in composite actions The event.json paths need to be copied over, since it the GithubContext is recreated from the composite RC. And that does read some value for the event file if available. * test: add test case * test: paste the test correctly and revert a line Co-authored-by: ChristopherHX Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/action_composite.go | 1 + act/runner/testdata/pull-request/main.yaml | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/act/runner/action_composite.go b/act/runner/action_composite.go index 9001b489..a7b41438 100644 --- a/act/runner/action_composite.go +++ b/act/runner/action_composite.go @@ -69,6 +69,7 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action Masks: parent.Masks, ExtraPath: parent.ExtraPath, Parent: parent, + EventJSON: parent.EventJSON, } return compositerc diff --git a/act/runner/testdata/pull-request/main.yaml b/act/runner/testdata/pull-request/main.yaml index b9946b94..eb81939b 100644 --- a/act/runner/testdata/pull-request/main.yaml +++ b/act/runner/testdata/pull-request/main.yaml @@ -5,6 +5,22 @@ jobs: build: runs-on: ubuntu-latest steps: - - run: echo '${{github.ref}}' + # test refs from event.json + - run: echo '${{github.ref}}' - run: echo '${{github.head_ref}}' | grep sample-head-ref - run: echo '${{github.base_ref}}' | grep sample-base-ref + # test main/composite context equality with data from event.json + - run: | + runs: + using: composite + steps: + - run: | + echo WORKFLOW_GITHUB_CONTEXT="$WORKFLOW_GITHUB_CONTEXT" + echo COMPOSITE_GITHUB_CONTEXT="$COMPOSITE_GITHUB_CONTEXT" + [[ "$WORKFLOW_GITHUB_CONTEXT" = "$COMPOSITE_GITHUB_CONTEXT" ]] + env: + WORKFLOW_GITHUB_CONTEXT: ${{ tojson(tojson(github.event)) }} + COMPOSITE_GITHUB_CONTEXT: ${{ '${{tojson(github.event)}}' }} + shell: bash + shell: cp {0} action.yml + - uses: ./ From ab1deb20a5ab6691b849b569ce29ceb0cc61fd89 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 16 Nov 2022 22:29:45 +0100 Subject: [PATCH 06/12] feat: Host environment (#1293) --- .github/workflows/checks.yml | 19 + act/container/docker_run.go | 3 +- act/container/docker_run_test.go | 3 + act/container/executions_environment.go | 13 + act/container/file_collector.go | 23 + act/container/host_environment.go | 470 ++++++++++++++++++ act/container/host_environment_test.go | 4 + .../linux_container_environment_extensions.go | 73 +++ ...x_container_environment_extensions_test.go | 71 +++ act/container/util.go | 26 + act/container/util_openbsd_mips64.go | 17 + act/container/util_plan9.go | 17 + act/container/util_windows.go | 15 + act/lookpath/LICENSE | 27 + act/lookpath/env.go | 18 + act/lookpath/error.go | 10 + act/lookpath/lp_js.go | 23 + act/lookpath/lp_plan9.go | 56 +++ act/lookpath/lp_unix.go | 59 +++ act/lookpath/lp_windows.go | 94 ++++ act/runner/action.go | 6 +- act/runner/container_mock_test.go | 1 + act/runner/expression.go | 57 +-- act/runner/expression_test.go | 1 - act/runner/job_executor.go | 14 + act/runner/job_executor_test.go | 1 + act/runner/run_context.go | 165 ++++-- act/runner/run_context_test.go | 2 - act/runner/runner.go | 49 -- act/runner/runner_test.go | 146 +++--- act/runner/step.go | 9 +- act/runner/step_action_remote.go | 2 +- act/runner/step_docker.go | 2 +- act/runner/step_docker_test.go | 2 +- act/runner/step_run.go | 6 +- act/runner/testdata/nix-prepend-path/push.yml | 26 + act/runner/testdata/windows-add-env/push.yml | 27 + .../testdata/windows-prepend-path/push.yml | 25 + 38 files changed, 1395 insertions(+), 187 deletions(-) create mode 100644 act/container/executions_environment.go create mode 100644 act/container/host_environment.go create mode 100644 act/container/host_environment_test.go create mode 100644 act/container/linux_container_environment_extensions.go create mode 100644 act/container/linux_container_environment_extensions_test.go create mode 100644 act/container/util.go create mode 100644 act/container/util_openbsd_mips64.go create mode 100644 act/container/util_plan9.go create mode 100644 act/container/util_windows.go create mode 100644 act/lookpath/LICENSE create mode 100644 act/lookpath/env.go create mode 100644 act/lookpath/error.go create mode 100644 act/lookpath/lp_js.go create mode 100644 act/lookpath/lp_plan9.go create mode 100644 act/lookpath/lp_unix.go create mode 100644 act/lookpath/lp_windows.go create mode 100644 act/runner/testdata/nix-prepend-path/push.yml create mode 100644 act/runner/testdata/windows-add-env/push.yml create mode 100644 act/runner/testdata/windows-prepend-path/push.yml diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d69a32d9..4a321285 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -57,6 +57,25 @@ jobs: files: coverage.txt fail_ci_if_error: true # optional (default = false) + test-host: + strategy: + matrix: + os: + - windows-latest + - macos-latest + name: test-${{matrix.os}} + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + - run: go test -v -run ^TestRunEventHostEnvironment$ ./... + # TODO merge coverage with test-linux + snapshot: name: snapshot runs-on: ubuntu-latest diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 896a269c..2740ad4d 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -84,7 +84,7 @@ type Container interface { } // NewContainer creates a reference to a container -func NewContainer(input *NewContainerInput) Container { +func NewContainer(input *NewContainerInput) ExecutionsEnvironment { cr := new(containerReference) cr.input = input return cr @@ -233,6 +233,7 @@ type containerReference struct { input *NewContainerInput UID int GID int + LinuxContainerEnvironmentExtensions } func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) { diff --git a/act/container/docker_run_test.go b/act/container/docker_run_test.go index 2e12fcfc..bc3ab4bd 100644 --- a/act/container/docker_run_test.go +++ b/act/container/docker_run_test.go @@ -163,3 +163,6 @@ func TestDockerExecFailure(t *testing.T) { conn.AssertExpectations(t) client.AssertExpectations(t) } + +// Type assert containerReference implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &containerReference{} diff --git a/act/container/executions_environment.go b/act/container/executions_environment.go new file mode 100644 index 00000000..1c21f943 --- /dev/null +++ b/act/container/executions_environment.go @@ -0,0 +1,13 @@ +package container + +import "context" + +type ExecutionsEnvironment interface { + Container + ToContainerPath(string) string + GetActPath() string + GetPathVariableName() string + DefaultPathVariable() string + JoinPathVariable(...string) string + GetRunnerContext(ctx context.Context) map[string]interface{} +} diff --git a/act/container/file_collector.go b/act/container/file_collector.go index 164bfe7d..a4143edc 100644 --- a/act/container/file_collector.go +++ b/act/container/file_collector.go @@ -59,6 +59,29 @@ func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, return nil } +type copyCollector struct { + DstDir string +} + +func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { + fdestpath := filepath.Join(cc.DstDir, fpath) + if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil { + return err + } + if f == nil { + return os.Symlink(linkName, fdestpath) + } + df, err := os.OpenFile(fdestpath, os.O_CREATE|os.O_WRONLY, fi.Mode()) + if err != nil { + return err + } + defer df.Close() + if _, err := io.Copy(df, f); err != nil { + return err + } + return nil +} + type fileCollector struct { Ignorer gitignore.Matcher SrcPath string diff --git a/act/container/host_environment.go b/act/container/host_environment.go new file mode 100644 index 00000000..b404e86d --- /dev/null +++ b/act/container/host_environment.go @@ -0,0 +1,470 @@ +package container + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "errors" + + "github.com/go-git/go-billy/v5/helper/polyfill" + "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/lookpath" + "golang.org/x/term" +) + +type HostEnvironment struct { + Path string + TmpDir string + ToolCache string + Workdir string + ActPath string + CleanUp func() + StdOut io.Writer +} + +func (e *HostEnvironment) Create(capAdd []string, capDrop []string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Close() common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor { + return func(ctx context.Context) error { + for _, f := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(destPath, f.Name)), 0777); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(destPath, f.Name), []byte(f.Body), fs.FileMode(f.Mode)); err != nil { + return err + } + } + return nil + } +} + +func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath) + var ignorer gitignore.Matcher + if useGitIgnore { + ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil) + if err != nil { + logger.Debugf("Error loading .gitignore: %v", err) + } + + ignorer = gitignore.NewMatcher(ps) + } + fc := &fileCollector{ + Fs: &defaultFs{}, + Ignorer: ignorer, + SrcPath: srcPath, + SrcPrefix: srcPrefix, + Handler: ©Collector{ + DstDir: destPath, + }, + } + return filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + } +} + +func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + defer tw.Close() + srcPath = filepath.Clean(srcPath) + fi, err := os.Lstat(srcPath) + if err != nil { + return nil, err + } + tc := &tarCollector{ + TarWriter: tw, + } + if fi.IsDir() { + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + fc := &fileCollector{ + Fs: &defaultFs{}, + SrcPath: srcPath, + SrcPrefix: srcPrefix, + Handler: tc, + } + err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + if err != nil { + return nil, err + } + } else { + var f io.ReadCloser + var linkname string + if fi.Mode()&fs.ModeSymlink != 0 { + linkname, err = os.Readlink(srcPath) + if err != nil { + return nil, err + } + } else { + f, err = os.Open(srcPath) + if err != nil { + return nil, err + } + defer f.Close() + } + err := tc.WriteFile(fi.Name(), fi, linkname, f) + if err != nil { + return nil, err + } + } + return io.NopCloser(buf), nil +} + +func (e *HostEnvironment) Pull(forcePull bool) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Start(attach bool) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +type ptyWriter struct { + Out io.Writer + AutoStop bool + dirtyLine bool +} + +func (w *ptyWriter) Write(buf []byte) (int, error) { + if w.AutoStop && len(buf) > 0 && buf[len(buf)-1] == 4 { + n, err := w.Out.Write(buf[:len(buf)-1]) + if err != nil { + return n, err + } + if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' { + _, _ = w.Out.Write([]byte("\n")) + return n, io.EOF + } + return n, io.EOF + } + w.dirtyLine = strings.LastIndex(string(buf), "\n") < len(buf)-1 + return w.Out.Write(buf) +} + +type localEnv struct { + env map[string]string +} + +func (l *localEnv) Getenv(name string) string { + if runtime.GOOS == "windows" { + for k, v := range l.env { + if strings.EqualFold(name, k) { + return v + } + } + return "" + } + return l.env[name] +} + +func lookupPathHost(cmd string, env map[string]string, writer io.Writer) (string, error) { + f, err := lookpath.LookPath2(cmd, &localEnv{env: env}) + if err != nil { + err := "Cannot find: " + fmt.Sprint(cmd) + " in PATH" + if _, _err := writer.Write([]byte(err + "\n")); _err != nil { + return "", fmt.Errorf("%v: %w", err, _err) + } + return "", errors.New(err) + } + return f, nil +} + +func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) { + ppty, tty, err := openPty() + if err != nil { + return nil, nil, err + } + if term.IsTerminal(int(tty.Fd())) { + _, err := term.MakeRaw(int(tty.Fd())) + if err != nil { + ppty.Close() + tty.Close() + return nil, nil, err + } + } + cmd.Stdin = tty + cmd.Stdout = tty + cmd.Stderr = tty + cmd.SysProcAttr = getSysProcAttr(cmdline, true) + return ppty, tty, nil +} + +func writeKeepAlive(ppty io.Writer) { + c := 1 + var err error + for c == 1 && err == nil { + c, err = ppty.Write([]byte{4}) + <-time.After(time.Second) + } +} + +func copyPtyOutput(writer io.Writer, ppty io.Reader, finishLog context.CancelFunc) { + defer func() { + finishLog() + }() + if _, err := io.Copy(writer, ppty); err != nil { + return + } +} + +func (e *HostEnvironment) UpdateFromImageEnv(env *map[string]string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func getEnvListFromMap(env map[string]string) []string { + envList := make([]string, 0) + for k, v := range env { + envList = append(envList, fmt.Sprintf("%s=%s", k, v)) + } + return envList +} + +func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, user, workdir string) error { + envList := getEnvListFromMap(env) + var wd string + if workdir != "" { + if filepath.IsAbs(workdir) { + wd = workdir + } else { + wd = filepath.Join(e.Path, workdir) + } + } else { + wd = e.Path + } + f, err := lookupPathHost(command[0], env, e.StdOut) + if err != nil { + return err + } + cmd := exec.CommandContext(ctx, f) + cmd.Path = f + cmd.Args = command + cmd.Stdin = nil + cmd.Stdout = e.StdOut + cmd.Env = envList + cmd.Stderr = e.StdOut + cmd.Dir = wd + cmd.SysProcAttr = getSysProcAttr(cmdline, false) + var ppty *os.File + var tty *os.File + defer func() { + if ppty != nil { + ppty.Close() + } + if tty != nil { + tty.Close() + } + }() + if true /* allocate Terminal */ { + var err error + ppty, tty, err = setupPty(cmd, cmdline) + if err != nil { + common.Logger(ctx).Debugf("Failed to setup Pty %v\n", err.Error()) + } + } + writer := &ptyWriter{Out: e.StdOut} + logctx, finishLog := context.WithCancel(context.Background()) + if ppty != nil { + go copyPtyOutput(writer, ppty, finishLog) + } else { + finishLog() + } + if ppty != nil { + go writeKeepAlive(ppty) + } + err = cmd.Run() + if err != nil { + return err + } + if tty != nil { + writer.AutoStop = true + if _, err := tty.Write([]byte("\x04")); err != nil { + common.Logger(ctx).Debug("Failed to write EOT") + } + } + <-logctx.Done() + + if ppty != nil { + ppty.Close() + ppty = nil + } + return err +} + +func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor { + return func(ctx context.Context) error { + if err := e.exec(ctx, command, "" /*cmdline*/, env, user, workdir); err != nil { + select { + case <-ctx.Done(): + return fmt.Errorf("this step has been cancelled: %w", err) + default: + return err + } + } + return nil + } +} + +func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor { + localEnv := *env + return func(ctx context.Context) error { + envTar, err := e.GetContainerArchive(ctx, srcPath) + if err != nil { + return nil + } + defer envTar.Close() + reader := tar.NewReader(envTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + for s.Scan() { + line := s.Text() + singleLineEnv := strings.Index(line, "=") + multiLineEnv := strings.Index(line, "<<") + if singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv) { + localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:] + } else if multiLineEnv != -1 { + multiLineEnvContent := "" + multiLineEnvDelimiter := line[multiLineEnv+2:] + delimiterFound := false + for s.Scan() { + content := s.Text() + if content == multiLineEnvDelimiter { + delimiterFound = true + break + } + if multiLineEnvContent != "" { + multiLineEnvContent += "\n" + } + multiLineEnvContent += content + } + if !delimiterFound { + return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter) + } + localEnv[line[:multiLineEnv]] = multiLineEnvContent + } else { + return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line) + } + } + env = &localEnv + return nil + } +} + +func (e *HostEnvironment) UpdateFromPath(env *map[string]string) common.Executor { + localEnv := *env + return func(ctx context.Context) error { + pathTar, err := e.GetContainerArchive(ctx, localEnv["GITHUB_PATH"]) + if err != nil { + return err + } + defer pathTar.Close() + + reader := tar.NewReader(pathTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + for s.Scan() { + line := s.Text() + pathSep := string(filepath.ListSeparator) + localEnv[e.GetPathVariableName()] = fmt.Sprintf("%s%s%s", line, pathSep, localEnv[e.GetPathVariableName()]) + } + + env = &localEnv + return nil + } +} + +func (e *HostEnvironment) Remove() common.Executor { + return func(ctx context.Context) error { + if e.CleanUp != nil { + e.CleanUp() + } + return os.RemoveAll(e.Path) + } +} + +func (e *HostEnvironment) ToContainerPath(path string) string { + if bp, err := filepath.Rel(e.Workdir, path); err != nil { + return filepath.Join(e.Path, bp) + } else if filepath.Clean(e.Workdir) == filepath.Clean(path) { + return e.Path + } + return path +} + +func (e *HostEnvironment) GetActPath() string { + return e.ActPath +} + +func (*HostEnvironment) GetPathVariableName() string { + if runtime.GOOS == "plan9" { + return "path" + } else if runtime.GOOS == "windows" { + return "Path" // Actually we need a case insensitive map + } + return "PATH" +} + +func (e *HostEnvironment) DefaultPathVariable() string { + v, _ := os.LookupEnv(e.GetPathVariableName()) + return v +} + +func (*HostEnvironment) JoinPathVariable(paths ...string) string { + return strings.Join(paths, string(filepath.ListSeparator)) +} + +func (e *HostEnvironment) GetRunnerContext(ctx context.Context) map[string]interface{} { + return map[string]interface{}{ + "os": runtime.GOOS, + "arch": runtime.GOARCH, + "temp": e.TmpDir, + "tool_cache": e.ToolCache, + } +} + +func (e *HostEnvironment) ReplaceLogWriter(stdout io.Writer, stderr io.Writer) (io.Writer, io.Writer) { + org := e.StdOut + e.StdOut = stdout + return org, org +} diff --git a/act/container/host_environment_test.go b/act/container/host_environment_test.go new file mode 100644 index 00000000..67787d95 --- /dev/null +++ b/act/container/host_environment_test.go @@ -0,0 +1,4 @@ +package container + +// Type assert HostEnvironment implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &HostEnvironment{} diff --git a/act/container/linux_container_environment_extensions.go b/act/container/linux_container_environment_extensions.go new file mode 100644 index 00000000..c369055d --- /dev/null +++ b/act/container/linux_container_environment_extensions.go @@ -0,0 +1,73 @@ +package container + +import ( + "context" + "path/filepath" + "regexp" + "runtime" + "strings" + + log "github.com/sirupsen/logrus" +) + +type LinuxContainerEnvironmentExtensions struct { +} + +// Resolves the equivalent host path inside the container +// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject +// For use in docker volumes and binds +func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string { + if runtime.GOOS == "windows" && strings.Contains(path, "/") { + log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.") + return "" + } + + abspath, err := filepath.Abs(path) + if err != nil { + log.Error(err) + return "" + } + + // Test if the path is a windows path + windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`) + windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath) + + // Return as-is if no match + if windowsPathComponents == nil { + return abspath + } + + // Convert to WSL2-compatible path if it is a windows path + // NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows + // based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work. + driveLetter := strings.ToLower(windowsPathComponents[1]) + translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`) + // Should make something like /mnt/c/Users/person/My Folder/MyActProject + result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`) + return result +} + +func (*LinuxContainerEnvironmentExtensions) GetActPath() string { + return "/var/run/act" +} + +func (*LinuxContainerEnvironmentExtensions) GetPathVariableName() string { + return "PATH" +} + +func (*LinuxContainerEnvironmentExtensions) DefaultPathVariable() string { + return "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +} + +func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) string { + return strings.Join(paths, ":") +} + +func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]interface{} { + return map[string]interface{}{ + "os": "Linux", + "arch": RunnerArch(ctx), + "temp": "/tmp", + "tool_cache": "/opt/hostedtoolcache", + } +} diff --git a/act/container/linux_container_environment_extensions_test.go b/act/container/linux_container_environment_extensions_test.go new file mode 100644 index 00000000..38111714 --- /dev/null +++ b/act/container/linux_container_environment_extensions_test.go @@ -0,0 +1,71 @@ +package container + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func TestContainerPath(t *testing.T) { + type containerPathJob struct { + destinationPath string + sourcePath string + workDir string + } + + linuxcontainerext := &LinuxContainerEnvironmentExtensions{} + + if runtime.GOOS == "windows" { + cwd, err := os.Getwd() + if err != nil { + log.Error(err) + } + + rootDrive := os.Getenv("SystemDrive") + rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") + for _, v := range []containerPathJob{ + {"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""}, + {"/mnt/f/work/dir", `F:\work\dir`, ""}, + {"/mnt/c/windows/to/unix", "windows\\to\\unix", fmt.Sprintf("%s\\", rootDrive)}, + {fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)}, + } { + if v.workDir != "" { + if err := os.Chdir(v.workDir); err != nil { + log.Error(err) + t.Fail() + } + } + + assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath)) + } + + if err := os.Chdir(cwd); err != nil { + log.Error(err) + } + } else { + cwd, err := os.Getwd() + if err != nil { + log.Error(err) + } + for _, v := range []containerPathJob{ + {"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""}, + {"/home/act", `/home/act/`, ""}, + {cwd, ".", ""}, + } { + assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath)) + } + } +} + +type typeAssertMockContainer struct { + Container + LinuxContainerEnvironmentExtensions +} + +// Type assert Container + LinuxContainerEnvironmentExtensions implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &typeAssertMockContainer{} diff --git a/act/container/util.go b/act/container/util.go new file mode 100644 index 00000000..eb7f46c6 --- /dev/null +++ b/act/container/util.go @@ -0,0 +1,26 @@ +//go:build (!windows && !plan9 && !openbsd) || (!windows && !plan9 && !mips64) + +package container + +import ( + "os" + "syscall" + + "github.com/creack/pty" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + if tty { + return &syscall.SysProcAttr{ + Setsid: true, + Setctty: true, + } + } + return &syscall.SysProcAttr{ + Setpgid: true, + } +} + +func openPty() (*os.File, *os.File, error) { + return pty.Open() +} diff --git a/act/container/util_openbsd_mips64.go b/act/container/util_openbsd_mips64.go new file mode 100644 index 00000000..b991d694 --- /dev/null +++ b/act/container/util_openbsd_mips64.go @@ -0,0 +1,17 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + Setpgid: true, + } +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/act/container/util_plan9.go b/act/container/util_plan9.go new file mode 100644 index 00000000..cd64b4ab --- /dev/null +++ b/act/container/util_plan9.go @@ -0,0 +1,17 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + Rfork: syscall.RFNOTEG, + } +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/act/container/util_windows.go b/act/container/util_windows.go new file mode 100644 index 00000000..0a94c83a --- /dev/null +++ b/act/container/util_windows.go @@ -0,0 +1,15 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{CmdLine: cmdLine, CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP} +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/act/lookpath/LICENSE b/act/lookpath/LICENSE new file mode 100644 index 00000000..83403eff --- /dev/null +++ b/act/lookpath/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/act/lookpath/env.go b/act/lookpath/env.go new file mode 100644 index 00000000..dc376e7b --- /dev/null +++ b/act/lookpath/env.go @@ -0,0 +1,18 @@ +package lookpath + +import "os" + +type Env interface { + Getenv(name string) string +} + +type defaultEnv struct { +} + +func (*defaultEnv) Getenv(name string) string { + return os.Getenv(name) +} + +func LookPath(file string) (string, error) { + return LookPath2(file, &defaultEnv{}) +} diff --git a/act/lookpath/error.go b/act/lookpath/error.go new file mode 100644 index 00000000..0e3a3735 --- /dev/null +++ b/act/lookpath/error.go @@ -0,0 +1,10 @@ +package lookpath + +type Error struct { + Name string + Err error +} + +func (e *Error) Error() string { + return e.Err.Error() +} diff --git a/act/lookpath/lp_js.go b/act/lookpath/lp_js.go new file mode 100644 index 00000000..a967b861 --- /dev/null +++ b/act/lookpath/lp_js.go @@ -0,0 +1,23 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build js && wasm + +package lookpath + +import ( + "errors" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $PATH") + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // Wasm can not execute processes, so act as if there are no executables at all. + return "", &Error{file, ErrNotFound} +} diff --git a/act/lookpath/lp_plan9.go b/act/lookpath/lp_plan9.go new file mode 100644 index 00000000..a201b715 --- /dev/null +++ b/act/lookpath/lp_plan9.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $path") + +func findExecutable(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if m := d.Mode(); !m.IsDir() && m&0111 != 0 { + return nil + } + return fs.ErrPermission +} + +// LookPath searches for an executable named file in the +// directories named by the path environment variable. +// If file begins with "/", "#", "./", or "../", it is tried +// directly and the path is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // skip the path lookup for these prefixes + skip := []string{"/", "#", "./", "../"} + + for _, p := range skip { + if strings.HasPrefix(file, p) { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + } + + path := lenv.Getenv("path") + for _, dir := range filepath.SplitList(path) { + path := filepath.Join(dir, file) + if err := findExecutable(path); err == nil { + return path, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/act/lookpath/lp_unix.go b/act/lookpath/lp_unix.go new file mode 100644 index 00000000..233e21f9 --- /dev/null +++ b/act/lookpath/lp_unix.go @@ -0,0 +1,59 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $PATH") + +func findExecutable(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if m := d.Mode(); !m.IsDir() && m&0111 != 0 { + return nil + } + return fs.ErrPermission +} + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // NOTE(rsc): I wish we could use the Plan 9 behavior here + // (only bypass the path if file begins with / or ./ or ../) + // but that would not match all the Unix shells. + + if strings.Contains(file, "/") { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + path := lenv.Getenv("PATH") + for _, dir := range filepath.SplitList(path) { + if dir == "" { + // Unix shell semantics: path element "" means "." + dir = "." + } + path := filepath.Join(dir, file) + if err := findExecutable(path); err == nil { + return path, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/act/lookpath/lp_windows.go b/act/lookpath/lp_windows.go new file mode 100644 index 00000000..48204a79 --- /dev/null +++ b/act/lookpath/lp_windows.go @@ -0,0 +1,94 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in %PATH%") + +func chkStat(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsDir() { + return fs.ErrPermission + } + return nil +} + +func hasExt(file string) bool { + i := strings.LastIndex(file, ".") + if i < 0 { + return false + } + return strings.LastIndexAny(file, `:\/`) < i +} + +func findExecutable(file string, exts []string) (string, error) { + if len(exts) == 0 { + return file, chkStat(file) + } + if hasExt(file) { + if chkStat(file) == nil { + return file, nil + } + } + for _, e := range exts { + if f := file + e; chkStat(f) == nil { + return f, nil + } + } + return "", fs.ErrNotExist +} + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// LookPath also uses PATHEXT environment variable to match +// a suitable candidate. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + var exts []string + x := lenv.Getenv(`PATHEXT`) + if x != "" { + for _, e := range strings.Split(strings.ToLower(x), `;`) { + if e == "" { + continue + } + if e[0] != '.' { + e = "." + e + } + exts = append(exts, e) + } + } else { + exts = []string{".com", ".exe", ".bat", ".cmd"} + } + + if strings.ContainsAny(file, `:\/`) { + if f, err := findExecutable(file, exts); err == nil { + return f, nil + } else { + return "", &Error{file, err} + } + } + if f, err := findExecutable(filepath.Join(".", file), exts); err == nil { + return f, nil + } + path := lenv.Getenv("path") + for _, dir := range filepath.SplitList(path) { + if f, err := findExecutable(filepath.Join(dir, file), exts); err == nil { + return f, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/act/runner/action.go b/act/runner/action.go index 2dee988c..d3b012db 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -352,7 +352,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd []string stepContainer := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: rc.JobContainer.ToContainerPath(rc.Config.Workdir), Image: image, Username: rc.Config.Secrets["DOCKER_USERNAME"], Password: rc.Config.Secrets["DOCKER_PASSWORD"], @@ -396,11 +396,11 @@ func getContainerActionPaths(step *model.Step, actionDir string, rc *RunContext) containerActionDir := "." if step.Type() != model.StepTypeUsesActionRemote { actionName = getOsSafeRelativePath(actionDir, rc.Config.Workdir) - containerActionDir = rc.Config.ContainerWorkdir() + "/" + actionName + containerActionDir = rc.JobContainer.ToContainerPath(rc.Config.Workdir) + "/" + actionName actionName = "./" + actionName } else if step.Type() == model.StepTypeUsesActionRemote { actionName = getOsSafeRelativePath(actionDir, rc.ActionCacheDir()) - containerActionDir = ActPath + "/actions/" + actionName + containerActionDir = rc.JobContainer.GetActPath() + "/actions/" + actionName } if actionName == "" { diff --git a/act/runner/container_mock_test.go b/act/runner/container_mock_test.go index a336d404..0de07815 100644 --- a/act/runner/container_mock_test.go +++ b/act/runner/container_mock_test.go @@ -11,6 +11,7 @@ import ( type containerMock struct { mock.Mock container.Container + container.LinuxContainerEnvironmentExtensions } func (cm *containerMock) Create(capAdd []string, capDrop []string) common.Executor { diff --git a/act/runner/expression.go b/act/runner/expression.go index 43699800..c2257b12 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/nektos/act/pkg/common" - "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/exprparser" "github.com/nektos/act/pkg/model" "gopkg.in/yaml.v3" @@ -23,20 +22,22 @@ type ExpressionEvaluator interface { // NewExpressionEvaluator creates a new evaluator func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEvaluator { // todo: cleanup EvaluationEnvironment creation - job := rc.Run.Job() - strategy := make(map[string]interface{}) - if job.Strategy != nil { - strategy["fail-fast"] = job.Strategy.FailFast - strategy["max-parallel"] = job.Strategy.MaxParallel - } - - jobs := rc.Run.Workflow.Jobs - jobNeeds := rc.Run.Job().Needs() - using := make(map[string]map[string]map[string]string) - for _, needs := range jobNeeds { - using[needs] = map[string]map[string]string{ - "outputs": jobs[needs].Outputs, + strategy := make(map[string]interface{}) + if rc.Run != nil { + job := rc.Run.Job() + if job != nil && job.Strategy != nil { + strategy["fail-fast"] = job.Strategy.FailFast + strategy["max-parallel"] = job.Strategy.MaxParallel + } + + jobs := rc.Run.Workflow.Jobs + jobNeeds := rc.Run.Job().Needs() + + for _, needs := range jobNeeds { + using[needs] = map[string]map[string]string{ + "outputs": jobs[needs].Outputs, + } } } @@ -49,19 +50,16 @@ func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEval Job: rc.getJobContext(), // todo: should be unavailable // but required to interpolate/evaluate the step outputs on the job - Steps: rc.getStepsContext(), - Runner: map[string]interface{}{ - "os": "Linux", - "arch": container.RunnerArch(ctx), - "temp": "/tmp", - "tool_cache": "/opt/hostedtoolcache", - }, + Steps: rc.getStepsContext(), Secrets: rc.Config.Secrets, Strategy: strategy, Matrix: rc.Matrix, Needs: using, Inputs: inputs, } + if rc.JobContainer != nil { + ee.Runner = rc.JobContainer.GetRunnerContext(ctx) + } return expressionEvaluator{ interpreter: exprparser.NewInterpeter(ee, exprparser.Config{ Run: rc.Run, @@ -95,16 +93,10 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) inputs := getEvaluatorInputs(ctx, rc, step, ghc) ee := &exprparser.EvaluationEnvironment{ - Github: step.getGithubContext(ctx), - Env: *step.getEnv(), - Job: rc.getJobContext(), - Steps: rc.getStepsContext(), - Runner: map[string]interface{}{ - "os": "Linux", - "arch": container.RunnerArch(ctx), - "temp": "/tmp", - "tool_cache": "/opt/hostedtoolcache", - }, + Github: step.getGithubContext(ctx), + Env: *step.getEnv(), + Job: rc.getJobContext(), + Steps: rc.getStepsContext(), Secrets: rc.Config.Secrets, Strategy: strategy, Matrix: rc.Matrix, @@ -113,6 +105,9 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) // but required to interpolate/evaluate the inputs in actions/composite Inputs: inputs, } + if rc.JobContainer != nil { + ee.Runner = rc.JobContainer.GetRunnerContext(ctx) + } return expressionEvaluator{ interpreter: exprparser.NewInterpeter(ee, exprparser.Config{ Run: rc.Run, diff --git a/act/runner/expression_test.go b/act/runner/expression_test.go index b784a9a7..283b6cf9 100644 --- a/act/runner/expression_test.go +++ b/act/runner/expression_test.go @@ -117,7 +117,6 @@ func TestEvaluateRunContext(t *testing.T) { {"github.run_id", "1", ""}, {"github.run_number", "1", ""}, {"job.status", "success", ""}, - {"runner.os", "Linux", ""}, {"matrix.os", "Linux", ""}, {"matrix.foo", "bar", ""}, {"env.key", "value", ""}, diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index 5c3e3b1d..547e234c 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -38,6 +38,20 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo return common.NewDebugExecutor("No steps found") } + preSteps = append(preSteps, func(ctx context.Context) error { + // Have to be skipped for some Tests + if rc.Run == nil { + return nil + } + rc.ExprEval = rc.NewExpressionEvaluator(ctx) + // evaluate environment variables since they can contain + // GitHub's special environment variables. + for k, v := range rc.GetEnv() { + rc.Env[k] = rc.ExprEval.Interpolate(ctx, v) + } + return nil + }) + for i, stepModel := range infoSteps { stepModel := stepModel if stepModel == nil { diff --git a/act/runner/job_executor_test.go b/act/runner/job_executor_test.go index 8299f63f..edcbfc1b 100644 --- a/act/runner/job_executor_test.go +++ b/act/runner/job_executor_test.go @@ -79,6 +79,7 @@ func (jim *jobInfoMock) result(result string) { type jobContainerMock struct { container.Container + container.LinuxContainerEnvironmentExtensions } func (jcm *jobContainerMock) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) { diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 2e945ad8..ce2532fa 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -2,7 +2,10 @@ package runner import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -21,26 +24,25 @@ import ( "github.com/nektos/act/pkg/model" ) -const ActPath string = "/var/run/act" - // RunContext contains info about current job type RunContext struct { - Name string - Config *Config - Matrix map[string]interface{} - Run *model.Run - EventJSON string - Env map[string]string - ExtraPath []string - CurrentStep string - StepResults map[string]*model.StepResult - ExprEval ExpressionEvaluator - JobContainer container.Container - OutputMappings map[MappableOutput]MappableOutput - JobName string - ActionPath string - Parent *RunContext - Masks []string + Name string + Config *Config + Matrix map[string]interface{} + Run *model.Run + EventJSON string + Env map[string]string + ExtraPath []string + CurrentStep string + StepResults map[string]*model.StepResult + ExprEval ExpressionEvaluator + JobContainer container.ExecutionsEnvironment + OutputMappings map[MappableOutput]MappableOutput + JobName string + ActionPath string + Parent *RunContext + Masks []string + cleanUpJobContainer common.Executor } func (rc *RunContext) AddMask(mask string) { @@ -59,7 +61,13 @@ func (rc *RunContext) String() string { // GetEnv returns the env for the context func (rc *RunContext) GetEnv() map[string]string { if rc.Env == nil { - rc.Env = mergeMaps(rc.Run.Workflow.Env, rc.Run.Job().Environment(), rc.Config.Env) + rc.Env = map[string]string{} + if rc.Run != nil && rc.Run.Workflow != nil && rc.Config != nil { + job := rc.Run.Job() + if job != nil { + rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env) + } + } } rc.Env["ACT"] = "true" return rc.Env @@ -81,9 +89,11 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { fmt.Sprintf("%s:%s", rc.Config.ContainerDaemonSocket, "/var/run/docker.sock"), } + ext := container.LinuxContainerEnvironmentExtensions{} + mounts := map[string]string{ "act-toolcache": "/toolcache", - name + "-env": ActPath, + name + "-env": ext.GetActPath(), } if job := rc.Run.Job(); job != nil { @@ -109,14 +119,84 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { if selinux.GetEnabled() { bindModifiers = ":z" } - binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, rc.Config.ContainerWorkdir(), bindModifiers)) + binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, ext.ToContainerPath(rc.Config.Workdir), bindModifiers)) } else { - mounts[name] = rc.Config.ContainerWorkdir() + mounts[name] = ext.ToContainerPath(rc.Config.Workdir) } return binds, mounts } +func (rc *RunContext) startHostEnvironment() common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + rawLogger := logger.WithField("raw_output", true) + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { + if rc.Config.LogOutput { + rawLogger.Infof("%s", s) + } else { + rawLogger.Debugf("%s", s) + } + return true + }) + cacheDir := rc.ActionCacheDir() + randBytes := make([]byte, 8) + _, _ = rand.Read(randBytes) + miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) + actPath := filepath.Join(miscpath, "act") + if err := os.MkdirAll(actPath, 0777); err != nil { + return err + } + path := filepath.Join(miscpath, "hostexecutor") + if err := os.MkdirAll(path, 0777); err != nil { + return err + } + runnerTmp := filepath.Join(miscpath, "tmp") + if err := os.MkdirAll(runnerTmp, 0777); err != nil { + return err + } + toolCache := filepath.Join(cacheDir, "tool_cache") + rc.JobContainer = &container.HostEnvironment{ + Path: path, + TmpDir: runnerTmp, + ToolCache: toolCache, + Workdir: rc.Config.Workdir, + ActPath: actPath, + CleanUp: func() { + os.RemoveAll(miscpath) + }, + StdOut: logWriter, + } + rc.cleanUpJobContainer = rc.JobContainer.Remove() + rc.Env["RUNNER_TOOL_CACHE"] = toolCache + rc.Env["RUNNER_OS"] = runtime.GOOS + rc.Env["RUNNER_ARCH"] = runtime.GOARCH + rc.Env["RUNNER_TEMP"] = runnerTmp + for _, env := range os.Environ() { + i := strings.Index(env, "=") + if i > 0 { + rc.Env[env[0:i]] = env[i+1:] + } + } + + return common.NewPipelineExecutor( + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ + Name: "workflow/event.json", + Mode: 0644, + Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "workflow/envs.txt", + Mode: 0666, + Body: "", + }, &container.FileEntry{ + Name: "workflow/paths.txt", + Mode: 0666, + Body: "", + }), + )(ctx) + } +} + func (rc *RunContext) startJobContainer() common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) @@ -146,12 +226,22 @@ func (rc *RunContext) startJobContainer() common.Executor { envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx))) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp")) + ext := container.LinuxContainerEnvironmentExtensions{} binds, mounts := rc.GetBindsAndMounts() + rc.cleanUpJobContainer = func(ctx context.Context) error { + if rc.JobContainer != nil && !rc.Config.ReuseContainers { + return rc.JobContainer.Remove(). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + } + return nil + } + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"}, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: ext.ToContainerPath(rc.Config.Workdir), Image: image, Username: username, Password: password, @@ -167,6 +257,9 @@ func (rc *RunContext) startJobContainer() common.Executor { Platform: rc.Config.ContainerArchitecture, Options: rc.options(ctx), }) + if rc.JobContainer == nil { + return errors.New("Failed to create job container") + } return common.NewPipelineExecutor( rc.JobContainer.Pull(rc.Config.ForcePull), @@ -175,7 +268,7 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.JobContainer.Start(false), rc.JobContainer.UpdateFromImageEnv(&rc.Env), rc.JobContainer.UpdateFromEnv("/etc/environment", &rc.Env), - rc.JobContainer.Copy(ActPath+"/", &container.FileEntry{ + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ Name: "workflow/event.json", Mode: 0644, Body: rc.EventJSON, @@ -201,10 +294,8 @@ func (rc *RunContext) execJobContainer(cmd []string, env map[string]string, user // stopJobContainer removes the job container (if it exists) and its volume (if it exists) if !rc.Config.ReuseContainers func (rc *RunContext) stopJobContainer() common.Executor { return func(ctx context.Context) error { - if rc.JobContainer != nil && !rc.Config.ReuseContainers { - return rc.JobContainer.Remove(). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + if rc.cleanUpJobContainer != nil && !rc.Config.ReuseContainers { + return rc.cleanUpJobContainer(ctx) } return nil } @@ -241,7 +332,13 @@ func (rc *RunContext) interpolateOutputs() common.Executor { } func (rc *RunContext) startContainer() common.Executor { - return rc.startJobContainer() + return func(ctx context.Context) error { + image := rc.platformImage(ctx) + if strings.EqualFold(image, "-self-hosted") { + return rc.startHostEnvironment()(ctx) + } + return rc.startJobContainer()(ctx) + } } func (rc *RunContext) stopContainer() common.Executor { @@ -409,13 +506,11 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext logger := common.Logger(ctx) ghc := &model.GithubContext{ Event: make(map[string]interface{}), - EventPath: ActPath + "/workflow/event.json", Workflow: rc.Run.Workflow.Name, RunID: rc.Config.Env["GITHUB_RUN_ID"], RunNumber: rc.Config.Env["GITHUB_RUN_NUMBER"], Actor: rc.Config.Actor, EventName: rc.Config.EventName, - Workspace: rc.Config.ContainerWorkdir(), Action: rc.CurrentStep, Token: rc.Config.Token, ActionPath: rc.ActionPath, @@ -424,6 +519,10 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext RunnerPerflog: rc.Config.Env["RUNNER_PERFLOG"], RunnerTrackingID: rc.Config.Env["RUNNER_TRACKING_ID"], } + if rc.JobContainer != nil { + ghc.EventPath = rc.JobContainer.GetActPath() + "/workflow/event.json" + ghc.Workspace = rc.JobContainer.ToContainerPath(rc.Config.Workdir) + } if ghc.RunID == "" { ghc.RunID = "1" @@ -538,8 +637,8 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubContext, env map[string]string) map[string]string { env["CI"] = "true" - env["GITHUB_ENV"] = ActPath + "/workflow/envs.txt" - env["GITHUB_PATH"] = ActPath + "/workflow/paths.txt" + env["GITHUB_ENV"] = rc.JobContainer.GetActPath() + "/workflow/envs.txt" + env["GITHUB_PATH"] = rc.JobContainer.GetActPath() + "/workflow/paths.txt" env["GITHUB_WORKFLOW"] = github.Workflow env["GITHUB_RUN_ID"] = github.RunID env["GITHUB_RUN_NUMBER"] = github.RunNumber diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index cdafb8ee..de8f177a 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -385,14 +385,12 @@ func TestGetGitHubContext(t *testing.T) { } assert.Equal(t, ghc.RunID, "1") - assert.Equal(t, ghc.Workspace, rc.Config.containerPath(cwd)) assert.Equal(t, ghc.RunNumber, "1") assert.Equal(t, ghc.RetentionDays, "0") assert.Equal(t, ghc.Actor, actor) assert.Equal(t, ghc.Repository, repo) assert.Equal(t, ghc.RepositoryOwner, owner) assert.Equal(t, ghc.RunnerPerflog, "/dev/null") - assert.Equal(t, ghc.EventPath, ActPath+"/workflow/event.json") assert.Equal(t, ghc.Token, rc.Config.Secrets["GITHUB_TOKEN"]) } diff --git a/act/runner/runner.go b/act/runner/runner.go index 9c10fbe4..564814bc 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -4,10 +4,6 @@ import ( "context" "fmt" "os" - "path/filepath" - "regexp" - "runtime" - "strings" "time" log "github.com/sirupsen/logrus" @@ -57,46 +53,6 @@ type Config struct { ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. } -// Resolves the equivalent host path inside the container -// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject -// For use in docker volumes and binds -func (config *Config) containerPath(path string) string { - if runtime.GOOS == "windows" && strings.Contains(path, "/") { - log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.") - return "" - } - - abspath, err := filepath.Abs(path) - if err != nil { - log.Error(err) - return "" - } - - // Test if the path is a windows path - windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`) - windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath) - - // Return as-is if no match - if windowsPathComponents == nil { - return abspath - } - - // Convert to WSL2-compatible path if it is a windows path - // NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows - // based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work. - driveLetter := strings.ToLower(windowsPathComponents[1]) - translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`) - // Should make something like /mnt/c/Users/person/My Folder/MyActProject - result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`) - return result -} - -// Resolves the equivalent host path inside the container -// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject -func (config *Config) ContainerWorkdir() string { - return config.containerPath(config.Workdir) -} - type runnerImpl struct { config *Config eventJSON string @@ -163,11 +119,6 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { if len(matrixes) > 1 { rc.Name = fmt.Sprintf("%s-%d", rc.Name, i+1) } - // evaluate environment variables since they can contain - // GitHub's special environment variables. - for k, v := range rc.GetEnv() { - rc.Env[k] = rc.ExprEval.Interpolate(ctx, v) - } if len(rc.String()) > maxJobNameLen { maxJobNameLen = len(rc.String()) } diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 232fc163..9cb4ff40 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -205,6 +205,95 @@ func TestRunEvent(t *testing.T) { } } +func TestRunEventHostEnvironment(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + ctx := context.Background() + + tables := []TestJobFileInfo{} + + if runtime.GOOS == "linux" { + platforms := map[string]string{ + "ubuntu-latest": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + // Shells + {workdir, "shells/defaults", "push", "", platforms}, + {workdir, "shells/pwsh", "push", "", platforms}, + {workdir, "shells/bash", "push", "", platforms}, + {workdir, "shells/python", "push", "", platforms}, + {workdir, "shells/sh", "push", "", platforms}, + + // Local action + {workdir, "local-action-js", "push", "", platforms}, + + // Uses + {workdir, "uses-composite", "push", "", platforms}, + {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms}, + {workdir, "uses-nested-composite", "push", "", platforms}, + {workdir, "act-composite-env-test", "push", "", platforms}, + + // Eval + {workdir, "evalmatrix", "push", "", platforms}, + {workdir, "evalmatrixneeds", "push", "", platforms}, + {workdir, "evalmatrixneeds2", "push", "", platforms}, + {workdir, "evalmatrix-merge-map", "push", "", platforms}, + {workdir, "evalmatrix-merge-array", "push", "", platforms}, + {workdir, "issue-1195", "push", "", platforms}, + + {workdir, "fail", "push", "exit with `FAILURE`: 1", platforms}, + {workdir, "runs-on", "push", "", platforms}, + {workdir, "checkout", "push", "", platforms}, + {workdir, "remote-action-js", "push", "", platforms}, + {workdir, "matrix", "push", "", platforms}, + {workdir, "matrix-include-exclude", "push", "", platforms}, + {workdir, "commands", "push", "", platforms}, + {workdir, "defaults-run", "push", "", platforms}, + {workdir, "composite-fail-with-output", "push", "", platforms}, + {workdir, "issue-597", "push", "", platforms}, + {workdir, "issue-598", "push", "", platforms}, + {workdir, "if-env-act", "push", "", platforms}, + {workdir, "env-and-path", "push", "", platforms}, + {workdir, "non-existent-action", "push", "Job 'nopanic' failed", platforms}, + {workdir, "outputs", "push", "", platforms}, + {workdir, "steps-context/conclusion", "push", "", platforms}, + {workdir, "steps-context/outcome", "push", "", platforms}, + {workdir, "job-status-check", "push", "job 'fail' failed", platforms}, + {workdir, "if-expressions", "push", "Job 'mytest' failed", platforms}, + {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms}, + {workdir, "evalenv", "push", "", platforms}, + {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms}, + }...) + } + if runtime.GOOS == "windows" { + platforms := map[string]string{ + "windows-latest": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + {workdir, "windows-prepend-path", "push", "", platforms}, + {workdir, "windows-add-env", "push", "", platforms}, + }...) + } else { + platforms := map[string]string{ + "self-hosted": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + {workdir, "nix-prepend-path", "push", "", platforms}, + }...) + } + + for _, table := range tables { + t.Run(table.workflowPath, func(t *testing.T) { + table.runTest(ctx, t, &Config{}) + }) + } +} + func TestDryrunEvent(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") @@ -320,60 +409,3 @@ func TestRunEventPullRequest(t *testing.T) { tjfi.runTest(context.Background(), t, &Config{EventPath: filepath.Join(workdir, workflowPath, "event.json")}) } - -func TestContainerPath(t *testing.T) { - type containerPathJob struct { - destinationPath string - sourcePath string - workDir string - } - - if runtime.GOOS == "windows" { - cwd, err := os.Getwd() - if err != nil { - log.Error(err) - } - - rootDrive := os.Getenv("SystemDrive") - rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") - for _, v := range []containerPathJob{ - {"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""}, - {"/mnt/f/work/dir", `F:\work\dir`, ""}, - {"/mnt/c/windows/to/unix", "windows\\to\\unix", fmt.Sprintf("%s\\", rootDrive)}, - {fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)}, - } { - if v.workDir != "" { - if err := os.Chdir(v.workDir); err != nil { - log.Error(err) - t.Fail() - } - } - - runnerConfig := &Config{ - Workdir: v.sourcePath, - } - - assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir)) - } - - if err := os.Chdir(cwd); err != nil { - log.Error(err) - } - } else { - cwd, err := os.Getwd() - if err != nil { - log.Error(err) - } - for _, v := range []containerPathJob{ - {"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""}, - {"/home/act", `/home/act/`, ""}, - {cwd, ".", ""}, - } { - runnerConfig := &Config{ - Workdir: v.sourcePath, - } - - assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir)) - } - } -} diff --git a/act/runner/step.go b/act/runner/step.go index 8682d582..4f7803e1 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -162,13 +162,12 @@ func mergeEnv(ctx context.Context, step step) { mergeIntoMap(env, rc.GetEnv()) } - if (*env)["PATH"] == "" { - (*env)["PATH"] = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` + path := rc.JobContainer.GetPathVariableName() + if (*env)[path] == "" { + (*env)[path] = rc.JobContainer.DefaultPathVariable() } if rc.ExtraPath != nil && len(rc.ExtraPath) > 0 { - p := (*env)["PATH"] - (*env)["PATH"] = strings.Join(rc.ExtraPath, `:`) - (*env)["PATH"] += `:` + p + (*env)[path] = rc.JobContainer.JoinPathVariable(append(rc.ExtraPath, (*env)[path])...) } rc.withGithubEnv(ctx, step.getGithubContext(ctx), *env) diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index 8afbe31b..00d9502a 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -118,7 +118,7 @@ func (sar *stepActionRemote) main() common.Executor { return nil } eval := sar.RunContext.NewExpressionEvaluator(ctx) - copyToPath := path.Join(sar.RunContext.Config.ContainerWorkdir(), eval.Interpolate(ctx, sar.Step.With["path"])) + copyToPath := path.Join(sar.RunContext.JobContainer.ToContainerPath(sar.RunContext.Config.Workdir), eval.Interpolate(ctx, sar.Step.With["path"])) return sar.RunContext.JobContainer.CopyDir(copyToPath, sar.RunContext.Config.Workdir+string(filepath.Separator)+".", sar.RunContext.Config.UseGitIgnore)(ctx) } diff --git a/act/runner/step_docker.go b/act/runner/step_docker.go index 4ac3b27f..c955b162 100644 --- a/act/runner/step_docker.go +++ b/act/runner/step_docker.go @@ -116,7 +116,7 @@ func (sd *stepDocker) newStepContainer(ctx context.Context, image string, cmd [] stepContainer := ContainerNewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: rc.JobContainer.ToContainerPath(rc.Config.Workdir), Image: image, Username: rc.Config.Secrets["DOCKER_USERNAME"], Password: rc.Config.Secrets["DOCKER_PASSWORD"], diff --git a/act/runner/step_docker_test.go b/act/runner/step_docker_test.go index c7482718..c6794b47 100644 --- a/act/runner/step_docker_test.go +++ b/act/runner/step_docker_test.go @@ -17,7 +17,7 @@ func TestStepDockerMain(t *testing.T) { // mock the new container call origContainerNewContainer := ContainerNewContainer - ContainerNewContainer = func(containerInput *container.NewContainerInput) container.Container { + ContainerNewContainer = func(containerInput *container.NewContainerInput) container.ExecutionsEnvironment { input = containerInput return cm } diff --git a/act/runner/step_run.go b/act/runner/step_run.go index 85dfed64..a74f781b 100644 --- a/act/runner/step_run.go +++ b/act/runner/step_run.go @@ -68,7 +68,8 @@ func (sr *stepRun) setupShellCommandExecutor() common.Executor { return err } - return sr.RunContext.JobContainer.Copy(ActPath, &container.FileEntry{ + rc := sr.getRunContext() + return rc.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{ Name: scriptName, Mode: 0755, Body: script, @@ -128,7 +129,8 @@ func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string, logger.Debugf("Wrote add-mask command to '%s'", name) } - scriptPath := fmt.Sprintf("%s/%s", ActPath, name) + rc := sr.getRunContext() + scriptPath := fmt.Sprintf("%s/%s", rc.JobContainer.GetActPath(), name) sr.cmd, err = shellquote.Split(strings.Replace(scCmd, `{0}`, scriptPath, 1)) return name, script, err diff --git a/act/runner/testdata/nix-prepend-path/push.yml b/act/runner/testdata/nix-prepend-path/push.yml new file mode 100644 index 00000000..71fd5bc7 --- /dev/null +++ b/act/runner/testdata/nix-prepend-path/push.yml @@ -0,0 +1,26 @@ +on: + push: +defaults: + run: + shell: sh +jobs: + test: + runs-on: self-hosted + steps: + - run: | + mkdir build + echo '#!/usr/bin/env sh' > build/testtool + echo 'echo Hi' >> build/testtool + chmod +x build/testtool + - run: | + echo '${{ tojson(runner) }}' + ls + echo '${{ github.workspace }}' + working-directory: ${{ github.workspace }}/build + - run: | + echo "$GITHUB_PATH" + echo '${{ github.workspace }}/build' > "$GITHUB_PATH" + cat "$GITHUB_PATH" + - run: | + echo "$PATH" + testtool diff --git a/act/runner/testdata/windows-add-env/push.yml b/act/runner/testdata/windows-add-env/push.yml new file mode 100644 index 00000000..275c5f1f --- /dev/null +++ b/act/runner/testdata/windows-add-env/push.yml @@ -0,0 +1,27 @@ +on: + push: +defaults: + run: + shell: pwsh +jobs: + test: + runs-on: windows-latest + steps: + - run: | + echo $env:GITHUB_ENV + echo "key=val" > $env:GITHUB_ENV + echo "key2<> $env:GITHUB_ENV + echo "line1" >> $env:GITHUB_ENV + echo "line2" >> $env:GITHUB_ENV + echo "EOF" >> $env:GITHUB_ENV + cat $env:GITHUB_ENV + - run: | + ls env: + if($env:key -ne 'val') { + echo "Unexpected value for `$env:key: $env:key" + exit 1 + } + if($env:key2 -ne "line1`nline2") { + echo "Unexpected value for `$env:key2: $env:key2" + exit 1 + } diff --git a/act/runner/testdata/windows-prepend-path/push.yml b/act/runner/testdata/windows-prepend-path/push.yml new file mode 100644 index 00000000..176de691 --- /dev/null +++ b/act/runner/testdata/windows-prepend-path/push.yml @@ -0,0 +1,25 @@ +on: + push: +defaults: + run: + shell: pwsh +jobs: + test: + runs-on: windows-latest + steps: + - run: | + mkdir build + echo '@echo off' > build/test.cmd + echo 'echo Hi' >> build/test.cmd + - run: | + echo '${{ tojson(runner) }}' + ls + echo '${{ github.workspace }}' + working-directory: ${{ github.workspace }}\build + - run: | + echo $env:GITHUB_PATH + echo '${{ github.workspace }}\build' > $env:GITHUB_PATH + cat $env:GITHUB_PATH + - run: | + echo $env:PATH + test From 082dbde25ca37e78b138692bbc1df3504fb70478 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 16 Nov 2022 22:42:57 +0100 Subject: [PATCH 07/12] feat: GITHUB_STATE and GITHUB_OUTPUT file commands (#1391) * feat: set-state and set-output file commands * increase test timeout from 10m to 15m * Prepare for HostExecutor PR Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- act/runner/step.go | 37 +++++++++++++++++++++++++++ act/runner/step_action_local_test.go | 29 +++++++++++++++++++-- act/runner/step_action_remote_test.go | 24 +++++++++++++++++ act/runner/step_docker_test.go | 12 +++++++++ act/runner/step_run_test.go | 12 +++++++++ 6 files changed, 113 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 4a321285..baf5196a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -50,7 +50,7 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- - - run: go test -v -cover -coverprofile=coverage.txt -covermode=atomic ./... + - run: go test -v -cover -coverprofile=coverage.txt -covermode=atomic -timeout 15m ./... - name: Upload Codecov report uses: codecov/codecov-action@v3.1.1 with: diff --git a/act/runner/step.go b/act/runner/step.go index 4f7803e1..8a9d2b6d 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -3,9 +3,11 @@ package runner import ( "context" "fmt" + "path" "strings" "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/exprparser" "github.com/nektos/act/pkg/model" ) @@ -94,6 +96,20 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo } logger.Infof("\u2B50 Run %s %s", stage, stepString) + // Prepare and clean Runner File Commands + actPath := rc.JobContainer.GetActPath() + outputFileCommand := path.Join("workflow", "outputcmd.txt") + stateFileCommand := path.Join("workflow", "statecmd.txt") + (*step.getEnv())["GITHUB_OUTPUT"] = path.Join(actPath, outputFileCommand) + (*step.getEnv())["GITHUB_STATE"] = path.Join(actPath, stateFileCommand) + _ = rc.JobContainer.Copy(actPath, &container.FileEntry{ + Name: outputFileCommand, + Mode: 0666, + }, &container.FileEntry{ + Name: stateFileCommand, + Mode: 0666, + })(ctx) + err = executor(ctx) if err == nil { @@ -117,6 +133,27 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo logger.WithField("stepResult", rc.StepResults[rc.CurrentStep].Outcome).Errorf(" \u274C Failure - %s %s", stage, stepString) } + // Process Runner File Commands + orgerr := err + state := map[string]string{} + err = rc.JobContainer.UpdateFromEnv(path.Join(actPath, stateFileCommand), &state)(ctx) + if err != nil { + return err + } + for k, v := range state { + rc.saveState(ctx, map[string]string{"name": k}, v) + } + output := map[string]string{} + err = rc.JobContainer.UpdateFromEnv(path.Join(actPath, outputFileCommand), &output)(ctx) + if err != nil { + return err + } + for k, v := range output { + rc.setOutput(ctx, map[string]string{"name": k}, v) + } + if orgerr != nil { + return orgerr + } return err } } diff --git a/act/runner/step_action_local_test.go b/act/runner/step_action_local_test.go index 8180d6de..6b14f3d2 100644 --- a/act/runner/step_action_local_test.go +++ b/act/runner/step_action_local_test.go @@ -2,6 +2,7 @@ package runner import ( "context" + "path/filepath" "strings" "testing" @@ -63,7 +64,7 @@ func TestStepActionLocalTest(t *testing.T) { }, } - salm.On("readAction", sal.Step, "/tmp/path/to/action", "", mock.Anything, mock.Anything). + salm.On("readAction", sal.Step, filepath.Clean("/tmp/path/to/action"), "", mock.Anything, mock.Anything). Return(&model.Action{}, nil) cm.On("UpdateFromImageEnv", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { @@ -78,7 +79,19 @@ func TestStepActionLocalTest(t *testing.T) { return nil }) - salm.On("runAction", sal, "/tmp/path/to/action", (*remoteAction)(nil)).Return(func(ctx context.Context) error { + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(func(ctx context.Context) error { return nil }) @@ -275,6 +288,18 @@ func TestStepActionLocalPost(t *testing.T) { }) } cm.On("Exec", suffixMatcher("pkg/runner/local/action/post.js"), sal.env, "", "").Return(func(ctx context.Context) error { return tt.err }) + + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) } err := sal.post()(ctx) diff --git a/act/runner/step_action_remote_test.go b/act/runner/step_action_remote_test.go index 1e117ea5..84fe0829 100644 --- a/act/runner/step_action_remote_test.go +++ b/act/runner/step_action_remote_test.go @@ -172,6 +172,18 @@ func TestStepActionRemote(t *testing.T) { } if tt.mocks.run { sarm.On("runAction", sar, suffixMatcher("act/remote-action@v1"), newRemoteAction(sar.Step.Uses)).Return(func(ctx context.Context) error { return tt.runError }) + + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) } err := sar.pre()(ctx) @@ -582,6 +594,18 @@ func TestStepActionRemotePost(t *testing.T) { } if tt.mocks.exec { cm.On("Exec", []string{"node", "/var/run/act/actions/remote-action@v1/post.js"}, sar.env, "", "").Return(func(ctx context.Context) error { return tt.err }) + + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) } err := sar.post()(ctx) diff --git a/act/runner/step_docker_test.go b/act/runner/step_docker_test.go index c6794b47..e0e8575f 100644 --- a/act/runner/step_docker_test.go +++ b/act/runner/step_docker_test.go @@ -86,6 +86,18 @@ func TestStepDockerMain(t *testing.T) { return nil }) + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + err := sd.main()(ctx) assert.Nil(t, err) diff --git a/act/runner/step_run_test.go b/act/runner/step_run_test.go index 081d8647..e5cde123 100644 --- a/act/runner/step_run_test.go +++ b/act/runner/step_run_test.go @@ -65,6 +65,18 @@ func TestStepRun(t *testing.T) { return nil }) + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + ctx := context.Background() err := sr.main()(ctx) From 49b7141194bc5420656ddc9146f4aae02da5683f Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Wed, 16 Nov 2022 22:55:23 +0100 Subject: [PATCH 08/12] feat: interpolate the step names (#1422) * feat: interpolate the step names Step names could contain expressions refering to event data. Fixes #1353 * test: add missing mock data * fix: setup composite expression evaluator The RunContext does contain a cached ExpressionEvaluator. This should be the case the composite RunContext as well. Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Casey Lee --- act/runner/action_composite.go | 1 + act/runner/job_executor.go | 2 +- act/runner/job_executor_test.go | 10 ++++++++++ act/runner/step.go | 2 +- act/runner/step_action_local_test.go | 1 + act/runner/step_action_remote_test.go | 2 ++ act/runner/step_docker_test.go | 5 +++-- 7 files changed, 19 insertions(+), 4 deletions(-) diff --git a/act/runner/action_composite.go b/act/runner/action_composite.go index a7b41438..c1e94fcd 100644 --- a/act/runner/action_composite.go +++ b/act/runner/action_composite.go @@ -71,6 +71,7 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action Parent: parent, EventJSON: parent.EventJSON, } + compositerc.ExprEval = compositerc.NewExpressionEvaluator(ctx) return compositerc } diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index 547e234c..e1de6cbc 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -133,7 +133,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo func useStepLogger(rc *RunContext, stepModel *model.Step, stage stepStage, executor common.Executor) common.Executor { return func(ctx context.Context) error { - ctx = withStepLogger(ctx, stepModel.ID, stepModel.String(), stage.String()) + ctx = withStepLogger(ctx, stepModel.ID, rc.ExprEval.Interpolate(ctx, stepModel.String()), stage.String()) rawLogger := common.Logger(ctx).WithField("raw_output", true) logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { diff --git a/act/runner/job_executor_test.go b/act/runner/job_executor_test.go index edcbfc1b..e00a4fd6 100644 --- a/act/runner/job_executor_test.go +++ b/act/runner/job_executor_test.go @@ -249,7 +249,17 @@ func TestNewJobExecutor(t *testing.T) { sfm := &stepFactoryMock{} rc := &RunContext{ JobContainer: &jobContainerMock{}, + Run: &model.Run{ + JobID: "test", + Workflow: &model.Workflow{ + Jobs: map[string]*model.Job{ + "test": {}, + }, + }, + }, + Config: &Config{}, } + rc.ExprEval = rc.NewExpressionEvaluator(ctx) executorOrder := make([]string, 0) jim.On("steps").Return(tt.steps) diff --git a/act/runner/step.go b/act/runner/step.go index 8a9d2b6d..f730ac1a 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -90,7 +90,7 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo return nil } - stepString := stepModel.String() + stepString := rc.ExprEval.Interpolate(ctx, stepModel.String()) if strings.Contains(stepString, "::add-mask::") { stepString = "add-mask command" } diff --git a/act/runner/step_action_local_test.go b/act/runner/step_action_local_test.go index 6b14f3d2..63902898 100644 --- a/act/runner/step_action_local_test.go +++ b/act/runner/step_action_local_test.go @@ -275,6 +275,7 @@ func TestStepActionLocalPost(t *testing.T) { Step: tt.stepModel, action: tt.actionModel, } + sal.RunContext.ExprEval = sal.RunContext.NewExpressionEvaluator(ctx) if tt.mocks.env { cm.On("UpdateFromImageEnv", &sal.env).Return(func(ctx context.Context) error { return nil }) diff --git a/act/runner/step_action_remote_test.go b/act/runner/step_action_remote_test.go index 84fe0829..72b0eeeb 100644 --- a/act/runner/step_action_remote_test.go +++ b/act/runner/step_action_remote_test.go @@ -155,6 +155,7 @@ func TestStepActionRemote(t *testing.T) { readAction: sarm.readAction, runAction: sarm.runAction, } + sar.RunContext.ExprEval = sar.RunContext.NewExpressionEvaluator(ctx) suffixMatcher := func(suffix string) interface{} { return mock.MatchedBy(func(actionDir string) bool { @@ -586,6 +587,7 @@ func TestStepActionRemotePost(t *testing.T) { Step: tt.stepModel, action: tt.actionModel, } + sar.RunContext.ExprEval = sar.RunContext.NewExpressionEvaluator(ctx) if tt.mocks.env { cm.On("UpdateFromImageEnv", &sar.env).Return(func(ctx context.Context) error { return nil }) diff --git a/act/runner/step_docker_test.go b/act/runner/step_docker_test.go index e0e8575f..2008357f 100644 --- a/act/runner/step_docker_test.go +++ b/act/runner/step_docker_test.go @@ -25,6 +25,8 @@ func TestStepDockerMain(t *testing.T) { ContainerNewContainer = origContainerNewContainer })() + ctx := context.Background() + sd := &stepDocker{ RunContext: &RunContext{ StepResults: map[string]*model.StepResult{}, @@ -51,8 +53,7 @@ func TestStepDockerMain(t *testing.T) { WorkingDirectory: "workdir", }, } - - ctx := context.Background() + sd.RunContext.ExprEval = sd.RunContext.NewExpressionEvaluator(ctx) cm.On("UpdateFromImageEnv", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil From a2c20c6cb7bec14b2233d5f249a8922ee9beccd0 Mon Sep 17 00:00:00 2001 From: Magnus Markling Date: Wed, 16 Nov 2022 23:12:00 +0100 Subject: [PATCH 09/12] Remove dead code (#1425) Co-authored-by: Casey Lee --- act/container/docker_logger.go | 53 ---------------------------------- 1 file changed, 53 deletions(-) diff --git a/act/container/docker_logger.go b/act/container/docker_logger.go index 5c85785c..b6b2f150 100644 --- a/act/container/docker_logger.go +++ b/act/container/docker_logger.go @@ -22,59 +22,6 @@ type dockerMessage struct { const logPrefix = " \U0001F433 " -/* -func logDockerOutput(ctx context.Context, dockerResponse io.Reader) { - logger := common.Logger(ctx) - if entry, ok := logger.(*logrus.Entry); ok { - w := entry.Writer() - _, err := stdcopy.StdCopy(w, w, dockerResponse) - if err != nil { - logrus.Error(err) - } - } else if lgr, ok := logger.(*logrus.Logger); ok { - w := lgr.Writer() - _, err := stdcopy.StdCopy(w, w, dockerResponse) - if err != nil { - logrus.Error(err) - } - } else { - logrus.Errorf("Unable to get writer from logger (type=%T)", logger) - } -} -*/ - -/* -func streamDockerOutput(ctx context.Context, dockerResponse io.Reader) { - /* - out := os.Stdout - go func() { - <-ctx.Done() - //fmt.Println() - }() - - _, err := io.Copy(out, dockerResponse) - if err != nil { - logrus.Error(err) - } - * / - - logger := common.Logger(ctx) - reader := bufio.NewReader(dockerResponse) - - for { - if ctx.Err() != nil { - break - } - line, _, err := reader.ReadLine() - if err == io.EOF { - break - } - logger.Debugf("%s\n", line) - } - -} -*/ - func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser, isError bool) error { if dockerResponse == nil { return nil From 76d766bcd1dcd309fe89b58c3e5da25ba4218753 Mon Sep 17 00:00:00 2001 From: Randolph Chung <112228989+avx-rchung@users.noreply.github.com> Date: Mon, 21 Nov 2022 06:47:38 -0800 Subject: [PATCH 10/12] fix: handle subdirectors and exclusions (#1012) (#1446) --- act/exprparser/functions.go | 32 +++++++++++++------ act/exprparser/functions_test.go | 6 +++- .../testdata/for-hashing-3/data.txt | 1 + .../for-hashing-3/nested/nested-data.txt | 1 + 4 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 act/exprparser/testdata/for-hashing-3/data.txt create mode 100644 act/exprparser/testdata/for-hashing-3/nested/nested-data.txt diff --git a/act/exprparser/functions.go b/act/exprparser/functions.go index 37a38db5..047a0e3c 100644 --- a/act/exprparser/functions.go +++ b/act/exprparser/functions.go @@ -6,12 +6,14 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "os" "path/filepath" "reflect" "strconv" "strings" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/nektos/act/pkg/model" "github.com/rhysd/actionlint" ) @@ -178,25 +180,37 @@ func (impl *interperterImpl) fromJSON(value reflect.Value) (interface{}, error) } func (impl *interperterImpl) hashFiles(paths ...reflect.Value) (string, error) { - var filepaths []string + var ps []gitignore.Pattern + const cwdPrefix = "." + string(filepath.Separator) + const excludeCwdPrefix = "!" + cwdPrefix for _, path := range paths { if path.Kind() == reflect.String { - filepaths = append(filepaths, path.String()) + cleanPath := path.String() + if strings.HasPrefix(cleanPath, cwdPrefix) { + cleanPath = cleanPath[len(cwdPrefix):] + } else if strings.HasPrefix(cleanPath, excludeCwdPrefix) { + cleanPath = "!" + cleanPath[len(excludeCwdPrefix):] + } + ps = append(ps, gitignore.ParsePattern(cleanPath, nil)) } else { return "", fmt.Errorf("Non-string path passed to hashFiles") } } + matcher := gitignore.NewMatcher(ps) + var files []string - - for i := range filepaths { - newFiles, err := filepath.Glob(filepath.Join(impl.config.WorkingDir, filepaths[i])) - if err != nil { - return "", fmt.Errorf("Unable to glob.Glob: %v", err) + if err := filepath.Walk(impl.config.WorkingDir, func(path string, fi fs.FileInfo, err error) error { + sansPrefix := strings.TrimPrefix(path, impl.config.WorkingDir+string(filepath.Separator)) + parts := strings.Split(sansPrefix, string(filepath.Separator)) + if fi.IsDir() || !matcher.Match(parts, fi.IsDir()) { + return nil } - - files = append(files, newFiles...) + files = append(files, path) + return nil + }); err != nil { + return "", fmt.Errorf("Unable to filepath.Walk: %v", err) } if len(files) == 0 { diff --git a/act/exprparser/functions_test.go b/act/exprparser/functions_test.go index 009e9113..3c6392c1 100644 --- a/act/exprparser/functions_test.go +++ b/act/exprparser/functions_test.go @@ -188,7 +188,11 @@ func TestFunctionHashFiles(t *testing.T) { {"hashFiles('**/non-extant-files') }}", "", "hash-non-existing-file"}, {"hashFiles('**/non-extant-files', '**/more-non-extant-files') }}", "", "hash-multiple-non-existing-files"}, {"hashFiles('./for-hashing-1.txt') }}", "66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18", "hash-single-file"}, - {"hashFiles('./for-hashing-*') }}", "8e5935e7e13368cd9688fe8f48a0955293676a021562582c7e848dafe13fb046", "hash-multiple-files"}, + {"hashFiles('./for-hashing-*.txt') }}", "8e5935e7e13368cd9688fe8f48a0955293676a021562582c7e848dafe13fb046", "hash-multiple-files"}, + {"hashFiles('./for-hashing-*.txt', '!./for-hashing-2.txt') }}", "66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18", "hash-negative-pattern"}, + {"hashFiles('./for-hashing-**') }}", "c418ba693753c84115ced0da77f876cddc662b9054f4b129b90f822597ee2f94", "hash-multiple-files-and-directories"}, + {"hashFiles('./for-hashing-3/**') }}", "6f5696b546a7a9d6d42a449dc9a56bef244aaa826601ef27466168846139d2c2", "hash-nested-directories"}, + {"hashFiles('./for-hashing-3/**/nested-data.txt') }}", "8ecadfb49f7f978d0a9f3a957e9c8da6cc9ab871f5203b5d9f9d1dc87d8af18c", "hash-nested-directories-2"}, } env := &EvaluationEnvironment{} diff --git a/act/exprparser/testdata/for-hashing-3/data.txt b/act/exprparser/testdata/for-hashing-3/data.txt new file mode 100644 index 00000000..5ac7bf9b --- /dev/null +++ b/act/exprparser/testdata/for-hashing-3/data.txt @@ -0,0 +1 @@ +Knock knock! diff --git a/act/exprparser/testdata/for-hashing-3/nested/nested-data.txt b/act/exprparser/testdata/for-hashing-3/nested/nested-data.txt new file mode 100644 index 00000000..ebe288b2 --- /dev/null +++ b/act/exprparser/testdata/for-hashing-3/nested/nested-data.txt @@ -0,0 +1 @@ +Anybody home? From fe754290e33e72d936ea3cec53251dc01b08cbb2 Mon Sep 17 00:00:00 2001 From: Lim Chun Leng Date: Fri, 25 Nov 2022 19:38:49 +0900 Subject: [PATCH 11/12] Fix shellcommand error on sh shell (#1464) Co-authored-by: Lim Chun Leng --- act/model/workflow.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/act/model/workflow.go b/act/model/workflow.go index 4e1d3496..f567a1f7 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -495,7 +495,7 @@ func (s *Step) ShellCommand() string { case "python": shellCommand = "python {0}" case "sh": - shellCommand = "sh -e -c {0}" + shellCommand = "sh -e {0}" case "cmd": shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" case "powershell": From 1566965d67804ee69ba7ed23544972a00a71bef4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Nov 2022 02:16:11 +0000 Subject: [PATCH 12/12] build(deps): bump megalinter/megalinter from 6.14.0 to 6.15.0 (#1475) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.14.0 to 6.15.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.14.0...v6.15.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index baf5196a..ffd9b7ec 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.3.1 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.14.0 + - uses: megalinter/megalinter/flavors/go@v6.15.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}