From b7905c0a84672c94ec2cee0463366c39c65cd42d Mon Sep 17 00:00:00 2001 From: Shubh Bapna <38372682+shubhbapna@users.noreply.github.com> Date: Sun, 19 Mar 2023 13:25:55 -0400 Subject: [PATCH 01/29] feat: specify matrix on command line (#1675) * added matrix option * select the correct subset of matrix configuration after producing all the matrix configuration * add tests * update readme * lint fix * remove matrix from readme --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/runner.go | 93 ++++++++++++------- act/runner/runner_test.go | 28 ++++++ .../matrix-with-user-inclusions/push.yml | 34 +++++++ cmd/input.go | 1 + cmd/root.go | 23 +++++ 5 files changed, 144 insertions(+), 35 deletions(-) create mode 100644 act/runner/testdata/matrix-with-user-inclusions/push.yml diff --git a/act/runner/runner.go b/act/runner/runner.go index 7e3f9b1b..715d1584 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -20,40 +20,41 @@ type Runner interface { // Config contains the config for a new runner type Config struct { - Actor string // the user that triggered the event - Workdir string // path to working directory - BindWorkdir bool // bind the workdir to the job container - EventName string // name of event to run - EventPath string // path to JSON file to use for event.json in containers - DefaultBranch string // name of the main branch for this repository - ReuseContainers bool // reuse containers to maintain state - ForcePull bool // force pulling of the image, even if already present - ForceRebuild bool // force rebuilding local docker image action - LogOutput bool // log the output from docker run - JSONLogger bool // use json or text logger - Env map[string]string // env for containers - Inputs map[string]string // manually passed action inputs - Secrets map[string]string // list of secrets - Token string // GitHub token - InsecureSecrets bool // switch hiding output when printing to terminal - Platforms map[string]string // list of platforms - Privileged bool // use privileged mode - UsernsMode string // user namespace to use - ContainerArchitecture string // Desired OS/architecture platform for running containers - ContainerDaemonSocket string // Path to Docker daemon socket - ContainerOptions string // Options for the job container - UseGitIgnore bool // controls if paths in .gitignore should not be copied into container, default true - GitHubInstance string // GitHub instance to use, default "github.com" - ContainerCapAdd []string // list of kernel capabilities to add to the containers - ContainerCapDrop []string // list of kernel capabilities to remove from the containers - AutoRemove bool // controls if the container is automatically removed upon workflow completion - ArtifactServerPath string // the path where the artifact server stores uploads - ArtifactServerAddr string // the address the artifact server binds to - ArtifactServerPort string // the port the artifact server binds to - NoSkipCheckout bool // do not skip actions/checkout - RemoteName string // remote name in local git repo config - ReplaceGheActionWithGithubCom []string // Use actions from GitHub Enterprise instance to GitHub - ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. + Actor string // the user that triggered the event + Workdir string // path to working directory + BindWorkdir bool // bind the workdir to the job container + EventName string // name of event to run + EventPath string // path to JSON file to use for event.json in containers + DefaultBranch string // name of the main branch for this repository + ReuseContainers bool // reuse containers to maintain state + ForcePull bool // force pulling of the image, even if already present + ForceRebuild bool // force rebuilding local docker image action + LogOutput bool // log the output from docker run + JSONLogger bool // use json or text logger + Env map[string]string // env for containers + Inputs map[string]string // manually passed action inputs + Secrets map[string]string // list of secrets + Token string // GitHub token + InsecureSecrets bool // switch hiding output when printing to terminal + Platforms map[string]string // list of platforms + Privileged bool // use privileged mode + UsernsMode string // user namespace to use + ContainerArchitecture string // Desired OS/architecture platform for running containers + ContainerDaemonSocket string // Path to Docker daemon socket + ContainerOptions string // Options for the job container + UseGitIgnore bool // controls if paths in .gitignore should not be copied into container, default true + GitHubInstance string // GitHub instance to use, default "github.com" + ContainerCapAdd []string // list of kernel capabilities to add to the containers + ContainerCapDrop []string // list of kernel capabilities to remove from the containers + AutoRemove bool // controls if the container is automatically removed upon workflow completion + ArtifactServerPath string // the path where the artifact server stores uploads + ArtifactServerAddr string // the address the artifact server binds to + ArtifactServerPort string // the port the artifact server binds to + NoSkipCheckout bool // do not skip actions/checkout + RemoteName string // remote name in local git repo config + ReplaceGheActionWithGithubCom []string // Use actions from GitHub Enterprise instance to GitHub + ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. + Matrix map[string]map[string]bool // Matrix config to run } type caller struct { @@ -116,7 +117,10 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { log.Errorf("Error while evaluating matrix: %v", err) } } - matrixes := job.GetMatrixes() + + matrixes := selectMatrixes(job.GetMatrixes(), runner.config.Matrix) + log.Debugf("Final matrix after applying user inclusions '%v'", matrixes) + maxParallel := 4 if job.Strategy != nil { maxParallel = job.Strategy.MaxParallel @@ -171,6 +175,25 @@ func handleFailure(plan *model.Plan) common.Executor { } } +func selectMatrixes(originalMatrixes []map[string]interface{}, targetMatrixValues map[string]map[string]bool) []map[string]interface{} { + matrixes := make([]map[string]interface{}, 0) + for _, original := range originalMatrixes { + flag := true + for key, val := range original { + if allowedVals, ok := targetMatrixValues[key]; ok { + valToString := fmt.Sprintf("%v", val) + if _, ok := allowedVals[valToString]; !ok { + flag = false + } + } + } + if flag { + matrixes = append(matrixes, original) + } + } + return matrixes +} + func (runner *runnerImpl) newRunContext(ctx context.Context, run *model.Run, matrix map[string]interface{}) *RunContext { rc := &RunContext{ Config: runner.config, diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 60a81937..f8468bbb 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -186,6 +186,7 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config Inputs: cfg.Inputs, GitHubInstance: "github.com", ContainerArchitecture: cfg.ContainerArchitecture, + Matrix: cfg.Matrix, } runner, err := New(runnerConfig) @@ -584,3 +585,30 @@ func TestRunEventPullRequest(t *testing.T) { tjfi.runTest(context.Background(), t, &Config{EventPath: filepath.Join(workdir, workflowPath, "event.json")}) } + +func TestRunMatrixWithUserDefinedInclusions(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + workflowPath := "matrix-with-user-inclusions" + + tjfi := TestJobFileInfo{ + workdir: workdir, + workflowPath: workflowPath, + eventName: "push", + errorMessage: "", + platforms: platforms, + } + + matrix := map[string]map[string]bool{ + "node": { + "8": true, + "8.x": true, + }, + "os": { + "ubuntu-18.04": true, + }, + } + + tjfi.runTest(context.Background(), t, &Config{Matrix: matrix}) +} diff --git a/act/runner/testdata/matrix-with-user-inclusions/push.yml b/act/runner/testdata/matrix-with-user-inclusions/push.yml new file mode 100644 index 00000000..2fd19b4e --- /dev/null +++ b/act/runner/testdata/matrix-with-user-inclusions/push.yml @@ -0,0 +1,34 @@ +name: matrix-with-user-inclusions +on: push + +jobs: + build: + name: PHP ${{ matrix.os }} ${{ matrix.node}} + runs-on: ubuntu-latest + steps: + - run: | + echo ${NODE_VERSION} | grep 8 + echo ${OS_VERSION} | grep ubuntu-18.04 + env: + NODE_VERSION: ${{ matrix.node }} + OS_VERSION: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-18.04, macos-latest] + node: [4, 6, 8, 10] + exclude: + - os: macos-latest + node: 4 + include: + - os: ubuntu-16.04 + node: 10 + + test: + runs-on: ubuntu-latest + strategy: + matrix: + node: [8.x, 10.x, 12.x, 13.x] + steps: + - run: echo ${NODE_VERSION} | grep 8.x + env: + NODE_VERSION: ${{ matrix.node }} diff --git a/cmd/input.go b/cmd/input.go index 37655a55..9327de2d 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -47,6 +47,7 @@ type Input struct { remoteName string replaceGheActionWithGithubCom []string replaceGheActionTokenWithGithubCom string + matrix []string } func (i *Input) resolve(path string) string { diff --git a/cmd/root.go b/cmd/root.go index e5c04791..9983a4b6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -66,6 +66,7 @@ func Execute(ctx context.Context, version string) { rootCmd.Flags().BoolVar(&input.autoRemove, "rm", false, "automatically remove container(s)/volume(s) after a workflow(s) failure") rootCmd.Flags().StringArrayVarP(&input.replaceGheActionWithGithubCom, "replace-ghe-action-with-github-com", "", []string{}, "If you are using GitHub Enterprise Server and allow specified actions from GitHub (github.com), you can set actions on this. (e.g. --replace-ghe-action-with-github-com =github/super-linter)") rootCmd.Flags().StringVar(&input.replaceGheActionTokenWithGithubCom, "replace-ghe-action-token-with-github-com", "", "If you are using replace-ghe-action-with-github-com and you want to use private actions on GitHub, you have to set personal access token") + rootCmd.Flags().StringArrayVarP(&input.matrix, "matrix", "", []string{}, "specify which matrix configuration to include (e.g. --matrix java:13") rootCmd.PersistentFlags().StringVarP(&input.actor, "actor", "a", "nektos/act", "user that triggered the event") rootCmd.PersistentFlags().StringVarP(&input.workflowsPath, "workflows", "W", "./.github/workflows/", "path to workflow file(s)") rootCmd.PersistentFlags().BoolVarP(&input.noWorkflowRecurse, "no-recurse", "", false, "Flag to disable running workflows from subdirectories of specified path in '--workflows'/'-W' flag") @@ -295,6 +296,24 @@ func readEnvs(path string, envs map[string]string) bool { return false } +func parseMatrix(matrix []string) map[string]map[string]bool { + // each matrix entry should be of the form - string:string + r := regexp.MustCompile(":") + matrixes := make(map[string]map[string]bool) + for _, m := range matrix { + matrix := r.Split(m, 2) + if len(matrix) < 2 { + log.Fatalf("Invalid matrix format. Failed to parse %s", m) + } else { + if _, ok := matrixes[matrix[0]]; !ok { + matrixes[matrix[0]] = make(map[string]bool) + } + matrixes[matrix[0]][matrix[1]] = true + } + } + return matrixes +} + //nolint:gocyclo func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []string) error { return func(cmd *cobra.Command, args []string) error { @@ -329,6 +348,9 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str secrets := newSecrets(input.secrets) _ = readEnvs(input.Secretfile(), secrets) + matrixes := parseMatrix(input.matrix) + log.Debugf("Evaluated matrix inclusions: %v", matrixes) + planner, err := model.NewWorkflowPlanner(input.WorkflowsPath(), input.noWorkflowRecurse) if err != nil { return err @@ -508,6 +530,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str RemoteName: input.remoteName, ReplaceGheActionWithGithubCom: input.replaceGheActionWithGithubCom, ReplaceGheActionTokenWithGithubCom: input.replaceGheActionTokenWithGithubCom, + Matrix: matrixes, } r, err := runner.New(config) if err != nil { From 9419713cf3acd04b36fe0cd0a67d3f6ad330f82e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Mar 2023 23:21:57 +0000 Subject: [PATCH 02/29] build(deps): bump fregante/setup-git-user from 1 to 2 (#1664) Bumps [fregante/setup-git-user](https://github.com/fregante/setup-git-user) from 1 to 2. - [Release notes](https://github.com/fregante/setup-git-user/releases) - [Commits](https://github.com/fregante/setup-git-user/compare/v1...v2) --- updated-dependencies: - dependency-name: fregante/setup-git-user dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/promote.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index a29f79b3..660ce555 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 ref: master token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} - - uses: fregante/setup-git-user@v1 + - uses: fregante/setup-git-user@v2 - uses: actions/setup-go@v3 with: go-version: ${{ env.GO_VERSION }} From 0207edafaae7c63a247a94b9153685bc3ac070ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 19 Mar 2023 23:37:58 +0000 Subject: [PATCH 03/29] build(deps): bump megalinter/megalinter from 6.20.0 to 6.20.1 (#1679) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.20.0 to 6.20.1. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.20.0...v6.20.1) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 69365dd5..bf156f88 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.4.0 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.20.0 + - uses: megalinter/megalinter/flavors/go@v6.20.1 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From cbaa7fb49a64979929a82d66916053c55022f3c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Mar 2023 03:31:00 +0000 Subject: [PATCH 04/29] build(deps): bump actions/setup-go from 3 to 4 (#1689) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3 to 4. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 8 ++++---- .github/workflows/promote.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index bf156f88..e4b67867 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ env.GO_VERSION }} check-latest: true @@ -39,7 +39,7 @@ jobs: fetch-depth: 2 - name: Set up QEMU uses: docker/setup-qemu-action@v2 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ env.GO_VERSION }} check-latest: true @@ -72,7 +72,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 2 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ env.GO_VERSION }} check-latest: true @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ env.GO_VERSION }} check-latest: true diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 660ce555..7ea6c2ab 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -18,7 +18,7 @@ jobs: ref: master token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} - uses: fregante/setup-git-user@v2 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ env.GO_VERSION }} check-latest: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69b5aecf..d990730d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v4 with: go-version: ${{ env.GO_VERSION }} check-latest: true From 114db8e32cc39fd55294af6d35d0fd75c0827724 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 03:13:54 +0000 Subject: [PATCH 05/29] build(deps): bump megalinter/megalinter from 6.20.1 to 6.21.0 (#1699) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.20.1 to 6.21.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.20.1...v6.21.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e4b67867..290fd0e7 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.4.0 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.20.1 + - uses: megalinter/megalinter/flavors/go@v6.21.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From d0a51942664eeef794f5794aadc840f896ea01b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Mar 2023 03:29:53 +0000 Subject: [PATCH 06/29] build(deps): bump actions/stale from 7 to 8 (#1700) Bumps [actions/stale](https://github.com/actions/stale) from 7 to 8. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v7...v8) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 91810627..88e815c1 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -8,7 +8,7 @@ jobs: name: Stale runs-on: ubuntu-latest steps: - - uses: actions/stale@v7 + - uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'Issue is stale and will be closed in 14 days unless there is new activity' From bd27c853b1a5d3e7afce3ae5356ce64372ce4dac Mon Sep 17 00:00:00 2001 From: Kris Date: Tue, 28 Mar 2023 14:24:03 +0200 Subject: [PATCH 07/29] Make sure working directory is respected when configured from matrix (#1686) * Make sure working directory is respected when configured from matrix * Fix regression by setting Workingdirectory on stepRun instead of step or too early --- act/runner/step_run.go | 24 ++++++++++++++---------- act/runner/testdata/workdir/push.yml | 10 ++++++++++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/act/runner/step_run.go b/act/runner/step_run.go index ca77d569..4d855fdb 100644 --- a/act/runner/step_run.go +++ b/act/runner/step_run.go @@ -13,10 +13,11 @@ import ( ) type stepRun struct { - Step *model.Step - RunContext *RunContext - cmd []string - env map[string]string + Step *model.Step + RunContext *RunContext + cmd []string + env map[string]string + WorkingDirectory string } func (sr *stepRun) pre() common.Executor { @@ -27,12 +28,11 @@ func (sr *stepRun) pre() common.Executor { func (sr *stepRun) main() common.Executor { sr.env = map[string]string{} - return runStepExecutor(sr, stepStageMain, common.NewPipelineExecutor( sr.setupShellCommandExecutor(), func(ctx context.Context) error { sr.getRunContext().ApplyExtraPath(ctx, &sr.env) - return sr.getRunContext().JobContainer.Exec(sr.cmd, sr.env, "", sr.Step.WorkingDirectory)(ctx) + return sr.getRunContext().JobContainer.Exec(sr.cmd, sr.env, "", sr.WorkingDirectory)(ctx) }, )) } @@ -167,16 +167,20 @@ func (sr *stepRun) setupShell(ctx context.Context) { func (sr *stepRun) setupWorkingDirectory(ctx context.Context) { rc := sr.RunContext step := sr.Step + workingdirectory := "" if step.WorkingDirectory == "" { - step.WorkingDirectory = rc.Run.Job().Defaults.Run.WorkingDirectory + workingdirectory = rc.Run.Job().Defaults.Run.WorkingDirectory + } else { + workingdirectory = step.WorkingDirectory } // jobs can receive context values, so we interpolate - step.WorkingDirectory = rc.NewExpressionEvaluator(ctx).Interpolate(ctx, step.WorkingDirectory) + workingdirectory = rc.NewExpressionEvaluator(ctx).Interpolate(ctx, workingdirectory) // but top level keys in workflow file like `defaults` or `env` can't - if step.WorkingDirectory == "" { - step.WorkingDirectory = rc.Run.Workflow.Defaults.Run.WorkingDirectory + if workingdirectory == "" { + workingdirectory = rc.Run.Workflow.Defaults.Run.WorkingDirectory } + sr.WorkingDirectory = workingdirectory } diff --git a/act/runner/testdata/workdir/push.yml b/act/runner/testdata/workdir/push.yml index c287d554..b76a9513 100644 --- a/act/runner/testdata/workdir/push.yml +++ b/act/runner/testdata/workdir/push.yml @@ -22,3 +22,13 @@ jobs: runs-on: ubuntu-latest steps: - run: '[[ "$(pwd)" == "/tmp" ]]' + + workdir-from-matrix: + runs-on: ubuntu-latest + strategy: + max-parallel: 1 + matrix: + work_dir: ["/tmp", "/root"] + steps: + - run: '[[ "$(pwd)" == "${{ matrix.work_dir }}" ]]' + working-directory: ${{ matrix.work_dir }} From 92da209c983992743052a98562eb2d5198195a05 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 31 Mar 2023 21:08:46 +0800 Subject: [PATCH 08/29] fix: use os.UserHomeDir (#1706) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 3 +-- cmd/notices.go | 3 +-- cmd/root.go | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 27bbe878..7883a0f1 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -17,7 +17,6 @@ import ( "runtime" "strings" - "github.com/mitchellh/go-homedir" "github.com/opencontainers/selinux/go-selinux" log "github.com/sirupsen/logrus" @@ -359,7 +358,7 @@ func (rc *RunContext) ActionCacheDir() string { var xdgCache string var ok bool if xdgCache, ok = os.LookupEnv("XDG_CACHE_HOME"); !ok || xdgCache == "" { - if home, err := homedir.Dir(); err == nil { + if home, err := os.UserHomeDir(); err == nil { xdgCache = filepath.Join(home, ".cache") } else if xdgCache, err = filepath.Abs("."); err != nil { log.Fatal(err) diff --git a/cmd/notices.go b/cmd/notices.go index bd03aa3e..9ddcf6fa 100644 --- a/cmd/notices.go +++ b/cmd/notices.go @@ -11,7 +11,6 @@ import ( "strings" "time" - "github.com/mitchellh/go-homedir" log "github.com/sirupsen/logrus" ) @@ -136,7 +135,7 @@ func etagPath() string { var xdgCache string var ok bool if xdgCache, ok = os.LookupEnv("XDG_CACHE_HOME"); !ok || xdgCache == "" { - if home, err := homedir.Dir(); err == nil { + if home, err := os.UserHomeDir(); err == nil { xdgCache = filepath.Join(home, ".cache") } else if xdgCache, err = filepath.Abs("."); err != nil { log.Fatal(err) diff --git a/cmd/root.go b/cmd/root.go index 9983a4b6..fc3e43e1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,7 +15,6 @@ import ( "github.com/adrg/xdg" "github.com/andreaskoch/go-fswatch" "github.com/joho/godotenv" - "github.com/mitchellh/go-homedir" gitignore "github.com/sabhiram/go-gitignore" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -95,7 +94,7 @@ func Execute(ctx context.Context, version string) { } func configLocations() []string { - home, err := homedir.Dir() + home, err := os.UserHomeDir() if err != nil { log.Fatal(err) } From 1dcd52ba08c8e05708bbecf0094b68faaa6e93e0 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 31 Mar 2023 21:28:08 +0800 Subject: [PATCH 09/29] feat: improve GetOutboundIP (#1707) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/common/outbound_ip.go | 72 ++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/act/common/outbound_ip.go b/act/common/outbound_ip.go index eaa4cce3..66e15e5d 100644 --- a/act/common/outbound_ip.go +++ b/act/common/outbound_ip.go @@ -2,20 +2,74 @@ package common import ( "net" - - log "github.com/sirupsen/logrus" + "sort" + "strings" ) -// https://stackoverflow.com/a/37382208 -// Get preferred outbound ip of this machine +// GetOutboundIP returns an outbound IP address of this machine. +// It tries to access the internet and returns the local IP address of the connection. +// If the machine cannot access the internet, it returns a preferred IP address from network interfaces. +// It returns nil if no IP address is found. func GetOutboundIP() net.IP { + // See https://stackoverflow.com/a/37382208 conn, err := net.Dial("udp", "8.8.8.8:80") - if err != nil { - log.Fatal(err) + if err == nil { + defer conn.Close() + return conn.LocalAddr().(*net.UDPAddr).IP } - defer conn.Close() - localAddr := conn.LocalAddr().(*net.UDPAddr) + // So the machine cannot access the internet. Pick an IP address from network interfaces. + if ifs, err := net.Interfaces(); err == nil { + type IP struct { + net.IP + net.Interface + } + var ips []IP + for _, i := range ifs { + if addrs, err := i.Addrs(); err == nil { + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + if ip.IsGlobalUnicast() { + ips = append(ips, IP{ip, i}) + } + } + } + } + if len(ips) > 1 { + sort.Slice(ips, func(i, j int) bool { + ifi := ips[i].Interface + ifj := ips[j].Interface - return localAddr.IP + // ethernet is preferred + if vi, vj := strings.HasPrefix(ifi.Name, "e"), strings.HasPrefix(ifj.Name, "e"); vi != vj { + return vi + } + + ipi := ips[i].IP + ipj := ips[j].IP + + // IPv4 is preferred + if vi, vj := ipi.To4() != nil, ipj.To4() != nil; vi != vj { + return vi + } + + // en0 is preferred to en1 + if ifi.Name != ifj.Name { + return ifi.Name < ifj.Name + } + + // fallback + return ipi.String() < ipj.String() + }) + return ips[0].IP + } + } + + return nil } From 3f8942c62d7cb6ee6c8ca497c8d059fad48f1be0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Apr 2023 03:13:25 +0000 Subject: [PATCH 10/29] build(deps): bump megalinter/megalinter from 6.21.0 to 6.22.1 (#1710) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.21.0 to 6.22.1. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.21.0...v6.22.1) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 290fd0e7..169e7cb6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.4.0 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.21.0 + - uses: megalinter/megalinter/flavors/go@v6.22.1 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 2d6c2844919905cde36794531ae5c6611fcf2eda Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Apr 2023 03:50:21 +0000 Subject: [PATCH 11/29] build(deps): bump megalinter/megalinter from 6.22.1 to 6.22.2 (#1720) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.22.1 to 6.22.2. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.22.1...v6.22.2) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 169e7cb6..06723787 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.4.0 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.22.1 + - uses: megalinter/megalinter/flavors/go@v6.22.2 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 8b25ad4dbef6a80f8d7db47e0e0317e6fde93670 Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Thu, 13 Apr 2023 15:09:28 +0200 Subject: [PATCH 12/29] fix: add `server_url` attribute to github context (#1727) * fix: add `server_url` attribute to github context The `server_urL` attribute was missing in the `github` context. Previously it was exposed as environment variable only. Closes #1726 * fix: also set `api_url` and `graphql_url` attributes --- act/model/github_context.go | 3 +++ act/runner/run_context.go | 46 +++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/act/model/github_context.go b/act/model/github_context.go index e4c31fcc..ceb54cdc 100644 --- a/act/model/github_context.go +++ b/act/model/github_context.go @@ -36,6 +36,9 @@ type GithubContext struct { RetentionDays string `json:"retention_days"` RunnerPerflog string `json:"runner_perflog"` RunnerTrackingID string `json:"runner_tracking_id"` + ServerURL string `json:"server_url "` + APIURL string `json:"api_url"` + GraphQLURL string `json:"graphql_url"` } func asString(v interface{}) string { diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 7883a0f1..7ab8d187 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -631,6 +631,27 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext ghc.SetRefTypeAndName() + // defaults + ghc.ServerURL = "https://github.com" + ghc.APIURL = "https://api.github.com" + ghc.GraphQLURL = "https://api.github.com/graphql" + // per GHES + if rc.Config.GitHubInstance != "github.com" { + ghc.ServerURL = fmt.Sprintf("https://%s", rc.Config.GitHubInstance) + ghc.APIURL = fmt.Sprintf("https://%s/api/v3", rc.Config.GitHubInstance) + ghc.GraphQLURL = fmt.Sprintf("https://%s/api/graphql", rc.Config.GitHubInstance) + } + // allow to be overridden by user + if rc.Config.Env["GITHUB_SERVER_URL"] != "" { + ghc.ServerURL = rc.Config.Env["GITHUB_SERVER_URL"] + } + if rc.Config.Env["GITHUB_API_URL"] != "" { + ghc.ServerURL = rc.Config.Env["GITHUB_API_URL"] + } + if rc.Config.Env["GITHUB_GRAPHQL_URL"] != "" { + ghc.ServerURL = rc.Config.Env["GITHUB_GRAPHQL_URL"] + } + return ghc } @@ -704,28 +725,9 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon env["RUNNER_TRACKING_ID"] = github.RunnerTrackingID env["GITHUB_BASE_REF"] = github.BaseRef env["GITHUB_HEAD_REF"] = github.HeadRef - - defaultServerURL := "https://github.com" - defaultAPIURL := "https://api.github.com" - defaultGraphqlURL := "https://api.github.com/graphql" - - if rc.Config.GitHubInstance != "github.com" { - defaultServerURL = fmt.Sprintf("https://%s", rc.Config.GitHubInstance) - defaultAPIURL = fmt.Sprintf("https://%s/api/v3", rc.Config.GitHubInstance) - defaultGraphqlURL = fmt.Sprintf("https://%s/api/graphql", rc.Config.GitHubInstance) - } - - if env["GITHUB_SERVER_URL"] == "" { - env["GITHUB_SERVER_URL"] = defaultServerURL - } - - if env["GITHUB_API_URL"] == "" { - env["GITHUB_API_URL"] = defaultAPIURL - } - - if env["GITHUB_GRAPHQL_URL"] == "" { - env["GITHUB_GRAPHQL_URL"] = defaultGraphqlURL - } + env["GITHUB_SERVER_URL"] = github.ServerURL + env["GITHUB_API_URL"] = github.APIURL + env["GITHUB_GRAPHQL_URL"] = github.GraphQLURL if rc.Config.ArtifactServerPath != "" { setActionRuntimeVars(rc, env) From e71161acde0bbc90beb91f4fd03a93f3b4efd37b Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Thu, 13 Apr 2023 15:47:59 +0200 Subject: [PATCH 13/29] fix: reusable workflow panic (#1728) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/model/workflow.go | 3 ++- .../local-reusable-workflow-no-inputs-array.yml | 10 ++++++++++ .../local-reusable-workflow-no-inputs-string.yml | 9 +++++++++ act/runner/testdata/uses-workflow/local-workflow.yml | 6 ++++++ 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 act/runner/testdata/.github/workflows/local-reusable-workflow-no-inputs-array.yml create mode 100644 act/runner/testdata/.github/workflows/local-reusable-workflow-no-inputs-string.yml diff --git a/act/model/workflow.go b/act/model/workflow.go index d7e2922b..35e4587b 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -123,7 +123,8 @@ type WorkflowCallResult struct { func (w *Workflow) WorkflowCallConfig() *WorkflowCall { if w.RawOn.Kind != yaml.MappingNode { - return nil + // The callers expect for "on: workflow_call" and "on: [ workflow_call ]" a non nil return value + return &WorkflowCall{} } var val map[string]yaml.Node diff --git a/act/runner/testdata/.github/workflows/local-reusable-workflow-no-inputs-array.yml b/act/runner/testdata/.github/workflows/local-reusable-workflow-no-inputs-array.yml new file mode 100644 index 00000000..3df4ae3a --- /dev/null +++ b/act/runner/testdata/.github/workflows/local-reusable-workflow-no-inputs-array.yml @@ -0,0 +1,10 @@ +name: reusable + +on: +- workflow_call + +jobs: + reusable_workflow_job: + runs-on: ubuntu-latest + steps: + - run: echo Test \ No newline at end of file diff --git a/act/runner/testdata/.github/workflows/local-reusable-workflow-no-inputs-string.yml b/act/runner/testdata/.github/workflows/local-reusable-workflow-no-inputs-string.yml new file mode 100644 index 00000000..7558c1c5 --- /dev/null +++ b/act/runner/testdata/.github/workflows/local-reusable-workflow-no-inputs-string.yml @@ -0,0 +1,9 @@ +name: reusable + +on: workflow_call + +jobs: + reusable_workflow_job: + runs-on: ubuntu-latest + steps: + - run: echo Test \ No newline at end of file diff --git a/act/runner/testdata/uses-workflow/local-workflow.yml b/act/runner/testdata/uses-workflow/local-workflow.yml index 070e4d0c..2e9a08d7 100644 --- a/act/runner/testdata/uses-workflow/local-workflow.yml +++ b/act/runner/testdata/uses-workflow/local-workflow.yml @@ -19,6 +19,12 @@ jobs: number_required: 1 secrets: inherit + reusable-workflow-with-on-string-notation: + uses: ./.github/workflows/local-reusable-workflow-no-inputs-string.yml + + reusable-workflow-with-on-array-notation: + uses: ./.github/workflows/local-reusable-workflow-no-inputs-array.yml + output-test: runs-on: ubuntu-latest needs: From 2e9189a6e29d0939e2c25108daccfc3102a4168d Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Thu, 13 Apr 2023 16:09:29 +0200 Subject: [PATCH 14/29] fix: ghc assignment typo (#1729) * fix: ghc assignment typo * fixup server_url --- act/model/github_context.go | 2 +- act/runner/run_context.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/act/model/github_context.go b/act/model/github_context.go index ceb54cdc..5ed3d962 100644 --- a/act/model/github_context.go +++ b/act/model/github_context.go @@ -36,7 +36,7 @@ type GithubContext struct { RetentionDays string `json:"retention_days"` RunnerPerflog string `json:"runner_perflog"` RunnerTrackingID string `json:"runner_tracking_id"` - ServerURL string `json:"server_url "` + ServerURL string `json:"server_url"` APIURL string `json:"api_url"` GraphQLURL string `json:"graphql_url"` } diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 7ab8d187..12738115 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -646,10 +646,10 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext ghc.ServerURL = rc.Config.Env["GITHUB_SERVER_URL"] } if rc.Config.Env["GITHUB_API_URL"] != "" { - ghc.ServerURL = rc.Config.Env["GITHUB_API_URL"] + ghc.APIURL = rc.Config.Env["GITHUB_API_URL"] } if rc.Config.Env["GITHUB_GRAPHQL_URL"] != "" { - ghc.ServerURL = rc.Config.Env["GITHUB_GRAPHQL_URL"] + ghc.GraphQLURL = rc.Config.Env["GITHUB_GRAPHQL_URL"] } return ghc From e95c7aba65aff6f77c8f84f2e5919888b46f9f2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Apr 2023 03:13:12 +0000 Subject: [PATCH 15/29] build(deps): bump codecov/codecov-action from 3.1.1 to 3.1.2 (#1735) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3.1.1 to 3.1.2. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3.1.1...v3.1.2) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 06723787..34ea1d2b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -55,7 +55,7 @@ jobs: with: upload-logs-name: logs-linux - name: Upload Codecov report - uses: codecov/codecov-action@v3.1.1 + uses: codecov/codecov-action@v3.1.2 with: files: coverage.txt fail_ci_if_error: true # optional (default = false) From 0539eb2ac52c8d29680b2a011d56257b8222270a Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 18 Apr 2023 15:35:31 +0200 Subject: [PATCH 16/29] feat: support yaml env/secrets/inputs file (#1733) * support yaml env/secrets/inputs file * Update root.go * read the docs again.. --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- cmd/root.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index fc3e43e1..02afb50b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -24,6 +24,7 @@ import ( "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/model" "github.com/nektos/act/pkg/runner" + "gopkg.in/yaml.v3" ) // Execute is the entry point to running the CLI @@ -281,9 +282,26 @@ func parseEnvs(env []string, envs map[string]string) bool { return false } +func readYamlFile(file string) (map[string]string, error) { + content, err := os.ReadFile(file) + if err != nil { + return nil, err + } + ret := map[string]string{} + if err = yaml.Unmarshal(content, &ret); err != nil { + return nil, err + } + return ret, nil +} + func readEnvs(path string, envs map[string]string) bool { if _, err := os.Stat(path); err == nil { - env, err := godotenv.Read(path) + var env map[string]string + if ext := filepath.Ext(path); ext == ".yml" || ext == ".yaml" { + env, err = readYamlFile(path) + } else { + env, err = godotenv.Read(path) + } if err != nil { log.Fatalf("Error loading from %s: %v", path, err) } From 8d2c320f7a015dc072bcea7ef65c7bb077b98523 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Tue, 18 Apr 2023 22:17:36 +0800 Subject: [PATCH 17/29] Avoid using `log.Fatal` in `pkg/*` (#1705) * fix: common * fix: in runner * fix: decodeNode * fix: GetMatrixes * Revert "fix: common" This reverts commit 6599803b6ae3b7adc168ef41b4afd4d89fc22f34. * fix: GetOutboundIP * test: fix cases --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/model/workflow.go | 58 ++++++++++++++++++++++---------------- act/model/workflow_test.go | 16 ++++++++--- act/runner/run_context.go | 4 +-- act/runner/runner.go | 7 ++++- 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/act/model/workflow.go b/act/model/workflow.go index 35e4587b..68f7b68a 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -248,15 +248,13 @@ func (j *Job) Container() *ContainerSpec { switch j.RawContainer.Kind { case yaml.ScalarNode: val = new(ContainerSpec) - err := j.RawContainer.Decode(&val.Image) - if err != nil { - log.Fatal(err) + if !decodeNode(j.RawContainer, &val.Image) { + return nil } case yaml.MappingNode: val = new(ContainerSpec) - err := j.RawContainer.Decode(val) - if err != nil { - log.Fatal(err) + if !decodeNode(j.RawContainer, val) { + return nil } } return val @@ -267,16 +265,14 @@ func (j *Job) Needs() []string { switch j.RawNeeds.Kind { case yaml.ScalarNode: var val string - err := j.RawNeeds.Decode(&val) - if err != nil { - log.Fatal(err) + if !decodeNode(j.RawNeeds, &val) { + return nil } return []string{val} case yaml.SequenceNode: var val []string - err := j.RawNeeds.Decode(&val) - if err != nil { - log.Fatal(err) + if !decodeNode(j.RawNeeds, &val) { + return nil } return val } @@ -288,16 +284,14 @@ func (j *Job) RunsOn() []string { switch j.RawRunsOn.Kind { case yaml.ScalarNode: var val string - err := j.RawRunsOn.Decode(&val) - if err != nil { - log.Fatal(err) + if !decodeNode(j.RawRunsOn, &val) { + return nil } return []string{val} case yaml.SequenceNode: var val []string - err := j.RawRunsOn.Decode(&val) - if err != nil { - log.Fatal(err) + if !decodeNode(j.RawRunsOn, &val) { + return nil } return val } @@ -307,8 +301,8 @@ func (j *Job) RunsOn() []string { func environment(yml yaml.Node) map[string]string { env := make(map[string]string) if yml.Kind == yaml.MappingNode { - if err := yml.Decode(&env); err != nil { - log.Fatal(err) + if !decodeNode(yml, &env) { + return nil } } return env @@ -323,8 +317,8 @@ func (j *Job) Environment() map[string]string { func (j *Job) Matrix() map[string][]interface{} { if j.Strategy.RawMatrix.Kind == yaml.MappingNode { var val map[string][]interface{} - if err := j.Strategy.RawMatrix.Decode(&val); err != nil { - log.Fatal(err) + if !decodeNode(j.Strategy.RawMatrix, &val) { + return nil } return val } @@ -335,7 +329,7 @@ func (j *Job) Matrix() map[string][]interface{} { // It skips includes and hard fails excludes for non-existing keys // //nolint:gocyclo -func (j *Job) GetMatrixes() []map[string]interface{} { +func (j *Job) GetMatrixes() ([]map[string]interface{}, error) { matrixes := make([]map[string]interface{}, 0) if j.Strategy != nil { j.Strategy.FailFast = j.Strategy.GetFailFast() @@ -386,7 +380,7 @@ func (j *Job) GetMatrixes() []map[string]interface{} { excludes = append(excludes, e) } else { // We fail completely here because that's what GitHub does for non-existing matrix keys, fail on exclude, silent skip on include - log.Fatalf("The workflow is not valid. Matrix exclude key '%s' does not match any key within the matrix", k) + return nil, fmt.Errorf("the workflow is not valid. Matrix exclude key %q does not match any key within the matrix", k) } } } @@ -431,7 +425,7 @@ func (j *Job) GetMatrixes() []map[string]interface{} { } else { matrixes = append(matrixes, make(map[string]interface{})) } - return matrixes + return matrixes, nil } func commonKeysMatch(a map[string]interface{}, b map[string]interface{}) bool { @@ -671,3 +665,17 @@ func (w *Workflow) GetJobIDs() []string { } return ids } + +var OnDecodeNodeError = func(node yaml.Node, out interface{}, err error) { + log.Fatalf("Failed to decode node %v into %T: %v", node, out, err) +} + +func decodeNode(node yaml.Node, out interface{}) bool { + if err := node.Decode(out); err != nil { + if OnDecodeNodeError != nil { + OnDecodeNodeError(node, out, err) + } + return false + } + return true +} diff --git a/act/model/workflow_test.go b/act/model/workflow_test.go index a7892338..8f8b7a9e 100644 --- a/act/model/workflow_test.go +++ b/act/model/workflow_test.go @@ -250,25 +250,33 @@ func TestReadWorkflow_Strategy(t *testing.T) { wf := p.Stages[0].Runs[0].Workflow job := wf.Jobs["strategy-only-max-parallel"] - assert.Equal(t, job.GetMatrixes(), []map[string]interface{}{{}}) + matrixes, err := job.GetMatrixes() + assert.NoError(t, err) + assert.Equal(t, matrixes, []map[string]interface{}{{}}) assert.Equal(t, job.Matrix(), map[string][]interface{}(nil)) assert.Equal(t, job.Strategy.MaxParallel, 2) assert.Equal(t, job.Strategy.FailFast, true) job = wf.Jobs["strategy-only-fail-fast"] - assert.Equal(t, job.GetMatrixes(), []map[string]interface{}{{}}) + matrixes, err = job.GetMatrixes() + assert.NoError(t, err) + assert.Equal(t, matrixes, []map[string]interface{}{{}}) assert.Equal(t, job.Matrix(), map[string][]interface{}(nil)) assert.Equal(t, job.Strategy.MaxParallel, 4) assert.Equal(t, job.Strategy.FailFast, false) job = wf.Jobs["strategy-no-matrix"] - assert.Equal(t, job.GetMatrixes(), []map[string]interface{}{{}}) + matrixes, err = job.GetMatrixes() + assert.NoError(t, err) + assert.Equal(t, matrixes, []map[string]interface{}{{}}) assert.Equal(t, job.Matrix(), map[string][]interface{}(nil)) assert.Equal(t, job.Strategy.MaxParallel, 2) assert.Equal(t, job.Strategy.FailFast, false) job = wf.Jobs["strategy-all"] - assert.Equal(t, job.GetMatrixes(), + matrixes, err = job.GetMatrixes() + assert.NoError(t, err) + assert.Equal(t, matrixes, []map[string]interface{}{ {"datacenter": "site-c", "node-version": "14.x", "site": "staging"}, {"datacenter": "site-c", "node-version": "16.x", "site": "staging"}, diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 12738115..bbd8ec4a 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -18,7 +18,6 @@ import ( "strings" "github.com/opencontainers/selinux/go-selinux" - log "github.com/sirupsen/logrus" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" @@ -361,7 +360,8 @@ func (rc *RunContext) ActionCacheDir() string { if home, err := os.UserHomeDir(); err == nil { xdgCache = filepath.Join(home, ".cache") } else if xdgCache, err = filepath.Abs("."); err != nil { - log.Fatal(err) + // It's almost impossible to get here, so the temp dir is a good fallback + xdgCache = os.TempDir() } } return filepath.Join(xdgCache, "act") diff --git a/act/runner/runner.go b/act/runner/runner.go index 715d1584..a47cf8b1 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -118,7 +118,12 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { } } - matrixes := selectMatrixes(job.GetMatrixes(), runner.config.Matrix) + var matrixes []map[string]interface{} + if m, err := job.GetMatrixes(); err != nil { + log.Errorf("Error while get job's matrix: %v", err) + } else { + matrixes = selectMatrixes(m, runner.config.Matrix) + } log.Debugf("Final matrix after applying user inclusions '%v'", matrixes) maxParallel := 4 From 1a239d7ab71e7303f9601b52b45a92399c052ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Brauer?= Date: Tue, 18 Apr 2023 16:37:59 +0200 Subject: [PATCH 18/29] fix: ensure networkmode "host" unless explicitly specified (#1739) act defaults network mode to "host", but when `--container-options` are passed on the CLI, it uses the docker CLI options parser, which fills empty values with defaults, in which case network mode is set to "default". Unless the user explicitly sets `--container-options="--network=xxx"`, we should always default to "host", to keep act's behaviour. Co-authored-by: Markus Wolf Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/docker_run.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 62390f6e..1b194eac 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -348,6 +348,12 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config return nil, nil, fmt.Errorf("Cannot parse container options: '%s': '%w'", input.Options, err) } + if len(copts.netMode.Value()) == 0 { + if err = copts.netMode.Set("host"); err != nil { + return nil, nil, fmt.Errorf("Cannot parse networkmode=host. This is an internal error and should not happen: '%w'", err) + } + } + containerConfig, err := parse(flags, copts, "") if err != nil { return nil, nil, fmt.Errorf("Cannot process container options: '%s': '%w'", input.Options, err) From b6ccb2fa98d3a96de4d622816dfd4c2f13a53180 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 18 Apr 2023 20:09:57 +0200 Subject: [PATCH 19/29] fix: environment handling windows (host mode) (#1732) * fix: environment handling windows (host mode) * fixup * fixup * add more tests * fixup * fix setenv * fixes * [skip ci] Apply suggestions from code review Co-authored-by: Jason Song * Update side effects --------- Co-authored-by: Jason Song Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/executions_environment.go | 2 + act/container/host_environment.go | 4 ++ .../linux_container_environment_extensions.go | 4 ++ act/runner/action.go | 4 +- act/runner/action_composite.go | 14 ++++--- act/runner/command.go | 12 ++++-- act/runner/run_context.go | 9 +++++ act/runner/step.go | 38 ++++++++++++++++--- act/runner/step_test.go | 4 +- .../testdata/windows-add-env/action.yml | 7 ++++ act/runner/testdata/windows-add-env/push.yml | 17 +++++++++ .../testdata/windows-prepend-path/push.yml | 9 +++++ 12 files changed, 107 insertions(+), 17 deletions(-) create mode 100644 act/runner/testdata/windows-add-env/action.yml diff --git a/act/container/executions_environment.go b/act/container/executions_environment.go index 1c21f943..41e3b57e 100644 --- a/act/container/executions_environment.go +++ b/act/container/executions_environment.go @@ -10,4 +10,6 @@ type ExecutionsEnvironment interface { DefaultPathVariable() string JoinPathVariable(...string) string GetRunnerContext(ctx context.Context) map[string]interface{} + // On windows PATH and Path are the same key + IsEnvironmentCaseInsensitive() bool } diff --git a/act/container/host_environment.go b/act/container/host_environment.go index 5d8c7dcd..abc1115a 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -419,3 +419,7 @@ func (e *HostEnvironment) ReplaceLogWriter(stdout io.Writer, stderr io.Writer) ( e.StdOut = stdout return org, org } + +func (*HostEnvironment) IsEnvironmentCaseInsensitive() bool { + return runtime.GOOS == "windows" +} diff --git a/act/container/linux_container_environment_extensions.go b/act/container/linux_container_environment_extensions.go index c369055d..d6734511 100644 --- a/act/container/linux_container_environment_extensions.go +++ b/act/container/linux_container_environment_extensions.go @@ -71,3 +71,7 @@ func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context "tool_cache": "/opt/hostedtoolcache", } } + +func (*LinuxContainerEnvironmentExtensions) IsEnvironmentCaseInsensitive() bool { + return false +} diff --git a/act/runner/action.go b/act/runner/action.go index a534170d..bd74b2e9 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -319,13 +319,13 @@ func evalDockerArgs(ctx context.Context, step step, action *model.Action, cmd *[ inputs[k] = eval.Interpolate(ctx, v) } } - mergeIntoMap(step.getEnv(), inputs) + mergeIntoMap(step, step.getEnv(), inputs) stepEE := rc.NewStepExpressionEvaluator(ctx, step) for i, v := range *cmd { (*cmd)[i] = stepEE.Interpolate(ctx, v) } - mergeIntoMap(step.getEnv(), action.Runs.Env) + mergeIntoMap(step, step.getEnv(), action.Runs.Env) ee := rc.NewStepExpressionEvaluator(ctx, step) for k, v := range *step.getEnv() { diff --git a/act/runner/action_composite.go b/act/runner/action_composite.go index b6ef58c8..0fc1fd89 100644 --- a/act/runner/action_composite.go +++ b/act/runner/action_composite.go @@ -105,13 +105,15 @@ func execAsComposite(step actionStep) common.Executor { rc.Masks = append(rc.Masks, compositeRC.Masks...) rc.ExtraPath = compositeRC.ExtraPath // compositeRC.Env is dirty, contains INPUT_ and merged step env, only rely on compositeRC.GlobalEnv - for k, v := range compositeRC.GlobalEnv { - rc.Env[k] = v - if rc.GlobalEnv == nil { - rc.GlobalEnv = map[string]string{} - } - rc.GlobalEnv[k] = v + mergeIntoMap := mergeIntoMapCaseSensitive + if rc.JobContainer.IsEnvironmentCaseInsensitive() { + mergeIntoMap = mergeIntoMapCaseInsensitive } + if rc.GlobalEnv == nil { + rc.GlobalEnv = map[string]string{} + } + mergeIntoMap(rc.GlobalEnv, compositeRC.GlobalEnv) + mergeIntoMap(rc.Env, compositeRC.GlobalEnv) return err } diff --git a/act/runner/command.go b/act/runner/command.go index f14eb7aa..e0abb645 100644 --- a/act/runner/command.go +++ b/act/runner/command.go @@ -87,12 +87,18 @@ func (rc *RunContext) setEnv(ctx context.Context, kvPairs map[string]string, arg if rc.Env == nil { rc.Env = make(map[string]string) } - rc.Env[name] = arg - // for composite action GITHUB_ENV and set-env passing if rc.GlobalEnv == nil { rc.GlobalEnv = map[string]string{} } - rc.GlobalEnv[name] = arg + newenv := map[string]string{ + name: arg, + } + mergeIntoMap := mergeIntoMapCaseSensitive + if rc.JobContainer != nil && rc.JobContainer.IsEnvironmentCaseInsensitive() { + mergeIntoMap = mergeIntoMapCaseInsensitive + } + mergeIntoMap(rc.Env, newenv) + mergeIntoMap(rc.GlobalEnv, newenv) } func (rc *RunContext) setOutput(ctx context.Context, kvPairs map[string]string, arg string) { logger := common.Logger(ctx) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index bbd8ec4a..fb04f89c 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -298,6 +298,15 @@ func (rc *RunContext) execJobContainer(cmd []string, env map[string]string, user func (rc *RunContext) ApplyExtraPath(ctx context.Context, env *map[string]string) { if rc.ExtraPath != nil && len(rc.ExtraPath) > 0 { path := rc.JobContainer.GetPathVariableName() + if rc.JobContainer.IsEnvironmentCaseInsensitive() { + // On windows system Path and PATH could also be in the map + for k := range *env { + if strings.EqualFold(path, k) { + path = k + break + } + } + } if (*env)[path] == "" { cenv := map[string]string{} var cpath string diff --git a/act/runner/step.go b/act/runner/step.go index 7cc355f4..cff48b1b 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -187,7 +187,7 @@ func setupEnv(ctx context.Context, step step) error { mergeEnv(ctx, step) // merge step env last, since it should not be overwritten - mergeIntoMap(step.getEnv(), step.getStepModel().GetEnv()) + mergeIntoMap(step, step.getEnv(), step.getStepModel().GetEnv()) exprEval := rc.NewExpressionEvaluator(ctx) for k, v := range *step.getEnv() { @@ -216,9 +216,9 @@ func mergeEnv(ctx context.Context, step step) { c := job.Container() if c != nil { - mergeIntoMap(env, rc.GetEnv(), c.Env) + mergeIntoMap(step, env, rc.GetEnv(), c.Env) } else { - mergeIntoMap(env, rc.GetEnv()) + mergeIntoMap(step, env, rc.GetEnv()) } rc.withGithubEnv(ctx, step.getGithubContext(ctx), *env) @@ -258,10 +258,38 @@ func isContinueOnError(ctx context.Context, expr string, step step, stage stepSt return continueOnError, nil } -func mergeIntoMap(target *map[string]string, maps ...map[string]string) { +func mergeIntoMap(step step, target *map[string]string, maps ...map[string]string) { + if rc := step.getRunContext(); rc != nil && rc.JobContainer != nil && rc.JobContainer.IsEnvironmentCaseInsensitive() { + mergeIntoMapCaseInsensitive(*target, maps...) + } else { + mergeIntoMapCaseSensitive(*target, maps...) + } +} + +func mergeIntoMapCaseSensitive(target map[string]string, maps ...map[string]string) { for _, m := range maps { for k, v := range m { - (*target)[k] = v + target[k] = v + } + } +} + +func mergeIntoMapCaseInsensitive(target map[string]string, maps ...map[string]string) { + foldKeys := make(map[string]string, len(target)) + for k := range target { + foldKeys[strings.ToLower(k)] = k + } + toKey := func(s string) string { + foldKey := strings.ToLower(s) + if k, ok := foldKeys[foldKey]; ok { + return k + } + foldKeys[strings.ToLower(foldKey)] = s + return s + } + for _, m := range maps { + for k, v := range m { + target[toKey(k)] = v } } } diff --git a/act/runner/step_test.go b/act/runner/step_test.go index 4fc77652..d08a1297 100644 --- a/act/runner/step_test.go +++ b/act/runner/step_test.go @@ -63,7 +63,9 @@ func TestMergeIntoMap(t *testing.T) { for _, tt := range table { t.Run(tt.name, func(t *testing.T) { - mergeIntoMap(&tt.target, tt.maps...) + mergeIntoMapCaseSensitive(tt.target, tt.maps...) + assert.Equal(t, tt.expected, tt.target) + mergeIntoMapCaseInsensitive(tt.target, tt.maps...) assert.Equal(t, tt.expected, tt.target) }) } diff --git a/act/runner/testdata/windows-add-env/action.yml b/act/runner/testdata/windows-add-env/action.yml new file mode 100644 index 00000000..a80684f3 --- /dev/null +++ b/act/runner/testdata/windows-add-env/action.yml @@ -0,0 +1,7 @@ +runs: + using: composite + steps: + - run: | + echo $env:GITHUB_ENV + echo "kEy=n/a" > $env:GITHUB_ENV + shell: pwsh \ No newline at end of file diff --git a/act/runner/testdata/windows-add-env/push.yml b/act/runner/testdata/windows-add-env/push.yml index 275c5f1f..bd233bb3 100644 --- a/act/runner/testdata/windows-add-env/push.yml +++ b/act/runner/testdata/windows-add-env/push.yml @@ -25,3 +25,20 @@ jobs: echo "Unexpected value for `$env:key2: $env:key2" exit 1 } + - run: | + echo $env:GITHUB_ENV + echo "KEY=test" > $env:GITHUB_ENV + echo "Key=expected" > $env:GITHUB_ENV + - name: Assert GITHUB_ENV is merged case insensitive + run: exit 1 + if: env.KEY != 'expected' || env.Key != 'expected' || env.key != 'expected' + - name: Assert step env is merged case insensitive + run: exit 1 + if: env.KEY != 'n/a' || env.Key != 'n/a' || env.key != 'n/a' + env: + KeY: 'n/a' + - uses: actions/checkout@v3 + - uses: ./windows-add-env + - name: Assert composite env is merged case insensitive + run: exit 1 + if: env.KEY != 'n/a' || env.Key != 'n/a' || env.key != 'n/a' \ No newline at end of file diff --git a/act/runner/testdata/windows-prepend-path/push.yml b/act/runner/testdata/windows-prepend-path/push.yml index 176de691..cf5026a6 100644 --- a/act/runner/testdata/windows-prepend-path/push.yml +++ b/act/runner/testdata/windows-prepend-path/push.yml @@ -11,6 +11,9 @@ jobs: mkdir build echo '@echo off' > build/test.cmd echo 'echo Hi' >> build/test.cmd + mkdir build2 + echo '@echo off' > build2/test2.cmd + echo 'echo test2' >> build2/test2.cmd - run: | echo '${{ tojson(runner) }}' ls @@ -23,3 +26,9 @@ jobs: - run: | echo $env:PATH test + - run: | + echo "PATH=$env:PATH;${{ github.workspace }}\build2" > $env:GITHUB_ENV + - run: | + echo $env:PATH + test + test2 \ No newline at end of file From b6cb81468be07b0872287e8facac100194f67165 Mon Sep 17 00:00:00 2001 From: Andy Wang <62269186+AnotiaWang@users.noreply.github.com> Date: Wed, 19 Apr 2023 11:00:33 +0800 Subject: [PATCH 20/29] typo: fix expression of warning message on macOS (#1693) Co-authored-by: Jason Song --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 02afb50b..fe4fe7c6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -348,7 +348,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str DisableQuote: true, DisableTimestamp: true, }) - l.Warnf(" \U000026A0 You are using Apple M1 chip and you have not specified container architecture, you might encounter issues while running act. If so, try running it with '--container-architecture linux/amd64'. \U000026A0 \n") + l.Warnf(" \U000026A0 You are using Apple M-series chip and you have not specified container architecture, you might encounter issues while running act. If so, try running it with '--container-architecture linux/amd64'. \U000026A0 \n") } log.Debugf("Loading environment from %s", input.Envfile()) From b7ebd96ec7fce0ba158b310094699a51eb315f4d Mon Sep 17 00:00:00 2001 From: "M.Yamashita" <4653960+M-Yamashita01@users.noreply.github.com> Date: Wed, 19 Apr 2023 12:46:00 +0900 Subject: [PATCH 21/29] Remove the comment-out code. (#1691) Co-authored-by: Jason Song --- act/runner/runner_test.go | 1 - act/runner/testdata/issue-228/main.yaml | 14 -------------- 2 files changed, 15 deletions(-) delete mode 100644 act/runner/testdata/issue-228/main.yaml diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index f8468bbb..db63fb94 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -294,7 +294,6 @@ func TestRunEvent(t *testing.T) { {workdir, "workflow_dispatch-scalar-composite-action", "workflow_dispatch", "", platforms, secrets}, {workdir, "job-needs-context-contains-result", "push", "", platforms, secrets}, {"../model/testdata", "strategy", "push", "", platforms, secrets}, // TODO: move all testdata into pkg so we can validate it with planner and runner - // {"testdata", "issue-228", "push", "", platforms, }, // TODO [igni]: Remove this once everything passes {"../model/testdata", "container-volumes", "push", "", platforms, secrets}, {workdir, "path-handling", "push", "", platforms, secrets}, {workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets}, diff --git a/act/runner/testdata/issue-228/main.yaml b/act/runner/testdata/issue-228/main.yaml deleted file mode 100644 index e0a0dfbf..00000000 --- a/act/runner/testdata/issue-228/main.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: issue-228 - -on: - - push - -jobs: - kind: - runs-on: ubuntu-latest - steps: - - run: apt-get update -y && apt-get install git -y # setup git credentials will fail otherwise - - name: Setup git credentials - uses: fusion-engineering/setup-git-credentials@v2 - with: - credentials: https://test@github.com/ From eb6b7f06094407e132d99a22f68c197d0747f51c Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 19 Apr 2023 12:19:40 +0800 Subject: [PATCH 22/29] Improve watchAndRun (#1743) * fix: improve watchAndRun * fix: lint * fix: lint --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- cmd/root.go | 54 +++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index fe4fe7c6..1742e8a1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -18,13 +18,13 @@ import ( gitignore "github.com/sabhiram/go-gitignore" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "gopkg.in/yaml.v3" "github.com/nektos/act/pkg/artifacts" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/model" "github.com/nektos/act/pkg/runner" - "gopkg.in/yaml.v3" ) // Execute is the entry point to running the CLI @@ -623,45 +623,47 @@ func defaultImageSurvey(actrc string) error { } func watchAndRun(ctx context.Context, fn common.Executor) error { - recurse := true - checkIntervalInSeconds := 2 dir, err := os.Getwd() if err != nil { return err } - var ignore *gitignore.GitIgnore - if _, err := os.Stat(filepath.Join(dir, ".gitignore")); !os.IsNotExist(err) { - ignore, _ = gitignore.CompileIgnoreFile(filepath.Join(dir, ".gitignore")) - } else { - ignore = &gitignore.GitIgnore{} + ignoreFile := filepath.Join(dir, ".gitignore") + ignore := &gitignore.GitIgnore{} + if info, err := os.Stat(ignoreFile); err == nil && !info.IsDir() { + ignore, err = gitignore.CompileIgnoreFile(ignoreFile) + if err != nil { + return fmt.Errorf("compile %q: %w", ignoreFile, err) + } } folderWatcher := fswatch.NewFolderWatcher( dir, - recurse, + true, ignore.MatchesPath, - checkIntervalInSeconds, + 2, // 2 seconds ) folderWatcher.Start() + defer folderWatcher.Stop() - go func() { - for folderWatcher.IsRunning() { - if err = fn(ctx); err != nil { - break - } - log.Debugf("Watching %s for changes", dir) - for changes := range folderWatcher.ChangeDetails() { - log.Debugf("%s", changes.String()) - if err = fn(ctx); err != nil { - break - } - log.Debugf("Watching %s for changes", dir) + // run once before watching + if err := fn(ctx); err != nil { + return err + } + + for folderWatcher.IsRunning() { + log.Debugf("Watching %s for changes", dir) + select { + case <-ctx.Done(): + return nil + case changes := <-folderWatcher.ChangeDetails(): + log.Debugf("%s", changes.String()) + if err := fn(ctx); err != nil { + return err } } - }() - <-ctx.Done() - folderWatcher.Stop() - return err + } + + return nil } From 9c3b242b12dc27d565346117296ec5d0b32ff1bd Mon Sep 17 00:00:00 2001 From: R Date: Sun, 23 Apr 2023 21:02:56 +0200 Subject: [PATCH 23/29] fix: try finding a socket, otherwise fail, respect user choice (#1745) * fix: try finding a socket, otherwise fail, respect user choice * Update cmd/root.go Co-authored-by: Jason Song * Update cmd/root.go Co-authored-by: Jason Song --------- Co-authored-by: Jason Song --- act/container/docker_run.go | 3 -- cmd/root.go | 68 +++++++++++++++++++++++++++---------- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 1b194eac..df296e0d 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -189,9 +189,6 @@ type containerReference struct { } func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) { - // TODO: this should maybe need to be a global option, not hidden in here? - // though i'm not sure how that works out when there's another Executor :D - // I really would like something that works on OSX native for eg dockerHost := os.Getenv("DOCKER_HOST") if strings.HasPrefix(dockerHost, "ssh://") { diff --git a/cmd/root.go b/cmd/root.go index 1742e8a1..55b55bb2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -80,7 +80,7 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().StringVarP(&input.envfile, "env-file", "", ".env", "environment file to read and use as env in the containers") rootCmd.PersistentFlags().StringVarP(&input.inputfile, "input-file", "", ".input", "input file to read and use as action input") rootCmd.PersistentFlags().StringVarP(&input.containerArchitecture, "container-architecture", "", "", "Architecture which should be used to run containers, e.g.: linux/amd64. If not specified, will use host default architecture. Requires Docker server API Version 1.41+. Ignored on earlier Docker server platforms.") - rootCmd.PersistentFlags().StringVarP(&input.containerDaemonSocket, "container-daemon-socket", "", "/var/run/docker.sock", "Path to Docker daemon socket which will be mounted to containers") + rootCmd.PersistentFlags().StringVarP(&input.containerDaemonSocket, "container-daemon-socket", "", "", "URI to Docker Engine socket (e.g.: unix://~/.docker/run/docker.sock)") rootCmd.PersistentFlags().StringVarP(&input.containerOptions, "container-options", "", "", "Custom docker container options for the job container without an options property in the job definition") rootCmd.PersistentFlags().StringVarP(&input.githubInstance, "github-instance", "", "github.com", "GitHub instance to use. Don't use this if you are not using GitHub Enterprise Server.") rootCmd.PersistentFlags().StringVarP(&input.artifactServerPath, "artifact-server-path", "", "", "Defines the path where the artifact server stores uploads and retrieves downloads from. If not specified the artifact server will not start.") @@ -118,6 +118,33 @@ func configLocations() []string { } } +var commonSocketPaths = []string{ + "/var/run/docker.sock", + "/var/run/podman/podman.sock", + "$HOME/.colima/docker.sock", + "$XDG_RUNTIME_DIR/docker.sock", + `\\.\pipe\docker_engine`, + "$HOME/.docker/run/docker.sock", +} + +// returns socket path or false if not found any +func socketLocation() (string, bool) { + if dockerHost, exists := os.LookupEnv("DOCKER_HOST"); exists { + return dockerHost, true + } + + for _, p := range commonSocketPaths { + if _, err := os.Lstat(os.ExpandEnv(p)); err == nil { + if strings.HasPrefix(p, `\\.\`) { + return "npipe://" + os.ExpandEnv(p), true + } + return "unix://" + os.ExpandEnv(p), true + } + } + + return "", false +} + func args() []string { actrc := configLocations() @@ -131,15 +158,6 @@ func args() []string { } func bugReport(ctx context.Context, version string) error { - var commonSocketPaths = []string{ - "/var/run/docker.sock", - "/var/run/podman/podman.sock", - "$HOME/.colima/docker.sock", - "$XDG_RUNTIME_DIR/docker.sock", - `\\.\pipe\docker_engine`, - "$HOME/.docker/run/docker.sock", - } - sprintf := func(key, val string) string { return fmt.Sprintf("%-24s%s\n", key, val) } @@ -150,19 +168,20 @@ func bugReport(ctx context.Context, version string) error { report += sprintf("NumCPU:", fmt.Sprint(runtime.NumCPU())) var dockerHost string - if dockerHost = os.Getenv("DOCKER_HOST"); dockerHost == "" { - dockerHost = "DOCKER_HOST environment variable is unset/empty." + var exists bool + if dockerHost, exists = os.LookupEnv("DOCKER_HOST"); !exists { + dockerHost = "DOCKER_HOST environment variable is not set" + } else if dockerHost == "" { + dockerHost = "DOCKER_HOST environment variable is empty." } report += sprintf("Docker host:", dockerHost) report += fmt.Sprintln("Sockets found:") for _, p := range commonSocketPaths { - if strings.HasPrefix(p, `$`) { - v := strings.Split(p, `/`)[0] - p = strings.Replace(p, v, os.Getenv(strings.TrimPrefix(v, `$`)), 1) - } - if _, err := os.Stat(p); err != nil { + if _, err := os.Lstat(os.ExpandEnv(p)); err != nil { continue + } else if _, err := os.Stat(os.ExpandEnv(p)); err != nil { + report += fmt.Sprintf("\t%s(broken)\n", p) } else { report += fmt.Sprintf("\t%s\n", p) } @@ -342,6 +361,19 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str return bugReport(ctx, cmd.Version) } + var socketPath string + if input.containerDaemonSocket != "" { + socketPath = input.containerDaemonSocket + } else { + socket, found := socketLocation() + if !found && input.containerDaemonSocket == "" { + log.Errorln("daemon Docker Engine socket not found and containerDaemonSocket option was not set") + } else { + socketPath = socket + } + } + os.Setenv("DOCKER_HOST", socketPath) + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && input.containerArchitecture == "" { l := log.New() l.SetFormatter(&log.TextFormatter{ @@ -533,7 +565,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str Privileged: input.privileged, UsernsMode: input.usernsMode, ContainerArchitecture: input.containerArchitecture, - ContainerDaemonSocket: input.containerDaemonSocket, + ContainerDaemonSocket: socketPath, ContainerOptions: input.containerOptions, UseGitIgnore: input.useGitIgnore, GitHubInstance: input.githubInstance, From 6764ffdd93e39e1b0d0f072235605a6eb314ea78 Mon Sep 17 00:00:00 2001 From: R Date: Sun, 23 Apr 2023 21:18:38 +0200 Subject: [PATCH 24/29] ci: deduplicate running workflows (#1751) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 34ea1d2b..b38ee34d 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,6 +1,10 @@ name: checks on: [pull_request, workflow_dispatch] +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + env: ACT_OWNER: ${{ github.repository_owner }} ACT_REPOSITORY: ${{ github.repository }} From 6abb05802ff2b0d4a165d2c33c02ea478245f6a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Apr 2023 03:13:14 +0000 Subject: [PATCH 25/29] build(deps): bump codecov/codecov-action from 3.1.2 to 3.1.3 (#1752) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3.1.2...v3.1.3) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b38ee34d..524471e2 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -59,7 +59,7 @@ jobs: with: upload-logs-name: logs-linux - name: Upload Codecov report - uses: codecov/codecov-action@v3.1.2 + uses: codecov/codecov-action@v3.1.3 with: files: coverage.txt fail_ci_if_error: true # optional (default = false) From 3d11b7686dfe409de6078f54339343806ba1bd18 Mon Sep 17 00:00:00 2001 From: Zettat123 Date: Tue, 25 Apr 2023 10:09:54 +0800 Subject: [PATCH 26/29] avoid using log.Fatal (#1759) --- act/model/workflow.go | 35 ++++++++++++++--------------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/act/model/workflow.go b/act/model/workflow.go index 68f7b68a..e7d43e4b 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -58,9 +58,8 @@ func (w *Workflow) On() []string { func (w *Workflow) OnEvent(event string) interface{} { if w.RawOn.Kind == yaml.MappingNode { var val map[string]interface{} - err := w.RawOn.Decode(&val) - if err != nil { - log.Fatal(err) + if !decodeNode(w.RawOn, &val) { + return nil } return val[event] } @@ -85,16 +84,14 @@ func (w *Workflow) WorkflowDispatchConfig() *WorkflowDispatch { } var val map[string]yaml.Node - err := w.RawOn.Decode(&val) - if err != nil { - log.Fatal(err) + if !decodeNode(w.RawOn, &val) { + return nil } var config WorkflowDispatch node := val["workflow_dispatch"] - err = node.Decode(&config) - if err != nil { - log.Fatal(err) + if !decodeNode(node, &config) { + return nil } return &config @@ -128,16 +125,14 @@ func (w *Workflow) WorkflowCallConfig() *WorkflowCall { } var val map[string]yaml.Node - err := w.RawOn.Decode(&val) - if err != nil { - log.Fatal(err) + if !decodeNode(w.RawOn, &val) { + return &WorkflowCall{} } var config WorkflowCall node := val["workflow_call"] - err = node.Decode(&config) - if err != nil { - log.Fatal(err) + if !decodeNode(node, &config) { + return &WorkflowCall{} } return &config @@ -220,9 +215,8 @@ func (j *Job) InheritSecrets() bool { } var val string - err := j.RawSecrets.Decode(&val) - if err != nil { - log.Fatal(err) + if !decodeNode(j.RawSecrets, &val) { + return false } return val == "inherit" @@ -234,9 +228,8 @@ func (j *Job) Secrets() map[string]string { } var val map[string]string - err := j.RawSecrets.Decode(&val) - if err != nil { - log.Fatal(err) + if !decodeNode(j.RawSecrets, &val) { + return nil } return val From bd467ec0ad6377b1da82b325231c9f2465d9ab27 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 25 Apr 2023 18:31:17 +0200 Subject: [PATCH 27/29] Revert breaking docker socket changes (#1763) * fix: rework docker socket changes * fixup * fixup * fixes * patch * ... * lint * Fix docker outputs windows * fix type * Revert containerDaemonSocket breaking change --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 24 ++++++++++++++++++-- cmd/root.go | 47 ++++++++++++++++++++++++++++----------- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index fb04f89c..e6f5dfc7 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -87,6 +87,24 @@ func (rc *RunContext) jobContainerName() string { return createContainerName("act", rc.String()) } +func getDockerDaemonSocketMountPath(daemonPath string) string { + if protoIndex := strings.Index(daemonPath, "://"); protoIndex != -1 { + scheme := daemonPath[:protoIndex] + if strings.EqualFold(scheme, "npipe") { + // linux container mount on windows, use the default socket path of the VM / wsl2 + return "/var/run/docker.sock" + } else if strings.EqualFold(scheme, "unix") { + return daemonPath[protoIndex+3:] + } else if strings.IndexFunc(scheme, func(r rune) bool { + return (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') + }) == -1 { + // unknown protocol use default + return "/var/run/docker.sock" + } + } + return daemonPath +} + // Returns the binds and mounts for the container, resolving paths as appopriate func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { name := rc.jobContainerName() @@ -95,8 +113,10 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { rc.Config.ContainerDaemonSocket = "/var/run/docker.sock" } - binds := []string{ - fmt.Sprintf("%s:%s", rc.Config.ContainerDaemonSocket, "/var/run/docker.sock"), + binds := []string{} + if rc.Config.ContainerDaemonSocket != "-" { + daemonPath := getDockerDaemonSocketMountPath(rc.Config.ContainerDaemonSocket) + binds = append(binds, fmt.Sprintf("%s:%s", daemonPath, "/var/run/docker.sock")) } ext := container.LinuxContainerEnvironmentExtensions{} diff --git a/cmd/root.go b/cmd/root.go index 55b55bb2..548d90cc 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -80,7 +80,7 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().StringVarP(&input.envfile, "env-file", "", ".env", "environment file to read and use as env in the containers") rootCmd.PersistentFlags().StringVarP(&input.inputfile, "input-file", "", ".input", "input file to read and use as action input") rootCmd.PersistentFlags().StringVarP(&input.containerArchitecture, "container-architecture", "", "", "Architecture which should be used to run containers, e.g.: linux/amd64. If not specified, will use host default architecture. Requires Docker server API Version 1.41+. Ignored on earlier Docker server platforms.") - rootCmd.PersistentFlags().StringVarP(&input.containerDaemonSocket, "container-daemon-socket", "", "", "URI to Docker Engine socket (e.g.: unix://~/.docker/run/docker.sock)") + rootCmd.PersistentFlags().StringVarP(&input.containerDaemonSocket, "container-daemon-socket", "", "", "URI to Docker Engine socket (e.g.: unix://~/.docker/run/docker.sock or - to disable bind mounting the socket)") rootCmd.PersistentFlags().StringVarP(&input.containerOptions, "container-options", "", "", "Custom docker container options for the job container without an options property in the job definition") rootCmd.PersistentFlags().StringVarP(&input.githubInstance, "github-instance", "", "github.com", "GitHub instance to use. Don't use this if you are not using GitHub Enterprise Server.") rootCmd.PersistentFlags().StringVarP(&input.artifactServerPath, "artifact-server-path", "", "", "Defines the path where the artifact server stores uploads and retrieves downloads from. If not specified the artifact server will not start.") @@ -136,9 +136,9 @@ func socketLocation() (string, bool) { for _, p := range commonSocketPaths { if _, err := os.Lstat(os.ExpandEnv(p)); err == nil { if strings.HasPrefix(p, `\\.\`) { - return "npipe://" + os.ExpandEnv(p), true + return "npipe://" + filepath.ToSlash(os.ExpandEnv(p)), true } - return "unix://" + os.ExpandEnv(p), true + return "unix://" + filepath.ToSlash(os.ExpandEnv(p)), true } } @@ -350,6 +350,18 @@ func parseMatrix(matrix []string) map[string]map[string]bool { return matrixes } +func isDockerHostURI(daemonPath string) bool { + if protoIndex := strings.Index(daemonPath, "://"); protoIndex != -1 { + scheme := daemonPath[:protoIndex] + if strings.IndexFunc(scheme, func(r rune) bool { + return (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') + }) == -1 { + return true + } + } + return false +} + //nolint:gocyclo func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []string) error { return func(cmd *cobra.Command, args []string) error { @@ -361,18 +373,27 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str return bugReport(ctx, cmd.Version) } - var socketPath string - if input.containerDaemonSocket != "" { - socketPath = input.containerDaemonSocket - } else { - socket, found := socketLocation() - if !found && input.containerDaemonSocket == "" { - log.Errorln("daemon Docker Engine socket not found and containerDaemonSocket option was not set") + // Prefer DOCKER_HOST, don't override it + socketPath, hasDockerHost := os.LookupEnv("DOCKER_HOST") + if !hasDockerHost { + // a - in containerDaemonSocket means don't mount, preserve this value + // otherwise if input.containerDaemonSocket is a filepath don't use it as socketPath + skipMount := input.containerDaemonSocket == "-" || !isDockerHostURI(input.containerDaemonSocket) + if input.containerDaemonSocket != "" && !skipMount { + socketPath = input.containerDaemonSocket } else { - socketPath = socket + socket, found := socketLocation() + if !found { + log.Errorln("daemon Docker Engine socket not found and containerDaemonSocket option was not set") + } else { + socketPath = socket + } + if !skipMount { + input.containerDaemonSocket = socketPath + } } + os.Setenv("DOCKER_HOST", socketPath) } - os.Setenv("DOCKER_HOST", socketPath) if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && input.containerArchitecture == "" { l := log.New() @@ -565,7 +586,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str Privileged: input.privileged, UsernsMode: input.usernsMode, ContainerArchitecture: input.containerArchitecture, - ContainerDaemonSocket: socketPath, + ContainerDaemonSocket: input.containerDaemonSocket, ContainerOptions: input.containerOptions, UseGitIgnore: input.useGitIgnore, GitHubInstance: input.githubInstance, From 364adb2acfdaa7f877c6e309fe512e2fa80169bf Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Tue, 25 Apr 2023 19:32:34 +0200 Subject: [PATCH 28/29] chore: run act from cli on linux (#1758) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: run act from cli on linux To prevent issues like #1756 in the future, we need to run act from the cli through all the root setup code. This ensures that the basic CLI setup can succeed. Co-authored-by: Björn Brauer * chore: set platform spec to use --------- Co-authored-by: Björn Brauer Co-authored-by: ChristopherHX --- .github/workflows/checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 524471e2..5d7cf8a6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -58,6 +58,8 @@ jobs: uses: ./.github/actions/run-tests with: upload-logs-name: logs-linux + - name: Run act from cli + run: go run main.go -P ubuntu-latest=node:16-buster-slim -C ./pkg/runner/testdata/ -W ./basic/push.yml - name: Upload Codecov report uses: codecov/codecov-action@v3.1.3 with: From b51f608660a542de1e1376441ea686a60223fde1 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 28 Apr 2023 23:57:40 +0800 Subject: [PATCH 29/29] Support cache (#1770) * feat: port * fix: use httprouter * fix: WriteHeader * fix: bolthold * fix: bugs * chore: one less file * test: test handler * fix: bug in id * test: fix cases * chore: tidy * fix: use atomic.Int32 * fix: use atomic.Store * feat: support close * chore: lint * fix: cache keys are case insensitive * fix: options * fix: use options * fix: close * fix: ignore close error * Revert "fix: close" This reverts commit d53ea7568ba03908eb153031c435008fd47e7ccb. * fix: cacheUrlKey * fix: nil close * chore: lint code * fix: test key * test: case insensitive * chore: lint --- act/artifactcache/doc.go | 8 + act/artifactcache/handler.go | 488 ++++++++++++++++++ act/artifactcache/handler_test.go | 469 +++++++++++++++++ act/artifactcache/model.go | 38 ++ act/artifactcache/storage.go | 126 +++++ .../testdata/example/example.yaml | 30 ++ cmd/dir.go | 27 + cmd/input.go | 4 + cmd/notices.go | 11 +- cmd/root.go | 24 +- 10 files changed, 1209 insertions(+), 16 deletions(-) create mode 100644 act/artifactcache/doc.go create mode 100644 act/artifactcache/handler.go create mode 100644 act/artifactcache/handler_test.go create mode 100644 act/artifactcache/model.go create mode 100644 act/artifactcache/storage.go create mode 100644 act/artifactcache/testdata/example/example.yaml create mode 100644 cmd/dir.go diff --git a/act/artifactcache/doc.go b/act/artifactcache/doc.go new file mode 100644 index 00000000..13d2644d --- /dev/null +++ b/act/artifactcache/doc.go @@ -0,0 +1,8 @@ +// Package artifactcache provides a cache handler for the runner. +// +// Inspired by https://github.com/sp-ricard-valverde/github-act-cache-server +// +// TODO: Authorization +// TODO: Restrictions for accessing a cache, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache +// TODO: Force deleting cache entries, see https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries +package artifactcache diff --git a/act/artifactcache/handler.go b/act/artifactcache/handler.go new file mode 100644 index 00000000..f11def68 --- /dev/null +++ b/act/artifactcache/handler.go @@ -0,0 +1,488 @@ +package artifactcache + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/julienschmidt/httprouter" + "github.com/sirupsen/logrus" + "github.com/timshannon/bolthold" + "go.etcd.io/bbolt" + + "github.com/nektos/act/pkg/common" +) + +const ( + urlBase = "/_apis/artifactcache" +) + +type Handler struct { + db *bolthold.Store + storage *Storage + router *httprouter.Router + listener net.Listener + server *http.Server + logger logrus.FieldLogger + + gcing int32 // TODO: use atomic.Bool when we can use Go 1.19 + gcAt time.Time + + outboundIP string +} + +func StartHandler(dir, outboundIP string, port uint16, logger logrus.FieldLogger) (*Handler, error) { + h := &Handler{} + + if logger == nil { + discard := logrus.New() + discard.Out = io.Discard + logger = discard + } + logger = logger.WithField("module", "artifactcache") + h.logger = logger + + if dir == "" { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + dir = filepath.Join(home, ".cache", "actcache") + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + + db, err := bolthold.Open(filepath.Join(dir, "bolt.db"), 0o644, &bolthold.Options{ + Encoder: json.Marshal, + Decoder: json.Unmarshal, + Options: &bbolt.Options{ + Timeout: 5 * time.Second, + NoGrowSync: bbolt.DefaultOptions.NoGrowSync, + FreelistType: bbolt.DefaultOptions.FreelistType, + }, + }) + if err != nil { + return nil, err + } + h.db = db + + storage, err := NewStorage(filepath.Join(dir, "cache")) + if err != nil { + return nil, err + } + h.storage = storage + + if outboundIP != "" { + h.outboundIP = outboundIP + } else if ip := common.GetOutboundIP(); ip == nil { + return nil, fmt.Errorf("unable to determine outbound IP address") + } else { + h.outboundIP = ip.String() + } + + router := httprouter.New() + router.GET(urlBase+"/cache", h.middleware(h.find)) + router.POST(urlBase+"/caches", h.middleware(h.reserve)) + router.PATCH(urlBase+"/caches/:id", h.middleware(h.upload)) + router.POST(urlBase+"/caches/:id", h.middleware(h.commit)) + router.GET(urlBase+"/artifacts/:id", h.middleware(h.get)) + router.POST(urlBase+"/clean", h.middleware(h.clean)) + + h.router = router + + h.gcCache() + + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) // listen on all interfaces + if err != nil { + return nil, err + } + server := &http.Server{ + ReadHeaderTimeout: 2 * time.Second, + Handler: router, + } + go func() { + if err := server.Serve(listener); err != nil && errors.Is(err, net.ErrClosed) { + logger.Errorf("http serve: %v", err) + } + }() + h.listener = listener + h.server = server + + return h, nil +} + +func (h *Handler) ExternalURL() string { + // TODO: make the external url configurable if necessary + return fmt.Sprintf("http://%s:%d", + h.outboundIP, + h.listener.Addr().(*net.TCPAddr).Port) +} + +func (h *Handler) Close() error { + if h == nil { + return nil + } + var retErr error + if h.server != nil { + err := h.server.Close() + if err != nil { + retErr = err + } + h.server = nil + } + if h.listener != nil { + err := h.listener.Close() + if errors.Is(err, net.ErrClosed) { + err = nil + } + if err != nil { + retErr = err + } + h.listener = nil + } + if h.db != nil { + err := h.db.Close() + if err != nil { + retErr = err + } + h.db = nil + } + return retErr +} + +// GET /_apis/artifactcache/cache +func (h *Handler) find(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + keys := strings.Split(r.URL.Query().Get("keys"), ",") + // cache keys are case insensitive + for i, key := range keys { + keys[i] = strings.ToLower(key) + } + version := r.URL.Query().Get("version") + + cache, err := h.findCache(keys, version) + if err != nil { + h.responseJSON(w, r, 500, err) + return + } + if cache == nil { + h.responseJSON(w, r, 204) + return + } + + if ok, err := h.storage.Exist(cache.ID); err != nil { + h.responseJSON(w, r, 500, err) + return + } else if !ok { + _ = h.db.Delete(cache.ID, cache) + h.responseJSON(w, r, 204) + return + } + h.responseJSON(w, r, 200, map[string]any{ + "result": "hit", + "archiveLocation": fmt.Sprintf("%s%s/artifacts/%d", h.ExternalURL(), urlBase, cache.ID), + "cacheKey": cache.Key, + }) +} + +// POST /_apis/artifactcache/caches +func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + api := &Request{} + if err := json.NewDecoder(r.Body).Decode(api); err != nil { + h.responseJSON(w, r, 400, err) + return + } + // cache keys are case insensitive + api.Key = strings.ToLower(api.Key) + + cache := api.ToCache() + cache.FillKeyVersionHash() + if err := h.db.FindOne(cache, bolthold.Where("KeyVersionHash").Eq(cache.KeyVersionHash)); err != nil { + if !errors.Is(err, bolthold.ErrNotFound) { + h.responseJSON(w, r, 500, err) + return + } + } else { + h.responseJSON(w, r, 400, fmt.Errorf("already exist")) + return + } + + now := time.Now().Unix() + cache.CreatedAt = now + cache.UsedAt = now + if err := h.db.Insert(bolthold.NextSequence(), cache); err != nil { + h.responseJSON(w, r, 500, err) + return + } + // write back id to db + if err := h.db.Update(cache.ID, cache); err != nil { + h.responseJSON(w, r, 500, err) + return + } + h.responseJSON(w, r, 200, map[string]any{ + "cacheId": cache.ID, + }) +} + +// PATCH /_apis/artifactcache/caches/:id +func (h *Handler) upload(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + id, err := strconv.ParseInt(params.ByName("id"), 10, 64) + if err != nil { + h.responseJSON(w, r, 400, err) + return + } + + cache := &Cache{} + if err := h.db.Get(id, cache); err != nil { + if errors.Is(err, bolthold.ErrNotFound) { + h.responseJSON(w, r, 400, fmt.Errorf("cache %d: not reserved", id)) + return + } + h.responseJSON(w, r, 500, err) + return + } + + if cache.Complete { + h.responseJSON(w, r, 400, fmt.Errorf("cache %v %q: already complete", cache.ID, cache.Key)) + return + } + start, _, err := parseContentRange(r.Header.Get("Content-Range")) + if err != nil { + h.responseJSON(w, r, 400, err) + return + } + if err := h.storage.Write(cache.ID, start, r.Body); err != nil { + h.responseJSON(w, r, 500, err) + } + h.useCache(id) + h.responseJSON(w, r, 200) +} + +// POST /_apis/artifactcache/caches/:id +func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + id, err := strconv.ParseInt(params.ByName("id"), 10, 64) + if err != nil { + h.responseJSON(w, r, 400, err) + return + } + + cache := &Cache{} + if err := h.db.Get(id, cache); err != nil { + if errors.Is(err, bolthold.ErrNotFound) { + h.responseJSON(w, r, 400, fmt.Errorf("cache %d: not reserved", id)) + return + } + h.responseJSON(w, r, 500, err) + return + } + + if cache.Complete { + h.responseJSON(w, r, 400, fmt.Errorf("cache %v %q: already complete", cache.ID, cache.Key)) + return + } + + if err := h.storage.Commit(cache.ID, cache.Size); err != nil { + h.responseJSON(w, r, 500, err) + return + } + + cache.Complete = true + if err := h.db.Update(cache.ID, cache); err != nil { + h.responseJSON(w, r, 500, err) + return + } + + h.responseJSON(w, r, 200) +} + +// GET /_apis/artifactcache/artifacts/:id +func (h *Handler) get(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + id, err := strconv.ParseInt(params.ByName("id"), 10, 64) + if err != nil { + h.responseJSON(w, r, 400, err) + return + } + h.useCache(id) + h.storage.Serve(w, r, uint64(id)) +} + +// POST /_apis/artifactcache/clean +func (h *Handler) clean(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { + // TODO: don't support force deleting cache entries + // see: https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries + + h.responseJSON(w, r, 200) +} + +func (h *Handler) middleware(handler httprouter.Handle) httprouter.Handle { + return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + h.logger.Debugf("%s %s", r.Method, r.RequestURI) + handler(w, r, params) + go h.gcCache() + } +} + +// if not found, return (nil, nil) instead of an error. +func (h *Handler) findCache(keys []string, version string) (*Cache, error) { + if len(keys) == 0 { + return nil, nil + } + key := keys[0] // the first key is for exact match. + + cache := &Cache{ + Key: key, + Version: version, + } + cache.FillKeyVersionHash() + + if err := h.db.FindOne(cache, bolthold.Where("KeyVersionHash").Eq(cache.KeyVersionHash)); err != nil { + if !errors.Is(err, bolthold.ErrNotFound) { + return nil, err + } + } else if cache.Complete { + return cache, nil + } + stop := fmt.Errorf("stop") + + for _, prefix := range keys[1:] { + found := false + if err := h.db.ForEach(bolthold.Where("Key").Ge(prefix).And("Version").Eq(version).SortBy("Key"), func(v *Cache) error { + if !strings.HasPrefix(v.Key, prefix) { + return stop + } + if v.Complete { + cache = v + found = true + return stop + } + return nil + }); err != nil { + if !errors.Is(err, stop) { + return nil, err + } + } + if found { + return cache, nil + } + } + return nil, nil +} + +func (h *Handler) useCache(id int64) { + cache := &Cache{} + if err := h.db.Get(id, cache); err != nil { + return + } + cache.UsedAt = time.Now().Unix() + _ = h.db.Update(cache.ID, cache) +} + +func (h *Handler) gcCache() { + if atomic.LoadInt32(&h.gcing) != 0 { + return + } + if !atomic.CompareAndSwapInt32(&h.gcing, 0, 1) { + return + } + defer atomic.StoreInt32(&h.gcing, 0) + + if time.Since(h.gcAt) < time.Hour { + h.logger.Debugf("skip gc: %v", h.gcAt.String()) + return + } + h.gcAt = time.Now() + h.logger.Debugf("gc: %v", h.gcAt.String()) + + const ( + keepUsed = 30 * 24 * time.Hour + keepUnused = 7 * 24 * time.Hour + keepTemp = 5 * time.Minute + ) + + var caches []*Cache + if err := h.db.Find(&caches, bolthold.Where("UsedAt").Lt(time.Now().Add(-keepTemp).Unix())); err != nil { + h.logger.Warnf("find caches: %v", err) + } else { + for _, cache := range caches { + if cache.Complete { + continue + } + h.storage.Remove(cache.ID) + if err := h.db.Delete(cache.ID, cache); err != nil { + h.logger.Warnf("delete cache: %v", err) + continue + } + h.logger.Infof("deleted cache: %+v", cache) + } + } + + caches = caches[:0] + if err := h.db.Find(&caches, bolthold.Where("UsedAt").Lt(time.Now().Add(-keepUnused).Unix())); err != nil { + h.logger.Warnf("find caches: %v", err) + } else { + for _, cache := range caches { + h.storage.Remove(cache.ID) + if err := h.db.Delete(cache.ID, cache); err != nil { + h.logger.Warnf("delete cache: %v", err) + continue + } + h.logger.Infof("deleted cache: %+v", cache) + } + } + + caches = caches[:0] + if err := h.db.Find(&caches, bolthold.Where("CreatedAt").Lt(time.Now().Add(-keepUsed).Unix())); err != nil { + h.logger.Warnf("find caches: %v", err) + } else { + for _, cache := range caches { + h.storage.Remove(cache.ID) + if err := h.db.Delete(cache.ID, cache); err != nil { + h.logger.Warnf("delete cache: %v", err) + continue + } + h.logger.Infof("deleted cache: %+v", cache) + } + } +} + +func (h *Handler) responseJSON(w http.ResponseWriter, r *http.Request, code int, v ...any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + var data []byte + if len(v) == 0 || v[0] == nil { + data, _ = json.Marshal(struct{}{}) + } else if err, ok := v[0].(error); ok { + h.logger.Errorf("%v %v: %v", r.Method, r.RequestURI, err) + data, _ = json.Marshal(map[string]any{ + "error": err.Error(), + }) + } else { + data, _ = json.Marshal(v[0]) + } + w.WriteHeader(code) + _, _ = w.Write(data) +} + +func parseContentRange(s string) (int64, int64, error) { + // support the format like "bytes 11-22/*" only + s, _, _ = strings.Cut(strings.TrimPrefix(s, "bytes "), "/") + s1, s2, _ := strings.Cut(s, "-") + + start, err := strconv.ParseInt(s1, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse %q: %w", s, err) + } + stop, err := strconv.ParseInt(s2, 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("parse %q: %w", s, err) + } + return start, stop, nil +} diff --git a/act/artifactcache/handler_test.go b/act/artifactcache/handler_test.go new file mode 100644 index 00000000..7c6840a1 --- /dev/null +++ b/act/artifactcache/handler_test.go @@ -0,0 +1,469 @@ +package artifactcache + +import ( + "bytes" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.etcd.io/bbolt" +) + +func TestHandler(t *testing.T) { + dir := filepath.Join(t.TempDir(), "artifactcache") + handler, err := StartHandler(dir, "", 0, nil) + require.NoError(t, err) + + base := fmt.Sprintf("%s%s", handler.ExternalURL(), urlBase) + + defer func() { + t.Run("inpect db", func(t *testing.T) { + require.NoError(t, handler.db.Bolt().View(func(tx *bbolt.Tx) error { + return tx.Bucket([]byte("Cache")).ForEach(func(k, v []byte) error { + t.Logf("%s: %s", k, v) + return nil + }) + })) + }) + t.Run("close", func(t *testing.T) { + require.NoError(t, handler.Close()) + assert.Nil(t, handler.server) + assert.Nil(t, handler.listener) + assert.Nil(t, handler.db) + _, err := http.Post(fmt.Sprintf("%s/caches/%d", base, 1), "", nil) + assert.Error(t, err) + }) + }() + + t.Run("get not exist", func(t *testing.T) { + key := strings.ToLower(t.Name()) + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + resp, err := http.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version)) + require.NoError(t, err) + require.Equal(t, 204, resp.StatusCode) + }) + + t.Run("reserve and upload", func(t *testing.T) { + key := strings.ToLower(t.Name()) + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + content := make([]byte, 100) + _, err := rand.Read(content) + require.NoError(t, err) + uploadCacheNormally(t, base, key, version, content) + }) + + t.Run("clean", func(t *testing.T) { + resp, err := http.Post(fmt.Sprintf("%s/clean", base), "", nil) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + }) + + t.Run("reserve with bad request", func(t *testing.T) { + body := []byte(`invalid json`) + require.NoError(t, err) + resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) + }) + + t.Run("duplicate reserve", func(t *testing.T) { + key := strings.ToLower(t.Name()) + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + { + body, err := json.Marshal(&Request{ + Key: key, + Version: version, + Size: 100, + }) + require.NoError(t, err) + resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + got := struct { + CacheID uint64 `json:"cacheId"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + } + { + body, err := json.Marshal(&Request{ + Key: key, + Version: version, + Size: 100, + }) + require.NoError(t, err) + resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) + } + }) + + t.Run("upload with bad id", func(t *testing.T) { + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/invalid_id", base), bytes.NewReader(nil)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes 0-99/*") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) + }) + + t.Run("upload without reserve", func(t *testing.T) { + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/%d", base, 1000), bytes.NewReader(nil)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes 0-99/*") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) + }) + + t.Run("upload with complete", func(t *testing.T) { + key := strings.ToLower(t.Name()) + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + var id uint64 + content := make([]byte, 100) + _, err := rand.Read(content) + require.NoError(t, err) + { + body, err := json.Marshal(&Request{ + Key: key, + Version: version, + Size: 100, + }) + require.NoError(t, err) + resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + got := struct { + CacheID uint64 `json:"cacheId"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + id = got.CacheID + } + { + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes 0-99/*") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + } + { + resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + } + { + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes 0-99/*") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) + } + }) + + t.Run("upload with invalid range", func(t *testing.T) { + key := strings.ToLower(t.Name()) + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + var id uint64 + content := make([]byte, 100) + _, err := rand.Read(content) + require.NoError(t, err) + { + body, err := json.Marshal(&Request{ + Key: key, + Version: version, + Size: 100, + }) + require.NoError(t, err) + resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + got := struct { + CacheID uint64 `json:"cacheId"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + id = got.CacheID + } + { + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes xx-99/*") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) + } + }) + + t.Run("commit with bad id", func(t *testing.T) { + { + resp, err := http.Post(fmt.Sprintf("%s/caches/invalid_id", base), "", nil) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) + } + }) + + t.Run("commit with not exist id", func(t *testing.T) { + { + resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, 100), "", nil) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) + } + }) + + t.Run("duplicate commit", func(t *testing.T) { + key := strings.ToLower(t.Name()) + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + var id uint64 + content := make([]byte, 100) + _, err := rand.Read(content) + require.NoError(t, err) + { + body, err := json.Marshal(&Request{ + Key: key, + Version: version, + Size: 100, + }) + require.NoError(t, err) + resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + got := struct { + CacheID uint64 `json:"cacheId"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + id = got.CacheID + } + { + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes 0-99/*") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + } + { + resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + } + { + resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil) + require.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) + } + }) + + t.Run("commit early", func(t *testing.T) { + key := strings.ToLower(t.Name()) + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + var id uint64 + content := make([]byte, 100) + _, err := rand.Read(content) + require.NoError(t, err) + { + body, err := json.Marshal(&Request{ + Key: key, + Version: version, + Size: 100, + }) + require.NoError(t, err) + resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + got := struct { + CacheID uint64 `json:"cacheId"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + id = got.CacheID + } + { + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content[:50])) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes 0-59/*") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + } + { + resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil) + require.NoError(t, err) + assert.Equal(t, 500, resp.StatusCode) + } + }) + + t.Run("get with bad id", func(t *testing.T) { + resp, err := http.Get(fmt.Sprintf("%s/artifacts/invalid_id", base)) + require.NoError(t, err) + require.Equal(t, 400, resp.StatusCode) + }) + + t.Run("get with not exist id", func(t *testing.T) { + resp, err := http.Get(fmt.Sprintf("%s/artifacts/%d", base, 100)) + require.NoError(t, err) + require.Equal(t, 404, resp.StatusCode) + }) + + t.Run("get with not exist id", func(t *testing.T) { + resp, err := http.Get(fmt.Sprintf("%s/artifacts/%d", base, 100)) + require.NoError(t, err) + require.Equal(t, 404, resp.StatusCode) + }) + + t.Run("get with multiple keys", func(t *testing.T) { + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + key := strings.ToLower(t.Name()) + keys := [3]string{ + key + "_a", + key + "_a_b", + key + "_a_b_c", + } + contents := [3][]byte{ + make([]byte, 100), + make([]byte, 200), + make([]byte, 300), + } + for i := range contents { + _, err := rand.Read(contents[i]) + require.NoError(t, err) + uploadCacheNormally(t, base, keys[i], version, contents[i]) + } + + reqKeys := strings.Join([]string{ + key + "_a_b_x", + key + "_a_b", + key + "_a", + }, ",") + var archiveLocation string + { + resp, err := http.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, reqKeys, version)) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + got := struct { + Result string `json:"result"` + ArchiveLocation string `json:"archiveLocation"` + CacheKey string `json:"cacheKey"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + assert.Equal(t, "hit", got.Result) + assert.Equal(t, keys[1], got.CacheKey) + archiveLocation = got.ArchiveLocation + } + { + resp, err := http.Get(archiveLocation) //nolint:gosec + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + got, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, contents[1], got) + } + }) + + t.Run("case insensitive", func(t *testing.T) { + version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" + key := strings.ToLower(t.Name()) + content := make([]byte, 100) + _, err := rand.Read(content) + require.NoError(t, err) + uploadCacheNormally(t, base, key+"_ABC", version, content) + + { + reqKey := key + "_aBc" + resp, err := http.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, reqKey, version)) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + got := struct { + Result string `json:"result"` + ArchiveLocation string `json:"archiveLocation"` + CacheKey string `json:"cacheKey"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + assert.Equal(t, "hit", got.Result) + assert.Equal(t, key+"_abc", got.CacheKey) + } + }) +} + +func uploadCacheNormally(t *testing.T, base, key, version string, content []byte) { + var id uint64 + { + body, err := json.Marshal(&Request{ + Key: key, + Version: version, + Size: int64(len(content)), + }) + require.NoError(t, err) + resp, err := http.Post(fmt.Sprintf("%s/caches", base), "application/json", bytes.NewReader(body)) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + got := struct { + CacheID uint64 `json:"cacheId"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + id = got.CacheID + } + { + req, err := http.NewRequest(http.MethodPatch, + fmt.Sprintf("%s/caches/%d", base, id), bytes.NewReader(content)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Range", "bytes 0-99/*") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + } + { + resp, err := http.Post(fmt.Sprintf("%s/caches/%d", base, id), "", nil) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + } + var archiveLocation string + { + resp, err := http.Get(fmt.Sprintf("%s/cache?keys=%s&version=%s", base, key, version)) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + got := struct { + Result string `json:"result"` + ArchiveLocation string `json:"archiveLocation"` + CacheKey string `json:"cacheKey"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + assert.Equal(t, "hit", got.Result) + assert.Equal(t, strings.ToLower(key), got.CacheKey) + archiveLocation = got.ArchiveLocation + } + { + resp, err := http.Get(archiveLocation) //nolint:gosec + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + got, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, content, got) + } +} diff --git a/act/artifactcache/model.go b/act/artifactcache/model.go new file mode 100644 index 00000000..5c288995 --- /dev/null +++ b/act/artifactcache/model.go @@ -0,0 +1,38 @@ +package artifactcache + +import ( + "crypto/sha256" + "fmt" +) + +type Request struct { + Key string `json:"key" ` + Version string `json:"version"` + Size int64 `json:"cacheSize"` +} + +func (c *Request) ToCache() *Cache { + if c == nil { + return nil + } + return &Cache{ + Key: c.Key, + Version: c.Version, + Size: c.Size, + } +} + +type Cache struct { + ID uint64 `json:"id" boltholdKey:"ID"` + Key string `json:"key" boltholdIndex:"Key"` + Version string `json:"version" boltholdIndex:"Version"` + KeyVersionHash string `json:"keyVersionHash" boltholdUnique:"KeyVersionHash"` + Size int64 `json:"cacheSize"` + Complete bool `json:"complete"` + UsedAt int64 `json:"usedAt" boltholdIndex:"UsedAt"` + CreatedAt int64 `json:"createdAt" boltholdIndex:"CreatedAt"` +} + +func (c *Cache) FillKeyVersionHash() { + c.KeyVersionHash = fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%s:%s", c.Key, c.Version)))) +} diff --git a/act/artifactcache/storage.go b/act/artifactcache/storage.go new file mode 100644 index 00000000..a49c94e3 --- /dev/null +++ b/act/artifactcache/storage.go @@ -0,0 +1,126 @@ +package artifactcache + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" +) + +type Storage struct { + rootDir string +} + +func NewStorage(rootDir string) (*Storage, error) { + if err := os.MkdirAll(rootDir, 0o755); err != nil { + return nil, err + } + return &Storage{ + rootDir: rootDir, + }, nil +} + +func (s *Storage) Exist(id uint64) (bool, error) { + name := s.filename(id) + if _, err := os.Stat(name); os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + +func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error { + name := s.tempName(id, offset) + if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil { + return err + } + file, err := os.Create(name) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(file, reader) + return err +} + +func (s *Storage) Commit(id uint64, size int64) error { + defer func() { + _ = os.RemoveAll(s.tempDir(id)) + }() + + name := s.filename(id) + tempNames, err := s.tempNames(id) + if err != nil { + return err + } + + if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil { + return err + } + file, err := os.Create(name) + if err != nil { + return err + } + defer file.Close() + + var written int64 + for _, v := range tempNames { + f, err := os.Open(v) + if err != nil { + return err + } + n, err := io.Copy(file, f) + _ = f.Close() + if err != nil { + return err + } + written += n + } + + if written != size { + _ = file.Close() + _ = os.Remove(name) + return fmt.Errorf("broken file: %v != %v", written, size) + } + return nil +} + +func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id uint64) { + name := s.filename(id) + http.ServeFile(w, r, name) +} + +func (s *Storage) Remove(id uint64) { + _ = os.Remove(s.filename(id)) + _ = os.RemoveAll(s.tempDir(id)) +} + +func (s *Storage) filename(id uint64) string { + return filepath.Join(s.rootDir, fmt.Sprintf("%02x", id%0xff), fmt.Sprint(id)) +} + +func (s *Storage) tempDir(id uint64) string { + return filepath.Join(s.rootDir, "tmp", fmt.Sprint(id)) +} + +func (s *Storage) tempName(id uint64, offset int64) string { + return filepath.Join(s.tempDir(id), fmt.Sprintf("%016x", offset)) +} + +func (s *Storage) tempNames(id uint64) ([]string, error) { + dir := s.tempDir(id) + files, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var names []string + for _, v := range files { + if !v.IsDir() { + names = append(names, filepath.Join(dir, v.Name())) + } + } + return names, nil +} diff --git a/act/artifactcache/testdata/example/example.yaml b/act/artifactcache/testdata/example/example.yaml new file mode 100644 index 00000000..5332e723 --- /dev/null +++ b/act/artifactcache/testdata/example/example.yaml @@ -0,0 +1,30 @@ +# Copied from https://github.com/actions/cache#example-cache-workflow +name: Caching Primes + +on: push + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - run: env + + - uses: actions/checkout@v3 + + - name: Cache Primes + id: cache-primes + uses: actions/cache@v3 + with: + path: prime-numbers + key: ${{ runner.os }}-primes-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-primes + ${{ runner.os }} + + - name: Generate Prime Numbers + if: steps.cache-primes.outputs.cache-hit != 'true' + run: cat /proc/sys/kernel/random/uuid > prime-numbers + + - name: Use Prime Numbers + run: cat prime-numbers diff --git a/cmd/dir.go b/cmd/dir.go new file mode 100644 index 00000000..e1d24e9a --- /dev/null +++ b/cmd/dir.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" +) + +var ( + UserHomeDir string + CacheHomeDir string +) + +func init() { + home, err := os.UserHomeDir() + if err != nil { + log.Fatal(err) + } + UserHomeDir = home + + if v := os.Getenv("XDG_CACHE_HOME"); v != "" { + CacheHomeDir = v + } else { + CacheHomeDir = filepath.Join(UserHomeDir, ".cache") + } +} diff --git a/cmd/input.go b/cmd/input.go index 9327de2d..ed9655c0 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -42,6 +42,10 @@ type Input struct { artifactServerPath string artifactServerAddr string artifactServerPort string + noCacheServer bool + cacheServerPath string + cacheServerAddr string + cacheServerPort uint16 jsonLogger bool noSkipCheckout bool remoteName string diff --git a/cmd/notices.go b/cmd/notices.go index 9ddcf6fa..a912bd9f 100644 --- a/cmd/notices.go +++ b/cmd/notices.go @@ -132,16 +132,7 @@ func saveNoticesEtag(etag string) { } func etagPath() string { - var xdgCache string - var ok bool - if xdgCache, ok = os.LookupEnv("XDG_CACHE_HOME"); !ok || xdgCache == "" { - if home, err := os.UserHomeDir(); err == nil { - xdgCache = filepath.Join(home, ".cache") - } else if xdgCache, err = filepath.Abs("."); err != nil { - log.Fatal(err) - } - } - dir := filepath.Join(xdgCache, "act") + dir := filepath.Join(CacheHomeDir, "act") if err := os.MkdirAll(dir, 0o777); err != nil { log.Fatal(err) } diff --git a/cmd/root.go b/cmd/root.go index 548d90cc..d5b8c398 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v3" + "github.com/nektos/act/pkg/artifactcache" "github.com/nektos/act/pkg/artifacts" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" @@ -87,6 +88,10 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().StringVarP(&input.artifactServerAddr, "artifact-server-addr", "", common.GetOutboundIP().String(), "Defines the address to which the artifact server binds.") rootCmd.PersistentFlags().StringVarP(&input.artifactServerPort, "artifact-server-port", "", "34567", "Defines the port where the artifact server listens.") rootCmd.PersistentFlags().BoolVarP(&input.noSkipCheckout, "no-skip-checkout", "", false, "Do not skip actions/checkout") + rootCmd.PersistentFlags().BoolVarP(&input.noCacheServer, "no-cache-server", "", false, "Disable cache server") + rootCmd.PersistentFlags().StringVarP(&input.cacheServerPath, "cache-server-path", "", filepath.Join(CacheHomeDir, "actcache"), "Defines the path where the cache server stores caches.") + rootCmd.PersistentFlags().StringVarP(&input.cacheServerAddr, "cache-server-addr", "", common.GetOutboundIP().String(), "Defines the address to which the cache server binds.") + rootCmd.PersistentFlags().Uint16VarP(&input.cacheServerPort, "cache-server-port", "", 0, "Defines the port where the artifact server listens. 0 means a randomly available port.") rootCmd.SetArgs(args()) if err := rootCmd.Execute(); err != nil { @@ -95,11 +100,6 @@ func Execute(ctx context.Context, version string) { } func configLocations() []string { - home, err := os.UserHomeDir() - if err != nil { - log.Fatal(err) - } - configFileName := ".actrc" // reference: https://specifications.freedesktop.org/basedir-spec/latest/ar01s03.html @@ -112,7 +112,7 @@ func configLocations() []string { } return []string{ - filepath.Join(home, configFileName), + filepath.Join(UserHomeDir, configFileName), actrcXdg, filepath.Join(".", configFileName), } @@ -609,6 +609,17 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str cancel := artifacts.Serve(ctx, input.artifactServerPath, input.artifactServerAddr, input.artifactServerPort) + const cacheURLKey = "ACTIONS_CACHE_URL" + var cacheHandler *artifactcache.Handler + if !input.noCacheServer && envs[cacheURLKey] == "" { + var err error + cacheHandler, err = artifactcache.StartHandler(input.cacheServerPath, input.cacheServerAddr, input.cacheServerPort, common.Logger(ctx)) + if err != nil { + return err + } + envs[cacheURLKey] = cacheHandler.ExternalURL() + "/" + } + ctx = common.WithDryrun(ctx, input.dryrun) if watch, err := cmd.Flags().GetBool("watch"); err != nil { return err @@ -622,6 +633,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str executor := r.NewPlanExecutor(plan).Finally(func(ctx context.Context) error { cancel() + _ = cacheHandler.Close() return nil }) err = executor(ctx)