From 4444ced98aae6f459fa29f789bb7b9bb98040239 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Tue, 6 Dec 2022 11:36:39 +0100 Subject: [PATCH 01/73] Pass `LANG=C.UTF-8` to environment (#1476) * fix: pass LANG=C.UTF-8 to environment Fixes: #1308 * fix: pass LANG=C.UTF-8 to environment in container only Fixes: #1308 Signed-off-by: Brice Dutheil Signed-off-by: Brice Dutheil Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 1 + 1 file changed, 1 insertion(+) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index ce2532fa..47662c95 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -225,6 +225,7 @@ func (rc *RunContext) startJobContainer() common.Executor { envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_OS", "Linux")) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx))) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp")) + envList = append(envList, fmt.Sprintf("%s=%s", "LANG", "C.UTF-8")) // Use same locale as GitHub Actions ext := container.LinuxContainerEnvironmentExtensions{} binds, mounts := rc.GetBindsAndMounts() From 633ec30a1cfbfb60b071875ed9e9ba37d6c7451e Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 6 Dec 2022 16:45:06 +0100 Subject: [PATCH 02/73] refactor: move autoremove into the jobexecutor (#1463) * refactor: move autoremove into the jobexecutor breaking: docker container are removed after job exit * reduce complexity * remove linter exception * reduce cyclic complexity * fix: always allow 1 min for stopping and removing the runner, even if we were cancelled Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/job_executor.go | 35 +++++++++++++++++++++-------------- act/runner/runner.go | 29 ++--------------------------- 2 files changed, 23 insertions(+), 41 deletions(-) diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index e1de6cbc..b4457085 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -95,21 +95,16 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo } postExecutor = postExecutor.Finally(func(ctx context.Context) error { - logger := common.Logger(ctx) jobError := common.JobError(ctx) - if jobError != nil { - info.result("failure") - logger.WithField("jobResult", "failure").Infof("\U0001F3C1 Job failed") - } else { - err := info.stopContainer()(ctx) - if err != nil { - return err - } - info.result("success") - logger.WithField("jobResult", "success").Infof("\U0001F3C1 Job succeeded") + var err error + if rc.Config.AutoRemove || jobError == nil { + // always allow 1 min for stopping and removing the runner, even if we were cancelled + ctx, cancel := context.WithTimeout(common.WithLogger(context.Background(), common.Logger(ctx)), time.Minute) + defer cancel() + err = info.stopContainer()(ctx) } - - return nil + setJobResult(ctx, info, rc, jobError == nil) + return err }) pipeline := make([]common.Executor, 0) @@ -122,7 +117,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo if ctx.Err() == context.Canceled { // in case of an aborted run, we still should execute the // post steps to allow cleanup. - ctx, cancel = context.WithTimeout(WithJobLogger(context.Background(), rc.Run.JobID, rc.String(), rc.Config, &rc.Masks, rc.Matrix), 5*time.Minute) + ctx, cancel = context.WithTimeout(common.WithLogger(context.Background(), common.Logger(ctx)), 5*time.Minute) defer cancel() } return postExecutor(ctx) @@ -131,6 +126,18 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo Finally(info.closeContainer())) } +func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success bool) { + logger := common.Logger(ctx) + jobResult := "success" + jobResultMessage := "succeeded" + if !success { + jobResult = "failure" + jobResultMessage = "failed" + } + info.result(jobResult) + logger.WithField("jobResult", jobResult).Infof("\U0001F3C1 Job %s", jobResultMessage) +} + func useStepLogger(rc *RunContext, stepModel *model.Step, stage stepStage, executor common.Executor) common.Executor { return func(ctx context.Context) error { ctx = withStepLogger(ctx, stepModel.ID, rc.ExprEval.Interpolate(ctx, stepModel.String()), stage.String()) diff --git a/act/runner/runner.go b/act/runner/runner.go index 564814bc..65f1897d 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "time" log "github.com/sirupsen/logrus" @@ -77,18 +76,15 @@ func New(runnerConfig *Config) (Runner, error) { } // NewPlanExecutor ... -// -//nolint:gocyclo func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { maxJobNameLen := 0 stagePipeline := make([]common.Executor, 0) for i := range plan.Stages { - s := i stage := plan.Stages[i] stagePipeline = append(stagePipeline, func(ctx context.Context) error { pipeline := make([]common.Executor, 0) - for r, run := range stage.Runs { + for _, run := range stage.Runs { stageExecutor := make([]common.Executor, 0) job := run.Job() @@ -123,29 +119,8 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { maxJobNameLen = len(rc.String()) } stageExecutor = append(stageExecutor, func(ctx context.Context) error { - logger := common.Logger(ctx) jobName := fmt.Sprintf("%-*s", maxJobNameLen, rc.String()) - return rc.Executor().Finally(func(ctx context.Context) error { - isLastRunningContainer := func(currentStage int, currentRun int) bool { - return currentStage == len(plan.Stages)-1 && currentRun == len(stage.Runs)-1 - } - - if runner.config.AutoRemove && isLastRunningContainer(s, r) { - var cancel context.CancelFunc - if ctx.Err() == context.Canceled { - ctx, cancel = context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - } - - log.Infof("Cleaning up container for job %s", rc.JobName) - - if err := rc.stopJobContainer()(ctx); err != nil { - logger.Errorf("Error while cleaning container: %v", err) - } - } - - return nil - })(common.WithJobErrorContainer(WithJobLogger(ctx, rc.Run.JobID, jobName, rc.Config, &rc.Masks, matrix))) + return rc.Executor()(common.WithJobErrorContainer(WithJobLogger(ctx, rc.Run.JobID, jobName, rc.Config, &rc.Masks, matrix))) }) } pipeline = append(pipeline, common.NewParallelExecutor(maxParallel, stageExecutor...)) From 5b3714bfa061a07d13e07eeb3b22ce7259d7bc9a Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 6 Dec 2022 16:58:47 +0100 Subject: [PATCH 03/73] feat: `--container-options` (#1462) * feat: `--container-options` This deprecates the following options - `--privileged` - `--container-cap-add` - `--container-cap-drop` - `--container-architecture` - `--userns` * Merge binds/mounts, add desc * avoid linter error * fix: apply options to step env / deprecate warning Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/docker_run.go | 6 ++++++ act/runner/action.go | 1 + act/runner/run_context.go | 2 +- act/runner/runner.go | 1 + cmd/input.go | 1 + cmd/root.go | 18 ++++++++++++++++++ 6 files changed, 28 insertions(+), 1 deletion(-) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 2740ad4d..f6e2743f 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -411,10 +411,16 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig) + hostConfig.Binds = append(hostConfig.Binds, containerConfig.HostConfig.Binds...) + hostConfig.Mounts = append(hostConfig.Mounts, containerConfig.HostConfig.Mounts...) + binds := hostConfig.Binds + mounts := hostConfig.Mounts err = mergo.Merge(hostConfig, containerConfig.HostConfig, mergo.WithOverride) if err != nil { return nil, nil, fmt.Errorf("Cannot merge container.HostConfig options: '%s': '%w'", input.Options, err) } + hostConfig.Binds = binds + hostConfig.Mounts = mounts logger.Debugf("Merged container.HostConfig ==> %+v", hostConfig) return config, hostConfig, nil diff --git a/act/runner/action.go b/act/runner/action.go index d3b012db..0174dd81 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -366,6 +366,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd []string Privileged: rc.Config.Privileged, UsernsMode: rc.Config.UsernsMode, Platform: rc.Config.ContainerArchitecture, + Options: rc.Config.ContainerOptions, }) return stepContainer } diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 47662c95..30548f55 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -410,7 +410,7 @@ func (rc *RunContext) options(ctx context.Context) string { job := rc.Run.Job() c := job.Container() if c == nil { - return "" + return rc.Config.ContainerOptions } return c.Options diff --git a/act/runner/runner.go b/act/runner/runner.go index 65f1897d..60819133 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -39,6 +39,7 @@ type Config struct { UsernsMode string // user namespace to use ContainerArchitecture string // Desired OS/architecture platform for running containers ContainerDaemonSocket string // Path to Docker daemon socket + ContainerOptions string // Options for the job container UseGitIgnore bool // controls if paths in .gitignore should not be copied into container, default true GitHubInstance string // GitHub instance to use, default "github.com" ContainerCapAdd []string // list of kernel capabilities to add to the containers diff --git a/cmd/input.go b/cmd/input.go index 2de0fd29..f17fdfce 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -30,6 +30,7 @@ type Input struct { usernsMode string containerArchitecture string containerDaemonSocket string + containerOptions string noWorkflowRecurse bool useGitIgnore bool githubInstance string diff --git a/cmd/root.go b/cmd/root.go index 3073b0f6..2a0baf0e 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -76,6 +76,7 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().StringVarP(&input.envfile, "env-file", "", ".env", "environment file to read and use as env in the containers") rootCmd.PersistentFlags().StringVarP(&input.containerArchitecture, "container-architecture", "", "", "Architecture which should be used to run containers, e.g.: linux/amd64. If not specified, will use host default architecture. Requires Docker server API Version 1.41+. Ignored on earlier Docker server platforms.") rootCmd.PersistentFlags().StringVarP(&input.containerDaemonSocket, "container-daemon-socket", "", "/var/run/docker.sock", "Path to Docker daemon socket which will be mounted to containers") + rootCmd.PersistentFlags().StringVarP(&input.containerOptions, "container-options", "", "", "Custom docker container options for the job container without an options property in the job definition") rootCmd.PersistentFlags().StringVarP(&input.githubInstance, "github-instance", "", "github.com", "GitHub instance to use. Don't use this if you are not using GitHub Enterprise Server.") rootCmd.PersistentFlags().StringVarP(&input.artifactServerPath, "artifact-server-path", "", "", "Defines the path where the artifact server stores uploads and retrieves downloads from. If not specified the artifact server will not start.") rootCmd.PersistentFlags().StringVarP(&input.artifactServerPort, "artifact-server-port", "", "34567", "Defines the port where the artifact server listens (will only bind to localhost).") @@ -414,6 +415,22 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str input.platforms = readArgsFile(cfgLocations[0], true) } } + deprecationWarning := "--%s is deprecated and will be removed soon, please switch to cli: `--container-options \"%[2]s\"` or `.actrc`: `--container-options %[2]s`." + if input.privileged { + log.Warnf(deprecationWarning, "privileged", "--privileged") + } + if len(input.usernsMode) > 0 { + log.Warnf(deprecationWarning, "userns", fmt.Sprintf("--userns=%s", input.usernsMode)) + } + if len(input.containerArchitecture) > 0 { + log.Warnf(deprecationWarning, "container-architecture", fmt.Sprintf("--platform=%s", input.containerArchitecture)) + } + if len(input.containerCapAdd) > 0 { + log.Warnf(deprecationWarning, "container-cap-add", fmt.Sprintf("--cap-add=%s", input.containerCapAdd)) + } + if len(input.containerCapDrop) > 0 { + log.Warnf(deprecationWarning, "container-cap-drop", fmt.Sprintf("--cap-drop=%s", input.containerCapDrop)) + } // run the plan config := &runner.Config{ @@ -437,6 +454,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str UsernsMode: input.usernsMode, ContainerArchitecture: input.containerArchitecture, ContainerDaemonSocket: input.containerDaemonSocket, + ContainerOptions: input.containerOptions, UseGitIgnore: input.useGitIgnore, GitHubInstance: input.githubInstance, ContainerCapAdd: input.containerCapAdd, From 1441baa591afc1d90fe289d431a6c72e16ede473 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 6 Dec 2022 17:19:27 +0100 Subject: [PATCH 04/73] refactor: share UpdateFromEnv logic (#1457) * refactor: share UpdateFromEnv logic * Add test for GITHUB_OUTPUT Co-authored-by: Ben Randall * Add GITHUB_STATE test * Add test for the old broken parser Co-authored-by: Ben Randall Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/docker_run.go | 55 +--------- act/container/host_environment.go | 46 +------- act/container/parse_env_file.go | 60 +++++++++++ act/runner/runner_test.go | 3 + act/runner/testdata/GITHUB_STATE/push.yml | 22 ++++ .../environment-files-parser-bug/push.yaml | 13 +++ .../testdata/environment-files/push.yaml | 101 ++++++++++++++++++ 7 files changed, 201 insertions(+), 99 deletions(-) create mode 100644 act/container/parse_env_file.go create mode 100644 act/runner/testdata/GITHUB_STATE/push.yml create mode 100644 act/runner/testdata/environment-files-parser-bug/push.yaml create mode 100644 act/runner/testdata/environment-files/push.yaml diff --git a/act/container/docker_run.go b/act/container/docker_run.go index f6e2743f..8e30a808 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -188,7 +188,7 @@ func (cr *containerReference) GetContainerArchive(ctx context.Context, srcPath s } func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor { - return cr.extractEnv(srcPath, env).IfNot(common.Dryrun) + return parseEnvFile(cr, srcPath, env).IfNot(common.Dryrun) } func (cr *containerReference) UpdateFromImageEnv(env *map[string]string) common.Executor { @@ -503,59 +503,6 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E } } -var singleLineEnvPattern, multiLineEnvPattern *regexp.Regexp - -func (cr *containerReference) extractEnv(srcPath string, env *map[string]string) common.Executor { - if singleLineEnvPattern == nil { - // Single line pattern matches: - // SOME_VAR=data=moredata - // SOME_VAR=datamoredata - singleLineEnvPattern = regexp.MustCompile(`^([^=]*)\=(.*)$`) - multiLineEnvPattern = regexp.MustCompile(`^([^<]+)<<([\w-]+)$`) - } - - localEnv := *env - return func(ctx context.Context) error { - envTar, _, err := cr.cli.CopyFromContainer(ctx, cr.id, srcPath) - if err != nil { - return nil - } - defer envTar.Close() - - reader := tar.NewReader(envTar) - _, err = reader.Next() - if err != nil && err != io.EOF { - return fmt.Errorf("failed to read tar archive: %w", err) - } - s := bufio.NewScanner(reader) - multiLineEnvKey := "" - multiLineEnvDelimiter := "" - multiLineEnvContent := "" - for s.Scan() { - line := s.Text() - if singleLineEnv := singleLineEnvPattern.FindStringSubmatch(line); singleLineEnv != nil { - localEnv[singleLineEnv[1]] = singleLineEnv[2] - } - if line == multiLineEnvDelimiter { - localEnv[multiLineEnvKey] = multiLineEnvContent - multiLineEnvKey, multiLineEnvDelimiter, multiLineEnvContent = "", "", "" - } - if multiLineEnvKey != "" && multiLineEnvDelimiter != "" { - if multiLineEnvContent != "" { - multiLineEnvContent += "\n" - } - multiLineEnvContent += line - } - if multiLineEnvStart := multiLineEnvPattern.FindStringSubmatch(line); multiLineEnvStart != nil { - multiLineEnvKey = multiLineEnvStart[1] - multiLineEnvDelimiter = multiLineEnvStart[2] - } - } - env = &localEnv - return nil - } -} - func (cr *containerReference) extractFromImageEnv(env *map[string]string) common.Executor { envMap := *env return func(ctx context.Context) error { diff --git a/act/container/host_environment.go b/act/container/host_environment.go index b404e86d..30cd5005 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -341,51 +341,7 @@ func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[st } func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor { - localEnv := *env - return func(ctx context.Context) error { - envTar, err := e.GetContainerArchive(ctx, srcPath) - if err != nil { - return nil - } - defer envTar.Close() - reader := tar.NewReader(envTar) - _, err = reader.Next() - if err != nil && err != io.EOF { - return err - } - s := bufio.NewScanner(reader) - for s.Scan() { - line := s.Text() - singleLineEnv := strings.Index(line, "=") - multiLineEnv := strings.Index(line, "<<") - if singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv) { - localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:] - } else if multiLineEnv != -1 { - multiLineEnvContent := "" - multiLineEnvDelimiter := line[multiLineEnv+2:] - delimiterFound := false - for s.Scan() { - content := s.Text() - if content == multiLineEnvDelimiter { - delimiterFound = true - break - } - if multiLineEnvContent != "" { - multiLineEnvContent += "\n" - } - multiLineEnvContent += content - } - if !delimiterFound { - return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter) - } - localEnv[line[:multiLineEnv]] = multiLineEnvContent - } else { - return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line) - } - } - env = &localEnv - return nil - } + return parseEnvFile(e, srcPath, env) } func (e *HostEnvironment) UpdateFromPath(env *map[string]string) common.Executor { diff --git a/act/container/parse_env_file.go b/act/container/parse_env_file.go new file mode 100644 index 00000000..ee79b7e7 --- /dev/null +++ b/act/container/parse_env_file.go @@ -0,0 +1,60 @@ +package container + +import ( + "archive/tar" + "bufio" + "context" + "fmt" + "io" + "strings" + + "github.com/nektos/act/pkg/common" +) + +func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Executor { + localEnv := *env + return func(ctx context.Context) error { + envTar, err := e.GetContainerArchive(ctx, srcPath) + if err != nil { + return nil + } + defer envTar.Close() + reader := tar.NewReader(envTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + for s.Scan() { + line := s.Text() + singleLineEnv := strings.Index(line, "=") + multiLineEnv := strings.Index(line, "<<") + if singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv) { + localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:] + } else if multiLineEnv != -1 { + multiLineEnvContent := "" + multiLineEnvDelimiter := line[multiLineEnv+2:] + delimiterFound := false + for s.Scan() { + content := s.Text() + if content == multiLineEnvDelimiter { + delimiterFound = true + break + } + if multiLineEnvContent != "" { + multiLineEnvContent += "\n" + } + multiLineEnvContent += content + } + if !delimiterFound { + return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter) + } + localEnv[line[:multiLineEnv]] = multiLineEnvContent + } else { + return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line) + } + } + env = &localEnv + return nil + } +} diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 9cb4ff40..812ae322 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -171,6 +171,9 @@ func TestRunEvent(t *testing.T) { {workdir, "issue-598", "push", "", platforms}, {workdir, "if-env-act", "push", "", platforms}, {workdir, "env-and-path", "push", "", platforms}, + {workdir, "environment-files", "push", "", platforms}, + {workdir, "GITHUB_STATE", "push", "", platforms}, + {workdir, "environment-files-parser-bug", "push", "", platforms}, {workdir, "non-existent-action", "push", "Job 'nopanic' failed", platforms}, {workdir, "outputs", "push", "", platforms}, {workdir, "networking", "push", "", platforms}, diff --git a/act/runner/testdata/GITHUB_STATE/push.yml b/act/runner/testdata/GITHUB_STATE/push.yml new file mode 100644 index 00000000..179c5a72 --- /dev/null +++ b/act/runner/testdata/GITHUB_STATE/push.yml @@ -0,0 +1,22 @@ +on: push +jobs: + _: + runs-on: ubuntu-latest + steps: + - uses: nektos/act-test-actions/script@main + with: + pre: | + env + echo mystate0=mystateval > $GITHUB_STATE + echo "::save-state name=mystate1::mystateval" + main: | + env + echo mystate2=mystateval > $GITHUB_STATE + echo "::save-state name=mystate3::mystateval" + post: | + env + # Enable once https://github.com/nektos/act/issues/1459 is fixed + # [ "$STATE_mystate0" = "mystateval" ] + # [ "$STATE_mystate1" = "mystateval" ] + [ "$STATE_mystate2" = "mystateval" ] + [ "$STATE_mystate3" = "mystateval" ] \ No newline at end of file diff --git a/act/runner/testdata/environment-files-parser-bug/push.yaml b/act/runner/testdata/environment-files-parser-bug/push.yaml new file mode 100644 index 00000000..a64546c0 --- /dev/null +++ b/act/runner/testdata/environment-files-parser-bug/push.yaml @@ -0,0 +1,13 @@ +on: push +jobs: + _: + runs-on: ubuntu-latest + steps: + - run: | + echo "test< $GITHUB_ENV + echo "x=Thats really Weird" >> $GITHUB_ENV + echo "World" >> $GITHUB_ENV + - if: env.test != 'x=Thats really Weird' + run: exit 1 + - if: env.x == 'Thats really Weird' # This assert is triggered by the broken impl of act + run: exit 1 \ No newline at end of file diff --git a/act/runner/testdata/environment-files/push.yaml b/act/runner/testdata/environment-files/push.yaml new file mode 100644 index 00000000..a6ac36c7 --- /dev/null +++ b/act/runner/testdata/environment-files/push.yaml @@ -0,0 +1,101 @@ +name: environment-files +on: push + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: "Append to $GITHUB_PATH" + run: | + echo "$HOME/someFolder" >> $GITHUB_PATH + - name: "Append some more to $GITHUB_PATH" + run: | + echo "$HOME/someOtherFolder" >> $GITHUB_PATH + - name: "Check PATH" + run: | + echo "${PATH}" + if [[ ! "${PATH}" =~ .*"$HOME/"someOtherFolder.*"$HOME/"someFolder.* ]]; then + echo "${PATH} doesn't match .*someOtherFolder.*someFolder.*" + exit 1 + fi + - name: "Prepend" + run: | + if ls | grep -q 'called ls' ; then + echo 'ls was overridden already?' + exit 2 + fi + path_add=$(mktemp -d) + cat > $path_add/ls <> $GITHUB_PATH + - name: "Verify prepend" + run: | + if ! ls | grep -q 'called ls' ; then + echo 'ls was not overridden' + exit 2 + fi + - name: "Write single line env to $GITHUB_ENV" + run: | + echo "KEY=value" >> $GITHUB_ENV + - name: "Check single line env" + run: | + if [[ "${KEY}" != "value" ]]; then + echo "${KEY} doesn't == 'value'" + exit 1 + fi + - name: "Write single line env with more than one 'equals' signs to $GITHUB_ENV" + run: | + echo "KEY=value=anothervalue" >> $GITHUB_ENV + - name: "Check single line env" + run: | + if [[ "${KEY}" != "value=anothervalue" ]]; then + echo "${KEY} doesn't == 'value=anothervalue'" + exit 1 + fi + - name: "Write multiline env to $GITHUB_ENV" + run: | + echo 'KEY2<> $GITHUB_ENV + echo value2 >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - name: "Check multiline line env" + run: | + if [[ "${KEY2}" != "value2" ]]; then + echo "${KEY2} doesn't == 'value'" + exit 1 + fi + - name: "Write multiline env with UUID to $GITHUB_ENV" + run: | + echo 'KEY3<> $GITHUB_ENV + echo value3 >> $GITHUB_ENV + echo 'ghadelimiter_b8273c6d-d535-419a-a010-b0aaac240e36' >> $GITHUB_ENV + - name: "Check multiline env with UUID to $GITHUB_ENV" + run: | + if [[ "${KEY3}" != "value3" ]]; then + echo "${KEY3} doesn't == 'value3'" + exit 1 + fi + - name: "Write single line output to $GITHUB_OUTPUT" + id: write-single-output + run: | + echo "KEY=value" >> $GITHUB_OUTPUT + - name: "Check single line output" + run: | + if [[ "${{ steps.write-single-output.outputs.KEY }}" != "value" ]]; then + echo "${{ steps.write-single-output.outputs.KEY }} doesn't == 'value'" + exit 1 + fi + - name: "Write multiline output to $GITHUB_OUTPUT" + id: write-multi-output + run: | + echo 'KEY2<> $GITHUB_OUTPUT + echo value2 >> $GITHUB_OUTPUT + echo 'EOF' >> $GITHUB_OUTPUT + - name: "Check multiline output" + run: | + if [[ "${{ steps.write-multi-output.outputs.KEY2 }}" != "value2" ]]; then + echo "${{ steps.write-multi-output.outputs.KEY2 }} doesn't == 'value2'" + exit 1 + fi \ No newline at end of file From 68e74447c3433d74ea7a2f2f744a3267803bb572 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 6 Dec 2022 17:46:20 +0100 Subject: [PATCH 05/73] fix: step env is unavailable in with property expr (#1458) * fix: step env is unavailable in with property expr * don't run the test on windows * fix: composite action add missing shell Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/expression.go | 6 +++++- act/runner/runner_test.go | 1 + act/runner/step.go | 12 +++++++++++- .../testdata/inputs-via-env-context/action.yml | 8 ++++++++ .../testdata/inputs-via-env-context/push.yml | 15 +++++++++++++++ 5 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 act/runner/testdata/inputs-via-env-context/action.yml create mode 100644 act/runner/testdata/inputs-via-env-context/push.yml diff --git a/act/runner/expression.go b/act/runner/expression.go index c2257b12..6a621f7b 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -21,6 +21,10 @@ type ExpressionEvaluator interface { // NewExpressionEvaluator creates a new evaluator func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEvaluator { + return rc.NewExpressionEvaluatorWithEnv(ctx, rc.GetEnv()) +} + +func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map[string]string) ExpressionEvaluator { // todo: cleanup EvaluationEnvironment creation using := make(map[string]map[string]map[string]string) strategy := make(map[string]interface{}) @@ -46,7 +50,7 @@ func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEval ee := &exprparser.EvaluationEnvironment{ Github: ghc, - Env: rc.GetEnv(), + Env: env, Job: rc.getJobContext(), // todo: should be unavailable // but required to interpolate/evaluate the step outputs on the job diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 812ae322..21b584f0 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -287,6 +287,7 @@ func TestRunEventHostEnvironment(t *testing.T) { tables = append(tables, []TestJobFileInfo{ {workdir, "nix-prepend-path", "push", "", platforms}, + {workdir, "inputs-via-env-context", "push", "", platforms}, }...) } diff --git a/act/runner/step.go b/act/runner/step.go index f730ac1a..560ba632 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -179,7 +179,17 @@ func setupEnv(ctx context.Context, step step) error { exprEval := rc.NewExpressionEvaluator(ctx) for k, v := range *step.getEnv() { - (*step.getEnv())[k] = exprEval.Interpolate(ctx, v) + if !strings.HasPrefix(k, "INPUT_") { + (*step.getEnv())[k] = exprEval.Interpolate(ctx, v) + } + } + // after we have an evaluated step context, update the expresson evaluator with a new env context + // you can use step level env in the with property of a uses construct + exprEval = rc.NewExpressionEvaluatorWithEnv(ctx, *step.getEnv()) + for k, v := range *step.getEnv() { + if strings.HasPrefix(k, "INPUT_") { + (*step.getEnv())[k] = exprEval.Interpolate(ctx, v) + } } common.Logger(ctx).Debugf("setupEnv => %v", *step.getEnv()) diff --git a/act/runner/testdata/inputs-via-env-context/action.yml b/act/runner/testdata/inputs-via-env-context/action.yml new file mode 100644 index 00000000..4ea270d4 --- /dev/null +++ b/act/runner/testdata/inputs-via-env-context/action.yml @@ -0,0 +1,8 @@ +inputs: + test-env-input: {} +runs: + using: composite + steps: + - run: | + exit ${{ inputs.test-env-input == env.test-env-input && '0' || '1'}} + shell: bash diff --git a/act/runner/testdata/inputs-via-env-context/push.yml b/act/runner/testdata/inputs-via-env-context/push.yml new file mode 100644 index 00000000..07fadeb1 --- /dev/null +++ b/act/runner/testdata/inputs-via-env-context/push.yml @@ -0,0 +1,15 @@ +on: push +jobs: + test-inputs-via-env-context: + runs-on: self-hosted + steps: + - uses: actions/checkout@v3 + - uses: ./inputs-via-env-context + with: + test-env-input: ${{ env.test-env-input }} + env: + test-env-input: ${{ github.event_name }}/${{ github.run_id }} + - run: | + exit ${{ env.test-env-input == format('{0}/{1}', github.event_name, github.run_id) && '0' || '1' }} + env: + test-env-input: ${{ github.event_name }}/${{ github.run_id }} \ No newline at end of file From 12c0f8eb8e17f4b416c0dbb33649fafefa5d73f9 Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Wed, 7 Dec 2022 16:31:33 +0100 Subject: [PATCH 06/73] fix: handle env-vars case sensitive (#1493) Closes #1488 --- act/model/workflow.go | 10 +----- act/runner/step.go | 2 +- .../testdata/environment-variables/push.yml | 33 +++++++++++++++++++ 3 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 act/runner/testdata/environment-variables/push.yml diff --git a/act/model/workflow.go b/act/model/workflow.go index f567a1f7..78914e82 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -458,16 +458,8 @@ func (s *Step) String() string { } // Environments returns string-based key=value map for a step -// Note: all keys are uppercase func (s *Step) Environment() map[string]string { - env := environment(s.Env) - - for k, v := range env { - delete(env, k) - env[strings.ToUpper(k)] = v - } - - return env + return environment(s.Env) } // GetEnv gets the env for a step diff --git a/act/runner/step.go b/act/runner/step.go index 560ba632..a3829fb9 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -183,7 +183,7 @@ func setupEnv(ctx context.Context, step step) error { (*step.getEnv())[k] = exprEval.Interpolate(ctx, v) } } - // after we have an evaluated step context, update the expresson evaluator with a new env context + // after we have an evaluated step context, update the expressions evaluator with a new env context // you can use step level env in the with property of a uses construct exprEval = rc.NewExpressionEvaluatorWithEnv(ctx, *step.getEnv()) for k, v := range *step.getEnv() { diff --git a/act/runner/testdata/environment-variables/push.yml b/act/runner/testdata/environment-variables/push.yml new file mode 100644 index 00000000..37218ad1 --- /dev/null +++ b/act/runner/testdata/environment-variables/push.yml @@ -0,0 +1,33 @@ +name: environment variables +on: push + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Test on job level + run: | + echo \$UPPER=$UPPER + echo \$upper=$upper + echo \$LOWER=$LOWER + echo \$lower=$lower + [[ "$UPPER" = "UPPER" ]] || exit 1 + [[ "$upper" = "" ]] || exit 1 + [[ "$LOWER" = "" ]] || exit 1 + [[ "$lower" = "lower" ]] || exit 1 + - name: Test on step level + run: | + echo \$UPPER=$UPPER + echo \$upper=$upper + echo \$LOWER=$LOWER + echo \$lower=$lower + [[ "$UPPER" = "upper" ]] || exit 1 + [[ "$upper" = "" ]] || exit 1 + [[ "$LOWER" = "" ]] || exit 1 + [[ "$lower" = "LOWER" ]] || exit 1 + env: + UPPER: upper + lower: LOWER + env: + UPPER: UPPER + lower: lower From 84b6e863efe185bad5e71be1c92fbdd1dc790698 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Fri, 9 Dec 2022 11:25:32 +0100 Subject: [PATCH 07/73] feat: JobLoggerFactory (#1496) Remove overriding io.Stdout in TestMaskValues to prevent deadlock in GitHub Actions --- act/runner/logger.go | 82 +++++++++++++++++++++++---------------- act/runner/runner_test.go | 19 +++++++-- 2 files changed, 64 insertions(+), 37 deletions(-) diff --git a/act/runner/logger.go b/act/runner/logger.go index 81cae21e..162fc57a 100644 --- a/act/runner/logger.go +++ b/act/runner/logger.go @@ -57,31 +57,48 @@ func WithMasks(ctx context.Context, masks *[]string) context.Context { return context.WithValue(ctx, masksContextKeyVal, masks) } +type JobLoggerFactory interface { + WithJobLogger() *logrus.Logger +} + +type jobLoggerFactoryContextKey string + +var jobLoggerFactoryContextKeyVal = (jobLoggerFactoryContextKey)("jobloggerkey") + +func WithJobLoggerFactory(ctx context.Context, factory JobLoggerFactory) context.Context { + return context.WithValue(ctx, jobLoggerFactoryContextKeyVal, factory) +} + // WithJobLogger attaches a new logger to context that is aware of steps func WithJobLogger(ctx context.Context, jobID string, jobName string, config *Config, masks *[]string, matrix map[string]interface{}) context.Context { - mux.Lock() - defer mux.Unlock() - - var formatter logrus.Formatter - if config.JSONLogger { - formatter = &jobLogJSONFormatter{ - formatter: &logrus.JSONFormatter{}, - masker: valueMasker(config.InsecureSecrets, config.Secrets), - } - } else { - formatter = &jobLogFormatter{ - color: colors[nextColor%len(colors)], - masker: valueMasker(config.InsecureSecrets, config.Secrets), - } - } - - nextColor++ ctx = WithMasks(ctx, masks) - logger := logrus.New() - logger.SetFormatter(formatter) - logger.SetOutput(os.Stdout) - logger.SetLevel(logrus.GetLevel()) + var logger *logrus.Logger + if jobLoggerFactory, ok := ctx.Value(jobLoggerFactoryContextKeyVal).(JobLoggerFactory); ok && jobLoggerFactory != nil { + logger = jobLoggerFactory.WithJobLogger() + } else { + var formatter logrus.Formatter + if config.JSONLogger { + formatter = &logrus.JSONFormatter{} + } else { + mux.Lock() + defer mux.Unlock() + nextColor++ + formatter = &jobLogFormatter{ + color: colors[nextColor%len(colors)], + } + } + + logger = logrus.New() + logger.SetOutput(os.Stdout) + logger.SetLevel(logrus.GetLevel()) + logger.SetFormatter(formatter) + } + + logger.SetFormatter(&maskedFormatter{ + Formatter: logger.Formatter, + masker: valueMasker(config.InsecureSecrets, config.Secrets), + }) rtn := logger.WithFields(logrus.Fields{ "job": jobName, "jobID": jobID, @@ -149,16 +166,22 @@ func valueMasker(insecureSecrets bool, secrets map[string]string) entryProcessor } } -type jobLogFormatter struct { - color int +type maskedFormatter struct { + logrus.Formatter masker entryProcessor } +func (f *maskedFormatter) Format(entry *logrus.Entry) ([]byte, error) { + return f.Formatter.Format(f.masker(entry)) +} + +type jobLogFormatter struct { + color int +} + func (f *jobLogFormatter) Format(entry *logrus.Entry) ([]byte, error) { b := &bytes.Buffer{} - entry = f.masker(entry) - if f.isColored(entry) { f.printColored(b, entry) } else { @@ -225,12 +248,3 @@ func checkIfTerminal(w io.Writer) bool { return false } } - -type jobLogJSONFormatter struct { - masker entryProcessor - formatter *logrus.JSONFormatter -} - -func (f *jobLogJSONFormatter) Format(entry *logrus.Entry) ([]byte, error) { - return f.formatter.Format(f.masker(entry)) -} diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 21b584f0..faa13ced 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -1,8 +1,10 @@ package runner import ( + "bytes" "context" "fmt" + "io" "os" "path/filepath" "runtime" @@ -343,6 +345,17 @@ func TestRunDifferentArchitecture(t *testing.T) { tjfi.runTest(context.Background(), t, &Config{ContainerArchitecture: "linux/arm64"}) } +type maskJobLoggerFactory struct { + Output bytes.Buffer +} + +func (f *maskJobLoggerFactory) WithJobLogger() *log.Logger { + logger := log.New() + logger.SetOutput(io.MultiWriter(&f.Output, os.Stdout)) + logger.SetLevel(log.DebugLevel) + return logger +} + func TestMaskValues(t *testing.T) { assertNoSecret := func(text string, secret string) { index := strings.Index(text, "composite secret") @@ -366,9 +379,9 @@ func TestMaskValues(t *testing.T) { platforms: platforms, } - output := captureOutput(t, func() { - tjfi.runTest(context.Background(), t, &Config{}) - }) + logger := &maskJobLoggerFactory{} + tjfi.runTest(WithJobLoggerFactory(common.WithLogger(context.Background(), logger.WithJobLogger()), logger), t, &Config{}) + output := logger.Output.String() assertNoSecret(output, "secret value") assertNoSecret(output, "YWJjCg==") From 4f1ccbd47adc3ebe8931d16f0a951bdaa8047354 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Fri, 9 Dec 2022 12:16:15 +0100 Subject: [PATCH 08/73] refactor: fix add-path / GITHUB_PATH commands (#1472) * fix: add-path / GITHUB_PATH commands * Disable old code * fix: missing mock * Update tests * fix tests * UpdateExtraPath skip on dryrun * patch test Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/action.go | 6 ++++ act/runner/command.go | 8 ++++- act/runner/command_test.go | 4 +-- act/runner/container_mock_test.go | 11 +++++++ act/runner/run_context.go | 47 ++++++++++++++++++++++----- act/runner/step.go | 21 +++++------- act/runner/step_action_local_test.go | 11 ++++--- act/runner/step_action_remote_test.go | 8 +++-- act/runner/step_docker_test.go | 8 ++--- act/runner/step_run.go | 1 + act/runner/step_run_test.go | 8 ++--- act/runner/step_test.go | 8 +---- 12 files changed, 95 insertions(+), 46 deletions(-) diff --git a/act/runner/action.go b/act/runner/action.go index 0174dd81..ec7ac7cd 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -154,6 +154,8 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction containerArgs := []string{"node", path.Join(containerActionDir, action.Runs.Main)} logger.Debugf("executing remote job container: %s", containerArgs) + rc.ApplyExtraPath(step.getEnv()) + return rc.execJobContainer(containerArgs, *step.getEnv(), "", "")(ctx) case model.ActionRunsUsingDocker: location := actionLocation @@ -486,6 +488,8 @@ func runPreStep(step actionStep) common.Executor { containerArgs := []string{"node", path.Join(containerActionDir, action.Runs.Pre)} logger.Debugf("executing remote job container: %s", containerArgs) + rc.ApplyExtraPath(step.getEnv()) + return rc.execJobContainer(containerArgs, *step.getEnv(), "", "")(ctx) case model.ActionRunsUsingComposite: @@ -573,6 +577,8 @@ func runPostStep(step actionStep) common.Executor { containerArgs := []string{"node", path.Join(containerActionDir, action.Runs.Post)} logger.Debugf("executing remote job container: %s", containerArgs) + rc.ApplyExtraPath(step.getEnv()) + return rc.execJobContainer(containerArgs, *step.getEnv(), "", "")(ctx) case model.ActionRunsUsingComposite: diff --git a/act/runner/command.go b/act/runner/command.go index a68be16d..3fa37151 100755 --- a/act/runner/command.go +++ b/act/runner/command.go @@ -101,7 +101,13 @@ func (rc *RunContext) setOutput(ctx context.Context, kvPairs map[string]string, } func (rc *RunContext) addPath(ctx context.Context, arg string) { common.Logger(ctx).Infof(" \U00002699 ::add-path:: %s", arg) - rc.ExtraPath = append(rc.ExtraPath, arg) + extraPath := []string{arg} + for _, v := range rc.ExtraPath { + if v != arg { + extraPath = append(extraPath, v) + } + } + rc.ExtraPath = extraPath } func parseKeyValuePairs(kvPairs string, separator string) map[string]string { diff --git a/act/runner/command_test.go b/act/runner/command_test.go index 0b6ec8cb..8f71af67 100644 --- a/act/runner/command_test.go +++ b/act/runner/command_test.go @@ -64,7 +64,7 @@ func TestAddpath(t *testing.T) { a.Equal("/zoo", rc.ExtraPath[0]) handler("::add-path::/boo\n") - a.Equal("/boo", rc.ExtraPath[1]) + a.Equal("/boo", rc.ExtraPath[0]) } func TestStopCommands(t *testing.T) { @@ -102,7 +102,7 @@ func TestAddpathADO(t *testing.T) { a.Equal("/zoo", rc.ExtraPath[0]) handler("##[add-path]/boo\n") - a.Equal("/boo", rc.ExtraPath[1]) + a.Equal("/boo", rc.ExtraPath[0]) } func TestAddmask(t *testing.T) { diff --git a/act/runner/container_mock_test.go b/act/runner/container_mock_test.go index 0de07815..19f89039 100644 --- a/act/runner/container_mock_test.go +++ b/act/runner/container_mock_test.go @@ -2,6 +2,7 @@ package runner import ( "context" + "io" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" @@ -63,7 +64,17 @@ func (cm *containerMock) CopyDir(destPath string, srcPath string, useGitIgnore b args := cm.Called(destPath, srcPath, useGitIgnore) return args.Get(0).(func(context.Context) error) } + func (cm *containerMock) Exec(command []string, env map[string]string, user, workdir string) common.Executor { args := cm.Called(command, env, user, workdir) return args.Get(0).(func(context.Context) error) } + +func (cm *containerMock) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { + args := cm.Called(ctx, srcPath) + err, hasErr := args.Get(1).(error) + if !hasErr { + err = nil + } + return args.Get(0).(io.ReadCloser), err +} diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 30548f55..c653cc7e 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -1,12 +1,15 @@ package runner import ( + "archive/tar" + "bufio" "context" "crypto/rand" "encoding/hex" "encoding/json" "errors" "fmt" + "io" "os" "path/filepath" "regexp" @@ -188,10 +191,6 @@ func (rc *RunContext) startHostEnvironment() common.Executor { Name: "workflow/envs.txt", Mode: 0666, Body: "", - }, &container.FileEntry{ - Name: "workflow/paths.txt", - Mode: 0666, - Body: "", }), )(ctx) } @@ -277,10 +276,6 @@ func (rc *RunContext) startJobContainer() common.Executor { Name: "workflow/envs.txt", Mode: 0666, Body: "", - }, &container.FileEntry{ - Name: "workflow/paths.txt", - Mode: 0666, - Body: "", }), )(ctx) } @@ -292,6 +287,41 @@ func (rc *RunContext) execJobContainer(cmd []string, env map[string]string, user } } +func (rc *RunContext) ApplyExtraPath(env *map[string]string) { + if rc.ExtraPath != nil && len(rc.ExtraPath) > 0 { + path := rc.JobContainer.GetPathVariableName() + if (*env)[path] == "" { + (*env)[path] = rc.JobContainer.DefaultPathVariable() + } + (*env)[path] = rc.JobContainer.JoinPathVariable(append(rc.ExtraPath, (*env)[path])...) + } +} + +func (rc *RunContext) UpdateExtraPath(ctx context.Context, githubEnvPath string) error { + if common.Dryrun(ctx) { + return nil + } + pathTar, err := rc.JobContainer.GetContainerArchive(ctx, githubEnvPath) + if err != nil { + return err + } + defer pathTar.Close() + + reader := tar.NewReader(pathTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + for s.Scan() { + line := s.Text() + if len(line) > 0 { + rc.addPath(ctx, line) + } + } + return nil +} + // stopJobContainer removes the job container (if it exists) and its volume (if it exists) if !rc.Config.ReuseContainers func (rc *RunContext) stopJobContainer() common.Executor { return func(ctx context.Context) error { @@ -639,7 +669,6 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubContext, env map[string]string) map[string]string { env["CI"] = "true" env["GITHUB_ENV"] = rc.JobContainer.GetActPath() + "/workflow/envs.txt" - env["GITHUB_PATH"] = rc.JobContainer.GetActPath() + "/workflow/paths.txt" env["GITHUB_WORKFLOW"] = github.Workflow env["GITHUB_RUN_ID"] = github.RunID env["GITHUB_RUN_NUMBER"] = github.RunNumber diff --git a/act/runner/step.go b/act/runner/step.go index a3829fb9..f8a192fd 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -100,14 +100,19 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo actPath := rc.JobContainer.GetActPath() outputFileCommand := path.Join("workflow", "outputcmd.txt") stateFileCommand := path.Join("workflow", "statecmd.txt") + pathFileCommand := path.Join("workflow", "pathcmd.txt") (*step.getEnv())["GITHUB_OUTPUT"] = path.Join(actPath, outputFileCommand) (*step.getEnv())["GITHUB_STATE"] = path.Join(actPath, stateFileCommand) + (*step.getEnv())["GITHUB_PATH"] = path.Join(actPath, pathFileCommand) _ = rc.JobContainer.Copy(actPath, &container.FileEntry{ Name: outputFileCommand, Mode: 0666, }, &container.FileEntry{ Name: stateFileCommand, Mode: 0666, + }, &container.FileEntry{ + Name: pathFileCommand, + Mode: 0666, })(ctx) err = executor(ctx) @@ -151,6 +156,10 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo for k, v := range output { rc.setOutput(ctx, map[string]string{"name": k}, v) } + err = rc.UpdateExtraPath(ctx, path.Join(actPath, pathFileCommand)) + if err != nil { + return err + } if orgerr != nil { return orgerr } @@ -170,10 +179,6 @@ func setupEnv(ctx context.Context, step step) error { if err != nil { return err } - err = rc.JobContainer.UpdateFromPath(step.getEnv())(ctx) - if err != nil { - return err - } // merge step env last, since it should not be overwritten mergeIntoMap(step.getEnv(), step.getStepModel().GetEnv()) @@ -209,14 +214,6 @@ func mergeEnv(ctx context.Context, step step) { mergeIntoMap(env, rc.GetEnv()) } - path := rc.JobContainer.GetPathVariableName() - if (*env)[path] == "" { - (*env)[path] = rc.JobContainer.DefaultPathVariable() - } - if rc.ExtraPath != nil && len(rc.ExtraPath) > 0 { - (*env)[path] = rc.JobContainer.JoinPathVariable(append(rc.ExtraPath, (*env)[path])...) - } - rc.withGithubEnv(ctx, step.getGithubContext(ctx), *env) } diff --git a/act/runner/step_action_local_test.go b/act/runner/step_action_local_test.go index 63902898..9ece06a4 100644 --- a/act/runner/step_action_local_test.go +++ b/act/runner/step_action_local_test.go @@ -1,7 +1,9 @@ package runner import ( + "bytes" "context" + "io" "path/filepath" "strings" "testing" @@ -75,10 +77,6 @@ func TestStepActionLocalTest(t *testing.T) { return nil }) - cm.On("UpdateFromPath", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { - return nil - }) - cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { return nil }) @@ -91,6 +89,8 @@ func TestStepActionLocalTest(t *testing.T) { return nil }) + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) + salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(func(ctx context.Context) error { return nil }) @@ -280,7 +280,6 @@ func TestStepActionLocalPost(t *testing.T) { if tt.mocks.env { cm.On("UpdateFromImageEnv", &sal.env).Return(func(ctx context.Context) error { return nil }) cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", &sal.env).Return(func(ctx context.Context) error { return nil }) - cm.On("UpdateFromPath", &sal.env).Return(func(ctx context.Context) error { return nil }) } if tt.mocks.exec { suffixMatcher := func(suffix string) interface{} { @@ -301,6 +300,8 @@ func TestStepActionLocalPost(t *testing.T) { cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil }) + + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) } err := sal.post()(ctx) diff --git a/act/runner/step_action_remote_test.go b/act/runner/step_action_remote_test.go index 72b0eeeb..829e3864 100644 --- a/act/runner/step_action_remote_test.go +++ b/act/runner/step_action_remote_test.go @@ -1,8 +1,10 @@ package runner import ( + "bytes" "context" "errors" + "io" "strings" "testing" @@ -166,7 +168,6 @@ func TestStepActionRemote(t *testing.T) { if tt.mocks.env { cm.On("UpdateFromImageEnv", &sar.env).Return(func(ctx context.Context) error { return nil }) cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", &sar.env).Return(func(ctx context.Context) error { return nil }) - cm.On("UpdateFromPath", &sar.env).Return(func(ctx context.Context) error { return nil }) } if tt.mocks.read { sarm.On("readAction", sar.Step, suffixMatcher("act/remote-action@v1"), "", mock.Anything, mock.Anything).Return(&model.Action{}, nil) @@ -185,6 +186,8 @@ func TestStepActionRemote(t *testing.T) { cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil }) + + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) } err := sar.pre()(ctx) @@ -592,7 +595,6 @@ func TestStepActionRemotePost(t *testing.T) { if tt.mocks.env { cm.On("UpdateFromImageEnv", &sar.env).Return(func(ctx context.Context) error { return nil }) cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", &sar.env).Return(func(ctx context.Context) error { return nil }) - cm.On("UpdateFromPath", &sar.env).Return(func(ctx context.Context) error { return nil }) } if tt.mocks.exec { cm.On("Exec", []string{"node", "/var/run/act/actions/remote-action@v1/post.js"}, sar.env, "", "").Return(func(ctx context.Context) error { return tt.err }) @@ -608,6 +610,8 @@ func TestStepActionRemotePost(t *testing.T) { cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil }) + + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) } err := sar.post()(ctx) diff --git a/act/runner/step_docker_test.go b/act/runner/step_docker_test.go index 2008357f..c26ffd68 100644 --- a/act/runner/step_docker_test.go +++ b/act/runner/step_docker_test.go @@ -1,7 +1,9 @@ package runner import ( + "bytes" "context" + "io" "testing" "github.com/nektos/act/pkg/container" @@ -63,10 +65,6 @@ func TestStepDockerMain(t *testing.T) { return nil }) - cm.On("UpdateFromPath", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { - return nil - }) - cm.On("Pull", false).Return(func(ctx context.Context) error { return nil }) @@ -99,6 +97,8 @@ func TestStepDockerMain(t *testing.T) { return nil }) + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) + err := sd.main()(ctx) assert.Nil(t, err) diff --git a/act/runner/step_run.go b/act/runner/step_run.go index a74f781b..17db77af 100644 --- a/act/runner/step_run.go +++ b/act/runner/step_run.go @@ -30,6 +30,7 @@ func (sr *stepRun) main() common.Executor { return runStepExecutor(sr, stepStageMain, common.NewPipelineExecutor( sr.setupShellCommandExecutor(), func(ctx context.Context) error { + sr.getRunContext().ApplyExtraPath(&sr.env) return sr.getRunContext().JobContainer.Exec(sr.cmd, sr.env, "", sr.Step.WorkingDirectory)(ctx) }, )) diff --git a/act/runner/step_run_test.go b/act/runner/step_run_test.go index e5cde123..324ed5fc 100644 --- a/act/runner/step_run_test.go +++ b/act/runner/step_run_test.go @@ -1,7 +1,9 @@ package runner import ( + "bytes" "context" + "io" "testing" "github.com/nektos/act/pkg/container" @@ -61,10 +63,6 @@ func TestStepRun(t *testing.T) { return nil }) - cm.On("UpdateFromPath", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { - return nil - }) - cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { return nil }) @@ -79,6 +77,8 @@ func TestStepRun(t *testing.T) { ctx := context.Background() + cm.On("GetContainerArchive", ctx, "/var/run/act/workflow/pathcmd.txt").Return(io.NopCloser(&bytes.Buffer{}), nil) + err := sr.main()(ctx) assert.Nil(t, err) diff --git a/act/runner/step_test.go b/act/runner/step_test.go index b72f9975..8930cca3 100644 --- a/act/runner/step_test.go +++ b/act/runner/step_test.go @@ -134,7 +134,6 @@ func TestSetupEnv(t *testing.T) { Env: map[string]string{ "RC_KEY": "rcvalue", }, - ExtraPath: []string{"/path/to/extra/file"}, JobContainer: cm, } step := &model.Step{ @@ -142,9 +141,7 @@ func TestSetupEnv(t *testing.T) { "STEP_WITH": "with-value", }, } - env := map[string]string{ - "PATH": "", - } + env := map[string]string{} sm.On("getRunContext").Return(rc) sm.On("getGithubContext").Return(rc) @@ -153,7 +150,6 @@ func TestSetupEnv(t *testing.T) { cm.On("UpdateFromImageEnv", &env).Return(func(ctx context.Context) error { return nil }) cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", &env).Return(func(ctx context.Context) error { return nil }) - cm.On("UpdateFromPath", &env).Return(func(ctx context.Context) error { return nil }) err := setupEnv(context.Background(), sm) assert.Nil(t, err) @@ -184,7 +180,6 @@ func TestSetupEnv(t *testing.T) { "GITHUB_GRAPHQL_URL": "https:///api/graphql", "GITHUB_HEAD_REF": "", "GITHUB_JOB": "", - "GITHUB_PATH": "/var/run/act/workflow/paths.txt", "GITHUB_RETENTION_DAYS": "0", "GITHUB_RUN_ID": "runId", "GITHUB_RUN_NUMBER": "1", @@ -192,7 +187,6 @@ func TestSetupEnv(t *testing.T) { "GITHUB_TOKEN": "", "GITHUB_WORKFLOW": "", "INPUT_STEP_WITH": "with-value", - "PATH": "/path/to/extra/file:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "RC_KEY": "rcvalue", "RUNNER_PERFLOG": "/dev/null", "RUNNER_TRACKING_ID": "", From 67aa5960082fb58609082e8aaec036e5af91425c Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Thu, 15 Dec 2022 17:45:22 +0100 Subject: [PATCH 09/73] feat: allow to spawn and run a local reusable workflow (#1423) * feat: allow to spawn and run a local reusable workflow This change contains the ability to parse/plan/run a local reusable workflow. There are still numerous things missing: - inputs - secrets - outputs * feat: add workflow_call inputs * test: improve inputs test * feat: add input defaults * feat: allow expressions in inputs * feat: use context specific expression evaluator * refactor: prepare for better re-usability * feat: add secrets for reusable workflows * test: use secrets during test run * feat: handle reusable workflow outputs Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/model/workflow.go | 68 +++++ act/runner/expression.go | 57 ++++- act/runner/job_executor.go | 20 ++ act/runner/job_executor_test.go | 18 +- act/runner/reusable_workflow.go | 45 ++++ act/runner/run_context.go | 32 ++- act/runner/runner.go | 17 +- act/runner/runner_test.go | 237 +++++++++--------- .../workflows/local-reusable-workflow.yml | 77 ++++++ .../testdata/uses-workflow/local-workflow.yml | 36 +++ 10 files changed, 470 insertions(+), 137 deletions(-) create mode 100644 act/runner/reusable_workflow.go create mode 100644 act/runner/testdata/.github/workflows/local-reusable-workflow.yml create mode 100644 act/runner/testdata/uses-workflow/local-workflow.yml diff --git a/act/model/workflow.go b/act/model/workflow.go index 78914e82..3da7a133 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -100,6 +100,44 @@ func (w *Workflow) WorkflowDispatchConfig() *WorkflowDispatch { return &config } +type WorkflowCallInput struct { + Description string `yaml:"description"` + Required bool `yaml:"required"` + Default string `yaml:"default"` + Type string `yaml:"type"` +} + +type WorkflowCallOutput struct { + Description string `yaml:"description"` + Value string `yaml:"value"` +} + +type WorkflowCall struct { + Inputs map[string]WorkflowCallInput `yaml:"inputs"` + Outputs map[string]WorkflowCallOutput `yaml:"outputs"` +} + +func (w *Workflow) WorkflowCallConfig() *WorkflowCall { + if w.RawOn.Kind != yaml.MappingNode { + return nil + } + + var val map[string]yaml.Node + err := w.RawOn.Decode(&val) + if err != nil { + log.Fatal(err) + } + + var config WorkflowCall + node := val["workflow_call"] + err = node.Decode(&config) + if err != nil { + log.Fatal(err) + } + + return &config +} + // Job is the structure of one job in a workflow type Job struct { Name string `yaml:"name"` @@ -115,6 +153,8 @@ type Job struct { Defaults Defaults `yaml:"defaults"` Outputs map[string]string `yaml:"outputs"` Uses string `yaml:"uses"` + With map[string]interface{} `yaml:"with"` + RawSecrets yaml.Node `yaml:"secrets"` Result string } @@ -169,6 +209,34 @@ func (s Strategy) GetFailFast() bool { return failFast } +func (j *Job) InheritSecrets() bool { + if j.RawSecrets.Kind != yaml.ScalarNode { + return false + } + + var val string + err := j.RawSecrets.Decode(&val) + if err != nil { + log.Fatal(err) + } + + return val == "inherit" +} + +func (j *Job) Secrets() map[string]string { + if j.RawSecrets.Kind != yaml.MappingNode { + return nil + } + + var val map[string]string + err := j.RawSecrets.Decode(&val) + if err != nil { + log.Fatal(err) + } + + return val +} + // Container details for the job func (j *Job) Container() *ContainerSpec { var val *ContainerSpec diff --git a/act/runner/expression.go b/act/runner/expression.go index 6a621f7b..dc6b0e5b 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -55,7 +55,7 @@ func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map // todo: should be unavailable // but required to interpolate/evaluate the step outputs on the job Steps: rc.getStepsContext(), - Secrets: rc.Config.Secrets, + Secrets: getWorkflowSecrets(ctx, rc), Strategy: strategy, Matrix: rc.Matrix, Needs: using, @@ -101,7 +101,7 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) Env: *step.getEnv(), Job: rc.getJobContext(), Steps: rc.getStepsContext(), - Secrets: rc.Config.Secrets, + Secrets: getWorkflowSecrets(ctx, rc), Strategy: strategy, Matrix: rc.Matrix, Needs: using, @@ -315,6 +315,8 @@ func rewriteSubExpression(ctx context.Context, in string, forceFormat bool) (str func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *model.GithubContext) map[string]interface{} { inputs := map[string]interface{}{} + setupWorkflowInputs(ctx, &inputs, rc) + var env map[string]string if step != nil { env = *step.getEnv() @@ -347,3 +349,54 @@ func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *mod return inputs } + +func setupWorkflowInputs(ctx context.Context, inputs *map[string]interface{}, rc *RunContext) { + if rc.caller != nil { + config := rc.Run.Workflow.WorkflowCallConfig() + + for name, input := range config.Inputs { + value := rc.caller.runContext.Run.Job().With[name] + if value != nil { + if str, ok := value.(string); ok { + // evaluate using the calling RunContext (outside) + value = rc.caller.runContext.ExprEval.Interpolate(ctx, str) + } + } + + if value == nil && config != nil && config.Inputs != nil { + value = input.Default + if rc.ExprEval != nil { + if str, ok := value.(string); ok { + // evaluate using the called RunContext (inside) + value = rc.ExprEval.Interpolate(ctx, str) + } + } + } + + (*inputs)[name] = value + } + } +} + +func getWorkflowSecrets(ctx context.Context, rc *RunContext) map[string]string { + if rc.caller != nil { + job := rc.caller.runContext.Run.Job() + secrets := job.Secrets() + + if secrets == nil && job.InheritSecrets() { + secrets = rc.caller.runContext.Config.Secrets + } + + if secrets == nil { + secrets = map[string]string{} + } + + for k, v := range secrets { + secrets[k] = rc.caller.runContext.ExprEval.Interpolate(ctx, v) + } + + return secrets + } + + return rc.Config.Secrets +} diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index b4457085..4ae77879 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -104,6 +104,8 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo err = info.stopContainer()(ctx) } setJobResult(ctx, info, rc, jobError == nil) + setJobOutputs(ctx, rc) + return err }) @@ -135,9 +137,27 @@ func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success boo jobResultMessage = "failed" } info.result(jobResult) + if rc.caller != nil { + // set reusable workflow job result + rc.caller.runContext.result(jobResult) + } logger.WithField("jobResult", jobResult).Infof("\U0001F3C1 Job %s", jobResultMessage) } +func setJobOutputs(ctx context.Context, rc *RunContext) { + if rc.caller != nil { + // map outputs for reusable workflows + callerOutputs := make(map[string]string) + + ee := rc.NewExpressionEvaluator(ctx) + for k, v := range rc.Run.Job().Outputs { + callerOutputs[k] = ee.Interpolate(ctx, v) + } + + rc.caller.runContext.Run.Job().Outputs = callerOutputs + } +} + func useStepLogger(rc *RunContext, stepModel *model.Step, stage stepStage, executor common.Executor) common.Executor { return func(ctx context.Context) error { ctx = withStepLogger(ctx, stepModel.ID, rc.ExprEval.Interpolate(ctx, stepModel.String()), stage.String()) diff --git a/act/runner/job_executor_test.go b/act/runner/job_executor_test.go index e00a4fd6..87c58886 100644 --- a/act/runner/job_executor_test.go +++ b/act/runner/job_executor_test.go @@ -15,15 +15,15 @@ import ( func TestJobExecutor(t *testing.T) { tables := []TestJobFileInfo{ - {workdir, "uses-and-run-in-one-step", "push", "Invalid run/uses syntax for job:test step:Test", platforms}, - {workdir, "uses-github-empty", "push", "Expected format {org}/{repo}[/path]@ref", platforms}, - {workdir, "uses-github-noref", "push", "Expected format {org}/{repo}[/path]@ref", platforms}, - {workdir, "uses-github-root", "push", "", platforms}, - {workdir, "uses-github-path", "push", "", platforms}, - {workdir, "uses-docker-url", "push", "", platforms}, - {workdir, "uses-github-full-sha", "push", "", platforms}, - {workdir, "uses-github-short-sha", "push", "Unable to resolve action `actions/hello-world-docker-action@b136eb8`, the provided ref `b136eb8` is the shortened version of a commit SHA, which is not supported. Please use the full commit SHA `b136eb8894c5cb1dd5807da824be97ccdf9b5423` instead", platforms}, - {workdir, "job-nil-step", "push", "invalid Step 0: missing run or uses key", platforms}, + {workdir, "uses-and-run-in-one-step", "push", "Invalid run/uses syntax for job:test step:Test", platforms, secrets}, + {workdir, "uses-github-empty", "push", "Expected format {org}/{repo}[/path]@ref", platforms, secrets}, + {workdir, "uses-github-noref", "push", "Expected format {org}/{repo}[/path]@ref", platforms, secrets}, + {workdir, "uses-github-root", "push", "", platforms, secrets}, + {workdir, "uses-github-path", "push", "", platforms, secrets}, + {workdir, "uses-docker-url", "push", "", platforms, secrets}, + {workdir, "uses-github-full-sha", "push", "", platforms, secrets}, + {workdir, "uses-github-short-sha", "push", "Unable to resolve action `actions/hello-world-docker-action@b136eb8`, the provided ref `b136eb8` is the shortened version of a commit SHA, which is not supported. Please use the full commit SHA `b136eb8894c5cb1dd5807da824be97ccdf9b5423` instead", platforms, secrets}, + {workdir, "job-nil-step", "push", "invalid Step 0: missing run or uses key", platforms, secrets}, } // These tests are sufficient to only check syntax. ctx := common.WithDryrun(context.Background(), true) diff --git a/act/runner/reusable_workflow.go b/act/runner/reusable_workflow.go new file mode 100644 index 00000000..87b7bde9 --- /dev/null +++ b/act/runner/reusable_workflow.go @@ -0,0 +1,45 @@ +package runner + +import ( + "fmt" + "path" + + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/model" +) + +func newLocalReusableWorkflowExecutor(rc *RunContext) common.Executor { + return newReusableWorkflowExecutor(rc, rc.Config.Workdir) +} + +func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor { + return common.NewErrorExecutor(fmt.Errorf("remote reusable workflows are currently not supported (see https://github.com/nektos/act/issues/826 for updates)")) +} + +func newReusableWorkflowExecutor(rc *RunContext, directory string) common.Executor { + planner, err := model.NewWorkflowPlanner(path.Join(directory, rc.Run.Job().Uses), true) + if err != nil { + return common.NewErrorExecutor(err) + } + + plan := planner.PlanEvent("workflow_call") + + runner, err := NewReusableWorkflowRunner(rc) + if err != nil { + return common.NewErrorExecutor(err) + } + + return runner.NewPlanExecutor(plan) +} + +func NewReusableWorkflowRunner(rc *RunContext) (Runner, error) { + runner := &runnerImpl{ + config: rc.Config, + eventJSON: rc.EventJSON, + caller: &caller{ + runContext: rc, + }, + } + + return runner.configure() +} diff --git a/act/runner/run_context.go b/act/runner/run_context.go index c653cc7e..854e628d 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -46,6 +46,7 @@ type RunContext struct { Parent *RunContext Masks []string cleanUpJobContainer common.Executor + caller *caller // job calling this RunContext (reusable workflows) } func (rc *RunContext) AddMask(mask string) { @@ -58,7 +59,13 @@ type MappableOutput struct { } func (rc *RunContext) String() string { - return fmt.Sprintf("%s/%s", rc.Run.Workflow.Name, rc.Name) + name := fmt.Sprintf("%s/%s", rc.Run.Workflow.Name, rc.Name) + if rc.caller != nil { + // prefix the reusable workflow with the caller job + // this is required to create unique container names + name = fmt.Sprintf("%s/%s", rc.caller.runContext.Run.JobID, name) + } + return name } // GetEnv returns the env for the context @@ -399,16 +406,25 @@ func (rc *RunContext) steps() []*model.Step { // Executor returns a pipeline executor for all the steps in the job func (rc *RunContext) Executor() common.Executor { + var executor common.Executor + + switch rc.Run.Job().Type() { + case model.JobTypeDefault: + executor = newJobExecutor(rc, &stepFactoryImpl{}, rc) + case model.JobTypeReusableWorkflowLocal: + executor = newLocalReusableWorkflowExecutor(rc) + case model.JobTypeReusableWorkflowRemote: + executor = newRemoteReusableWorkflowExecutor(rc) + } + return func(ctx context.Context) error { - isEnabled, err := rc.isEnabled(ctx) + res, err := rc.isEnabled(ctx) if err != nil { return err } - - if isEnabled { - return newJobExecutor(rc, &stepFactoryImpl{}, rc)(ctx) + if res { + return executor(ctx) } - return nil } } @@ -458,6 +474,10 @@ func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) { return false, nil } + if job.Type() != model.JobTypeDefault { + return true, nil + } + img := rc.platformImage(ctx) if img == "" { if job.RunsOn() == nil { diff --git a/act/runner/runner.go b/act/runner/runner.go index 60819133..2ed967db 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -53,9 +53,14 @@ type Config struct { ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. } +type caller struct { + runContext *RunContext +} + type runnerImpl struct { config *Config eventJSON string + caller *caller // the job calling this runner (caller of a reusable workflow) } // New Creates a new Runner @@ -64,8 +69,12 @@ func New(runnerConfig *Config) (Runner, error) { config: runnerConfig, } + return runner.configure() +} + +func (runner *runnerImpl) configure() (Runner, error) { runner.eventJSON = "{}" - if runnerConfig.EventPath != "" { + if runner.config.EventPath != "" { log.Debugf("Reading event.json from %s", runner.config.EventPath) eventJSONBytes, err := os.ReadFile(runner.config.EventPath) if err != nil { @@ -89,10 +98,6 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { stageExecutor := make([]common.Executor, 0) job := run.Job() - if job.Uses != "" { - return fmt.Errorf("reusable workflows are currently not supported (see https://github.com/nektos/act/issues/826 for updates)") - } - if job.Strategy != nil { strategyRc := runner.newRunContext(ctx, run, nil) if err := strategyRc.NewExpressionEvaluator(ctx).EvaluateYamlNode(ctx, &job.Strategy.RawMatrix); err != nil { @@ -161,8 +166,10 @@ func (runner *runnerImpl) newRunContext(ctx context.Context, run *model.Run, mat EventJSON: runner.eventJSON, StepResults: make(map[string]*model.StepResult), Matrix: matrix, + caller: runner.caller, } rc.ExprEval = rc.NewExpressionEvaluator(ctx) rc.Name = rc.ExprEval.Interpolate(ctx, run.String()) + return rc } diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index faa13ced..1f614117 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -24,6 +24,7 @@ var ( platforms map[string]string logLevel = log.DebugLevel workdir = "testdata" + secrets map[string]string ) func init() { @@ -44,6 +45,8 @@ func init() { if wd, err := filepath.Abs(workdir); err == nil { workdir = wd } + + secrets = map[string]string{} } func TestGraphEvent(t *testing.T) { @@ -70,6 +73,7 @@ type TestJobFileInfo struct { eventName string errorMessage string platforms map[string]string + secrets map[string]string } func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config) { @@ -121,84 +125,87 @@ func TestRunEvent(t *testing.T) { tables := []TestJobFileInfo{ // Shells - {workdir, "shells/defaults", "push", "", platforms}, + {workdir, "shells/defaults", "push", "", platforms, secrets}, // TODO: figure out why it fails // {workdir, "shells/custom", "push", "", map[string]string{"ubuntu-latest": "catthehacker/ubuntu:pwsh-latest"}, }, // custom image with pwsh - {workdir, "shells/pwsh", "push", "", map[string]string{"ubuntu-latest": "catthehacker/ubuntu:pwsh-latest"}}, // custom image with pwsh - {workdir, "shells/bash", "push", "", platforms}, - {workdir, "shells/python", "push", "", map[string]string{"ubuntu-latest": "node:16-buster"}}, // slim doesn't have python - {workdir, "shells/sh", "push", "", platforms}, + {workdir, "shells/pwsh", "push", "", map[string]string{"ubuntu-latest": "catthehacker/ubuntu:pwsh-latest"}, secrets}, // custom image with pwsh + {workdir, "shells/bash", "push", "", platforms, secrets}, + {workdir, "shells/python", "push", "", map[string]string{"ubuntu-latest": "node:16-buster"}, secrets}, // slim doesn't have python + {workdir, "shells/sh", "push", "", platforms, secrets}, // Local action - {workdir, "local-action-docker-url", "push", "", platforms}, - {workdir, "local-action-dockerfile", "push", "", platforms}, - {workdir, "local-action-via-composite-dockerfile", "push", "", platforms}, - {workdir, "local-action-js", "push", "", platforms}, + {workdir, "local-action-docker-url", "push", "", platforms, secrets}, + {workdir, "local-action-dockerfile", "push", "", platforms, secrets}, + {workdir, "local-action-via-composite-dockerfile", "push", "", platforms, secrets}, + {workdir, "local-action-js", "push", "", platforms, secrets}, // Uses - {workdir, "uses-composite", "push", "", platforms}, - {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms}, - {workdir, "uses-nested-composite", "push", "", platforms}, - {workdir, "remote-action-composite-js-pre-with-defaults", "push", "", platforms}, - {workdir, "uses-workflow", "push", "reusable workflows are currently not supported (see https://github.com/nektos/act/issues/826 for updates)", platforms}, - {workdir, "uses-docker-url", "push", "", platforms}, - {workdir, "act-composite-env-test", "push", "", platforms}, + {workdir, "uses-composite", "push", "", platforms, secrets}, + {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms, secrets}, + {workdir, "uses-nested-composite", "push", "", platforms, secrets}, + {workdir, "remote-action-composite-js-pre-with-defaults", "push", "", platforms, secrets}, + {workdir, "uses-workflow", "push", "reusable workflows are currently not supported (see https://github.com/nektos/act/issues/826 for updates)", platforms, secrets}, + {workdir, "uses-workflow", "pull_request", "", platforms, map[string]string{"secret": "keep_it_private"}}, + {workdir, "uses-docker-url", "push", "", platforms, secrets}, + {workdir, "act-composite-env-test", "push", "", platforms, secrets}, // Eval - {workdir, "evalmatrix", "push", "", platforms}, - {workdir, "evalmatrixneeds", "push", "", platforms}, - {workdir, "evalmatrixneeds2", "push", "", platforms}, - {workdir, "evalmatrix-merge-map", "push", "", platforms}, - {workdir, "evalmatrix-merge-array", "push", "", platforms}, - {workdir, "issue-1195", "push", "", platforms}, + {workdir, "evalmatrix", "push", "", platforms, secrets}, + {workdir, "evalmatrixneeds", "push", "", platforms, secrets}, + {workdir, "evalmatrixneeds2", "push", "", platforms, secrets}, + {workdir, "evalmatrix-merge-map", "push", "", platforms, secrets}, + {workdir, "evalmatrix-merge-array", "push", "", platforms, secrets}, + {workdir, "issue-1195", "push", "", platforms, secrets}, - {workdir, "basic", "push", "", platforms}, - {workdir, "fail", "push", "exit with `FAILURE`: 1", platforms}, - {workdir, "runs-on", "push", "", platforms}, - {workdir, "checkout", "push", "", platforms}, - {workdir, "job-container", "push", "", platforms}, - {workdir, "job-container-non-root", "push", "", platforms}, - {workdir, "job-container-invalid-credentials", "push", "failed to handle credentials: failed to interpolate container.credentials.password", platforms}, - {workdir, "container-hostname", "push", "", platforms}, - {workdir, "remote-action-docker", "push", "", platforms}, - {workdir, "remote-action-js", "push", "", platforms}, - {workdir, "remote-action-js", "push", "", map[string]string{"ubuntu-latest": "catthehacker/ubuntu:runner-latest"}}, // Test if this works with non root container - {workdir, "matrix", "push", "", platforms}, - {workdir, "matrix-include-exclude", "push", "", platforms}, - {workdir, "commands", "push", "", platforms}, - {workdir, "workdir", "push", "", platforms}, - {workdir, "defaults-run", "push", "", platforms}, - {workdir, "composite-fail-with-output", "push", "", platforms}, - {workdir, "issue-597", "push", "", platforms}, - {workdir, "issue-598", "push", "", platforms}, - {workdir, "if-env-act", "push", "", platforms}, - {workdir, "env-and-path", "push", "", platforms}, - {workdir, "environment-files", "push", "", platforms}, - {workdir, "GITHUB_STATE", "push", "", platforms}, - {workdir, "environment-files-parser-bug", "push", "", platforms}, - {workdir, "non-existent-action", "push", "Job 'nopanic' failed", platforms}, - {workdir, "outputs", "push", "", platforms}, - {workdir, "networking", "push", "", platforms}, - {workdir, "steps-context/conclusion", "push", "", platforms}, - {workdir, "steps-context/outcome", "push", "", platforms}, - {workdir, "job-status-check", "push", "job 'fail' failed", platforms}, - {workdir, "if-expressions", "push", "Job 'mytest' failed", platforms}, - {workdir, "actions-environment-and-context-tests", "push", "", platforms}, - {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms}, - {workdir, "evalenv", "push", "", platforms}, - {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms}, - {workdir, "workflow_dispatch", "workflow_dispatch", "", platforms}, - {workdir, "workflow_dispatch_no_inputs_mapping", "workflow_dispatch", "", platforms}, - {workdir, "workflow_dispatch-scalar", "workflow_dispatch", "", platforms}, - {workdir, "workflow_dispatch-scalar-composite-action", "workflow_dispatch", "", platforms}, - {"../model/testdata", "strategy", "push", "", platforms}, // TODO: move all testdata into pkg so we can validate it with planner and runner + {workdir, "basic", "push", "", platforms, secrets}, + {workdir, "fail", "push", "exit with `FAILURE`: 1", platforms, secrets}, + {workdir, "runs-on", "push", "", platforms, secrets}, + {workdir, "checkout", "push", "", platforms, secrets}, + {workdir, "job-container", "push", "", platforms, secrets}, + {workdir, "job-container-non-root", "push", "", platforms, secrets}, + {workdir, "job-container-invalid-credentials", "push", "failed to handle credentials: failed to interpolate container.credentials.password", platforms, secrets}, + {workdir, "container-hostname", "push", "", platforms, secrets}, + {workdir, "remote-action-docker", "push", "", platforms, secrets}, + {workdir, "remote-action-js", "push", "", platforms, secrets}, + {workdir, "remote-action-js", "push", "", map[string]string{"ubuntu-latest": "catthehacker/ubuntu:runner-latest"}, secrets}, // Test if this works with non root container + {workdir, "matrix", "push", "", platforms, secrets}, + {workdir, "matrix-include-exclude", "push", "", platforms, secrets}, + {workdir, "commands", "push", "", platforms, secrets}, + {workdir, "workdir", "push", "", platforms, secrets}, + {workdir, "defaults-run", "push", "", platforms, secrets}, + {workdir, "composite-fail-with-output", "push", "", platforms, secrets}, + {workdir, "issue-597", "push", "", platforms, secrets}, + {workdir, "issue-598", "push", "", platforms, secrets}, + {workdir, "if-env-act", "push", "", platforms, secrets}, + {workdir, "env-and-path", "push", "", platforms, secrets}, + {workdir, "environment-files", "push", "", platforms, secrets}, + {workdir, "GITHUB_STATE", "push", "", platforms, secrets}, + {workdir, "environment-files-parser-bug", "push", "", platforms, secrets}, + {workdir, "non-existent-action", "push", "Job 'nopanic' failed", platforms, secrets}, + {workdir, "outputs", "push", "", platforms, secrets}, + {workdir, "networking", "push", "", platforms, secrets}, + {workdir, "steps-context/conclusion", "push", "", platforms, secrets}, + {workdir, "steps-context/outcome", "push", "", platforms, secrets}, + {workdir, "job-status-check", "push", "job 'fail' failed", platforms, secrets}, + {workdir, "if-expressions", "push", "Job 'mytest' failed", platforms, secrets}, + {workdir, "actions-environment-and-context-tests", "push", "", platforms, secrets}, + {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms, secrets}, + {workdir, "evalenv", "push", "", platforms, secrets}, + {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms, secrets}, + {workdir, "workflow_dispatch", "workflow_dispatch", "", platforms, secrets}, + {workdir, "workflow_dispatch_no_inputs_mapping", "workflow_dispatch", "", platforms, secrets}, + {workdir, "workflow_dispatch-scalar", "workflow_dispatch", "", platforms, secrets}, + {workdir, "workflow_dispatch-scalar-composite-action", "workflow_dispatch", "", platforms, secrets}, + {"../model/testdata", "strategy", "push", "", platforms, secrets}, // TODO: move all testdata into pkg so we can validate it with planner and runner // {"testdata", "issue-228", "push", "", platforms, }, // TODO [igni]: Remove this once everything passes - {"../model/testdata", "container-volumes", "push", "", platforms}, + {"../model/testdata", "container-volumes", "push", "", platforms, secrets}, } for _, table := range tables { t.Run(table.workflowPath, func(t *testing.T) { - config := &Config{} + config := &Config{ + Secrets: table.secrets, + } eventFile := filepath.Join(workdir, table.workflowPath, "event.json") if _, err := os.Stat(eventFile); err == nil { @@ -226,51 +233,51 @@ func TestRunEventHostEnvironment(t *testing.T) { tables = append(tables, []TestJobFileInfo{ // Shells - {workdir, "shells/defaults", "push", "", platforms}, - {workdir, "shells/pwsh", "push", "", platforms}, - {workdir, "shells/bash", "push", "", platforms}, - {workdir, "shells/python", "push", "", platforms}, - {workdir, "shells/sh", "push", "", platforms}, + {workdir, "shells/defaults", "push", "", platforms, secrets}, + {workdir, "shells/pwsh", "push", "", platforms, secrets}, + {workdir, "shells/bash", "push", "", platforms, secrets}, + {workdir, "shells/python", "push", "", platforms, secrets}, + {workdir, "shells/sh", "push", "", platforms, secrets}, // Local action - {workdir, "local-action-js", "push", "", platforms}, + {workdir, "local-action-js", "push", "", platforms, secrets}, // Uses - {workdir, "uses-composite", "push", "", platforms}, - {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms}, - {workdir, "uses-nested-composite", "push", "", platforms}, - {workdir, "act-composite-env-test", "push", "", platforms}, + {workdir, "uses-composite", "push", "", platforms, secrets}, + {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms, secrets}, + {workdir, "uses-nested-composite", "push", "", platforms, secrets}, + {workdir, "act-composite-env-test", "push", "", platforms, secrets}, // Eval - {workdir, "evalmatrix", "push", "", platforms}, - {workdir, "evalmatrixneeds", "push", "", platforms}, - {workdir, "evalmatrixneeds2", "push", "", platforms}, - {workdir, "evalmatrix-merge-map", "push", "", platforms}, - {workdir, "evalmatrix-merge-array", "push", "", platforms}, - {workdir, "issue-1195", "push", "", platforms}, + {workdir, "evalmatrix", "push", "", platforms, secrets}, + {workdir, "evalmatrixneeds", "push", "", platforms, secrets}, + {workdir, "evalmatrixneeds2", "push", "", platforms, secrets}, + {workdir, "evalmatrix-merge-map", "push", "", platforms, secrets}, + {workdir, "evalmatrix-merge-array", "push", "", platforms, secrets}, + {workdir, "issue-1195", "push", "", platforms, secrets}, - {workdir, "fail", "push", "exit with `FAILURE`: 1", platforms}, - {workdir, "runs-on", "push", "", platforms}, - {workdir, "checkout", "push", "", platforms}, - {workdir, "remote-action-js", "push", "", platforms}, - {workdir, "matrix", "push", "", platforms}, - {workdir, "matrix-include-exclude", "push", "", platforms}, - {workdir, "commands", "push", "", platforms}, - {workdir, "defaults-run", "push", "", platforms}, - {workdir, "composite-fail-with-output", "push", "", platforms}, - {workdir, "issue-597", "push", "", platforms}, - {workdir, "issue-598", "push", "", platforms}, - {workdir, "if-env-act", "push", "", platforms}, - {workdir, "env-and-path", "push", "", platforms}, - {workdir, "non-existent-action", "push", "Job 'nopanic' failed", platforms}, - {workdir, "outputs", "push", "", platforms}, - {workdir, "steps-context/conclusion", "push", "", platforms}, - {workdir, "steps-context/outcome", "push", "", platforms}, - {workdir, "job-status-check", "push", "job 'fail' failed", platforms}, - {workdir, "if-expressions", "push", "Job 'mytest' failed", platforms}, - {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms}, - {workdir, "evalenv", "push", "", platforms}, - {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms}, + {workdir, "fail", "push", "exit with `FAILURE`: 1", platforms, secrets}, + {workdir, "runs-on", "push", "", platforms, secrets}, + {workdir, "checkout", "push", "", platforms, secrets}, + {workdir, "remote-action-js", "push", "", platforms, secrets}, + {workdir, "matrix", "push", "", platforms, secrets}, + {workdir, "matrix-include-exclude", "push", "", platforms, secrets}, + {workdir, "commands", "push", "", platforms, secrets}, + {workdir, "defaults-run", "push", "", platforms, secrets}, + {workdir, "composite-fail-with-output", "push", "", platforms, secrets}, + {workdir, "issue-597", "push", "", platforms, secrets}, + {workdir, "issue-598", "push", "", platforms, secrets}, + {workdir, "if-env-act", "push", "", platforms, secrets}, + {workdir, "env-and-path", "push", "", platforms, secrets}, + {workdir, "non-existent-action", "push", "Job 'nopanic' failed", platforms, secrets}, + {workdir, "outputs", "push", "", platforms, secrets}, + {workdir, "steps-context/conclusion", "push", "", platforms, secrets}, + {workdir, "steps-context/outcome", "push", "", platforms, secrets}, + {workdir, "job-status-check", "push", "job 'fail' failed", platforms, secrets}, + {workdir, "if-expressions", "push", "Job 'mytest' failed", platforms, secrets}, + {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms, secrets}, + {workdir, "evalenv", "push", "", platforms, secrets}, + {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms, secrets}, }...) } if runtime.GOOS == "windows" { @@ -279,8 +286,8 @@ func TestRunEventHostEnvironment(t *testing.T) { } tables = append(tables, []TestJobFileInfo{ - {workdir, "windows-prepend-path", "push", "", platforms}, - {workdir, "windows-add-env", "push", "", platforms}, + {workdir, "windows-prepend-path", "push", "", platforms, secrets}, + {workdir, "windows-add-env", "push", "", platforms, secrets}, }...) } else { platforms := map[string]string{ @@ -288,8 +295,8 @@ func TestRunEventHostEnvironment(t *testing.T) { } tables = append(tables, []TestJobFileInfo{ - {workdir, "nix-prepend-path", "push", "", platforms}, - {workdir, "inputs-via-env-context", "push", "", platforms}, + {workdir, "nix-prepend-path", "push", "", platforms, secrets}, + {workdir, "inputs-via-env-context", "push", "", platforms, secrets}, }...) } @@ -309,17 +316,17 @@ func TestDryrunEvent(t *testing.T) { tables := []TestJobFileInfo{ // Shells - {workdir, "shells/defaults", "push", "", platforms}, - {workdir, "shells/pwsh", "push", "", map[string]string{"ubuntu-latest": "catthehacker/ubuntu:pwsh-latest"}}, // custom image with pwsh - {workdir, "shells/bash", "push", "", platforms}, - {workdir, "shells/python", "push", "", map[string]string{"ubuntu-latest": "node:16-buster"}}, // slim doesn't have python - {workdir, "shells/sh", "push", "", platforms}, + {workdir, "shells/defaults", "push", "", platforms, secrets}, + {workdir, "shells/pwsh", "push", "", map[string]string{"ubuntu-latest": "catthehacker/ubuntu:pwsh-latest"}, secrets}, // custom image with pwsh + {workdir, "shells/bash", "push", "", platforms, secrets}, + {workdir, "shells/python", "push", "", map[string]string{"ubuntu-latest": "node:16-buster"}, secrets}, // slim doesn't have python + {workdir, "shells/sh", "push", "", platforms, secrets}, // Local action - {workdir, "local-action-docker-url", "push", "", platforms}, - {workdir, "local-action-dockerfile", "push", "", platforms}, - {workdir, "local-action-via-composite-dockerfile", "push", "", platforms}, - {workdir, "local-action-js", "push", "", platforms}, + {workdir, "local-action-docker-url", "push", "", platforms, secrets}, + {workdir, "local-action-dockerfile", "push", "", platforms, secrets}, + {workdir, "local-action-via-composite-dockerfile", "push", "", platforms, secrets}, + {workdir, "local-action-js", "push", "", platforms, secrets}, } for _, table := range tables { diff --git a/act/runner/testdata/.github/workflows/local-reusable-workflow.yml b/act/runner/testdata/.github/workflows/local-reusable-workflow.yml new file mode 100644 index 00000000..b04fe72d --- /dev/null +++ b/act/runner/testdata/.github/workflows/local-reusable-workflow.yml @@ -0,0 +1,77 @@ +name: reusable + +on: + workflow_call: + inputs: + string_required: + required: true + type: string + string_optional: + required: false + type: string + default: string + bool_required: + required: true + type: boolean + bool_optional: + required: false + type: boolean + default: true + number_required: + required: true + type: number + number_optional: + required: false + type: number + default: ${{ 1 }} + outputs: + output: + description: "A workflow output" + value: ${{ jobs.reusable_workflow_job.outputs.output }} + +jobs: + reusable_workflow_job: + runs-on: ubuntu-latest + steps: + - name: test required string + run: | + echo inputs.string_required=${{ inputs.string_required }} + [[ "${{ inputs.string_required == 'string' }}" = "true" ]] || exit 1 + + - name: test optional string + run: | + echo inputs.string_optional=${{ inputs.string_optional }} + [[ "${{ inputs.string_optional == 'string' }}" = "true" ]] || exit 1 + + - name: test required bool + run: | + echo inputs.bool_required=${{ inputs.bool_required }} + [[ "${{ inputs.bool_required }}" = "true" ]] || exit 1 + + - name: test optional bool + run: | + echo inputs.bool_optional=${{ inputs.bool_optional }} + [[ "${{ inputs.bool_optional }}" = "true" ]] || exit 1 + + - name: test required number + run: | + echo inputs.number_required=${{ inputs.number_required }} + [[ "${{ inputs.number_required == 1 }}" = "true" ]] || exit 1 + + - name: test optional number + run: | + echo inputs.number_optional=${{ inputs.number_optional }} + [[ "${{ inputs.number_optional == 1 }}" = "true" ]] || exit 1 + + - name: test secret + run: | + echo secrets.secret=${{ secrets.secret }} + [[ "${{ secrets.secret == 'keep_it_private' }}" = "true" ]] || exit 1 + + - name: test output + id: output_test + run: | + echo "value=${{ inputs.string_required }}" >> $GITHUB_OUTPUT + + outputs: + output: ${{ steps.output_test.outputs.value }} diff --git a/act/runner/testdata/uses-workflow/local-workflow.yml b/act/runner/testdata/uses-workflow/local-workflow.yml new file mode 100644 index 00000000..070e4d0c --- /dev/null +++ b/act/runner/testdata/uses-workflow/local-workflow.yml @@ -0,0 +1,36 @@ +name: local-reusable-workflows +on: pull_request + +jobs: + reusable-workflow: + uses: ./.github/workflows/local-reusable-workflow.yml + with: + string_required: string + bool_required: ${{ true }} + number_required: 1 + secrets: + secret: keep_it_private + + reusable-workflow-with-inherited-secrets: + uses: ./.github/workflows/local-reusable-workflow.yml + with: + string_required: string + bool_required: ${{ true }} + number_required: 1 + secrets: inherit + + output-test: + runs-on: ubuntu-latest + needs: + - reusable-workflow + - reusable-workflow-with-inherited-secrets + steps: + - name: output with secrets map + run: | + echo reusable-workflow.output=${{ needs.reusable-workflow.outputs.output }} + [[ "${{ needs.reusable-workflow.outputs.output == 'string' }}" = "true" ]] || exit 1 + + - name: output with inherited secrets + run: | + echo reusable-workflow-with-inherited-secrets.output=${{ needs.reusable-workflow-with-inherited-secrets.outputs.output }} + [[ "${{ needs.reusable-workflow-with-inherited-secrets.outputs.output == 'string' }}" = "true" ]] || exit 1 From 29756ec8f3359fcba1289f08c475361c940a96d4 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Thu, 15 Dec 2022 18:08:31 +0100 Subject: [PATCH 10/73] refactor: fix savestate in pre steps (#1466) * refactor: fix savestate in pre steps * fix pre steps collision * fix tests * remove * enable tests * Update pkg/runner/action.go * Rename InterActionState to IntraActionState Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/model/step_result.go | 1 - act/runner/action.go | 6 +-- act/runner/action_test.go | 7 ++-- act/runner/command.go | 17 +++++---- act/runner/command_test.go | 8 +--- act/runner/run_context.go | 1 + act/runner/step.go | 41 ++++++++------------ act/runner/step_action_local_test.go | 31 ++++----------- act/runner/step_action_remote_test.go | 46 +++++++++-------------- act/runner/testdata/GITHUB_STATE/push.yml | 34 +++++++++++++++-- 10 files changed, 90 insertions(+), 102 deletions(-) diff --git a/act/model/step_result.go b/act/model/step_result.go index 49b7705d..86e5ebf3 100644 --- a/act/model/step_result.go +++ b/act/model/step_result.go @@ -42,5 +42,4 @@ type StepResult struct { Outputs map[string]string `json:"outputs"` Conclusion stepStatus `json:"conclusion"` Outcome stepStatus `json:"outcome"` - State map[string]string } diff --git a/act/runner/action.go b/act/runner/action.go index ec7ac7cd..5614414e 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -374,9 +374,9 @@ func newStepContainer(ctx context.Context, step step, image string, cmd []string } func populateEnvsFromSavedState(env *map[string]string, step actionStep, rc *RunContext) { - stepResult := rc.StepResults[step.getStepModel().ID] - if stepResult != nil { - for name, value := range stepResult.State { + state, ok := rc.IntraActionState[step.getStepModel().ID] + if ok { + for name, value := range state { envName := fmt.Sprintf("STATE_%s", name) (*env)[envName] = value } diff --git a/act/runner/action_test.go b/act/runner/action_test.go index 0b230855..36ee14f7 100644 --- a/act/runner/action_test.go +++ b/act/runner/action_test.go @@ -201,10 +201,11 @@ func TestActionRunner(t *testing.T) { }, CurrentStep: "post-step", StepResults: map[string]*model.StepResult{ + "step": {}, + }, + IntraActionState: map[string]map[string]string{ "step": { - State: map[string]string{ - "name": "state value", - }, + "name": "state value", }, }, }, diff --git a/act/runner/command.go b/act/runner/command.go index 3fa37151..0b0ba2fe 100755 --- a/act/runner/command.go +++ b/act/runner/command.go @@ -153,13 +153,16 @@ func unescapeKvPairs(kvPairs map[string]string) map[string]string { } func (rc *RunContext) saveState(ctx context.Context, kvPairs map[string]string, arg string) { - if rc.CurrentStep != "" { - stepResult := rc.StepResults[rc.CurrentStep] - if stepResult != nil { - if stepResult.State == nil { - stepResult.State = map[string]string{} - } - stepResult.State[kvPairs["name"]] = arg + stepID := rc.CurrentStep + if stepID != "" { + if rc.IntraActionState == nil { + rc.IntraActionState = map[string]map[string]string{} } + state, ok := rc.IntraActionState[stepID] + if !ok { + state = map[string]string{} + rc.IntraActionState[stepID] = state + } + state[kvPairs["name"]] = arg } } diff --git a/act/runner/command_test.go b/act/runner/command_test.go index 8f71af67..57c7de5f 100644 --- a/act/runner/command_test.go +++ b/act/runner/command_test.go @@ -177,11 +177,7 @@ func TestAddmaskUsemask(t *testing.T) { func TestSaveState(t *testing.T) { rc := &RunContext{ CurrentStep: "step", - StepResults: map[string]*model.StepResult{ - "step": { - State: map[string]string{}, - }, - }, + StepResults: map[string]*model.StepResult{}, } ctx := context.Background() @@ -189,5 +185,5 @@ func TestSaveState(t *testing.T) { handler := rc.commandHandler(ctx) handler("::save-state name=state-name::state-value\n") - assert.Equal(t, "state-value", rc.StepResults["step"].State["state-name"]) + assert.Equal(t, "state-value", rc.IntraActionState["step"]["state-name"]) } diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 854e628d..b579e3e5 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -38,6 +38,7 @@ type RunContext struct { ExtraPath []string CurrentStep string StepResults map[string]*model.StepResult + IntraActionState map[string]map[string]string ExprEval ExpressionEvaluator JobContainer container.ExecutionsEnvironment OutputMappings map[MappableOutput]MappableOutput diff --git a/act/runner/step.go b/act/runner/step.go index f8a192fd..2e211a30 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -44,18 +44,6 @@ func (s stepStage) String() string { return "Unknown" } -func (s stepStage) getStepName(stepModel *model.Step) string { - switch s { - case stepStagePre: - return fmt.Sprintf("pre-%s", stepModel.ID) - case stepStageMain: - return stepModel.ID - case stepStagePost: - return fmt.Sprintf("post-%s", stepModel.ID) - } - return "unknown" -} - func runStepExecutor(step step, stage stepStage, executor common.Executor) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) @@ -63,13 +51,16 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo stepModel := step.getStepModel() ifExpression := step.getIfExpression(ctx, stage) - rc.CurrentStep = stage.getStepName(stepModel) + rc.CurrentStep = stepModel.ID - rc.StepResults[rc.CurrentStep] = &model.StepResult{ + stepResult := &model.StepResult{ Outcome: model.StepStatusSuccess, Conclusion: model.StepStatusSuccess, Outputs: make(map[string]string), } + if stage == stepStageMain { + rc.StepResults[rc.CurrentStep] = stepResult + } err := setupEnv(ctx, step) if err != nil { @@ -78,15 +69,15 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo runStep, err := isStepEnabled(ctx, ifExpression, step, stage) if err != nil { - rc.StepResults[rc.CurrentStep].Conclusion = model.StepStatusFailure - rc.StepResults[rc.CurrentStep].Outcome = model.StepStatusFailure + stepResult.Conclusion = model.StepStatusFailure + stepResult.Outcome = model.StepStatusFailure return err } if !runStep { - rc.StepResults[rc.CurrentStep].Conclusion = model.StepStatusSkipped - rc.StepResults[rc.CurrentStep].Outcome = model.StepStatusSkipped - logger.WithField("stepResult", rc.StepResults[rc.CurrentStep].Outcome).Debugf("Skipping step '%s' due to '%s'", stepModel, ifExpression) + stepResult.Conclusion = model.StepStatusSkipped + stepResult.Outcome = model.StepStatusSkipped + logger.WithField("stepResult", stepResult.Outcome).Debugf("Skipping step '%s' due to '%s'", stepModel, ifExpression) return nil } @@ -118,25 +109,25 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo err = executor(ctx) if err == nil { - logger.WithField("stepResult", rc.StepResults[rc.CurrentStep].Outcome).Infof(" \u2705 Success - %s %s", stage, stepString) + logger.WithField("stepResult", stepResult.Outcome).Infof(" \u2705 Success - %s %s", stage, stepString) } else { - rc.StepResults[rc.CurrentStep].Outcome = model.StepStatusFailure + stepResult.Outcome = model.StepStatusFailure continueOnError, parseErr := isContinueOnError(ctx, stepModel.RawContinueOnError, step, stage) if parseErr != nil { - rc.StepResults[rc.CurrentStep].Conclusion = model.StepStatusFailure + stepResult.Conclusion = model.StepStatusFailure return parseErr } if continueOnError { logger.Infof("Failed but continue next step") err = nil - rc.StepResults[rc.CurrentStep].Conclusion = model.StepStatusSuccess + stepResult.Conclusion = model.StepStatusSuccess } else { - rc.StepResults[rc.CurrentStep].Conclusion = model.StepStatusFailure + stepResult.Conclusion = model.StepStatusFailure } - logger.WithField("stepResult", rc.StepResults[rc.CurrentStep].Outcome).Errorf(" \u274C Failure - %s %s", stage, stepString) + logger.WithField("stepResult", stepResult.Outcome).Errorf(" \u274C Failure - %s %s", stage, stepString) } // Process Runner File Commands orgerr := err diff --git a/act/runner/step_action_local_test.go b/act/runner/step_action_local_test.go index 9ece06a4..023f7018 100644 --- a/act/runner/step_action_local_test.go +++ b/act/runner/step_action_local_test.go @@ -107,13 +107,12 @@ func TestStepActionLocalTest(t *testing.T) { func TestStepActionLocalPost(t *testing.T) { table := []struct { - name string - stepModel *model.Step - actionModel *model.Action - initialStepResults map[string]*model.StepResult - expectedPostStepResult *model.StepResult - err error - mocks struct { + name string + stepModel *model.Step + actionModel *model.Action + initialStepResults map[string]*model.StepResult + err error + mocks struct { env bool exec bool } @@ -138,11 +137,6 @@ func TestStepActionLocalPost(t *testing.T) { Outputs: map[string]string{}, }, }, - expectedPostStepResult: &model.StepResult{ - Conclusion: model.StepStatusSuccess, - Outcome: model.StepStatusSuccess, - Outputs: map[string]string{}, - }, mocks: struct { env bool exec bool @@ -171,11 +165,6 @@ func TestStepActionLocalPost(t *testing.T) { Outputs: map[string]string{}, }, }, - expectedPostStepResult: &model.StepResult{ - Conclusion: model.StepStatusSuccess, - Outcome: model.StepStatusSuccess, - Outputs: map[string]string{}, - }, mocks: struct { env bool exec bool @@ -204,11 +193,6 @@ func TestStepActionLocalPost(t *testing.T) { Outputs: map[string]string{}, }, }, - expectedPostStepResult: &model.StepResult{ - Conclusion: model.StepStatusSkipped, - Outcome: model.StepStatusSkipped, - Outputs: map[string]string{}, - }, mocks: struct { env bool exec bool @@ -238,7 +222,6 @@ func TestStepActionLocalPost(t *testing.T) { Outputs: map[string]string{}, }, }, - expectedPostStepResult: nil, mocks: struct { env bool exec bool @@ -307,7 +290,7 @@ func TestStepActionLocalPost(t *testing.T) { err := sal.post()(ctx) assert.Equal(t, tt.err, err) - assert.Equal(t, tt.expectedPostStepResult, sal.RunContext.StepResults["post-step"]) + assert.Equal(t, sal.RunContext.StepResults["post-step"], (*model.StepResult)(nil)) cm.AssertExpectations(t) }) } diff --git a/act/runner/step_action_remote_test.go b/act/runner/step_action_remote_test.go index 829e3864..a6653f7d 100644 --- a/act/runner/step_action_remote_test.go +++ b/act/runner/step_action_remote_test.go @@ -415,14 +415,14 @@ func TestStepActionRemotePreThroughActionToken(t *testing.T) { func TestStepActionRemotePost(t *testing.T) { table := []struct { - name string - stepModel *model.Step - actionModel *model.Action - initialStepResults map[string]*model.StepResult - expectedEnv map[string]string - expectedPostStepResult *model.StepResult - err error - mocks struct { + name string + stepModel *model.Step + actionModel *model.Action + initialStepResults map[string]*model.StepResult + IntraActionState map[string]map[string]string + expectedEnv map[string]string + err error + mocks struct { env bool exec bool } @@ -445,19 +445,16 @@ func TestStepActionRemotePost(t *testing.T) { Conclusion: model.StepStatusSuccess, Outcome: model.StepStatusSuccess, Outputs: map[string]string{}, - State: map[string]string{ - "key": "value", - }, + }, + }, + IntraActionState: map[string]map[string]string{ + "step": { + "key": "value", }, }, expectedEnv: map[string]string{ "STATE_key": "value", }, - expectedPostStepResult: &model.StepResult{ - Conclusion: model.StepStatusSuccess, - Outcome: model.StepStatusSuccess, - Outputs: map[string]string{}, - }, mocks: struct { env bool exec bool @@ -486,11 +483,6 @@ func TestStepActionRemotePost(t *testing.T) { Outputs: map[string]string{}, }, }, - expectedPostStepResult: &model.StepResult{ - Conclusion: model.StepStatusSuccess, - Outcome: model.StepStatusSuccess, - Outputs: map[string]string{}, - }, mocks: struct { env bool exec bool @@ -519,11 +511,6 @@ func TestStepActionRemotePost(t *testing.T) { Outputs: map[string]string{}, }, }, - expectedPostStepResult: &model.StepResult{ - Conclusion: model.StepStatusSkipped, - Outcome: model.StepStatusSkipped, - Outputs: map[string]string{}, - }, mocks: struct { env bool exec bool @@ -553,7 +540,6 @@ func TestStepActionRemotePost(t *testing.T) { Outputs: map[string]string{}, }, }, - expectedPostStepResult: nil, mocks: struct { env bool exec bool @@ -585,7 +571,8 @@ func TestStepActionRemotePost(t *testing.T) { }, }, }, - StepResults: tt.initialStepResults, + StepResults: tt.initialStepResults, + IntraActionState: tt.IntraActionState, }, Step: tt.stepModel, action: tt.actionModel, @@ -622,7 +609,8 @@ func TestStepActionRemotePost(t *testing.T) { assert.Equal(t, value, sar.env[key]) } } - assert.Equal(t, tt.expectedPostStepResult, sar.RunContext.StepResults["post-step"]) + // Enshure that StepResults is nil in this test + assert.Equal(t, sar.RunContext.StepResults["post-step"], (*model.StepResult)(nil)) cm.AssertExpectations(t) }) } diff --git a/act/runner/testdata/GITHUB_STATE/push.yml b/act/runner/testdata/GITHUB_STATE/push.yml index 179c5a72..61afc07c 100644 --- a/act/runner/testdata/GITHUB_STATE/push.yml +++ b/act/runner/testdata/GITHUB_STATE/push.yml @@ -15,8 +15,34 @@ jobs: echo "::save-state name=mystate3::mystateval" post: | env - # Enable once https://github.com/nektos/act/issues/1459 is fixed - # [ "$STATE_mystate0" = "mystateval" ] - # [ "$STATE_mystate1" = "mystateval" ] + [ "$STATE_mystate0" = "mystateval" ] + [ "$STATE_mystate1" = "mystateval" ] [ "$STATE_mystate2" = "mystateval" ] - [ "$STATE_mystate3" = "mystateval" ] \ No newline at end of file + [ "$STATE_mystate3" = "mystateval" ] + test-id-collision-bug: + runs-on: ubuntu-latest + steps: + - uses: nektos/act-test-actions/script@main + id: script + with: + pre: | + env + echo mystate0=mystateval > $GITHUB_STATE + echo "::save-state name=mystate1::mystateval" + main: | + env + echo mystate2=mystateval > $GITHUB_STATE + echo "::save-state name=mystate3::mystateval" + post: | + env + [ "$STATE_mystate0" = "mystateval" ] + [ "$STATE_mystate1" = "mystateval" ] + [ "$STATE_mystate2" = "mystateval" ] + [ "$STATE_mystate3" = "mystateval" ] + - uses: nektos/act-test-actions/script@main + id: pre-script + with: + main: | + env + echo mystate0=mystateerror > $GITHUB_STATE + echo "::save-state name=mystate1::mystateerror" \ No newline at end of file From 27dd7dc3f4d19be948206c9a0431e5d5fb44a0e4 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Thu, 15 Dec 2022 18:31:59 +0100 Subject: [PATCH 11/73] fix: tail (not absolute) as entrypoint of job container (#1506) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index b579e3e5..538e75c9 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -248,7 +248,7 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, - Entrypoint: []string{"/usr/bin/tail", "-f", "/dev/null"}, + Entrypoint: []string{"tail", "-f", "/dev/null"}, WorkingDir: ext.ToContainerPath(rc.Config.Workdir), Image: image, Username: username, From 5f0cee8ce1e9d7d816a52087e94f6aad4e1e90b2 Mon Sep 17 00:00:00 2001 From: Aaron Holmes Date: Mon, 19 Dec 2022 00:37:53 -0800 Subject: [PATCH 12/73] feat: Support "result" on "needs" context. (#1497) * Support "result" on "needs" context. This change adds "result" to a job's "needs" context, as documented [here](https://docs.github.com/en/actions/learn-github-actions/contexts#needs-context). `act` currently tracks the success/failure/cancelled status of a job, but does not include this value the `needs` context. Fixes #1367 * Change `Needs` to use a new struct rather than the open type `interface{}`. Related #1497 Fixes #1367 * Add integration test to "needs" context change. Relates: #1497 * feat: allow to spawn and run a local reusable workflow (#1423) * feat: allow to spawn and run a local reusable workflow This change contains the ability to parse/plan/run a local reusable workflow. There are still numerous things missing: - inputs - secrets - outputs * feat: add workflow_call inputs * test: improve inputs test * feat: add input defaults * feat: allow expressions in inputs * feat: use context specific expression evaluator * refactor: prepare for better re-usability * feat: add secrets for reusable workflows * test: use secrets during test run * feat: handle reusable workflow outputs Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * refactor: fix savestate in pre steps (#1466) * refactor: fix savestate in pre steps * fix pre steps collision * fix tests * remove * enable tests * Update pkg/runner/action.go * Rename InterActionState to IntraActionState Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * fix: tail (not absolute) as entrypoint of job container (#1506) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> * Fix conflict in merge. Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/exprparser/interpreter.go | 7 ++++++- act/exprparser/interpreter_test.go | 6 ++++-- act/runner/expression.go | 14 ++++++++------ act/runner/runner_test.go | 1 + .../job-needs-context-contains-result/push.yml | 15 +++++++++++++++ 5 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 act/runner/testdata/job-needs-context-contains-result/push.yml diff --git a/act/exprparser/interpreter.go b/act/exprparser/interpreter.go index 7b76f3bb..6b276fd1 100644 --- a/act/exprparser/interpreter.go +++ b/act/exprparser/interpreter.go @@ -20,10 +20,15 @@ type EvaluationEnvironment struct { Secrets map[string]string Strategy map[string]interface{} Matrix map[string]interface{} - Needs map[string]map[string]map[string]string + Needs map[string]Needs Inputs map[string]interface{} } +type Needs struct { + Outputs map[string]string `json:"outputs"` + Result string `json:"result"` +} + type Config struct { Run *model.Run WorkingDir string diff --git a/act/exprparser/interpreter_test.go b/act/exprparser/interpreter_test.go index 2547aae5..d6f58a7c 100644 --- a/act/exprparser/interpreter_test.go +++ b/act/exprparser/interpreter_test.go @@ -555,6 +555,7 @@ func TestContexts(t *testing.T) { {"strategy.fail-fast", true, "strategy-context"}, {"matrix.os", "Linux", "matrix-context"}, {"needs.job-id.outputs.output-name", "value", "needs-context"}, + {"needs.job-id.result", "success", "needs-context"}, {"inputs.name", "value", "inputs-context"}, } @@ -593,11 +594,12 @@ func TestContexts(t *testing.T) { Matrix: map[string]interface{}{ "os": "Linux", }, - Needs: map[string]map[string]map[string]string{ + Needs: map[string]Needs{ "job-id": { - "outputs": { + Outputs: map[string]string{ "output-name": "value", }, + Result: "success", }, }, Inputs: map[string]interface{}{ diff --git a/act/runner/expression.go b/act/runner/expression.go index dc6b0e5b..a8d506ea 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -26,7 +26,7 @@ func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEval func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map[string]string) ExpressionEvaluator { // todo: cleanup EvaluationEnvironment creation - using := make(map[string]map[string]map[string]string) + using := make(map[string]exprparser.Needs) strategy := make(map[string]interface{}) if rc.Run != nil { job := rc.Run.Job() @@ -39,8 +39,9 @@ func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map jobNeeds := rc.Run.Job().Needs() for _, needs := range jobNeeds { - using[needs] = map[string]map[string]string{ - "outputs": jobs[needs].Outputs, + using[needs] = exprparser.Needs{ + Outputs: jobs[needs].Outputs, + Result: jobs[needs].Result, } } } @@ -86,10 +87,11 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) jobs := rc.Run.Workflow.Jobs jobNeeds := rc.Run.Job().Needs() - using := make(map[string]map[string]map[string]string) + using := make(map[string]exprparser.Needs) for _, needs := range jobNeeds { - using[needs] = map[string]map[string]string{ - "outputs": jobs[needs].Outputs, + using[needs] = exprparser.Needs{ + Outputs: jobs[needs].Outputs, + Result: jobs[needs].Result, } } diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 1f614117..5cf48375 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -196,6 +196,7 @@ func TestRunEvent(t *testing.T) { {workdir, "workflow_dispatch_no_inputs_mapping", "workflow_dispatch", "", platforms, secrets}, {workdir, "workflow_dispatch-scalar", "workflow_dispatch", "", platforms, secrets}, {workdir, "workflow_dispatch-scalar-composite-action", "workflow_dispatch", "", platforms, secrets}, + {workdir, "job-needs-context-contains-result", "push", "", platforms, secrets}, {"../model/testdata", "strategy", "push", "", platforms, secrets}, // TODO: move all testdata into pkg so we can validate it with planner and runner // {"testdata", "issue-228", "push", "", platforms, }, // TODO [igni]: Remove this once everything passes {"../model/testdata", "container-volumes", "push", "", platforms, secrets}, diff --git a/act/runner/testdata/job-needs-context-contains-result/push.yml b/act/runner/testdata/job-needs-context-contains-result/push.yml new file mode 100644 index 00000000..0ecbcea1 --- /dev/null +++ b/act/runner/testdata/job-needs-context-contains-result/push.yml @@ -0,0 +1,15 @@ +on: + push: +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: exit 0 + assert: + needs: test + if: | + ( always() && !cancelled() ) && ( + ( needs.test.result != 'success' || !success() ) ) + runs-on: ubuntu-latest + steps: + - run: exit 1 From f3490ecaf761e86a2addf973f1460e1d87f32cd9 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Mon, 19 Dec 2022 15:58:55 +0100 Subject: [PATCH 13/73] fix: align runner.os / runner.arch to known values (#1510) * fix: align runner.os / runner.arch to known values * . * . Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/host_environment.go | 26 ++++++++++++++++++++++++-- act/runner/run_context.go | 9 +++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/act/container/host_environment.go b/act/container/host_environment.go index 30cd5005..ff21b0ad 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -410,10 +410,32 @@ func (*HostEnvironment) JoinPathVariable(paths ...string) string { return strings.Join(paths, string(filepath.ListSeparator)) } +func goArchToActionArch(arch string) string { + archMapper := map[string]string{ + "x86_64": "X64", + "386": "x86", + "aarch64": "arm64", + } + if arch, ok := archMapper[arch]; ok { + return arch + } + return arch +} + +func goOsToActionOs(os string) string { + osMapper := map[string]string{ + "darwin": "macOS", + } + if os, ok := osMapper[os]; ok { + return os + } + return os +} + func (e *HostEnvironment) GetRunnerContext(ctx context.Context) map[string]interface{} { return map[string]interface{}{ - "os": runtime.GOOS, - "arch": runtime.GOARCH, + "os": goOsToActionOs(runtime.GOOS), + "arch": goArchToActionArch(runtime.GOARCH), "temp": e.TmpDir, "tool_cache": e.ToolCache, } diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 538e75c9..c9c8ea1d 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -179,10 +179,11 @@ func (rc *RunContext) startHostEnvironment() common.Executor { StdOut: logWriter, } rc.cleanUpJobContainer = rc.JobContainer.Remove() - rc.Env["RUNNER_TOOL_CACHE"] = toolCache - rc.Env["RUNNER_OS"] = runtime.GOOS - rc.Env["RUNNER_ARCH"] = runtime.GOARCH - rc.Env["RUNNER_TEMP"] = runnerTmp + for k, v := range rc.JobContainer.GetRunnerContext(ctx) { + if v, ok := v.(string); ok { + rc.Env[fmt.Sprintf("RUNNER_%s", strings.ToUpper(k))] = v + } + } for _, env := range os.Environ() { i := strings.Index(env, "=") if i > 0 { From 972bded7a292ae44fcb4f363f6cfcfb59436f8c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 15:46:21 +0000 Subject: [PATCH 14/73] build(deps): bump goreleaser/goreleaser-action from 3 to 4 (#1515) Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 3 to 4. - [Release notes](https://github.com/goreleaser/goreleaser-action/releases) - [Commits](https://github.com/goreleaser/goreleaser-action/compare/v3...v4) --- updated-dependencies: - dependency-name: goreleaser/goreleaser-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ffd9b7ec..44dba166 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -93,7 +93,7 @@ jobs: restore-keys: | ${{ runner.os }}-go- - name: GoReleaser - uses: goreleaser/goreleaser-action@v3 + uses: goreleaser/goreleaser-action@v4 with: version: latest args: release --snapshot --rm-dist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cda8a0d7..2dce51b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: restore-keys: | ${{ runner.os }}-go- - name: GoReleaser - uses: goreleaser/goreleaser-action@v3 + uses: goreleaser/goreleaser-action@v4 with: version: latest args: release --rm-dist From 691dafd7e7cbe91cc9b54b772b5a1d3c4f408131 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Mon, 19 Dec 2022 22:24:05 +0100 Subject: [PATCH 15/73] revert: deprecation of containerArchitecture (#1514) * fix: ci snaphot job * revert: deprecation of containerArchitecture This option isn't part of parsed docker cli flags Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/actions/choco/Dockerfile | 2 +- cmd/root.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/actions/choco/Dockerfile b/.github/actions/choco/Dockerfile index d301f0aa..aabcb3a6 100644 --- a/.github/actions/choco/Dockerfile +++ b/.github/actions/choco/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.16 +FROM alpine:3.17 ARG CHOCOVERSION=1.1.0 diff --git a/cmd/root.go b/cmd/root.go index 2a0baf0e..92ae8732 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -422,9 +422,6 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str if len(input.usernsMode) > 0 { log.Warnf(deprecationWarning, "userns", fmt.Sprintf("--userns=%s", input.usernsMode)) } - if len(input.containerArchitecture) > 0 { - log.Warnf(deprecationWarning, "container-architecture", fmt.Sprintf("--platform=%s", input.containerArchitecture)) - } if len(input.containerCapAdd) > 0 { log.Warnf(deprecationWarning, "container-cap-add", fmt.Sprintf("--cap-add=%s", input.containerCapAdd)) } From a69bd3578ec0e73624bf96b5725cd11c99dafe97 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 02:13:17 +0000 Subject: [PATCH 16/73] build(deps): bump actions/stale from 6 to 7 (#1535) Bumps [actions/stale](https://github.com/actions/stale) from 6 to 7. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index dd2f07c4..2102cf3d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -8,7 +8,7 @@ jobs: name: Stale runs-on: ubuntu-latest steps: - - uses: actions/stale@v6 + - uses: actions/stale@v7 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'Issue is stale and will be closed in 14 days unless there is new activity' From d0a51b569bd6944ef7700b2f1ec579893df7563a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 02:28:46 +0000 Subject: [PATCH 17/73] build(deps): bump megalinter/megalinter from 6.15.0 to 6.16.0 (#1534) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.15.0 to 6.16.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.15.0...v6.16.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 44dba166..bbcfbdbc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.3.1 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.15.0 + - uses: megalinter/megalinter/flavors/go@v6.16.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 799b296ed842a8f9a17f4b4674c6fbc0d029d156 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 02:43:57 +0000 Subject: [PATCH 18/73] build(deps): bump megalinter/megalinter from 6.16.0 to 6.17.0 (#1540) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.16.0 to 6.17.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.16.0...v6.17.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index bbcfbdbc..95077c45 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.3.1 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.16.0 + - uses: megalinter/megalinter/flavors/go@v6.17.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From bc22ccbaef4b88ffd4cdcb5034d3ba251bc3cd39 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 21:16:52 +0000 Subject: [PATCH 19/73] build(deps): bump megalinter/megalinter from 6.17.0 to 6.18.0 (#1550) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.17.0 to 6.18.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.17.0...v6.18.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 95077c45..781675fc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.3.1 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.17.0 + - uses: megalinter/megalinter/flavors/go@v6.18.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f4c69c8b842aa315dfc8d8b7996eadea8bf507ac Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Tue, 10 Jan 2023 22:31:12 +0100 Subject: [PATCH 20/73] fix: preserve job result state in case of failure (#1519) * fix: preserve job result state in case of failure There is just one job field for the job result. This is also true for matrix jobs. We need to preserve the failure state of a job to have the whole job failing in case of one permuation of the matrix failed. Closes #1518 * test: remove continue-on-error on job level This feature is not yet supported by act and if implemented would make this test invalid Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/job_executor.go | 16 ++++++++++++++-- act/runner/runner_test.go | 1 + act/runner/testdata/matrix-exitcode/push.yml | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 act/runner/testdata/matrix-exitcode/push.yml diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index 4ae77879..88c227fe 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -130,17 +130,29 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo func setJobResult(ctx context.Context, info jobInfo, rc *RunContext, success bool) { logger := common.Logger(ctx) + jobResult := "success" - jobResultMessage := "succeeded" + // we have only one result for a whole matrix build, so we need + // to keep an existing result state if we run a matrix + if len(info.matrix()) > 0 && rc.Run.Job().Result != "" { + jobResult = rc.Run.Job().Result + } + if !success { jobResult = "failure" - jobResultMessage = "failed" } + info.result(jobResult) if rc.caller != nil { // set reusable workflow job result rc.caller.runContext.result(jobResult) } + + jobResultMessage := "succeeded" + if jobResult != "success" { + jobResultMessage = "failed" + } + logger.WithField("jobResult", jobResult).Infof("\U0001F3C1 Job %s", jobResultMessage) } diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 5cf48375..ceae70c5 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -170,6 +170,7 @@ func TestRunEvent(t *testing.T) { {workdir, "remote-action-js", "push", "", map[string]string{"ubuntu-latest": "catthehacker/ubuntu:runner-latest"}, secrets}, // Test if this works with non root container {workdir, "matrix", "push", "", platforms, secrets}, {workdir, "matrix-include-exclude", "push", "", platforms, secrets}, + {workdir, "matrix-exitcode", "push", "Job 'test' failed", platforms, secrets}, {workdir, "commands", "push", "", platforms, secrets}, {workdir, "workdir", "push", "", platforms, secrets}, {workdir, "defaults-run", "push", "", platforms, secrets}, diff --git a/act/runner/testdata/matrix-exitcode/push.yml b/act/runner/testdata/matrix-exitcode/push.yml new file mode 100644 index 00000000..0f5d3352 --- /dev/null +++ b/act/runner/testdata/matrix-exitcode/push.yml @@ -0,0 +1,16 @@ +name: test + +on: push + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + val: ["success", "failure"] + fail-fast: false + steps: + - name: test + run: | + echo "Expected job result: ${{ matrix.val }}" + [[ "${{ matrix.val }}" = "success" ]] || exit 1 From 21a2eb0d83621c41a1df58c71e4cbd0f6f72edfb Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Tue, 10 Jan 2023 22:43:12 +0100 Subject: [PATCH 21/73] test: make sure workflow_call is not a github event calling our workflow (#1520) Since reusable workflows are defining inputs and ouputs using the on.workflow_call syntax, this could also be triggered by a workflow_call event. That event does not exist within GitHub and we should make sure our worklow is not called by that kind of 'synthetic' event. See https://github.com/nektos/act/pull/1423/files/74da5b085c0c4d08c5e5bf53501e555cb585b26c#r1042413431 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../testdata/.github/workflows/local-reusable-workflow.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/act/runner/testdata/.github/workflows/local-reusable-workflow.yml b/act/runner/testdata/.github/workflows/local-reusable-workflow.yml index b04fe72d..a52fdcf3 100644 --- a/act/runner/testdata/.github/workflows/local-reusable-workflow.yml +++ b/act/runner/testdata/.github/workflows/local-reusable-workflow.yml @@ -68,6 +68,11 @@ jobs: echo secrets.secret=${{ secrets.secret }} [[ "${{ secrets.secret == 'keep_it_private' }}" = "true" ]] || exit 1 + - name: test github.event_name is never workflow_call + run: | + echo github.event_name=${{ github.event_name }} + [[ "${{ github.event_name != 'workflow_call' }}" = "true" ]] || exit 1 + - name: test output id: output_test run: | From fd250664e34381a2c5cb3d68c6e2b78a30c0e311 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 10 Jan 2023 22:55:05 +0100 Subject: [PATCH 22/73] fix: extra path lost in composite actions (#1531) * test: define test case of path issues Test case for #1528 * test: add multi arch grep * fix: Always use current ExtraPath * replace setup-node with run step * Update push.yml * yaml mistake Co-authored-by: Markus Wolf Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/runner_test.go | 1 + act/runner/step_action_remote.go | 1 + act/runner/testdata/path-handling/action.yml | 21 +++++++++++ act/runner/testdata/path-handling/push.yml | 39 ++++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 act/runner/testdata/path-handling/action.yml create mode 100644 act/runner/testdata/path-handling/push.yml diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index ceae70c5..95fbbcc2 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -201,6 +201,7 @@ func TestRunEvent(t *testing.T) { {"../model/testdata", "strategy", "push", "", platforms, secrets}, // TODO: move all testdata into pkg so we can validate it with planner and runner // {"testdata", "issue-228", "push", "", platforms, }, // TODO [igni]: Remove this once everything passes {"../model/testdata", "container-volumes", "push", "", platforms, secrets}, + {workdir, "path-handling", "push", "", platforms, secrets}, } for _, table := range tables { diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index 00d9502a..17834866 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -196,6 +196,7 @@ func (sar *stepActionRemote) getCompositeRunContext(ctx context.Context) *RunCon // was already created during the pre stage) env := evaluateCompositeInputAndEnv(ctx, sar.RunContext, sar) sar.compositeRunContext.Env = env + sar.compositeRunContext.ExtraPath = sar.RunContext.ExtraPath } return sar.compositeRunContext } diff --git a/act/runner/testdata/path-handling/action.yml b/act/runner/testdata/path-handling/action.yml new file mode 100644 index 00000000..8db98c52 --- /dev/null +++ b/act/runner/testdata/path-handling/action.yml @@ -0,0 +1,21 @@ +name: output action +description: output action + +inputs: + input: + description: some input + required: false + +outputs: + job-output: + description: some output + value: ${{ steps.gen-out.outputs.step-output }} + +runs: + using: composite + steps: + - name: run step + id: gen-out + run: | + echo "::set-output name=step-output::" + shell: bash diff --git a/act/runner/testdata/path-handling/push.yml b/act/runner/testdata/path-handling/push.yml new file mode 100644 index 00000000..812c8b8a --- /dev/null +++ b/act/runner/testdata/path-handling/push.yml @@ -0,0 +1,39 @@ +name: path tests +on: push +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: "Append to $GITHUB_PATH" + run: | + echo "/opt/hostedtoolcache/node/18.99/x64/bin" >> $GITHUB_PATH + + - name: test path (after setup) + run: | + if ! echo "$PATH" |grep "/opt/hostedtoolcache/node/18.*/\(x64\|arm64\)/bin" ; then + echo "Node binaries not in path: $PATH" + exit 1 + fi + + - id: action-with-output + uses: ./path-handling/ + + - name: test path (after local action) + run: | + if ! echo "$PATH" |grep "/opt/hostedtoolcache/node/18.*/\(x64\|arm64\)/bin" ; then + echo "Node binaries not in path: $PATH" + exit 1 + fi + + - uses: nektos/act-test-actions/composite@main + with: + input: some input + + - name: test path (after remote action) + run: | + if ! echo "$PATH" |grep "/opt/hostedtoolcache/node/18.*/\(x64\|arm64\)/bin" ; then + echo "Node binaries not in path: $PATH" + exit 1 + fi From ea8ae90674dc63933d59ee8c465e2e90bf0f4256 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 10 Jan 2023 23:08:57 +0100 Subject: [PATCH 23/73] feat: Allow building without docker support (#1507) * feat: Allow building without docker support * fix macos build tag Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/container_types.go | 70 ++++++++++++++++++++++++++++++++ act/container/docker_auth.go | 2 + act/container/docker_build.go | 10 +---- act/container/docker_cli.go | 2 + act/container/docker_images.go | 2 + act/container/docker_logger.go | 2 + act/container/docker_pull.go | 11 +---- act/container/docker_run.go | 47 +-------------------- act/container/docker_stub.go | 57 ++++++++++++++++++++++++++ act/container/docker_volume.go | 2 + 10 files changed, 143 insertions(+), 62 deletions(-) create mode 100644 act/container/container_types.go create mode 100644 act/container/docker_stub.go diff --git a/act/container/container_types.go b/act/container/container_types.go new file mode 100644 index 00000000..c83ec35b --- /dev/null +++ b/act/container/container_types.go @@ -0,0 +1,70 @@ +package container + +import ( + "context" + "io" + + "github.com/nektos/act/pkg/common" +) + +// NewContainerInput the input for the New function +type NewContainerInput struct { + Image string + Username string + Password string + Entrypoint []string + Cmd []string + WorkingDir string + Env []string + Binds []string + Mounts map[string]string + Name string + Stdout io.Writer + Stderr io.Writer + NetworkMode string + Privileged bool + UsernsMode string + Platform string + Options string +} + +// FileEntry is a file to copy to a container +type FileEntry struct { + Name string + Mode int64 + Body string +} + +// Container for managing docker run containers +type Container interface { + Create(capAdd []string, capDrop []string) common.Executor + Copy(destPath string, files ...*FileEntry) common.Executor + CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor + GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) + Pull(forcePull bool) common.Executor + Start(attach bool) common.Executor + Exec(command []string, env map[string]string, user, workdir string) common.Executor + UpdateFromEnv(srcPath string, env *map[string]string) common.Executor + UpdateFromImageEnv(env *map[string]string) common.Executor + UpdateFromPath(env *map[string]string) common.Executor + Remove() common.Executor + Close() common.Executor + ReplaceLogWriter(io.Writer, io.Writer) (io.Writer, io.Writer) +} + +// NewDockerBuildExecutorInput the input for the NewDockerBuildExecutor function +type NewDockerBuildExecutorInput struct { + ContextDir string + Container Container + ImageTag string + Platform string +} + +// NewDockerPullExecutorInput the input for the NewDockerPullExecutor function +type NewDockerPullExecutorInput struct { + Image string + ForcePull bool + Platform string + Username string + Password string +} diff --git a/act/container/docker_auth.go b/act/container/docker_auth.go index 7d2fc4a3..466c0033 100644 --- a/act/container/docker_auth.go +++ b/act/container/docker_auth.go @@ -1,3 +1,5 @@ +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) + package container import ( diff --git a/act/container/docker_build.go b/act/container/docker_build.go index 17e2c7b7..2c972f4f 100644 --- a/act/container/docker_build.go +++ b/act/container/docker_build.go @@ -1,3 +1,5 @@ +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) + package container import ( @@ -16,14 +18,6 @@ import ( "github.com/nektos/act/pkg/common" ) -// NewDockerBuildExecutorInput the input for the NewDockerBuildExecutor function -type NewDockerBuildExecutorInput struct { - ContextDir string - Container Container - ImageTag string - Platform string -} - // NewDockerBuildExecutor function to create a run executor for the container func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor { return func(ctx context.Context) error { diff --git a/act/container/docker_cli.go b/act/container/docker_cli.go index 60c9fe83..a1481c36 100644 --- a/act/container/docker_cli.go +++ b/act/container/docker_cli.go @@ -1,3 +1,5 @@ +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) + // This file is exact copy of https://github.com/docker/cli/blob/9ac8584acfd501c3f4da0e845e3a40ed15c85041/cli/command/container/opts.go // appended with license information. // diff --git a/act/container/docker_images.go b/act/container/docker_images.go index e23699e1..676d4ce5 100644 --- a/act/container/docker_images.go +++ b/act/container/docker_images.go @@ -1,3 +1,5 @@ +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) + package container import ( diff --git a/act/container/docker_logger.go b/act/container/docker_logger.go index b6b2f150..f2c21e6c 100644 --- a/act/container/docker_logger.go +++ b/act/container/docker_logger.go @@ -1,3 +1,5 @@ +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) + package container import ( diff --git a/act/container/docker_pull.go b/act/container/docker_pull.go index 1eb04e1c..75bfed16 100644 --- a/act/container/docker_pull.go +++ b/act/container/docker_pull.go @@ -1,3 +1,5 @@ +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) + package container import ( @@ -12,15 +14,6 @@ import ( "github.com/nektos/act/pkg/common" ) -// NewDockerPullExecutorInput the input for the NewDockerPullExecutor function -type NewDockerPullExecutorInput struct { - Image string - ForcePull bool - Platform string - Username string - Password string -} - // NewDockerPullExecutor function to create a run executor for the container func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor { return func(ctx context.Context) error { diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 8e30a808..4ef68602 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -1,3 +1,5 @@ +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) + package container import ( @@ -38,51 +40,6 @@ import ( "github.com/nektos/act/pkg/common" ) -// NewContainerInput the input for the New function -type NewContainerInput struct { - Image string - Username string - Password string - Entrypoint []string - Cmd []string - WorkingDir string - Env []string - Binds []string - Mounts map[string]string - Name string - Stdout io.Writer - Stderr io.Writer - NetworkMode string - Privileged bool - UsernsMode string - Platform string - Options string -} - -// FileEntry is a file to copy to a container -type FileEntry struct { - Name string - Mode int64 - Body string -} - -// Container for managing docker run containers -type Container interface { - Create(capAdd []string, capDrop []string) common.Executor - Copy(destPath string, files ...*FileEntry) common.Executor - CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor - GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) - Pull(forcePull bool) common.Executor - Start(attach bool) common.Executor - Exec(command []string, env map[string]string, user, workdir string) common.Executor - UpdateFromEnv(srcPath string, env *map[string]string) common.Executor - UpdateFromImageEnv(env *map[string]string) common.Executor - UpdateFromPath(env *map[string]string) common.Executor - Remove() common.Executor - Close() common.Executor - ReplaceLogWriter(io.Writer, io.Writer) (io.Writer, io.Writer) -} - // NewContainer creates a reference to a container func NewContainer(input *NewContainerInput) ExecutionsEnvironment { cr := new(containerReference) diff --git a/act/container/docker_stub.go b/act/container/docker_stub.go new file mode 100644 index 00000000..b28c90de --- /dev/null +++ b/act/container/docker_stub.go @@ -0,0 +1,57 @@ +//go:build WITHOUT_DOCKER || !(linux || darwin || windows) + +package container + +import ( + "context" + "runtime" + + "github.com/docker/docker/api/types" + "github.com/nektos/act/pkg/common" + "github.com/pkg/errors" +) + +// ImageExistsLocally returns a boolean indicating if an image with the +// requested name, tag and architecture exists in the local docker image store +func ImageExistsLocally(ctx context.Context, imageName string, platform string) (bool, error) { + return false, errors.New("Unsupported Operation") +} + +// RemoveImage removes image from local store, the function is used to run different +// container image architectures +func RemoveImage(ctx context.Context, imageName string, force bool, pruneChildren bool) (bool, error) { + return false, errors.New("Unsupported Operation") +} + +// NewDockerBuildExecutor function to create a run executor for the container +func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor { + return func(ctx context.Context) error { + return errors.New("Unsupported Operation") + } +} + +// NewDockerPullExecutor function to create a run executor for the container +func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor { + return func(ctx context.Context) error { + return errors.New("Unsupported Operation") + } +} + +// NewContainer creates a reference to a container +func NewContainer(input *NewContainerInput) ExecutionsEnvironment { + return nil +} + +func RunnerArch(ctx context.Context) string { + return runtime.GOOS +} + +func GetHostInfo(ctx context.Context) (info types.Info, err error) { + return types.Info{}, nil +} + +func NewDockerVolumeRemoveExecutor(volume string, force bool) common.Executor { + return func(ctx context.Context) error { + return nil + } +} diff --git a/act/container/docker_volume.go b/act/container/docker_volume.go index 5a6d4764..6eafd33c 100644 --- a/act/container/docker_volume.go +++ b/act/container/docker_volume.go @@ -1,3 +1,5 @@ +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) + package container import ( From 79833c689f63843544b29aebb465bb4feb28b00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Brauer?= Date: Thu, 12 Jan 2023 22:29:30 +0100 Subject: [PATCH 24/73] feat: add support for building docker actions with private registries (#1557) This commit adds a new `LoadDockerAuthConfigs` function, which loads all registry auths that are configured on the host and sends them with the build command to the docker daemon. This is needed in case act builds a docker action and the images referenced in that docker action are located on private registries or otherwise require authentication (e.g. to get a higher rate limit). The code is adapted from how the docker cli works: https://github.com/docker/cli/blob/257ff41304bf121bdf1acdf00a1c7a896ed038d1/cli/command/image/build.go#L323-L332 Co-authored-by: Markus Wolf Co-authored-by: Markus Wolf --- act/container/docker_auth.go | 21 +++++++++++++++++++++ act/container/docker_build.go | 7 ++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/act/container/docker_auth.go b/act/container/docker_auth.go index 466c0033..e47fe64a 100644 --- a/act/container/docker_auth.go +++ b/act/container/docker_auth.go @@ -38,3 +38,24 @@ func LoadDockerAuthConfig(ctx context.Context, image string) (types.AuthConfig, return types.AuthConfig(authConfig), nil } + +func LoadDockerAuthConfigs(ctx context.Context) map[string]types.AuthConfig { + logger := common.Logger(ctx) + config, err := config.Load(config.Dir()) + if err != nil { + logger.Warnf("Could not load docker config: %v", err) + return nil + } + + if !config.ContainsAuth() { + config.CredentialsStore = credentials.DetectDefaultStore(config.CredentialsStore) + } + + creds, _ := config.GetAllCredentials() + authConfigs := make(map[string]types.AuthConfig, len(creds)) + for k, v := range creds { + authConfigs[k] = types.AuthConfig(v) + } + + return authConfigs +} diff --git a/act/container/docker_build.go b/act/container/docker_build.go index 2c972f4f..f05a513f 100644 --- a/act/container/docker_build.go +++ b/act/container/docker_build.go @@ -41,9 +41,10 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor { tags := []string{input.ImageTag} options := types.ImageBuildOptions{ - Tags: tags, - Remove: true, - Platform: input.Platform, + Tags: tags, + Remove: true, + Platform: input.Platform, + AuthConfigs: LoadDockerAuthConfigs(ctx), } var buildContext io.ReadCloser if input.Container != nil { From 7e25783091532048d64b460b5ddc8dffdca8fc76 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Fri, 13 Jan 2023 18:01:40 +0100 Subject: [PATCH 25/73] fix: add-matcher fails github workflow (#1532) * fix: add-matcher fails github workflow * make linter happy --- act/runner/command.go | 31 ++++++++++------- .../composite_action2/action.yml | 33 +++++++++---------- 2 files changed, 35 insertions(+), 29 deletions(-) mode change 100755 => 100644 act/runner/command.go diff --git a/act/runner/command.go b/act/runner/command.go old mode 100755 new mode 100644 index 0b0ba2fe..53e167cd --- a/act/runner/command.go +++ b/act/runner/command.go @@ -16,22 +16,27 @@ func init() { commandPatternADO = regexp.MustCompile("^##\\[([^ ]+)( (.+))?]([^\r\n]*)[\r\n]+$") } +func tryParseRawActionCommand(line string) (command string, kvPairs map[string]string, arg string, ok bool) { + if m := commandPatternGA.FindStringSubmatch(line); m != nil { + command = m[1] + kvPairs = parseKeyValuePairs(m[3], ",") + arg = m[4] + ok = true + } else if m := commandPatternADO.FindStringSubmatch(line); m != nil { + command = m[1] + kvPairs = parseKeyValuePairs(m[3], ";") + arg = m[4] + ok = true + } + return +} + func (rc *RunContext) commandHandler(ctx context.Context) common.LineHandler { logger := common.Logger(ctx) resumeCommand := "" return func(line string) bool { - var command string - var kvPairs map[string]string - var arg string - if m := commandPatternGA.FindStringSubmatch(line); m != nil { - command = m[1] - kvPairs = parseKeyValuePairs(m[3], ",") - arg = m[4] - } else if m := commandPatternADO.FindStringSubmatch(line); m != nil { - command = m[1] - kvPairs = parseKeyValuePairs(m[3], ";") - arg = m[4] - } else { + command, kvPairs, arg, ok := tryParseRawActionCommand(line) + if !ok { return true } @@ -66,6 +71,8 @@ func (rc *RunContext) commandHandler(ctx context.Context) common.LineHandler { case "save-state": logger.Infof(" \U0001f4be %s", line) rc.saveState(ctx, kvPairs, arg) + case "add-matcher": + logger.Infof(" \U00002753 add-matcher %s", arg) default: logger.Infof(" \U00002753 %s", line) } diff --git a/act/runner/testdata/uses-nested-composite/composite_action2/action.yml b/act/runner/testdata/uses-nested-composite/composite_action2/action.yml index 2fae40cd..4aec9a8c 100644 --- a/act/runner/testdata/uses-nested-composite/composite_action2/action.yml +++ b/act/runner/testdata/uses-nested-composite/composite_action2/action.yml @@ -9,23 +9,22 @@ inputs: runs: using: "composite" steps: -# The output of actions/setup-node@v2 seems to fail the workflow -# - uses: actions/setup-node@v2 -# with: -# node-version: '16' -# - run: | -# console.log(process.version); -# console.log("Hi from node"); -# console.log("${{ inputs.test_input_optional }}"); -# if("${{ inputs.test_input_optional }}" !== "Test") { -# console.log("Invalid input test_input_optional expected \"Test\" as value"); -# process.exit(1); -# } -# if(!process.version.startsWith('v16')) { -# console.log("Expected node v16, but got " + process.version); -# process.exit(1); -# } -# shell: node {0} + - uses: actions/setup-node@v3 + with: + node-version: '16' + - run: | + console.log(process.version); + console.log("Hi from node"); + console.log("${{ inputs.test_input_optional }}"); + if("${{ inputs.test_input_optional }}" !== "Test") { + console.log("Invalid input test_input_optional expected \"Test\" as value"); + process.exit(1); + } + if(!process.version.startsWith('v16')) { + console.log("Expected node v16, but got " + process.version); + process.exit(1); + } + shell: node {0} - uses: ./uses-composite/composite_action id: composite with: From f6f191d9e0fa2cf07b183c37b66bacf8287ff8d0 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Fri, 13 Jan 2023 18:52:54 +0100 Subject: [PATCH 26/73] refactor: remove docker image list reference filter (#1501) * refactor: remove docker reference filter * make it work * solve logic failure * Another mistake * another one * revert signature of ImageExistsLocally It is better to keep two return values --- act/container/docker_images.go | 67 +++++++++------------------------- 1 file changed, 18 insertions(+), 49 deletions(-) diff --git a/act/container/docker_images.go b/act/container/docker_images.go index 676d4ce5..22772307 100644 --- a/act/container/docker_images.go +++ b/act/container/docker_images.go @@ -7,7 +7,7 @@ import ( "fmt" "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" ) // ImageExistsLocally returns a boolean indicating if an image with the @@ -19,33 +19,15 @@ func ImageExistsLocally(ctx context.Context, imageName string, platform string) } defer cli.Close() - filters := filters.NewArgs() - filters.Add("reference", imageName) - - imageListOptions := types.ImageListOptions{ - Filters: filters, - } - - images, err := cli.ImageList(ctx, imageListOptions) - if err != nil { + inspectImage, _, err := cli.ImageInspectWithRaw(ctx, imageName) + if client.IsErrNotFound(err) { + return false, nil + } else if err != nil { return false, err } - if len(images) > 0 { - if platform == "any" || platform == "" { - return true, nil - } - for _, v := range images { - inspectImage, _, err := cli.ImageInspectWithRaw(ctx, v.ID) - if err != nil { - return false, err - } - - if fmt.Sprintf("%s/%s", inspectImage.Os, inspectImage.Architecture) == platform { - return true, nil - } - } - return false, nil + if platform == "" || platform == "any" || fmt.Sprintf("%s/%s", inspectImage.Os, inspectImage.Architecture) == platform { + return true, nil } return false, nil @@ -54,38 +36,25 @@ func ImageExistsLocally(ctx context.Context, imageName string, platform string) // RemoveImage removes image from local store, the function is used to run different // container image architectures func RemoveImage(ctx context.Context, imageName string, force bool, pruneChildren bool) (bool, error) { - if exists, err := ImageExistsLocally(ctx, imageName, "any"); !exists { - return false, err - } - cli, err := GetDockerClient(ctx) if err != nil { return false, err } + defer cli.Close() - filters := filters.NewArgs() - filters.Add("reference", imageName) - - imageListOptions := types.ImageListOptions{ - Filters: filters, - } - - images, err := cli.ImageList(ctx, imageListOptions) - if err != nil { + inspectImage, _, err := cli.ImageInspectWithRaw(ctx, imageName) + if client.IsErrNotFound(err) { + return false, nil + } else if err != nil { return false, err } - if len(images) > 0 { - for _, v := range images { - if _, err = cli.ImageRemove(ctx, v.ID, types.ImageRemoveOptions{ - Force: force, - PruneChildren: pruneChildren, - }); err != nil { - return false, err - } - } - return true, nil + if _, err = cli.ImageRemove(ctx, inspectImage.ID, types.ImageRemoveOptions{ + Force: force, + PruneChildren: pruneChildren, + }); err != nil { + return false, err } - return false, nil + return true, nil } From 409d161ed11f4d46207ba807cedb61e543631db0 Mon Sep 17 00:00:00 2001 From: Shubh Bapna <38372682+shubhbapna@users.noreply.github.com> Date: Sat, 14 Jan 2023 00:58:17 +0530 Subject: [PATCH 27/73] Input (#1524) * added input flags * added input as part of the action event and added test cases * updated readme Co-authored-by: ChristopherHX --- act/runner/runner.go | 11 +++++++ act/runner/runner_test.go | 22 +++++++++++++ act/runner/testdata/input-from-cli/input.yml | 21 ++++++++++++ cmd/input.go | 7 ++++ cmd/root.go | 34 ++++++++++++++------ 5 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 act/runner/testdata/input-from-cli/input.yml diff --git a/act/runner/runner.go b/act/runner/runner.go index 2ed967db..9eb225fb 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -2,6 +2,7 @@ package runner import ( "context" + "encoding/json" "fmt" "os" @@ -31,6 +32,7 @@ type Config struct { LogOutput bool // log the output from docker run JSONLogger bool // use json or text logger Env map[string]string // env for containers + Inputs map[string]string // manually passed action inputs Secrets map[string]string // list of secrets Token string // GitHub token InsecureSecrets bool // switch hiding output when printing to terminal @@ -81,6 +83,15 @@ func (runner *runnerImpl) configure() (Runner, error) { return nil, err } runner.eventJSON = string(eventJSONBytes) + } else if len(runner.config.Inputs) != 0 { + eventMap := map[string]map[string]string{ + "inputs": runner.config.Inputs, + } + eventJSON, err := json.Marshal(eventMap) + if err != nil { + return nil, err + } + runner.eventJSON = string(eventJSON) } return runner, nil } diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 95fbbcc2..6096abe1 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -94,6 +94,7 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config ReuseContainers: false, Env: cfg.Env, Secrets: cfg.Secrets, + Inputs: cfg.Inputs, GitHubInstance: "github.com", ContainerArchitecture: cfg.ContainerArchitecture, } @@ -419,6 +420,27 @@ func TestRunEventSecrets(t *testing.T) { tjfi.runTest(context.Background(), t, &Config{Secrets: secrets, Env: env}) } +func TestRunActionInputs(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + workflowPath := "input-from-cli" + + tjfi := TestJobFileInfo{ + workdir: workdir, + workflowPath: workflowPath, + eventName: "workflow_dispatch", + errorMessage: "", + platforms: platforms, + } + + inputs := map[string]string{ + "SOME_INPUT": "input", + } + + tjfi.runTest(context.Background(), t, &Config{Inputs: inputs}) +} + func TestRunEventPullRequest(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") diff --git a/act/runner/testdata/input-from-cli/input.yml b/act/runner/testdata/input-from-cli/input.yml new file mode 100644 index 00000000..42d3460f --- /dev/null +++ b/act/runner/testdata/input-from-cli/input.yml @@ -0,0 +1,21 @@ +on: + workflow_dispatch: + inputs: + NAME: + description: "A random input name for the workflow" + type: string + required: true + SOME_VALUE: + description: "Some other input to pass" + type: string + required: true + +jobs: + test: + name: Test + runs-on: ubuntu-latest + + steps: + - name: Test with inputs + run: | + [ -z "${{ github.event.inputs.SOME_INPUT }}" ] && exit 1 || exit 0 diff --git a/cmd/input.go b/cmd/input.go index f17fdfce..1caeccbb 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -17,12 +17,14 @@ type Input struct { bindWorkdir bool secrets []string envs []string + inputs []string platforms []string dryrun bool forcePull bool forceRebuild bool noOutput bool envfile string + inputfile string secretfile string insecureSecrets bool defaultBranch string @@ -84,3 +86,8 @@ func (i *Input) WorkflowsPath() string { func (i *Input) EventPath() string { return i.resolve(i.eventPath) } + +// Inputfile returns the path to the input file +func (i *Input) Inputfile() string { + return i.resolve(i.inputfile) +} diff --git a/cmd/root.go b/cmd/root.go index 92ae8732..0aa56e78 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -47,6 +47,7 @@ func Execute(ctx context.Context, version string) { rootCmd.Flags().StringVar(&input.remoteName, "remote-name", "origin", "git remote name that will be used to retrieve url of git repo") rootCmd.Flags().StringArrayVarP(&input.secrets, "secret", "s", []string{}, "secret to make available to actions with optional value (e.g. -s mysecret=foo or -s mysecret)") rootCmd.Flags().StringArrayVarP(&input.envs, "env", "", []string{}, "env to make available to actions with optional value (e.g. --env myenv=foo or --env myenv)") + rootCmd.Flags().StringArrayVarP(&input.inputs, "input", "", []string{}, "action input to make available to actions (e.g. --input myinput=foo)") rootCmd.Flags().StringArrayVarP(&input.platforms, "platform", "P", []string{}, "custom image to use per platform (e.g. -P ubuntu-18.04=nektos/act-environments-ubuntu:18.04)") rootCmd.Flags().BoolVarP(&input.reuseContainers, "reuse", "r", false, "don't remove container(s) on successfully completed workflow(s) to maintain state between runs") rootCmd.Flags().BoolVarP(&input.bindWorkdir, "bind", "b", false, "bind working directory to container, rather than copy") @@ -74,6 +75,7 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().StringVarP(&input.secretfile, "secret-file", "", ".secrets", "file with list of secrets to read from (e.g. --secret-file .secrets)") rootCmd.PersistentFlags().BoolVarP(&input.insecureSecrets, "insecure-secrets", "", false, "NOT RECOMMENDED! Doesn't hide secrets while printing logs.") rootCmd.PersistentFlags().StringVarP(&input.envfile, "env-file", "", ".env", "environment file to read and use as env in the containers") + rootCmd.PersistentFlags().StringVarP(&input.inputfile, "input-file", "", ".input", "input file to read and use as action input") rootCmd.PersistentFlags().StringVarP(&input.containerArchitecture, "container-architecture", "", "", "Architecture which should be used to run containers, e.g.: linux/amd64. If not specified, will use host default architecture. Requires Docker server API Version 1.41+. Ignored on earlier Docker server platforms.") rootCmd.PersistentFlags().StringVarP(&input.containerDaemonSocket, "container-daemon-socket", "", "/var/run/docker.sock", "Path to Docker daemon socket which will be mounted to containers") rootCmd.PersistentFlags().StringVarP(&input.containerOptions, "container-options", "", "", "Custom docker container options for the job container without an options property in the job definition") @@ -249,6 +251,21 @@ func setupLogging(cmd *cobra.Command, _ []string) { } } +func parseEnvs(env []string, envs map[string]string) bool { + if env != nil { + for _, envVar := range env { + e := strings.SplitN(envVar, `=`, 2) + if len(e) == 2 { + envs[e[0]] = e[1] + } else { + envs[e[0]] = "" + } + } + return true + } + return false +} + func readEnvs(path string, envs map[string]string) bool { if _, err := os.Stat(path); err == nil { env, err := godotenv.Read(path) @@ -285,18 +302,14 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str log.Debugf("Loading environment from %s", input.Envfile()) envs := make(map[string]string) - if input.envs != nil { - for _, envVar := range input.envs { - e := strings.SplitN(envVar, `=`, 2) - if len(e) == 2 { - envs[e[0]] = e[1] - } else { - envs[e[0]] = "" - } - } - } + _ = parseEnvs(input.envs, envs) _ = readEnvs(input.Envfile(), envs) + log.Debugf("Loading action inputs from %s", input.Inputfile()) + inputs := make(map[string]string) + _ = parseEnvs(input.inputs, inputs) + _ = readEnvs(input.Inputfile(), inputs) + log.Debugf("Loading secrets from %s", input.Secretfile()) secrets := newSecrets(input.secrets) _ = readEnvs(input.Secretfile(), secrets) @@ -444,6 +457,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str JSONLogger: input.jsonLogger, Env: envs, Secrets: secrets, + Inputs: inputs, Token: secrets["GITHUB_TOKEN"], InsecureSecrets: input.insecureSecrets, Platforms: input.newPlatforms(), From e83ef12e1bd4621e679e31a198b5dcac8fdb2c7f Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Sun, 15 Jan 2023 02:30:41 -0800 Subject: [PATCH 28/73] feat: add check for newer versions (#1562) * feat: add check for newer versions * fix: support JSON logger and rever updates to go.mod * fix: keep version updated in source code * fix: lint errors * fix: revert go.* --- cmd/notices.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 32 ++++++++++++------ main.go | 4 ++- 3 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 cmd/notices.go diff --git a/cmd/notices.go b/cmd/notices.go new file mode 100644 index 00000000..4ad32d73 --- /dev/null +++ b/cmd/notices.go @@ -0,0 +1,90 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "runtime" + "time" + + log "github.com/sirupsen/logrus" +) + +type Notice struct { + Level string `json:"level"` + Message string `json:"message"` +} + +func displayNotices(input *Input) { + select { + case notices := <-noticesLoaded: + if len(notices) > 0 { + noticeLogger := log.New() + if input.jsonLogger { + noticeLogger.SetFormatter(&log.JSONFormatter{}) + } else { + noticeLogger.SetFormatter(&log.TextFormatter{ + DisableQuote: true, + DisableTimestamp: true, + PadLevelText: true, + }) + } + + fmt.Printf("\n") + for _, notice := range notices { + level, err := log.ParseLevel(notice.Level) + if err != nil { + level = log.InfoLevel + } + noticeLogger.Log(level, notice.Message) + } + } + case <-time.After(time.Second * 1): + log.Debugf("Timeout waiting for notices") + } +} + +var noticesLoaded = make(chan []Notice) + +func loadVersionNotices(version string) { + go func() { + noticesLoaded <- getVersionNotices(version) + }() +} + +const NoticeURL = "https://api.nektosact.com/notices" + +func getVersionNotices(version string) []Notice { + if os.Getenv("ACT_DISABLE_VERSION_CHECK") == "1" { + return nil + } + + noticeURL, err := url.Parse(NoticeURL) + if err != nil { + log.Error(err) + return nil + } + query := noticeURL.Query() + query.Add("os", runtime.GOOS) + query.Add("arch", runtime.GOARCH) + query.Add("version", version) + + noticeURL.RawQuery = query.Encode() + + resp, err := http.Get(noticeURL.String()) + if err != nil { + log.Debug(err) + return nil + } + + defer resp.Body.Close() + notices := []Notice{} + if err := json.NewDecoder(resp.Body).Decode(¬ices); err != nil { + log.Debug(err) + return nil + } + + return notices +} diff --git a/cmd/root.go b/cmd/root.go index 0aa56e78..1d153abb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -30,13 +30,14 @@ import ( func Execute(ctx context.Context, version string) { input := new(Input) var rootCmd = &cobra.Command{ - Use: "act [event name to run] [flags]\n\nIf no event name passed, will default to \"on: push\"\nIf actions handles only one event it will be used as default instead of \"on: push\"", - Short: "Run GitHub actions locally by specifying the event name (e.g. `push`) or an action name directly.", - Args: cobra.MaximumNArgs(1), - RunE: newRunCommand(ctx, input), - PersistentPreRun: setupLogging, - Version: version, - SilenceUsage: true, + Use: "act [event name to run] [flags]\n\nIf no event name passed, will default to \"on: push\"\nIf actions handles only one event it will be used as default instead of \"on: push\"", + Short: "Run GitHub actions locally by specifying the event name (e.g. `push`) or an action name directly.", + Args: cobra.MaximumNArgs(1), + RunE: newRunCommand(ctx, input), + PersistentPreRun: setup(input), + PersistentPostRun: cleanup(input), + Version: version, + SilenceUsage: true, } rootCmd.Flags().BoolP("watch", "w", false, "watch the contents of the local repo and run when files change") rootCmd.Flags().BoolP("list", "l", false, "list workflows") @@ -244,10 +245,19 @@ func readArgsFile(file string, split bool) []string { return args } -func setupLogging(cmd *cobra.Command, _ []string) { - verbose, _ := cmd.Flags().GetBool("verbose") - if verbose { - log.SetLevel(log.DebugLevel) +func setup(inputs *Input) func(*cobra.Command, []string) { + return func(cmd *cobra.Command, _ []string) { + verbose, _ := cmd.Flags().GetBool("verbose") + if verbose { + log.SetLevel(log.DebugLevel) + } + loadVersionNotices(cmd.Version) + } +} + +func cleanup(inputs *Input) func(*cobra.Command, []string) { + return func(cmd *cobra.Command, _ []string) { + displayNotices(inputs) } } diff --git a/main.go b/main.go index 41cf7c47..37b0fece 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "context" + _ "embed" "os" "os/signal" "syscall" @@ -9,7 +10,8 @@ import ( "github.com/nektos/act/cmd" ) -var version = "v0.2.27-dev" // Manually bump after tagging next release +//go:embed VERSION +var version string func main() { ctx := context.Background() From cf233000b53d62a2fd2750b321ac4182c3fa12ba Mon Sep 17 00:00:00 2001 From: Robin Breathe Date: Mon, 16 Jan 2023 15:12:20 +0100 Subject: [PATCH 29/73] fix: allow override of artifact server bind address (#1560) * Prior to this change, the artifact server always binds to the detected "outbound IP", breaks functionality when that IP is unroutable. For example, Zscaler assigns the host a local CGNAT address, 100.64.0.1, which is unreachable from Docker Desktop. * Add the `--artifact-server-addr` flag to allow override of the address to which the artifact server binds, defaulting to the existing behaviour. Fixes: #1559 --- act/artifacts/server.go | 7 +++---- act/artifacts/server_test.go | 10 ++++++---- act/runner/run_context.go | 2 +- act/runner/runner.go | 1 + cmd/input.go | 1 + cmd/root.go | 6 ++++-- 6 files changed, 16 insertions(+), 11 deletions(-) diff --git a/act/artifacts/server.go b/act/artifacts/server.go index 06a77061..9b114573 100644 --- a/act/artifacts/server.go +++ b/act/artifacts/server.go @@ -262,7 +262,7 @@ func downloads(router *httprouter.Router, fsys fs.FS) { }) } -func Serve(ctx context.Context, artifactPath string, port string) context.CancelFunc { +func Serve(ctx context.Context, artifactPath string, addr string, port string) context.CancelFunc { serverContext, cancel := context.WithCancel(ctx) logger := common.Logger(serverContext) @@ -276,17 +276,16 @@ func Serve(ctx context.Context, artifactPath string, port string) context.Cancel fs := os.DirFS(artifactPath) uploads(router, MkdirFsImpl{artifactPath, fs}) downloads(router, fs) - ip := common.GetOutboundIP().String() server := &http.Server{ - Addr: fmt.Sprintf("%s:%s", ip, port), + Addr: fmt.Sprintf("%s:%s", addr, port), ReadHeaderTimeout: 2 * time.Second, Handler: router, } // run server go func() { - logger.Infof("Start server on http://%s:%s", ip, port) + logger.Infof("Start server on http://%s:%s", addr, port) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { logger.Fatal(err) } diff --git a/act/artifacts/server_test.go b/act/artifacts/server_test.go index f1c09a30..6f865dc2 100644 --- a/act/artifacts/server_test.go +++ b/act/artifacts/server_test.go @@ -240,7 +240,8 @@ type TestJobFileInfo struct { containerArchitecture string } -var aritfactsPath = path.Join(os.TempDir(), "test-artifacts") +var artifactsPath = path.Join(os.TempDir(), "test-artifacts") +var artifactsAddr = "127.0.0.1" var artifactsPort = "12345" func TestArtifactFlow(t *testing.T) { @@ -250,7 +251,7 @@ func TestArtifactFlow(t *testing.T) { ctx := context.Background() - cancel := Serve(ctx, aritfactsPath, artifactsPort) + cancel := Serve(ctx, artifactsPath, artifactsAddr, artifactsPort) defer cancel() platforms := map[string]string{ @@ -271,7 +272,7 @@ func runTestJobFile(ctx context.Context, t *testing.T, tjfi TestJobFileInfo) { t.Run(tjfi.workflowPath, func(t *testing.T) { fmt.Printf("::group::%s\n", tjfi.workflowPath) - if err := os.RemoveAll(aritfactsPath); err != nil { + if err := os.RemoveAll(artifactsPath); err != nil { panic(err) } @@ -286,7 +287,8 @@ func runTestJobFile(ctx context.Context, t *testing.T, tjfi TestJobFileInfo) { ReuseContainers: false, ContainerArchitecture: tjfi.containerArchitecture, GitHubInstance: "github.com", - ArtifactServerPath: aritfactsPath, + ArtifactServerPath: artifactsPath, + ArtifactServerAddr: artifactsAddr, ArtifactServerPort: artifactsPort, } diff --git a/act/runner/run_context.go b/act/runner/run_context.go index c9c8ea1d..fa2da05d 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -751,7 +751,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon func setActionRuntimeVars(rc *RunContext, env map[string]string) { actionsRuntimeURL := os.Getenv("ACTIONS_RUNTIME_URL") if actionsRuntimeURL == "" { - actionsRuntimeURL = fmt.Sprintf("http://%s:%s/", common.GetOutboundIP().String(), rc.Config.ArtifactServerPort) + actionsRuntimeURL = fmt.Sprintf("http://%s:%s/", rc.Config.ArtifactServerAddr, rc.Config.ArtifactServerPort) } env["ACTIONS_RUNTIME_URL"] = actionsRuntimeURL diff --git a/act/runner/runner.go b/act/runner/runner.go index 9eb225fb..7e3f9b1b 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -48,6 +48,7 @@ type Config struct { ContainerCapDrop []string // list of kernel capabilities to remove from the containers AutoRemove bool // controls if the container is automatically removed upon workflow completion ArtifactServerPath string // the path where the artifact server stores uploads + ArtifactServerAddr string // the address the artifact server binds to ArtifactServerPort string // the port the artifact server binds to NoSkipCheckout bool // do not skip actions/checkout RemoteName string // remote name in local git repo config diff --git a/cmd/input.go b/cmd/input.go index 1caeccbb..37655a55 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -40,6 +40,7 @@ type Input struct { containerCapDrop []string autoRemove bool artifactServerPath string + artifactServerAddr string artifactServerPort string jsonLogger bool noSkipCheckout bool diff --git a/cmd/root.go b/cmd/root.go index 1d153abb..78f3a72d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -82,7 +82,8 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().StringVarP(&input.containerOptions, "container-options", "", "", "Custom docker container options for the job container without an options property in the job definition") rootCmd.PersistentFlags().StringVarP(&input.githubInstance, "github-instance", "", "github.com", "GitHub instance to use. Don't use this if you are not using GitHub Enterprise Server.") rootCmd.PersistentFlags().StringVarP(&input.artifactServerPath, "artifact-server-path", "", "", "Defines the path where the artifact server stores uploads and retrieves downloads from. If not specified the artifact server will not start.") - rootCmd.PersistentFlags().StringVarP(&input.artifactServerPort, "artifact-server-port", "", "34567", "Defines the port where the artifact server listens (will only bind to localhost).") + rootCmd.PersistentFlags().StringVarP(&input.artifactServerAddr, "artifact-server-addr", "", common.GetOutboundIP().String(), "Defines the address to which the artifact server binds.") + rootCmd.PersistentFlags().StringVarP(&input.artifactServerPort, "artifact-server-port", "", "34567", "Defines the port where the artifact server listens.") rootCmd.PersistentFlags().BoolVarP(&input.noSkipCheckout, "no-skip-checkout", "", false, "Do not skip actions/checkout") rootCmd.SetArgs(args()) @@ -482,6 +483,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str ContainerCapDrop: input.containerCapDrop, AutoRemove: input.autoRemove, ArtifactServerPath: input.artifactServerPath, + ArtifactServerAddr: input.artifactServerAddr, ArtifactServerPort: input.artifactServerPort, NoSkipCheckout: input.noSkipCheckout, RemoteName: input.remoteName, @@ -493,7 +495,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str return err } - cancel := artifacts.Serve(ctx, input.artifactServerPath, input.artifactServerPort) + cancel := artifacts.Serve(ctx, input.artifactServerPath, input.artifactServerAddr, input.artifactServerPort) ctx = common.WithDryrun(ctx, input.dryrun) if watch, err := cmd.Flags().GetBool("watch"); err != nil { From c282c7df54abf8abacd0e95eacba4332a2817ad4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jan 2023 17:26:22 +0000 Subject: [PATCH 30/73] build(deps): bump github.com/moby/buildkit from 0.10.6 to 0.11.0 (#1563) * build(deps): bump github.com/moby/buildkit from 0.10.6 to 0.11.0 Bumps [github.com/moby/buildkit](https://github.com/moby/buildkit) from 0.10.6 to 0.11.0. - [Release notes](https://github.com/moby/buildkit/releases) - [Commits](https://github.com/moby/buildkit/compare/v0.10.6...v0.11.0) --- updated-dependencies: - dependency-name: github.com/moby/buildkit dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * chore: use new patternmatcher.Matches Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Casey Lee Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/docker_build.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/act/container/docker_build.go b/act/container/docker_build.go index f05a513f..0c87cfdf 100644 --- a/act/container/docker_build.go +++ b/act/container/docker_build.go @@ -10,10 +10,10 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/pkg/fileutils" // github.com/docker/docker/builder/dockerignore is deprecated "github.com/moby/buildkit/frontend/dockerfile/dockerignore" + "github.com/moby/patternmatcher" "github.com/nektos/act/pkg/common" ) @@ -96,8 +96,8 @@ func createBuildContext(ctx context.Context, contextDir string, relDockerfile st // parses the Dockerfile. Ignore errors here, as they will have been // caught by validateContextDirectory above. var includes = []string{"."} - keepThem1, _ := fileutils.Matches(".dockerignore", excludes) - keepThem2, _ := fileutils.Matches(relDockerfile, excludes) + keepThem1, _ := patternmatcher.Matches(".dockerignore", excludes) + keepThem2, _ := patternmatcher.Matches(relDockerfile, excludes) if keepThem1 || keepThem2 { includes = append(includes, ".dockerignore", relDockerfile) } From f70639d09ebf138bf2ad2f67bcd685033854ac23 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Mon, 16 Jan 2023 13:01:54 -0800 Subject: [PATCH 31/73] fix: update artifact server to address GHSL-2023-004 (#1565) --- act/artifacts/server.go | 82 +++++---- act/artifacts/server_test.go | 161 +++++++++++++----- .../testdata/GHSL-2023-004/artifacts.yml | 43 +++++ 3 files changed, 207 insertions(+), 79 deletions(-) create mode 100644 act/artifacts/testdata/GHSL-2023-004/artifacts.yml diff --git a/act/artifacts/server.go b/act/artifacts/server.go index 9b114573..7c3df56c 100644 --- a/act/artifacts/server.go +++ b/act/artifacts/server.go @@ -9,7 +9,6 @@ import ( "io/fs" "net/http" "os" - "path" "path/filepath" "strings" "time" @@ -46,28 +45,34 @@ type ResponseMessage struct { Message string `json:"message"` } -type MkdirFS interface { - fs.FS - MkdirAll(path string, perm fs.FileMode) error - Open(name string) (fs.File, error) - OpenAtEnd(name string) (fs.File, error) +type WritableFile interface { + io.WriteCloser } -type MkdirFsImpl struct { - dir string - fs.FS +type WriteFS interface { + OpenWritable(name string) (WritableFile, error) + OpenAppendable(name string) (WritableFile, error) } -func (fsys MkdirFsImpl) MkdirAll(path string, perm fs.FileMode) error { - return os.MkdirAll(fsys.dir+"/"+path, perm) +type readWriteFSImpl struct { } -func (fsys MkdirFsImpl) Open(name string) (fs.File, error) { - return os.OpenFile(fsys.dir+"/"+name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) +func (fwfs readWriteFSImpl) Open(name string) (fs.File, error) { + return os.Open(name) } -func (fsys MkdirFsImpl) OpenAtEnd(name string) (fs.File, error) { - file, err := os.OpenFile(fsys.dir+"/"+name, os.O_CREATE|os.O_RDWR, 0644) +func (fwfs readWriteFSImpl) OpenWritable(name string) (WritableFile, error) { + if err := os.MkdirAll(filepath.Dir(name), os.ModePerm); err != nil { + return nil, err + } + return os.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) +} + +func (fwfs readWriteFSImpl) OpenAppendable(name string) (WritableFile, error) { + if err := os.MkdirAll(filepath.Dir(name), os.ModePerm); err != nil { + return nil, err + } + file, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644) if err != nil { return nil, err @@ -77,13 +82,16 @@ func (fsys MkdirFsImpl) OpenAtEnd(name string) (fs.File, error) { if err != nil { return nil, err } - return file, nil } var gzipExtension = ".gz__" -func uploads(router *httprouter.Router, fsys MkdirFS) { +func safeResolve(baseDir string, relPath string) string { + return filepath.Join(baseDir, filepath.Clean(filepath.Join(string(os.PathSeparator), relPath))) +} + +func uploads(router *httprouter.Router, baseDir string, fsys WriteFS) { router.POST("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { runID := params.ByName("runId") @@ -108,19 +116,15 @@ func uploads(router *httprouter.Router, fsys MkdirFS) { itemPath += gzipExtension } - filePath := fmt.Sprintf("%s/%s", runID, itemPath) + safeRunPath := safeResolve(baseDir, runID) + safePath := safeResolve(safeRunPath, itemPath) - err := fsys.MkdirAll(path.Dir(filePath), os.ModePerm) - if err != nil { - panic(err) - } - - file, err := func() (fs.File, error) { + file, err := func() (WritableFile, error) { contentRange := req.Header.Get("Content-Range") if contentRange != "" && !strings.HasPrefix(contentRange, "bytes 0-") { - return fsys.OpenAtEnd(filePath) + return fsys.OpenAppendable(safePath) } - return fsys.Open(filePath) + return fsys.OpenWritable(safePath) }() if err != nil { @@ -170,11 +174,13 @@ func uploads(router *httprouter.Router, fsys MkdirFS) { }) } -func downloads(router *httprouter.Router, fsys fs.FS) { +func downloads(router *httprouter.Router, baseDir string, fsys fs.FS) { router.GET("/_apis/pipelines/workflows/:runId/artifacts", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { runID := params.ByName("runId") - entries, err := fs.ReadDir(fsys, runID) + safePath := safeResolve(baseDir, runID) + + entries, err := fs.ReadDir(fsys, safePath) if err != nil { panic(err) } @@ -204,12 +210,12 @@ func downloads(router *httprouter.Router, fsys fs.FS) { router.GET("/download/:container", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { container := params.ByName("container") itemPath := req.URL.Query().Get("itemPath") - dirPath := fmt.Sprintf("%s/%s", container, itemPath) + safePath := safeResolve(baseDir, filepath.Join(container, itemPath)) var files []ContainerItem - err := fs.WalkDir(fsys, dirPath, func(path string, entry fs.DirEntry, err error) error { + err := fs.WalkDir(fsys, safePath, func(path string, entry fs.DirEntry, err error) error { if !entry.IsDir() { - rel, err := filepath.Rel(dirPath, path) + rel, err := filepath.Rel(safePath, path) if err != nil { panic(err) } @@ -218,7 +224,7 @@ func downloads(router *httprouter.Router, fsys fs.FS) { rel = strings.TrimSuffix(rel, gzipExtension) files = append(files, ContainerItem{ - Path: fmt.Sprintf("%s/%s", itemPath, rel), + Path: filepath.Join(itemPath, rel), ItemType: "file", ContentLocation: fmt.Sprintf("http://%s/artifact/%s/%s/%s", req.Host, container, itemPath, rel), }) @@ -245,10 +251,12 @@ func downloads(router *httprouter.Router, fsys fs.FS) { router.GET("/artifact/*path", func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { path := params.ByName("path")[1:] - file, err := fsys.Open(path) + safePath := safeResolve(baseDir, path) + + file, err := fsys.Open(safePath) if err != nil { // try gzip file - file, err = fsys.Open(path + gzipExtension) + file, err = fsys.Open(safePath + gzipExtension) if err != nil { panic(err) } @@ -273,9 +281,9 @@ func Serve(ctx context.Context, artifactPath string, addr string, port string) c router := httprouter.New() logger.Debugf("Artifacts base path '%s'", artifactPath) - fs := os.DirFS(artifactPath) - uploads(router, MkdirFsImpl{artifactPath, fs}) - downloads(router, fs) + fsys := readWriteFSImpl{} + uploads(router, artifactPath, fsys) + downloads(router, artifactPath, fsys) server := &http.Server{ Addr: fmt.Sprintf("%s:%s", addr, port), diff --git a/act/artifacts/server_test.go b/act/artifacts/server_test.go index 6f865dc2..259c9542 100644 --- a/act/artifacts/server_test.go +++ b/act/artifacts/server_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "io/fs" "net/http" "net/http/httptest" "os" @@ -21,44 +20,43 @@ import ( "github.com/stretchr/testify/assert" ) -type MapFsImpl struct { - fstest.MapFS +type writableMapFile struct { + fstest.MapFile } -func (fsys MapFsImpl) MkdirAll(path string, perm fs.FileMode) error { - // mocked no-op - return nil -} - -type WritableFile struct { - fs.File - fsys fstest.MapFS - path string -} - -func (file WritableFile) Write(data []byte) (int, error) { - file.fsys[file.path].Data = data +func (f *writableMapFile) Write(data []byte) (int, error) { + f.Data = data return len(data), nil } -func (fsys MapFsImpl) Open(path string) (fs.File, error) { - var file = fstest.MapFile{ - Data: []byte("content2"), - } - fsys.MapFS[path] = &file - - result, err := fsys.MapFS.Open(path) - return WritableFile{result, fsys.MapFS, path}, err +func (f *writableMapFile) Close() error { + return nil } -func (fsys MapFsImpl) OpenAtEnd(path string) (fs.File, error) { - var file = fstest.MapFile{ - Data: []byte("content2"), - } - fsys.MapFS[path] = &file +type writeMapFS struct { + fstest.MapFS +} - result, err := fsys.MapFS.Open(path) - return WritableFile{result, fsys.MapFS, path}, err +func (fsys writeMapFS) OpenWritable(name string) (WritableFile, error) { + var file = &writableMapFile{ + MapFile: fstest.MapFile{ + Data: []byte("content2"), + }, + } + fsys.MapFS[name] = &file.MapFile + + return file, nil +} + +func (fsys writeMapFS) OpenAppendable(name string) (WritableFile, error) { + var file = &writableMapFile{ + MapFile: fstest.MapFile{ + Data: []byte("content2"), + }, + } + fsys.MapFS[name] = &file.MapFile + + return file, nil } func TestNewArtifactUploadPrepare(t *testing.T) { @@ -67,7 +65,7 @@ func TestNewArtifactUploadPrepare(t *testing.T) { var memfs = fstest.MapFS(map[string]*fstest.MapFile{}) router := httprouter.New() - uploads(router, MapFsImpl{memfs}) + uploads(router, "artifact/server/path", writeMapFS{memfs}) req, _ := http.NewRequest("POST", "http://localhost/_apis/pipelines/workflows/1/artifacts", nil) rr := httptest.NewRecorder() @@ -93,7 +91,7 @@ func TestArtifactUploadBlob(t *testing.T) { var memfs = fstest.MapFS(map[string]*fstest.MapFile{}) router := httprouter.New() - uploads(router, MapFsImpl{memfs}) + uploads(router, "artifact/server/path", writeMapFS{memfs}) req, _ := http.NewRequest("PUT", "http://localhost/upload/1?itemPath=some/file", strings.NewReader("content")) rr := httptest.NewRecorder() @@ -111,7 +109,7 @@ func TestArtifactUploadBlob(t *testing.T) { } assert.Equal("success", response.Message) - assert.Equal("content", string(memfs["1/some/file"].Data)) + assert.Equal("content", string(memfs["artifact/server/path/1/some/file"].Data)) } func TestFinalizeArtifactUpload(t *testing.T) { @@ -120,7 +118,7 @@ func TestFinalizeArtifactUpload(t *testing.T) { var memfs = fstest.MapFS(map[string]*fstest.MapFile{}) router := httprouter.New() - uploads(router, MapFsImpl{memfs}) + uploads(router, "artifact/server/path", writeMapFS{memfs}) req, _ := http.NewRequest("PATCH", "http://localhost/_apis/pipelines/workflows/1/artifacts", nil) rr := httptest.NewRecorder() @@ -144,13 +142,13 @@ func TestListArtifacts(t *testing.T) { assert := assert.New(t) var memfs = fstest.MapFS(map[string]*fstest.MapFile{ - "1/file.txt": { + "artifact/server/path/1/file.txt": { Data: []byte(""), }, }) router := httprouter.New() - downloads(router, memfs) + downloads(router, "artifact/server/path", memfs) req, _ := http.NewRequest("GET", "http://localhost/_apis/pipelines/workflows/1/artifacts", nil) rr := httptest.NewRecorder() @@ -176,13 +174,13 @@ func TestListArtifactContainer(t *testing.T) { assert := assert.New(t) var memfs = fstest.MapFS(map[string]*fstest.MapFile{ - "1/some/file": { + "artifact/server/path/1/some/file": { Data: []byte(""), }, }) router := httprouter.New() - downloads(router, memfs) + downloads(router, "artifact/server/path", memfs) req, _ := http.NewRequest("GET", "http://localhost/download/1?itemPath=some/file", nil) rr := httptest.NewRecorder() @@ -200,7 +198,7 @@ func TestListArtifactContainer(t *testing.T) { } assert.Equal(1, len(response.Value)) - assert.Equal("some/file/.", response.Value[0].Path) + assert.Equal("some/file", response.Value[0].Path) assert.Equal("file", response.Value[0].ItemType) assert.Equal("http://localhost/artifact/1/some/file/.", response.Value[0].ContentLocation) } @@ -209,13 +207,13 @@ func TestDownloadArtifactFile(t *testing.T) { assert := assert.New(t) var memfs = fstest.MapFS(map[string]*fstest.MapFile{ - "1/some/file": { + "artifact/server/path/1/some/file": { Data: []byte("content"), }, }) router := httprouter.New() - downloads(router, memfs) + downloads(router, "artifact/server/path", memfs) req, _ := http.NewRequest("GET", "http://localhost/artifact/1/some/file", nil) rr := httptest.NewRecorder() @@ -260,6 +258,7 @@ func TestArtifactFlow(t *testing.T) { tables := []TestJobFileInfo{ {"testdata", "upload-and-download", "push", "", platforms, ""}, + {"testdata", "GHSL-2023-004", "push", "", platforms, ""}, } log.SetLevel(log.DebugLevel) @@ -310,3 +309,81 @@ func runTestJobFile(ctx context.Context, t *testing.T, tjfi TestJobFileInfo) { fmt.Println("::endgroup::") }) } + +func TestMkdirFsImplSafeResolve(t *testing.T) { + assert := assert.New(t) + + baseDir := "/foo/bar" + + tests := map[string]struct { + input string + want string + }{ + "simple": {input: "baz", want: "/foo/bar/baz"}, + "nested": {input: "baz/blue", want: "/foo/bar/baz/blue"}, + "dots in middle": {input: "baz/../../blue", want: "/foo/bar/blue"}, + "leading dots": {input: "../../parent", want: "/foo/bar/parent"}, + "root path": {input: "/root", want: "/foo/bar/root"}, + "root": {input: "/", want: "/foo/bar"}, + "empty": {input: "", want: "/foo/bar"}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(tc.want, safeResolve(baseDir, tc.input)) + }) + } +} + +func TestDownloadArtifactFileUnsafePath(t *testing.T) { + assert := assert.New(t) + + var memfs = fstest.MapFS(map[string]*fstest.MapFile{ + "artifact/server/path/some/file": { + Data: []byte("content"), + }, + }) + + router := httprouter.New() + downloads(router, "artifact/server/path", memfs) + + req, _ := http.NewRequest("GET", "http://localhost/artifact/2/../../some/file", nil) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + if status := rr.Code; status != http.StatusOK { + assert.FailNow(fmt.Sprintf("Wrong status: %d", status)) + } + + data := rr.Body.Bytes() + + assert.Equal("content", string(data)) +} + +func TestArtifactUploadBlobUnsafePath(t *testing.T) { + assert := assert.New(t) + + var memfs = fstest.MapFS(map[string]*fstest.MapFile{}) + + router := httprouter.New() + uploads(router, "artifact/server/path", writeMapFS{memfs}) + + req, _ := http.NewRequest("PUT", "http://localhost/upload/1?itemPath=../../some/file", strings.NewReader("content")) + rr := httptest.NewRecorder() + + router.ServeHTTP(rr, req) + + if status := rr.Code; status != http.StatusOK { + assert.Fail("Wrong status") + } + + response := ResponseMessage{} + err := json.Unmarshal(rr.Body.Bytes(), &response) + if err != nil { + panic(err) + } + + assert.Equal("success", response.Message) + assert.Equal("content", string(memfs["artifact/server/path/1/some/file"].Data)) +} diff --git a/act/artifacts/testdata/GHSL-2023-004/artifacts.yml b/act/artifacts/testdata/GHSL-2023-004/artifacts.yml new file mode 100644 index 00000000..e717f141 --- /dev/null +++ b/act/artifacts/testdata/GHSL-2023-004/artifacts.yml @@ -0,0 +1,43 @@ + +name: "GHSL-2023-0004" +on: push + +jobs: + test-artifacts: + runs-on: ubuntu-latest + steps: + - run: echo "hello world" > test.txt + - name: curl upload + uses: wei/curl@v1 + with: + args: -s --fail ${ACTIONS_RUNTIME_URL}upload/1?itemPath=../../my-artifact/secret.txt --upload-file test.txt + - uses: actions/download-artifact@v2 + with: + name: my-artifact + path: test-artifacts + - name: 'Verify Artifact #1' + run: | + file="test-artifacts/secret.txt" + if [ ! -f $file ] ; then + echo "Expected file does not exist" + exit 1 + fi + if [ "$(cat $file)" != "hello world" ] ; then + echo "File contents of downloaded artifact are incorrect" + exit 1 + fi + - name: Verify download should work by clean extra dots + uses: wei/curl@v1 + with: + args: --path-as-is -s -o out.txt --fail ${ACTIONS_RUNTIME_URL}artifact/1/../../../1/my-artifact/secret.txt + - name: 'Verify download content' + run: | + file="out.txt" + if [ ! -f $file ] ; then + echo "Expected file does not exist" + exit 1 + fi + if [ "$(cat $file)" != "hello world" ] ; then + echo "File contents of downloaded artifact are incorrect" + exit 1 + fi From aeebe9f1d5be031c6212521735be304548b7982a Mon Sep 17 00:00:00 2001 From: Alexandre Lavigne Date: Thu, 19 Jan 2023 07:29:23 +0100 Subject: [PATCH 32/73] Feature/allow worktrees (#1530) * Use go-git to find remote URL * Use go-git package to resolve HEAD revision (commit sha1) * Use go-git to find checked-out reference * Remove unused functions --- act/common/git/git.go | 206 ++++++++++++++++--------------------- act/common/git/git_test.go | 9 +- 2 files changed, 95 insertions(+), 120 deletions(-) diff --git a/act/common/git/git.go b/act/common/git/git.go index d03dada3..74df3c0f 100644 --- a/act/common/git/git.go +++ b/act/common/git/git.go @@ -7,7 +7,6 @@ import ( "io" "os" "path" - "path/filepath" "regexp" "strings" "sync" @@ -17,8 +16,8 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/storer" "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-ini/ini" "github.com/mattn/go-isatty" log "github.com/sirupsen/logrus" ) @@ -55,41 +54,40 @@ func (e *Error) Commit() string { // FindGitRevision get the current git revision func FindGitRevision(ctx context.Context, file string) (shortSha string, sha string, err error) { logger := common.Logger(ctx) - gitDir, err := findGitDirectory(file) + + gitDir, err := git.PlainOpenWithOptions( + file, + &git.PlainOpenOptions{ + DetectDotGit: true, + EnableDotGitCommonDir: true, + }, + ) + + if err != nil { + logger.WithError(err).Error("path", file, "not located inside a git repository") + return "", "", err + } + + head, err := gitDir.Reference(plumbing.HEAD, true) if err != nil { return "", "", err } - bts, err := os.ReadFile(filepath.Join(gitDir, "HEAD")) - if err != nil { - return "", "", err + if head.Hash().IsZero() { + return "", "", fmt.Errorf("HEAD sha1 could not be resolved") } - var ref = strings.TrimSpace(strings.TrimPrefix(string(bts), "ref:")) - var refBuf []byte - if strings.HasPrefix(ref, "refs/") { - // load commitid ref - refBuf, err = os.ReadFile(filepath.Join(gitDir, ref)) - if err != nil { - return "", "", err - } - } else { - refBuf = []byte(ref) - } + hash := head.Hash().String() - logger.Debugf("Found revision: %s", refBuf) - return string(refBuf[:7]), strings.TrimSpace(string(refBuf)), nil + logger.Debugf("Found revision: %s", hash) + return hash[:7], strings.TrimSpace(hash), nil } // FindGitRef get the current git ref func FindGitRef(ctx context.Context, file string) (string, error) { logger := common.Logger(ctx) - gitDir, err := findGitDirectory(file) - if err != nil { - return "", err - } - logger.Debugf("Loading revision from git directory '%s'", gitDir) + logger.Debugf("Loading revision from git directory") _, ref, err := FindGitRevision(ctx, file) if err != nil { return "", err @@ -100,28 +98,58 @@ func FindGitRef(ctx context.Context, file string) (string, error) { // Prefer the git library to iterate over the references and find a matching tag or branch. var refTag = "" var refBranch = "" - r, err := git.PlainOpen(filepath.Join(gitDir, "..")) - if err == nil { - iter, err := r.References() - if err == nil { - for { - r, err := iter.Next() - if r == nil || err != nil { - break - } - // logger.Debugf("Reference: name=%s sha=%s", r.Name().String(), r.Hash().String()) - if r.Hash().String() == ref { - if r.Name().IsTag() { - refTag = r.Name().String() - } - if r.Name().IsBranch() { - refBranch = r.Name().String() - } - } - } - iter.Close() - } + repo, err := git.PlainOpenWithOptions( + file, + &git.PlainOpenOptions{ + DetectDotGit: true, + EnableDotGitCommonDir: true, + }, + ) + + if err != nil { + return "", err } + + iter, err := repo.References() + if err != nil { + return "", err + } + + // find the reference that matches the revision's has + err = iter.ForEach(func(r *plumbing.Reference) error { + /* tags and branches will have the same hash + * when a user checks out a tag, it is not mentioned explicitly + * in the go-git package, we must identify the revision + * then check if any tag matches that revision, + * if so then we checked out a tag + * else we look for branches and if matches, + * it means we checked out a branch + * + * If a branches matches first we must continue and check all tags (all references) + * in case we match with a tag later in the interation + */ + if r.Hash().String() == ref { + if r.Name().IsTag() { + refTag = r.Name().String() + } + if r.Name().IsBranch() { + refBranch = r.Name().String() + } + } + + // we found what we where looking for + if refTag != "" && refBranch != "" { + return storer.ErrStop + } + + return nil + }) + + if err != nil { + return "", err + } + + // order matters here see above comment. if refTag != "" { return refTag, nil } @@ -129,39 +157,7 @@ func FindGitRef(ctx context.Context, file string) (string, error) { return refBranch, nil } - // If the above doesn't work, fall back to the old way - - // try tags first - tag, err := findGitPrettyRef(ctx, ref, gitDir, "refs/tags") - if err != nil || tag != "" { - return tag, err - } - // and then branches - return findGitPrettyRef(ctx, ref, gitDir, "refs/heads") -} - -func findGitPrettyRef(ctx context.Context, head, root, sub string) (string, error) { - var name string - var err = filepath.Walk(filepath.Join(root, sub), func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if name != "" || info.IsDir() { - return nil - } - var bts []byte - if bts, err = os.ReadFile(path); err != nil { - return err - } - var pointsTo = strings.TrimSpace(string(bts)) - if head == pointsTo { - // On Windows paths are separated with backslash character so they should be replaced to provide proper git refs format - name = strings.TrimPrefix(strings.ReplaceAll(strings.Replace(path, root, "", 1), `\`, `/`), "/") - common.Logger(ctx).Debugf("HEAD matches %s", name) - } - return nil - }) - return name, err + return "", fmt.Errorf("failed to identify reference (tag/branch) for the checked-out revision '%s'", ref) } // FindGithubRepo get the repo @@ -179,26 +175,27 @@ func FindGithubRepo(ctx context.Context, file, githubInstance, remoteName string } func findGitRemoteURL(ctx context.Context, file, remoteName string) (string, error) { - gitDir, err := findGitDirectory(file) + repo, err := git.PlainOpenWithOptions( + file, + &git.PlainOpenOptions{ + DetectDotGit: true, + EnableDotGitCommonDir: true, + }, + ) if err != nil { return "", err } - common.Logger(ctx).Debugf("Loading slug from git directory '%s'", gitDir) - gitconfig, err := ini.InsensitiveLoad(fmt.Sprintf("%s/config", gitDir)) + remote, err := repo.Remote(remoteName) if err != nil { return "", err } - remote, err := gitconfig.GetSection(fmt.Sprintf(`remote "%s"`, remoteName)) - if err != nil { - return "", err + + if len(remote.Config().URLs) < 1 { + return "", fmt.Errorf("remote '%s' exists but has no URL", remoteName) } - urlKey, err := remote.GetKey("url") - if err != nil { - return "", err - } - url := urlKey.String() - return url, nil + + return remote.Config().URLs[0], nil } func findGitSlug(url string, githubInstance string) (string, string, error) { @@ -222,35 +219,6 @@ func findGitSlug(url string, githubInstance string) (string, string, error) { return "", url, nil } -func findGitDirectory(fromFile string) (string, error) { - absPath, err := filepath.Abs(fromFile) - if err != nil { - return "", err - } - - fi, err := os.Stat(absPath) - if err != nil { - return "", err - } - - var dir string - if fi.Mode().IsDir() { - dir = absPath - } else { - dir = filepath.Dir(absPath) - } - - gitPath := filepath.Join(dir, ".git") - fi, err = os.Stat(gitPath) - if err == nil && fi.Mode().IsDir() { - return gitPath, nil - } else if dir == "/" || dir == "C:\\" || dir == "c:\\" { - return "", &Error{err: ErrNoRepo} - } - - return findGitDirectory(filepath.Dir(dir)) -} - // NewGitCloneExecutorInput the input for the NewGitCloneExecutor type NewGitCloneExecutorInput struct { URL string diff --git a/act/common/git/git_test.go b/act/common/git/git_test.go index 9798193e..5d3e5886 100644 --- a/act/common/git/git_test.go +++ b/act/common/git/git_test.go @@ -82,12 +82,19 @@ func TestFindGitRemoteURL(t *testing.T) { assert.NoError(err) remoteURL := "https://git-codecommit.us-east-1.amazonaws.com/v1/repos/my-repo-name" - err = gitCmd("config", "-f", fmt.Sprintf("%s/.git/config", basedir), "--add", "remote.origin.url", remoteURL) + err = gitCmd("-C", basedir, "remote", "add", "origin", remoteURL) assert.NoError(err) u, err := findGitRemoteURL(context.Background(), basedir, "origin") assert.NoError(err) assert.Equal(remoteURL, u) + + remoteURL = "git@github.com/AwesomeOwner/MyAwesomeRepo.git" + err = gitCmd("-C", basedir, "remote", "add", "upstream", remoteURL) + assert.NoError(err) + u, err = findGitRemoteURL(context.Background(), basedir, "upstream") + assert.NoError(err) + assert.Equal(remoteURL, u) } func TestGitFindRef(t *testing.T) { From b59e15740911ef1170bdfae59f23d15071f3c784 Mon Sep 17 00:00:00 2001 From: Yoshiaki Yoshida Date: Thu, 19 Jan 2023 15:49:16 +0900 Subject: [PATCH 33/73] Fixed auto-generated platform configuration with Micro size image (#1566) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- cmd/root.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 78f3a72d..d60e4c87 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -533,7 +533,7 @@ func defaultImageSurvey(actrc string) error { case "Medium": option = "-P ubuntu-latest=catthehacker/ubuntu:act-latest\n-P ubuntu-22.04=catthehacker/ubuntu:act-22.04\n-P ubuntu-20.04=catthehacker/ubuntu:act-20.04\n-P ubuntu-18.04=catthehacker/ubuntu:act-18.04\n" case "Micro": - option = "-P ubuntu-latest=node:16-buster-slim\n-P -P ubuntu-22.04=node:16-bullseye-slim\n ubuntu-20.04=node:16-buster-slim\n-P ubuntu-18.04=node:16-buster-slim\n" + option = "-P ubuntu-latest=node:16-buster-slim\n-P ubuntu-22.04=node:16-bullseye-slim\n-P ubuntu-20.04=node:16-buster-slim\n-P ubuntu-18.04=node:16-buster-slim\n" } f, err := os.Create(actrc) From 67bb6970555752566ac6e83224c627ee58141442 Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Thu, 19 Jan 2023 21:49:11 +0100 Subject: [PATCH 34/73] feat: add remote reusable workflows (#1525) * feat: add remote reusable workflows This changes adds cloning of a remote repository to run a workflow included in it. Closes #826 * fix: defer plan creation until clone is done We need wait for the full clone (and only clone once) before we start to plan the execution for a remote workflow Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/reusable_workflow.go | 106 ++++++++++++++++++--- act/runner/runner_test.go | 2 +- act/runner/testdata/uses-workflow/push.yml | 32 ++++++- 3 files changed, 124 insertions(+), 16 deletions(-) diff --git a/act/runner/reusable_workflow.go b/act/runner/reusable_workflow.go index 87b7bde9..b080b4db 100644 --- a/act/runner/reusable_workflow.go +++ b/act/runner/reusable_workflow.go @@ -1,35 +1,88 @@ package runner import ( + "context" + "errors" "fmt" + "io/fs" + "os" "path" + "regexp" + "strings" + "sync" "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/common/git" "github.com/nektos/act/pkg/model" ) func newLocalReusableWorkflowExecutor(rc *RunContext) common.Executor { - return newReusableWorkflowExecutor(rc, rc.Config.Workdir) + return newReusableWorkflowExecutor(rc, rc.Config.Workdir, rc.Run.Job().Uses) } func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor { - return common.NewErrorExecutor(fmt.Errorf("remote reusable workflows are currently not supported (see https://github.com/nektos/act/issues/826 for updates)")) + uses := rc.Run.Job().Uses + + remoteReusableWorkflow := newRemoteReusableWorkflow(uses) + if remoteReusableWorkflow == nil { + return common.NewErrorExecutor(fmt.Errorf("expected format {owner}/{repo}/.github/workflows/{filename}@{ref}. Actual '%s' Input string was not in a correct format", uses)) + } + remoteReusableWorkflow.URL = rc.Config.GitHubInstance + + workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), strings.ReplaceAll(uses, "/", "-")) + + return common.NewPipelineExecutor( + newMutexExecutor(cloneIfRequired(rc, *remoteReusableWorkflow, workflowDir)), + newReusableWorkflowExecutor(rc, workflowDir, fmt.Sprintf("./.github/workflows/%s", remoteReusableWorkflow.Filename)), + ) } -func newReusableWorkflowExecutor(rc *RunContext, directory string) common.Executor { - planner, err := model.NewWorkflowPlanner(path.Join(directory, rc.Run.Job().Uses), true) - if err != nil { - return common.NewErrorExecutor(err) +var ( + executorLock sync.Mutex +) + +func newMutexExecutor(executor common.Executor) common.Executor { + return func(ctx context.Context) error { + executorLock.Lock() + defer executorLock.Unlock() + + return executor(ctx) } +} - plan := planner.PlanEvent("workflow_call") +func cloneIfRequired(rc *RunContext, remoteReusableWorkflow remoteReusableWorkflow, targetDirectory string) common.Executor { + return common.NewConditionalExecutor( + func(ctx context.Context) bool { + _, err := os.Stat(targetDirectory) + notExists := errors.Is(err, fs.ErrNotExist) + return notExists + }, + git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{ + URL: remoteReusableWorkflow.CloneURL(), + Ref: remoteReusableWorkflow.Ref, + Dir: targetDirectory, + Token: rc.Config.Token, + }), + nil, + ) +} - runner, err := NewReusableWorkflowRunner(rc) - if err != nil { - return common.NewErrorExecutor(err) +func newReusableWorkflowExecutor(rc *RunContext, directory string, workflow string) common.Executor { + return func(ctx context.Context) error { + planner, err := model.NewWorkflowPlanner(path.Join(directory, workflow), true) + if err != nil { + return err + } + + plan := planner.PlanEvent("workflow_call") + + runner, err := NewReusableWorkflowRunner(rc) + if err != nil { + return err + } + + return runner.NewPlanExecutor(plan)(ctx) } - - return runner.NewPlanExecutor(plan) } func NewReusableWorkflowRunner(rc *RunContext) (Runner, error) { @@ -43,3 +96,32 @@ func NewReusableWorkflowRunner(rc *RunContext) (Runner, error) { return runner.configure() } + +type remoteReusableWorkflow struct { + URL string + Org string + Repo string + Filename string + Ref string +} + +func (r *remoteReusableWorkflow) CloneURL() string { + return fmt.Sprintf("https://%s/%s/%s", r.URL, r.Org, r.Repo) +} + +func newRemoteReusableWorkflow(uses string) *remoteReusableWorkflow { + // GitHub docs: + // https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_iduses + r := regexp.MustCompile(`^([^/]+)/([^/]+)/.github/workflows/([^@]+)@(.*)$`) + matches := r.FindStringSubmatch(uses) + if len(matches) != 5 { + return nil + } + return &remoteReusableWorkflow{ + Org: matches[1], + Repo: matches[2], + Filename: matches[3], + Ref: matches[4], + URL: "github.com", + } +} diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 6096abe1..9298afcc 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -145,7 +145,7 @@ func TestRunEvent(t *testing.T) { {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms, secrets}, {workdir, "uses-nested-composite", "push", "", platforms, secrets}, {workdir, "remote-action-composite-js-pre-with-defaults", "push", "", platforms, secrets}, - {workdir, "uses-workflow", "push", "reusable workflows are currently not supported (see https://github.com/nektos/act/issues/826 for updates)", platforms, secrets}, + {workdir, "uses-workflow", "push", "", platforms, map[string]string{"secret": "keep_it_private"}}, {workdir, "uses-workflow", "pull_request", "", platforms, map[string]string{"secret": "keep_it_private"}}, {workdir, "uses-docker-url", "push", "", platforms, secrets}, {workdir, "act-composite-env-test", "push", "", platforms, secrets}, diff --git a/act/runner/testdata/uses-workflow/push.yml b/act/runner/testdata/uses-workflow/push.yml index 855dacfe..ddc37b86 100644 --- a/act/runner/testdata/uses-workflow/push.yml +++ b/act/runner/testdata/uses-workflow/push.yml @@ -2,8 +2,34 @@ on: push jobs: reusable-workflow: - uses: nektos/act-tests/.github/workflows/reusable-workflow.yml@master + uses: nektos/act-test-actions/.github/workflows/reusable-workflow.yml@main with: - username: mona + string_required: string + bool_required: ${{ true }} + number_required: 1 secrets: - envPATH: ${{ secrets.envPAT }} + secret: keep_it_private + + reusable-workflow-with-inherited-secrets: + uses: nektos/act-test-actions/.github/workflows/reusable-workflow.yml@main + with: + string_required: string + bool_required: ${{ true }} + number_required: 1 + secrets: inherit + + output-test: + runs-on: ubuntu-latest + needs: + - reusable-workflow + - reusable-workflow-with-inherited-secrets + steps: + - name: output with secrets map + run: | + echo reusable-workflow.output=${{ needs.reusable-workflow.outputs.output }} + [[ "${{ needs.reusable-workflow.outputs.output == 'string' }}" = "true" ]] || exit 1 + + - name: output with inherited secrets + run: | + echo reusable-workflow-with-inherited-secrets.output=${{ needs.reusable-workflow-with-inherited-secrets.outputs.output }} + [[ "${{ needs.reusable-workflow-with-inherited-secrets.outputs.output == 'string' }}" = "true" ]] || exit 1 From 06249a9225db3407369bfb95e861fbbf8cec0fa2 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Fri, 20 Jan 2023 16:46:43 +0100 Subject: [PATCH 35/73] refactor: pull and rebuild docker by default (#1569) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- cmd/root.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index d60e4c87..23f411c9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -52,8 +52,8 @@ func Execute(ctx context.Context, version string) { rootCmd.Flags().StringArrayVarP(&input.platforms, "platform", "P", []string{}, "custom image to use per platform (e.g. -P ubuntu-18.04=nektos/act-environments-ubuntu:18.04)") rootCmd.Flags().BoolVarP(&input.reuseContainers, "reuse", "r", false, "don't remove container(s) on successfully completed workflow(s) to maintain state between runs") rootCmd.Flags().BoolVarP(&input.bindWorkdir, "bind", "b", false, "bind working directory to container, rather than copy") - rootCmd.Flags().BoolVarP(&input.forcePull, "pull", "p", false, "pull docker image(s) even if already present") - rootCmd.Flags().BoolVarP(&input.forceRebuild, "rebuild", "", false, "rebuild local action docker image(s) even if already present") + rootCmd.Flags().BoolVarP(&input.forcePull, "pull", "p", true, "pull docker image(s) even if already present") + rootCmd.Flags().BoolVarP(&input.forceRebuild, "rebuild", "", true, "rebuild local action docker image(s) even if already present") rootCmd.Flags().BoolVarP(&input.autodetectEvent, "detect-event", "", false, "Use first event type from workflow as event that triggered the workflow") rootCmd.Flags().StringVarP(&input.eventPath, "eventpath", "e", "", "path to event JSON file") rootCmd.Flags().StringVar(&input.defaultBranch, "defaultbranch", "", "the name of the main branch") From d5753c760d93e193ae8e97403e535d4e2e96ed05 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Fri, 20 Jan 2023 10:10:20 -0800 Subject: [PATCH 36/73] feat: release extension --- .github/workflows/gh-extension.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/gh-extension.yml diff --git a/.github/workflows/gh-extension.yml b/.github/workflows/gh-extension.yml new file mode 100644 index 00000000..46a4024f --- /dev/null +++ b/.github/workflows/gh-extension.yml @@ -0,0 +1,25 @@ +name: gh-extension +on: + workflow_dispatch + +jobs: + release: + name: release + runs-on: ubuntu-latest + steps: + - name: Release extension + uses: actions/github-script@v5 + with: + github-token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} + script: | + const mainRef = github.rest.git.getRef({ + owner: 'nektos', + repo: 'gh-act', + ref: 'heads/main', + }); + github.rest.git.createRef({ + owner: 'nektos', + repo: 'gh-act', + ref: 'refs/tags/v0.2.39', + sha: mainRef.object.sha, + }); From 5b30187a52a068a32a26b2502027d17f62c81a48 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Fri, 20 Jan 2023 10:11:34 -0800 Subject: [PATCH 37/73] feat: release extension --- .github/workflows/gh-extension.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gh-extension.yml b/.github/workflows/gh-extension.yml index 46a4024f..4580ce79 100644 --- a/.github/workflows/gh-extension.yml +++ b/.github/workflows/gh-extension.yml @@ -12,11 +12,12 @@ jobs: with: github-token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} script: | - const mainRef = github.rest.git.getRef({ + const mainRef = await github.rest.git.getRef({ owner: 'nektos', repo: 'gh-act', ref: 'heads/main', }); + console.log(mainRef); github.rest.git.createRef({ owner: 'nektos', repo: 'gh-act', From a34ad7e64833c7d90d2a23bc40bbbc319107c45b Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Fri, 20 Jan 2023 10:13:18 -0800 Subject: [PATCH 38/73] feat: release extension --- .github/workflows/gh-extension.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh-extension.yml b/.github/workflows/gh-extension.yml index 4580ce79..e42bdeb2 100644 --- a/.github/workflows/gh-extension.yml +++ b/.github/workflows/gh-extension.yml @@ -12,11 +12,11 @@ jobs: with: github-token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} script: | - const mainRef = await github.rest.git.getRef({ + const mainRef = (await github.rest.git.getRef({ owner: 'nektos', repo: 'gh-act', ref: 'heads/main', - }); + })).data; console.log(mainRef); github.rest.git.createRef({ owner: 'nektos', From d70ed0dbbf9d3eca14a4e9b4f801a5842332f1fc Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Fri, 20 Jan 2023 10:22:58 -0800 Subject: [PATCH 39/73] chore: update docs for installing act as GH CLI extension --- .github/workflows/gh-extension.yml | 26 -------------------------- .github/workflows/release.yml | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 26 deletions(-) delete mode 100644 .github/workflows/gh-extension.yml diff --git a/.github/workflows/gh-extension.yml b/.github/workflows/gh-extension.yml deleted file mode 100644 index e42bdeb2..00000000 --- a/.github/workflows/gh-extension.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: gh-extension -on: - workflow_dispatch - -jobs: - release: - name: release - runs-on: ubuntu-latest - steps: - - name: Release extension - uses: actions/github-script@v5 - with: - github-token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} - script: | - const mainRef = (await github.rest.git.getRef({ - owner: 'nektos', - repo: 'gh-act', - ref: 'heads/main', - })).data; - console.log(mainRef); - github.rest.git.createRef({ - owner: 'nektos', - repo: 'gh-act', - ref: 'refs/tags/v0.2.39', - sha: mainRef.object.sha, - }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2dce51b0..a08896d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,3 +39,20 @@ jobs: version: ${{ github.ref }} apiKey: ${{ secrets.CHOCO_APIKEY }} push: true + - name: GitHub CLI extension + uses: actions/github-script@v5 + with: + github-token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} + script: | + const mainRef = (await github.rest.git.getRef({ + owner: 'nektos', + repo: 'gh-act', + ref: 'heads/main', + })).data; + console.log(mainRef); + github.rest.git.createRef({ + owner: 'nektos', + repo: 'gh-act', + ref: `refs/tags/${context.ref}`, + sha: mainRef.object.sha, + }); From 2e6ebbac55de0ef2973fc6f87156aafe8f271579 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jan 2023 02:56:38 +0000 Subject: [PATCH 40/73] build(deps): bump actions/github-script from 5 to 6 (#1572) Bumps [actions/github-script](https://github.com/actions/github-script) from 5 to 6. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/github-script dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a08896d9..b5543cc8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,7 +40,7 @@ jobs: apiKey: ${{ secrets.CHOCO_APIKEY }} push: true - name: GitHub CLI extension - uses: actions/github-script@v5 + uses: actions/github-script@v6 with: github-token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} script: | From 97fb7ba580730ca348edb59319502132e5c4a1f0 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 25 Jan 2023 09:14:51 +0100 Subject: [PATCH 41/73] feat: step summary of test results (#1580) * feat: step summary of test results * fix: indent style * fix: handle failed tests * fix upload / create a logs artifact * Update checks.yml * fix: always upload logs * fix: run success * Move steps into a composite action * use args and not the hardcoded ones * format composite action * format --- .github/actions/run-tests/action.yml | 77 ++++++++++++++++++++++++++++ .github/workflows/checks.yml | 12 +++-- 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 .github/actions/run-tests/action.yml diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml new file mode 100644 index 00000000..09cb3dae --- /dev/null +++ b/.github/actions/run-tests/action.yml @@ -0,0 +1,77 @@ +name: 'run-tests' +description: 'Runs go test and upload a step summary' +inputs: + filter: + description: 'The go test pattern for the tests to run' + required: false + default: '' + upload-logs-name: + description: 'Choose the name of the log artifact' + required: false + default: logs-${{ github.job }}-${{ strategy.job-index }} + upload-logs: + description: 'If true uploads logs of each tests as an artifact' + required: false + default: 'true' +runs: + using: composite + steps: + - uses: actions/github-script@v6 + with: + github-token: none # No reason to grant access to the GITHUB_TOKEN + script: | + let myOutput = ''; + var fs = require('fs'); + var uploadLogs = process.env.UPLOAD_LOGS === 'true'; + if(uploadLogs) { + await io.mkdirP('logs'); + } + var filename = null; + const options = {}; + options.ignoreReturnCode = true; + options.env = Object.assign({}, process.env); + delete options.env.ACTIONS_RUNTIME_URL; + delete options.env.ACTIONS_RUNTIME_TOKEN; + delete options.env.ACTIONS_CACHE_URL; + options.listeners = { + stdout: (data) => { + for(line of data.toString().split('\n')) { + if(/^\s*(===\s[^\s]+\s|---\s[^\s]+:\s)/.test(line)) { + if(uploadLogs) { + var runprefix = "=== RUN "; + if(line.startsWith(runprefix)) { + filename = "logs/" + line.substring(runprefix.length).replace(/[^A-Za-z0-9]/g, '-') + ".txt"; + fs.writeFileSync(filename, line + "\n"); + } else if(filename) { + fs.appendFileSync(filename, line + "\n"); + filename = null; + } + } + myOutput += line + "\n"; + } else if(filename) { + fs.appendFileSync(filename, line + "\n"); + } + } + } + }; + var args = ['test', '-v', '-cover', '-coverprofile=coverage.txt', '-covermode=atomic', '-timeout', '15m']; + var filter = process.env.FILTER; + if(filter) { + args.push('-run'); + args.push(filter); + } + args.push('./...'); + var exitcode = await exec.exec('go', args, options); + if(process.env.GITHUB_STEP_SUMMARY) { + core.summary.addCodeBlock(myOutput); + await core.summary.write(); + } + process.exit(exitcode); + env: + FILTER: ${{ inputs.filter }} + UPLOAD_LOGS: ${{ inputs.upload-logs }} + - uses: actions/upload-artifact@v3 + if: always() && inputs.upload-logs == 'true' && !env.ACT + with: + name: ${{ inputs.upload-logs-name }} + path: logs diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 781675fc..561d00c1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -50,7 +50,10 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- - - run: go test -v -cover -coverprofile=coverage.txt -covermode=atomic -timeout 15m ./... + - name: Run Tests + uses: ./.github/actions/run-tests + with: + upload-logs-name: logs-linux - name: Upload Codecov report uses: codecov/codecov-action@v3.1.1 with: @@ -73,8 +76,11 @@ jobs: with: go-version: ${{ env.GO_VERSION }} check-latest: true - - run: go test -v -run ^TestRunEventHostEnvironment$ ./... - # TODO merge coverage with test-linux + - name: Run Tests + uses: ./.github/actions/run-tests + with: + filter: '^TestRunEventHostEnvironment$' + upload-logs-name: logs-${{ matrix.os }} snapshot: name: snapshot From 4dd6cc398605b4be9d8a0701ea40724a1a07dbb0 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sun, 29 Jan 2023 15:47:56 +0100 Subject: [PATCH 42/73] test: Do not leak step env in composite (#1585) * test: Do not leak step env in composite To prevent merging regressions. * Update runner_test.go --- act/runner/runner_test.go | 5 ++++- .../do-not-leak-step-env-in-composite/push.yml | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 act/runner/testdata/do-not-leak-step-env-in-composite/push.yml diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 9298afcc..0b2f9399 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -203,6 +203,7 @@ func TestRunEvent(t *testing.T) { // {"testdata", "issue-228", "push", "", platforms, }, // TODO [igni]: Remove this once everything passes {"../model/testdata", "container-volumes", "push", "", platforms, secrets}, {workdir, "path-handling", "push", "", platforms, secrets}, + {workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets}, } for _, table := range tables { @@ -295,12 +296,14 @@ func TestRunEventHostEnvironment(t *testing.T) { }...) } else { platforms := map[string]string{ - "self-hosted": "-self-hosted", + "self-hosted": "-self-hosted", + "ubuntu-latest": "-self-hosted", } tables = append(tables, []TestJobFileInfo{ {workdir, "nix-prepend-path", "push", "", platforms, secrets}, {workdir, "inputs-via-env-context", "push", "", platforms, secrets}, + {workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets}, }...) } diff --git a/act/runner/testdata/do-not-leak-step-env-in-composite/push.yml b/act/runner/testdata/do-not-leak-step-env-in-composite/push.yml new file mode 100644 index 00000000..df5aab7f --- /dev/null +++ b/act/runner/testdata/do-not-leak-step-env-in-composite/push.yml @@ -0,0 +1,17 @@ +on: push +jobs: + _: + runs-on: ubuntu-latest + steps: + - run: | + runs: + using: composite + steps: + - run: exit 1 + if: env.LEAK_ENV != 'val' + shell: cp {0} action.yml + - uses: ./ + env: + LEAK_ENV: val + - run: exit 1 + if: env.LEAK_ENV == 'val' \ No newline at end of file From a54e2029af0cb5620bed73f444d96e9992ca5af3 Mon Sep 17 00:00:00 2001 From: R Date: Sun, 29 Jan 2023 17:44:53 +0100 Subject: [PATCH 43/73] cI: make stalebot less annoying (#1587) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 2102cf3d..91810627 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -19,5 +19,5 @@ jobs: exempt-pr-labels: 'stale-exempt' remove-stale-when-updated: 'True' operations-per-run: 500 - days-before-stale: 30 + days-before-stale: 180 days-before-close: 14 From 2969fb1f4b4b295ee34085356c535d82bf0cd1e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jan 2023 02:26:24 +0000 Subject: [PATCH 44/73] build(deps): bump golangci/golangci-lint-action from 3.3.1 to 3.4.0 (#1590) Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.3.1 to 3.4.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v3.3.1...v3.4.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 561d00c1..38514270 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,7 +19,7 @@ jobs: with: go-version: ${{ env.GO_VERSION }} check-latest: true - - uses: golangci/golangci-lint-action@v3.3.1 + - uses: golangci/golangci-lint-action@v3.4.0 with: version: v1.47.2 - uses: megalinter/megalinter/flavors/go@v6.18.0 From 4510ea8edd8eed02a49baf5ac4efff47c97d94af Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Tue, 31 Jan 2023 18:55:22 -0800 Subject: [PATCH 45/73] chore: fix release script to trigger gh-act --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5543cc8..3ad8665d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,6 +53,6 @@ jobs: github.rest.git.createRef({ owner: 'nektos', repo: 'gh-act', - ref: `refs/tags/${context.ref}`, + ref: context.ref, sha: mainRef.object.sha, }); From 4028d0c1ad207da8527d4ea326cccfaab08bc0c1 Mon Sep 17 00:00:00 2001 From: Casey Lee Date: Wed, 1 Feb 2023 16:54:57 -0800 Subject: [PATCH 46/73] feat: cache notices to reduce frequency of upgrade notifications (#1592) * feat: cache notices to reduce frequency of upgrade notifications * fix: reduce WriteFile permissions * fix: remove reference to deprecated lib * fix: handle HTTP status 304 --- cmd/notices.go | 62 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/cmd/notices.go b/cmd/notices.go index 4ad32d73..ffa84312 100644 --- a/cmd/notices.go +++ b/cmd/notices.go @@ -6,9 +6,12 @@ import ( "net/http" "net/url" "os" + "path/filepath" "runtime" + "strings" "time" + "github.com/mitchellh/go-homedir" log "github.com/sirupsen/logrus" ) @@ -73,14 +76,37 @@ func getVersionNotices(version string) []Notice { noticeURL.RawQuery = query.Encode() - resp, err := http.Get(noticeURL.String()) + client := &http.Client{} + req, err := http.NewRequest("GET", noticeURL.String(), nil) if err != nil { log.Debug(err) return nil } + etag := loadNoticesEtag() + if etag != "" { + log.Debugf("Conditional GET for notices etag=%s", etag) + req.Header.Set("If-None-Match", etag) + } + + resp, err := client.Do(req) + if err != nil { + log.Debug(err) + return nil + } + + newEtag := resp.Header.Get("Etag") + if newEtag != "" { + log.Debugf("Saving notices etag=%s", newEtag) + saveNoticesEtag(newEtag) + } + defer resp.Body.Close() notices := []Notice{} + if resp.StatusCode == 304 { + log.Debug("No new notices") + return nil + } if err := json.NewDecoder(resp.Body).Decode(¬ices); err != nil { log.Debug(err) return nil @@ -88,3 +114,37 @@ func getVersionNotices(version string) []Notice { return notices } + +func loadNoticesEtag() string { + p := etagPath() + content, err := os.ReadFile(p) + if err != nil { + log.Debugf("Unable to load etag from %s: %e", p, err) + } + return strings.TrimSuffix(string(content), "\n") +} + +func saveNoticesEtag(etag string) { + p := etagPath() + err := os.WriteFile(p, []byte(strings.TrimSuffix(etag, "\n")), 0600) + if err != nil { + log.Debugf("Unable to save etag to %s: %e", p, err) + } +} + +func etagPath() string { + var xdgCache string + var ok bool + if xdgCache, ok = os.LookupEnv("XDG_CACHE_HOME"); !ok || xdgCache == "" { + if home, err := homedir.Dir(); err == nil { + xdgCache = filepath.Join(home, ".cache") + } else if xdgCache, err = filepath.Abs("."); err != nil { + log.Fatal(err) + } + } + dir := filepath.Join(xdgCache, "act") + if err := os.MkdirAll(dir, 0777); err != nil { + log.Fatal(err) + } + return filepath.Join(dir, ".notices.etag") +} From 596c0899f553b9ad1581c936c5c534246bf3c2a2 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Thu, 2 Feb 2023 18:24:35 +0100 Subject: [PATCH 47/73] fix: Apply forcePull only for prebuild docker actions (#1599) --- act/runner/action.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/act/runner/action.go b/act/runner/action.go index 5614414e..347544ed 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -221,8 +221,11 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based var prepImage common.Executor var image string + forcePull := false if strings.HasPrefix(action.Runs.Image, "docker://") { image = strings.TrimPrefix(action.Runs.Image, "docker://") + // Apply forcePull only for prebuild docker images + forcePull = rc.Config.ForcePull } else { // "-dockeraction" enshures that "./", "./test " won't get converted to "act-:latest", "act-test-:latest" which are invalid docker image names image = fmt.Sprintf("%s-dockeraction:%s", regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(actionName, "-"), "latest") @@ -289,7 +292,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based stepContainer := newStepContainer(ctx, step, image, cmd, entrypoint) return common.NewPipelineExecutor( prepImage, - stepContainer.Pull(rc.Config.ForcePull), + stepContainer.Pull(forcePull), stepContainer.Remove().IfBool(!rc.Config.ReuseContainers), stepContainer.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop), stepContainer.Start(true), From 2164524d6209bd1c6ec4039feead1fff48140e63 Mon Sep 17 00:00:00 2001 From: Aidan Date: Fri, 3 Feb 2023 01:07:16 -0800 Subject: [PATCH 48/73] Docker build fixes (#1596) - Join relative path and split dockerfile off to get context Signed-off-by: Aidan Jensen Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/action.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/act/runner/action.go b/act/runner/action.go index 347544ed..01e56f1b 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -231,7 +231,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based image = fmt.Sprintf("%s-dockeraction:%s", regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(actionName, "-"), "latest") image = fmt.Sprintf("act-%s", strings.TrimLeft(image, "-")) image = strings.ToLower(image) - contextDir := filepath.Join(basedir, action.Runs.Main) + contextDir, _ := filepath.Split(filepath.Join(basedir, action.Runs.Image)) anyArchExists, err := container.ImageExistsLocally(ctx, image, "any") if err != nil { From e8f2d13c434836f0343ad2556edaaa608964cc55 Mon Sep 17 00:00:00 2001 From: Shubh Bapna <38372682+shubhbapna@users.noreply.github.com> Date: Fri, 3 Feb 2023 14:35:49 -0500 Subject: [PATCH 49/73] feat: allow overriding of `GITHUB_` env variables (#1582) * allow overriding of GITHUB_ env variables * bug fix for overriding env vars with empty string * revert step.go * refactor github_context to prevent lint failures. added more setters * added ability to override github env variables * handled base and head ref --- act/model/github_context.go | 71 ++++++++++++++++-- act/model/github_context_test.go | 119 ++++++++++++++++++++++++------- act/runner/run_context.go | 83 +++++++++++---------- 3 files changed, 199 insertions(+), 74 deletions(-) diff --git a/act/model/github_context.go b/act/model/github_context.go index 86172dfa..9ce8d08a 100644 --- a/act/model/github_context.go +++ b/act/model/github_context.go @@ -3,6 +3,7 @@ package model import ( "context" "fmt" + "strings" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/common/git" @@ -89,26 +90,22 @@ func withDefaultBranch(ctx context.Context, b string, event map[string]interface var findGitRef = git.FindGitRef var findGitRevision = git.FindGitRevision -func (ghc *GithubContext) SetRefAndSha(ctx context.Context, defaultBranch string, repoPath string) { +func (ghc *GithubContext) SetRef(ctx context.Context, defaultBranch string, repoPath string) { logger := common.Logger(ctx) + // https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads switch ghc.EventName { case "pull_request_target": ghc.Ref = fmt.Sprintf("refs/heads/%s", ghc.BaseRef) - ghc.Sha = asString(nestedMapLookup(ghc.Event, "pull_request", "base", "sha")) case "pull_request", "pull_request_review", "pull_request_review_comment": ghc.Ref = fmt.Sprintf("refs/pull/%.0f/merge", ghc.Event["number"]) case "deployment", "deployment_status": ghc.Ref = asString(nestedMapLookup(ghc.Event, "deployment", "ref")) - ghc.Sha = asString(nestedMapLookup(ghc.Event, "deployment", "sha")) case "release": ghc.Ref = asString(nestedMapLookup(ghc.Event, "release", "tag_name")) case "push", "create", "workflow_dispatch": ghc.Ref = asString(ghc.Event["ref"]) - if deleted, ok := ghc.Event["deleted"].(bool); ok && !deleted { - ghc.Sha = asString(ghc.Event["after"]) - } default: defaultBranch := asString(nestedMapLookup(ghc.Event, "repository", "default_branch")) if defaultBranch != "" { @@ -136,6 +133,23 @@ func (ghc *GithubContext) SetRefAndSha(ctx context.Context, defaultBranch string ghc.Ref = fmt.Sprintf("refs/heads/%s", asString(nestedMapLookup(ghc.Event, "repository", "default_branch"))) } } +} + +func (ghc *GithubContext) SetSha(ctx context.Context, repoPath string) { + logger := common.Logger(ctx) + + // https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows + // https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads + switch ghc.EventName { + case "pull_request_target": + ghc.Sha = asString(nestedMapLookup(ghc.Event, "pull_request", "base", "sha")) + case "deployment", "deployment_status": + ghc.Sha = asString(nestedMapLookup(ghc.Event, "deployment", "sha")) + case "push", "create", "workflow_dispatch": + if deleted, ok := ghc.Event["deleted"].(bool); ok && !deleted { + ghc.Sha = asString(ghc.Event["after"]) + } + } if ghc.Sha == "" { _, sha, err := findGitRevision(ctx, repoPath) @@ -146,3 +160,48 @@ func (ghc *GithubContext) SetRefAndSha(ctx context.Context, defaultBranch string } } } + +func (ghc *GithubContext) SetRepositoryAndOwner(ctx context.Context, githubInstance string, remoteName string, repoPath string) { + if ghc.Repository == "" { + repo, err := git.FindGithubRepo(ctx, repoPath, githubInstance, remoteName) + if err != nil { + common.Logger(ctx).Warningf("unable to get git repo: %v", err) + return + } + ghc.Repository = repo + } + ghc.RepositoryOwner = strings.Split(ghc.Repository, "/")[0] +} + +func (ghc *GithubContext) SetRefTypeAndName() { + var refType, refName string + + // https://docs.github.com/en/actions/learn-github-actions/environment-variables + if strings.HasPrefix(ghc.Ref, "refs/tags/") { + refType = "tag" + refName = ghc.Ref[len("refs/tags/"):] + } else if strings.HasPrefix(ghc.Ref, "refs/heads/") { + refType = "branch" + refName = ghc.Ref[len("refs/heads/"):] + } + + if ghc.RefType == "" { + ghc.RefType = refType + } + + if ghc.RefName == "" { + ghc.RefName = refName + } +} + +func (ghc *GithubContext) SetBaseAndHeadRef() { + if ghc.EventName == "pull_request" || ghc.EventName == "pull_request_target" { + if ghc.BaseRef == "" { + ghc.BaseRef = asString(nestedMapLookup(ghc.Event, "pull_request", "base", "ref")) + } + + if ghc.HeadRef == "" { + ghc.HeadRef = asString(nestedMapLookup(ghc.Event, "pull_request", "head", "ref")) + } + } +} diff --git a/act/model/github_context_test.go b/act/model/github_context_test.go index a2900944..29bb546a 100644 --- a/act/model/github_context_test.go +++ b/act/model/github_context_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestSetRefAndSha(t *testing.T) { +func TestSetRef(t *testing.T) { log.SetLevel(log.DebugLevel) oldFindGitRef := findGitRef @@ -29,19 +29,11 @@ func TestSetRefAndSha(t *testing.T) { eventName string event map[string]interface{} ref string - sha string }{ { eventName: "pull_request_target", - event: map[string]interface{}{ - "pull_request": map[string]interface{}{ - "base": map[string]interface{}{ - "sha": "pr-base-sha", - }, - }, - }, - ref: "refs/heads/master", - sha: "pr-base-sha", + event: map[string]interface{}{}, + ref: "refs/heads/master", }, { eventName: "pull_request", @@ -49,18 +41,15 @@ func TestSetRefAndSha(t *testing.T) { "number": 1234., }, ref: "refs/pull/1234/merge", - sha: "1234fakesha", }, { eventName: "deployment", event: map[string]interface{}{ "deployment": map[string]interface{}{ "ref": "refs/heads/somebranch", - "sha": "deployment-sha", }, }, ref: "refs/heads/somebranch", - sha: "deployment-sha", }, { eventName: "release", @@ -70,17 +59,13 @@ func TestSetRefAndSha(t *testing.T) { }, }, ref: "v1.0.0", - sha: "1234fakesha", }, { eventName: "push", event: map[string]interface{}{ - "ref": "refs/heads/somebranch", - "after": "push-sha", - "deleted": false, + "ref": "refs/heads/somebranch", }, ref: "refs/heads/somebranch", - sha: "push-sha", }, { eventName: "unknown", @@ -90,13 +75,11 @@ func TestSetRefAndSha(t *testing.T) { }, }, ref: "refs/heads/main", - sha: "1234fakesha", }, { eventName: "no-event", event: map[string]interface{}{}, ref: "refs/heads/master", - sha: "1234fakesha", }, } @@ -108,10 +91,9 @@ func TestSetRefAndSha(t *testing.T) { Event: table.event, } - ghc.SetRefAndSha(context.Background(), "main", "/some/dir") + ghc.SetRef(context.Background(), "main", "/some/dir") assert.Equal(t, table.ref, ghc.Ref) - assert.Equal(t, table.sha, ghc.Sha) }) } @@ -125,9 +107,96 @@ func TestSetRefAndSha(t *testing.T) { Event: map[string]interface{}{}, } - ghc.SetRefAndSha(context.Background(), "", "/some/dir") + ghc.SetRef(context.Background(), "", "/some/dir") assert.Equal(t, "refs/heads/master", ghc.Ref) - assert.Equal(t, "1234fakesha", ghc.Sha) }) } + +func TestSetSha(t *testing.T) { + log.SetLevel(log.DebugLevel) + + oldFindGitRef := findGitRef + oldFindGitRevision := findGitRevision + defer func() { findGitRef = oldFindGitRef }() + defer func() { findGitRevision = oldFindGitRevision }() + + findGitRef = func(ctx context.Context, file string) (string, error) { + return "refs/heads/master", nil + } + + findGitRevision = func(ctx context.Context, file string) (string, string, error) { + return "", "1234fakesha", nil + } + + tables := []struct { + eventName string + event map[string]interface{} + sha string + }{ + { + eventName: "pull_request_target", + event: map[string]interface{}{ + "pull_request": map[string]interface{}{ + "base": map[string]interface{}{ + "sha": "pr-base-sha", + }, + }, + }, + sha: "pr-base-sha", + }, + { + eventName: "pull_request", + event: map[string]interface{}{ + "number": 1234., + }, + sha: "1234fakesha", + }, + { + eventName: "deployment", + event: map[string]interface{}{ + "deployment": map[string]interface{}{ + "sha": "deployment-sha", + }, + }, + sha: "deployment-sha", + }, + { + eventName: "release", + event: map[string]interface{}{}, + sha: "1234fakesha", + }, + { + eventName: "push", + event: map[string]interface{}{ + "after": "push-sha", + "deleted": false, + }, + sha: "push-sha", + }, + { + eventName: "unknown", + event: map[string]interface{}{}, + sha: "1234fakesha", + }, + { + eventName: "no-event", + event: map[string]interface{}{}, + sha: "1234fakesha", + }, + } + + for _, table := range tables { + t.Run(table.eventName, func(t *testing.T) { + ghc := &GithubContext{ + EventName: table.eventName, + BaseRef: "master", + Event: table.event, + } + + ghc.SetSha(context.Background(), "/some/dir") + + assert.Equal(t, table.sha, ghc.Sha) + }) + } +} diff --git a/act/runner/run_context.go b/act/runner/run_context.go index fa2da05d..db283287 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -21,7 +21,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/nektos/act/pkg/common" - "github.com/nektos/act/pkg/common/git" "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/exprparser" "github.com/nektos/act/pkg/model" @@ -571,6 +570,14 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext RetentionDays: rc.Config.Env["GITHUB_RETENTION_DAYS"], RunnerPerflog: rc.Config.Env["RUNNER_PERFLOG"], RunnerTrackingID: rc.Config.Env["RUNNER_TRACKING_ID"], + Repository: rc.Config.Env["GITHUB_REPOSITORY"], + Ref: rc.Config.Env["GITHUB_REF"], + Sha: rc.Config.Env["SHA_REF"], + RefName: rc.Config.Env["GITHUB_REF_NAME"], + RefType: rc.Config.Env["GITHUB_REF_TYPE"], + BaseRef: rc.Config.Env["GITHUB_BASE_REF"], + HeadRef: rc.Config.Env["GITHUB_HEAD_REF"], + Workspace: rc.Config.Env["GITHUB_WORKSPACE"], } if rc.JobContainer != nil { ghc.EventPath = rc.JobContainer.GetActPath() + "/workflow/event.json" @@ -599,39 +606,24 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext ghc.Actor = "nektos/act" } - repoPath := rc.Config.Workdir - repo, err := git.FindGithubRepo(ctx, repoPath, rc.Config.GitHubInstance, rc.Config.RemoteName) - if err != nil { - logger.Warningf("unable to get git repo: %v", err) - } else { - ghc.Repository = repo - if ghc.RepositoryOwner == "" { - ghc.RepositoryOwner = strings.Split(repo, "/")[0] - } - } - if rc.EventJSON != "" { - err = json.Unmarshal([]byte(rc.EventJSON), &ghc.Event) + err := json.Unmarshal([]byte(rc.EventJSON), &ghc.Event) if err != nil { logger.Errorf("Unable to Unmarshal event '%s': %v", rc.EventJSON, err) } } - if ghc.EventName == "pull_request" || ghc.EventName == "pull_request_target" { - ghc.BaseRef = asString(nestedMapLookup(ghc.Event, "pull_request", "base", "ref")) - ghc.HeadRef = asString(nestedMapLookup(ghc.Event, "pull_request", "head", "ref")) + ghc.SetBaseAndHeadRef() + repoPath := rc.Config.Workdir + ghc.SetRepositoryAndOwner(ctx, rc.Config.GitHubInstance, rc.Config.RemoteName, repoPath) + if ghc.Ref == "" { + ghc.SetRef(ctx, rc.Config.DefaultBranch, repoPath) + } + if ghc.Sha == "" { + ghc.SetSha(ctx, repoPath) } - ghc.SetRefAndSha(ctx, rc.Config.DefaultBranch, repoPath) - - // https://docs.github.com/en/actions/learn-github-actions/environment-variables - if strings.HasPrefix(ghc.Ref, "refs/tags/") { - ghc.RefType = "tag" - ghc.RefName = ghc.Ref[len("refs/tags/"):] - } else if strings.HasPrefix(ghc.Ref, "refs/heads/") { - ghc.RefType = "branch" - ghc.RefName = ghc.Ref[len("refs/heads/"):] - } + ghc.SetRefTypeAndName() return ghc } @@ -662,15 +654,6 @@ func isLocalCheckout(ghc *model.GithubContext, step *model.Step) bool { return true } -func asString(v interface{}) string { - if v == nil { - return "" - } else if s, ok := v.(string); ok { - return s - } - return "" -} - func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) { var ok bool @@ -709,20 +692,34 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon env["GITHUB_REF_NAME"] = github.RefName env["GITHUB_REF_TYPE"] = github.RefType env["GITHUB_TOKEN"] = github.Token - env["GITHUB_SERVER_URL"] = "https://github.com" - env["GITHUB_API_URL"] = "https://api.github.com" - env["GITHUB_GRAPHQL_URL"] = "https://api.github.com/graphql" - env["GITHUB_BASE_REF"] = github.BaseRef - env["GITHUB_HEAD_REF"] = github.HeadRef env["GITHUB_JOB"] = rc.JobName env["GITHUB_REPOSITORY_OWNER"] = github.RepositoryOwner env["GITHUB_RETENTION_DAYS"] = github.RetentionDays env["RUNNER_PERFLOG"] = github.RunnerPerflog env["RUNNER_TRACKING_ID"] = github.RunnerTrackingID + env["GITHUB_BASE_REF"] = github.BaseRef + env["GITHUB_HEAD_REF"] = github.HeadRef + + defaultServerURL := "https://github.com" + defaultAPIURL := "https://api.github.com" + defaultGraphqlURL := "https://api.github.com/graphql" + if rc.Config.GitHubInstance != "github.com" { - env["GITHUB_SERVER_URL"] = fmt.Sprintf("https://%s", rc.Config.GitHubInstance) - env["GITHUB_API_URL"] = fmt.Sprintf("https://%s/api/v3", rc.Config.GitHubInstance) - env["GITHUB_GRAPHQL_URL"] = fmt.Sprintf("https://%s/api/graphql", rc.Config.GitHubInstance) + defaultServerURL = fmt.Sprintf("https://%s", rc.Config.GitHubInstance) + defaultAPIURL = fmt.Sprintf("https://%s/api/v3", rc.Config.GitHubInstance) + defaultGraphqlURL = fmt.Sprintf("https://%s/api/graphql", rc.Config.GitHubInstance) + } + + if env["GITHUB_SERVER_URL"] == "" { + env["GITHUB_SERVER_URL"] = defaultServerURL + } + + if env["GITHUB_API_URL"] == "" { + env["GITHUB_API_URL"] = defaultAPIURL + } + + if env["GITHUB_GRAPHQL_URL"] == "" { + env["GITHUB_GRAPHQL_URL"] = defaultGraphqlURL } if rc.Config.ArtifactServerPath != "" { From 89a74e2a7429faee163147d1b9606fe355ddbe84 Mon Sep 17 00:00:00 2001 From: Aidan Date: Fri, 3 Feb 2023 11:54:19 -0800 Subject: [PATCH 50/73] Update max container name length (#1597) * Update max container name length Signed-off-by: Aidan Jensen * Use hashed name instead to prevent conflicts Signed-off-by: Aidan Jensen --------- Signed-off-by: Aidan Jensen Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index db283287..501eb71e 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -5,6 +5,7 @@ import ( "bufio" "context" "crypto/rand" + "crypto/sha256" "encoding/hex" "encoding/json" "errors" @@ -505,26 +506,16 @@ func mergeMaps(maps ...map[string]string) map[string]string { } func createContainerName(parts ...string) string { - name := make([]string, 0) + name := strings.Join(parts, "-") pattern := regexp.MustCompile("[^a-zA-Z0-9]") - partLen := (30 / len(parts)) - 1 - for i, part := range parts { - if i == len(parts)-1 { - name = append(name, pattern.ReplaceAllString(part, "-")) - } else { - // If any part has a '-' on the end it is likely part of a matrix job. - // Let's preserve the number to prevent clashes in container names. - re := regexp.MustCompile("-[0-9]+$") - num := re.FindStringSubmatch(part) - if len(num) > 0 { - name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen-len(num[0]))) - name = append(name, num[0]) - } else { - name = append(name, trimToLen(pattern.ReplaceAllString(part, "-"), partLen)) - } - } - } - return strings.ReplaceAll(strings.Trim(strings.Join(name, "-"), "-"), "--", "-") + name = pattern.ReplaceAllString(name, "-") + name = strings.ReplaceAll(name, "--", "-") + hash := sha256.Sum256([]byte(name)) + + // SHA256 is 64 hex characters. So trim name to 63 characters to make room for the hash and separator + trimmedName := strings.Trim(trimToLen(name, 63), "-") + + return fmt.Sprintf("%s-%x", trimmedName, hash) } func trimToLen(s string, l int) string { From 02e21de560692ab8aff4d3b4ad58f9bf9648fa1c Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sat, 4 Feb 2023 14:35:13 +0100 Subject: [PATCH 51/73] refactor: GITHUB_ENV command / remove env.PATH (#1503) * fix: GITHUB_ENV / PATH handling * apply workaround * add ctx to ApplyExtraPath * fix: Do not leak step env in composite See https://github.com/nektos/act/pull/1585 for a test * add more tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/action.go | 6 +-- act/runner/action_composite.go | 9 +++++ act/runner/command.go | 10 ++++- act/runner/run_context.go | 18 ++++++--- act/runner/runner_test.go | 8 +++- act/runner/step.go | 39 +++++++++++-------- act/runner/step_action_local_test.go | 16 +++----- act/runner/step_action_remote_test.go | 16 ++++---- act/runner/step_docker_test.go | 12 ++---- act/runner/step_run.go | 2 +- act/runner/step_run_test.go | 6 +-- act/runner/step_test.go | 4 -- .../GITHUB_ENV-use-in-env-ctx/push.yml | 27 +++++++++++++ .../push.yml | 1 + .../docker-action-custom-path/push.yml | 12 ++++++ .../remote-action-js-node-user/push.yml | 15 +++++++ .../set-env-new-env-file-per-step/push.yml | 15 +++++++ .../set-env-step-env-override/push.yml | 24 ++++++++++++ 18 files changed, 176 insertions(+), 64 deletions(-) create mode 100644 act/runner/testdata/GITHUB_ENV-use-in-env-ctx/push.yml create mode 100644 act/runner/testdata/docker-action-custom-path/push.yml create mode 100644 act/runner/testdata/remote-action-js-node-user/push.yml create mode 100644 act/runner/testdata/set-env-new-env-file-per-step/push.yml create mode 100644 act/runner/testdata/set-env-step-env-override/push.yml diff --git a/act/runner/action.go b/act/runner/action.go index 01e56f1b..1da76959 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -154,7 +154,7 @@ func runActionImpl(step actionStep, actionDir string, remoteAction *remoteAction containerArgs := []string{"node", path.Join(containerActionDir, action.Runs.Main)} logger.Debugf("executing remote job container: %s", containerArgs) - rc.ApplyExtraPath(step.getEnv()) + rc.ApplyExtraPath(ctx, step.getEnv()) return rc.execJobContainer(containerArgs, *step.getEnv(), "", "")(ctx) case model.ActionRunsUsingDocker: @@ -491,7 +491,7 @@ func runPreStep(step actionStep) common.Executor { containerArgs := []string{"node", path.Join(containerActionDir, action.Runs.Pre)} logger.Debugf("executing remote job container: %s", containerArgs) - rc.ApplyExtraPath(step.getEnv()) + rc.ApplyExtraPath(ctx, step.getEnv()) return rc.execJobContainer(containerArgs, *step.getEnv(), "", "")(ctx) @@ -580,7 +580,7 @@ func runPostStep(step actionStep) common.Executor { containerArgs := []string{"node", path.Join(containerActionDir, action.Runs.Post)} logger.Debugf("executing remote job container: %s", containerArgs) - rc.ApplyExtraPath(step.getEnv()) + rc.ApplyExtraPath(ctx, step.getEnv()) return rc.execJobContainer(containerArgs, *step.getEnv(), "", "")(ctx) diff --git a/act/runner/action_composite.go b/act/runner/action_composite.go index c1e94fcd..0b6fbe55 100644 --- a/act/runner/action_composite.go +++ b/act/runner/action_composite.go @@ -66,6 +66,7 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action JobContainer: parent.JobContainer, ActionPath: actionPath, Env: env, + GlobalEnv: parent.GlobalEnv, Masks: parent.Masks, ExtraPath: parent.ExtraPath, Parent: parent, @@ -99,6 +100,14 @@ func execAsComposite(step actionStep) common.Executor { rc.Masks = append(rc.Masks, compositeRC.Masks...) rc.ExtraPath = compositeRC.ExtraPath + // compositeRC.Env is dirty, contains INPUT_ and merged step env, only rely on compositeRC.GlobalEnv + for k, v := range compositeRC.GlobalEnv { + rc.Env[k] = v + if rc.GlobalEnv == nil { + rc.GlobalEnv = map[string]string{} + } + rc.GlobalEnv[k] = v + } return err } diff --git a/act/runner/command.go b/act/runner/command.go index 53e167cd..f14eb7aa 100644 --- a/act/runner/command.go +++ b/act/runner/command.go @@ -82,11 +82,17 @@ func (rc *RunContext) commandHandler(ctx context.Context) common.LineHandler { } func (rc *RunContext) setEnv(ctx context.Context, kvPairs map[string]string, arg string) { - common.Logger(ctx).Infof(" \U00002699 ::set-env:: %s=%s", kvPairs["name"], arg) + name := kvPairs["name"] + common.Logger(ctx).Infof(" \U00002699 ::set-env:: %s=%s", name, arg) if rc.Env == nil { rc.Env = make(map[string]string) } - rc.Env[kvPairs["name"]] = arg + rc.Env[name] = arg + // for composite action GITHUB_ENV and set-env passing + if rc.GlobalEnv == nil { + rc.GlobalEnv = map[string]string{} + } + rc.GlobalEnv[name] = arg } func (rc *RunContext) setOutput(ctx context.Context, kvPairs map[string]string, arg string) { logger := common.Logger(ctx) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 501eb71e..bfd090ce 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -35,6 +35,7 @@ type RunContext struct { Run *model.Run EventJSON string Env map[string]string + GlobalEnv map[string]string // to pass env changes of GITHUB_ENV and set-env correctly, due to dirty Env field ExtraPath []string CurrentStep string StepResults map[string]*model.StepResult @@ -275,8 +276,6 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.stopJobContainer(), rc.JobContainer.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop), rc.JobContainer.Start(false), - rc.JobContainer.UpdateFromImageEnv(&rc.Env), - rc.JobContainer.UpdateFromEnv("/etc/environment", &rc.Env), rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ Name: "workflow/event.json", Mode: 0644, @@ -296,11 +295,21 @@ func (rc *RunContext) execJobContainer(cmd []string, env map[string]string, user } } -func (rc *RunContext) ApplyExtraPath(env *map[string]string) { +func (rc *RunContext) ApplyExtraPath(ctx context.Context, env *map[string]string) { if rc.ExtraPath != nil && len(rc.ExtraPath) > 0 { path := rc.JobContainer.GetPathVariableName() if (*env)[path] == "" { - (*env)[path] = rc.JobContainer.DefaultPathVariable() + cenv := map[string]string{} + var cpath string + if err := rc.JobContainer.UpdateFromImageEnv(&cenv)(ctx); err == nil { + if p, ok := cenv[path]; ok { + cpath = p + } + } + if len(cpath) == 0 { + cpath = rc.JobContainer.DefaultPathVariable() + } + (*env)[path] = cpath } (*env)[path] = rc.JobContainer.JoinPathVariable(append(rc.ExtraPath, (*env)[path])...) } @@ -664,7 +673,6 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubContext, env map[string]string) map[string]string { env["CI"] = "true" - env["GITHUB_ENV"] = rc.JobContainer.GetActPath() + "/workflow/envs.txt" env["GITHUB_WORKFLOW"] = github.Workflow env["GITHUB_RUN_ID"] = github.RunID env["GITHUB_RUN_NUMBER"] = github.RunNumber diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 0b2f9399..0a83537e 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -168,7 +168,7 @@ func TestRunEvent(t *testing.T) { {workdir, "container-hostname", "push", "", platforms, secrets}, {workdir, "remote-action-docker", "push", "", platforms, secrets}, {workdir, "remote-action-js", "push", "", platforms, secrets}, - {workdir, "remote-action-js", "push", "", map[string]string{"ubuntu-latest": "catthehacker/ubuntu:runner-latest"}, secrets}, // Test if this works with non root container + {workdir, "remote-action-js-node-user", "push", "", platforms, secrets}, // Test if this works with non root container {workdir, "matrix", "push", "", platforms, secrets}, {workdir, "matrix-include-exclude", "push", "", platforms, secrets}, {workdir, "matrix-exitcode", "push", "Job 'test' failed", platforms, secrets}, @@ -193,6 +193,8 @@ func TestRunEvent(t *testing.T) { {workdir, "actions-environment-and-context-tests", "push", "", platforms, secrets}, {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms, secrets}, {workdir, "evalenv", "push", "", platforms, secrets}, + {workdir, "docker-action-custom-path", "push", "", platforms, secrets}, + {workdir, "GITHUB_ENV-use-in-env-ctx", "push", "", platforms, secrets}, {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms, secrets}, {workdir, "workflow_dispatch", "workflow_dispatch", "", platforms, secrets}, {workdir, "workflow_dispatch_no_inputs_mapping", "workflow_dispatch", "", platforms, secrets}, @@ -204,6 +206,8 @@ func TestRunEvent(t *testing.T) { {"../model/testdata", "container-volumes", "push", "", platforms, secrets}, {workdir, "path-handling", "push", "", platforms, secrets}, {workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets}, + {workdir, "set-env-step-env-override", "push", "", platforms, secrets}, + {workdir, "set-env-new-env-file-per-step", "push", "", platforms, secrets}, } for _, table := range tables { @@ -304,6 +308,8 @@ func TestRunEventHostEnvironment(t *testing.T) { {workdir, "nix-prepend-path", "push", "", platforms, secrets}, {workdir, "inputs-via-env-context", "push", "", platforms, secrets}, {workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets}, + {workdir, "set-env-step-env-override", "push", "", platforms, secrets}, + {workdir, "set-env-new-env-file-per-step", "push", "", platforms, secrets}, }...) } diff --git a/act/runner/step.go b/act/runner/step.go index 2e211a30..d66827a9 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -44,6 +44,18 @@ func (s stepStage) String() string { return "Unknown" } +func processRunnerEnvFileCommand(ctx context.Context, fileName string, rc *RunContext, setter func(context.Context, map[string]string, string)) error { + env := map[string]string{} + err := rc.JobContainer.UpdateFromEnv(path.Join(rc.JobContainer.GetActPath(), fileName), &env)(ctx) + if err != nil { + return err + } + for k, v := range env { + setter(ctx, map[string]string{"name": k}, v) + } + return nil +} + func runStepExecutor(step step, stage stepStage, executor common.Executor) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) @@ -92,9 +104,11 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo outputFileCommand := path.Join("workflow", "outputcmd.txt") stateFileCommand := path.Join("workflow", "statecmd.txt") pathFileCommand := path.Join("workflow", "pathcmd.txt") + envFileCommand := path.Join("workflow", "envs.txt") (*step.getEnv())["GITHUB_OUTPUT"] = path.Join(actPath, outputFileCommand) (*step.getEnv())["GITHUB_STATE"] = path.Join(actPath, stateFileCommand) (*step.getEnv())["GITHUB_PATH"] = path.Join(actPath, pathFileCommand) + (*step.getEnv())["GITHUB_ENV"] = path.Join(actPath, envFileCommand) _ = rc.JobContainer.Copy(actPath, &container.FileEntry{ Name: outputFileCommand, Mode: 0666, @@ -104,6 +118,9 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo }, &container.FileEntry{ Name: pathFileCommand, Mode: 0666, + }, &container.FileEntry{ + Name: envFileCommand, + Mode: 0666, })(ctx) err = executor(ctx) @@ -131,21 +148,17 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo } // Process Runner File Commands orgerr := err - state := map[string]string{} - err = rc.JobContainer.UpdateFromEnv(path.Join(actPath, stateFileCommand), &state)(ctx) + err = processRunnerEnvFileCommand(ctx, envFileCommand, rc, rc.setEnv) if err != nil { return err } - for k, v := range state { - rc.saveState(ctx, map[string]string{"name": k}, v) - } - output := map[string]string{} - err = rc.JobContainer.UpdateFromEnv(path.Join(actPath, outputFileCommand), &output)(ctx) + err = processRunnerEnvFileCommand(ctx, stateFileCommand, rc, rc.saveState) if err != nil { return err } - for k, v := range output { - rc.setOutput(ctx, map[string]string{"name": k}, v) + err = processRunnerEnvFileCommand(ctx, outputFileCommand, rc, rc.setOutput) + if err != nil { + return err } err = rc.UpdateExtraPath(ctx, path.Join(actPath, pathFileCommand)) if err != nil { @@ -162,14 +175,6 @@ func setupEnv(ctx context.Context, step step) error { rc := step.getRunContext() mergeEnv(ctx, step) - err := rc.JobContainer.UpdateFromImageEnv(step.getEnv())(ctx) - if err != nil { - return err - } - err = rc.JobContainer.UpdateFromEnv((*step.getEnv())["GITHUB_ENV"], step.getEnv())(ctx) - if err != nil { - return err - } // merge step env last, since it should not be overwritten mergeIntoMap(step.getEnv(), step.getStepModel().GetEnv()) diff --git a/act/runner/step_action_local_test.go b/act/runner/step_action_local_test.go index 023f7018..5fe7f291 100644 --- a/act/runner/step_action_local_test.go +++ b/act/runner/step_action_local_test.go @@ -69,7 +69,7 @@ func TestStepActionLocalTest(t *testing.T) { salm.On("readAction", sal.Step, filepath.Clean("/tmp/path/to/action"), "", mock.Anything, mock.Anything). Return(&model.Action{}, nil) - cm.On("UpdateFromImageEnv", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { return nil }) @@ -77,10 +77,6 @@ func TestStepActionLocalTest(t *testing.T) { return nil }) - cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { - return nil - }) - cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil }) @@ -197,7 +193,7 @@ func TestStepActionLocalPost(t *testing.T) { env bool exec bool }{ - env: true, + env: false, exec: false, }, }, @@ -260,10 +256,6 @@ func TestStepActionLocalPost(t *testing.T) { } sal.RunContext.ExprEval = sal.RunContext.NewExpressionEvaluator(ctx) - if tt.mocks.env { - cm.On("UpdateFromImageEnv", &sal.env).Return(func(ctx context.Context) error { return nil }) - cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", &sal.env).Return(func(ctx context.Context) error { return nil }) - } if tt.mocks.exec { suffixMatcher := func(suffix string) interface{} { return mock.MatchedBy(func(array []string) bool { @@ -276,6 +268,10 @@ func TestStepActionLocalPost(t *testing.T) { return nil }) + cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil }) diff --git a/act/runner/step_action_remote_test.go b/act/runner/step_action_remote_test.go index a6653f7d..23d65545 100644 --- a/act/runner/step_action_remote_test.go +++ b/act/runner/step_action_remote_test.go @@ -165,10 +165,6 @@ func TestStepActionRemote(t *testing.T) { }) } - if tt.mocks.env { - cm.On("UpdateFromImageEnv", &sar.env).Return(func(ctx context.Context) error { return nil }) - cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", &sar.env).Return(func(ctx context.Context) error { return nil }) - } if tt.mocks.read { sarm.On("readAction", sar.Step, suffixMatcher("act/remote-action@v1"), "", mock.Anything, mock.Anything).Return(&model.Action{}, nil) } @@ -179,6 +175,10 @@ func TestStepActionRemote(t *testing.T) { return nil }) + cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil }) @@ -579,10 +579,6 @@ func TestStepActionRemotePost(t *testing.T) { } sar.RunContext.ExprEval = sar.RunContext.NewExpressionEvaluator(ctx) - if tt.mocks.env { - cm.On("UpdateFromImageEnv", &sar.env).Return(func(ctx context.Context) error { return nil }) - cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", &sar.env).Return(func(ctx context.Context) error { return nil }) - } if tt.mocks.exec { cm.On("Exec", []string{"node", "/var/run/act/actions/remote-action@v1/post.js"}, sar.env, "", "").Return(func(ctx context.Context) error { return tt.err }) @@ -590,6 +586,10 @@ func TestStepActionRemotePost(t *testing.T) { return nil }) + cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil }) diff --git a/act/runner/step_docker_test.go b/act/runner/step_docker_test.go index c26ffd68..3d90ac34 100644 --- a/act/runner/step_docker_test.go +++ b/act/runner/step_docker_test.go @@ -57,14 +57,6 @@ func TestStepDockerMain(t *testing.T) { } sd.RunContext.ExprEval = sd.RunContext.NewExpressionEvaluator(ctx) - cm.On("UpdateFromImageEnv", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { - return nil - }) - - cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { - return nil - }) - cm.On("Pull", false).Return(func(ctx context.Context) error { return nil }) @@ -89,6 +81,10 @@ func TestStepDockerMain(t *testing.T) { return nil }) + cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil }) diff --git a/act/runner/step_run.go b/act/runner/step_run.go index 17db77af..f833fc1a 100644 --- a/act/runner/step_run.go +++ b/act/runner/step_run.go @@ -30,7 +30,7 @@ func (sr *stepRun) main() common.Executor { return runStepExecutor(sr, stepStageMain, common.NewPipelineExecutor( sr.setupShellCommandExecutor(), func(ctx context.Context) error { - sr.getRunContext().ApplyExtraPath(&sr.env) + sr.getRunContext().ApplyExtraPath(ctx, &sr.env) return sr.getRunContext().JobContainer.Exec(sr.cmd, sr.env, "", sr.Step.WorkingDirectory)(ctx) }, )) diff --git a/act/runner/step_run_test.go b/act/runner/step_run_test.go index 324ed5fc..4ca2eb97 100644 --- a/act/runner/step_run_test.go +++ b/act/runner/step_run_test.go @@ -55,7 +55,7 @@ func TestStepRun(t *testing.T) { return nil }) - cm.On("UpdateFromImageEnv", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { return nil }) @@ -63,10 +63,6 @@ func TestStepRun(t *testing.T) { return nil }) - cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { - return nil - }) - cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil }) diff --git a/act/runner/step_test.go b/act/runner/step_test.go index 8930cca3..86e5acc4 100644 --- a/act/runner/step_test.go +++ b/act/runner/step_test.go @@ -148,9 +148,6 @@ func TestSetupEnv(t *testing.T) { sm.On("getStepModel").Return(step) sm.On("getEnv").Return(&env) - cm.On("UpdateFromImageEnv", &env).Return(func(ctx context.Context) error { return nil }) - cm.On("UpdateFromEnv", "/var/run/act/workflow/envs.txt", &env).Return(func(ctx context.Context) error { return nil }) - err := setupEnv(context.Background(), sm) assert.Nil(t, err) @@ -174,7 +171,6 @@ func TestSetupEnv(t *testing.T) { "GITHUB_ACTION_REPOSITORY": "", "GITHUB_API_URL": "https:///api/v3", "GITHUB_BASE_REF": "", - "GITHUB_ENV": "/var/run/act/workflow/envs.txt", "GITHUB_EVENT_NAME": "", "GITHUB_EVENT_PATH": "/var/run/act/workflow/event.json", "GITHUB_GRAPHQL_URL": "https:///api/graphql", diff --git a/act/runner/testdata/GITHUB_ENV-use-in-env-ctx/push.yml b/act/runner/testdata/GITHUB_ENV-use-in-env-ctx/push.yml new file mode 100644 index 00000000..c7b75a02 --- /dev/null +++ b/act/runner/testdata/GITHUB_ENV-use-in-env-ctx/push.yml @@ -0,0 +1,27 @@ +on: push +jobs: + _: + runs-on: ubuntu-latest + env: + MYGLOBALENV3: myglobalval3 + steps: + - run: | + echo MYGLOBALENV1=myglobalval1 > $GITHUB_ENV + echo "::set-env name=MYGLOBALENV2::myglobalval2" + - uses: nektos/act-test-actions/script@main + with: + main: | + env + [[ "$MYGLOBALENV1" = "${{ env.MYGLOBALENV1 }}" ]] + [[ "$MYGLOBALENV1" = "${{ env.MYGLOBALENV1ALIAS }}" ]] + [[ "$MYGLOBALENV1" = "$MYGLOBALENV1ALIAS" ]] + [[ "$MYGLOBALENV2" = "${{ env.MYGLOBALENV2 }}" ]] + [[ "$MYGLOBALENV2" = "${{ env.MYGLOBALENV2ALIAS }}" ]] + [[ "$MYGLOBALENV2" = "$MYGLOBALENV2ALIAS" ]] + [[ "$MYGLOBALENV3" = "${{ env.MYGLOBALENV3 }}" ]] + [[ "$MYGLOBALENV3" = "${{ env.MYGLOBALENV3ALIAS }}" ]] + [[ "$MYGLOBALENV3" = "$MYGLOBALENV3ALIAS" ]] + env: + MYGLOBALENV1ALIAS: ${{ env.MYGLOBALENV1 }} + MYGLOBALENV2ALIAS: ${{ env.MYGLOBALENV2 }} + MYGLOBALENV3ALIAS: ${{ env.MYGLOBALENV3 }} \ No newline at end of file diff --git a/act/runner/testdata/do-not-leak-step-env-in-composite/push.yml b/act/runner/testdata/do-not-leak-step-env-in-composite/push.yml index df5aab7f..1bebab0b 100644 --- a/act/runner/testdata/do-not-leak-step-env-in-composite/push.yml +++ b/act/runner/testdata/do-not-leak-step-env-in-composite/push.yml @@ -8,6 +8,7 @@ jobs: using: composite steps: - run: exit 1 + shell: bash if: env.LEAK_ENV != 'val' shell: cp {0} action.yml - uses: ./ diff --git a/act/runner/testdata/docker-action-custom-path/push.yml b/act/runner/testdata/docker-action-custom-path/push.yml new file mode 100644 index 00000000..37bbf417 --- /dev/null +++ b/act/runner/testdata/docker-action-custom-path/push.yml @@ -0,0 +1,12 @@ +on: push +jobs: + _: + runs-on: ubuntu-latest + steps: + - run: | + FROM ubuntu:latest + ENV PATH="/opt/texlive/texdir/bin/x86_64-linuxmusl:${PATH}" + ENV ORG_PATH="${PATH}" + ENTRYPOINT [ "bash", "-c", "echo \"PATH=$PATH\" && echo \"ORG_PATH=$ORG_PATH\" && [[ \"$PATH\" = \"$ORG_PATH\" ]]" ] + shell: mv {0} Dockerfile + - uses: ./ \ No newline at end of file diff --git a/act/runner/testdata/remote-action-js-node-user/push.yml b/act/runner/testdata/remote-action-js-node-user/push.yml new file mode 100644 index 00000000..cede7b0a --- /dev/null +++ b/act/runner/testdata/remote-action-js-node-user/push.yml @@ -0,0 +1,15 @@ +name: remote-action-js +on: push + +jobs: + test: + runs-on: ubuntu-latest + container: + image: node:16-buster-slim + options: --user node + steps: + - uses: actions/hello-world-javascript-action@v1 + with: + who-to-greet: 'Mona the Octocat' + + - uses: cloudposse/actions/github/slash-command-dispatch@0.14.0 diff --git a/act/runner/testdata/set-env-new-env-file-per-step/push.yml b/act/runner/testdata/set-env-new-env-file-per-step/push.yml new file mode 100644 index 00000000..34f4bad9 --- /dev/null +++ b/act/runner/testdata/set-env-new-env-file-per-step/push.yml @@ -0,0 +1,15 @@ +on: push +jobs: + _: + runs-on: ubuntu-latest + env: + MY_ENV: test + steps: + - run: exit 1 + if: env.MY_ENV != 'test' + - run: echo "MY_ENV=test2" > $GITHUB_ENV + - run: exit 1 + if: env.MY_ENV != 'test2' + - run: echo "MY_ENV=returnedenv" > $GITHUB_ENV + - run: exit 1 + if: env.MY_ENV != 'returnedenv' \ No newline at end of file diff --git a/act/runner/testdata/set-env-step-env-override/push.yml b/act/runner/testdata/set-env-step-env-override/push.yml new file mode 100644 index 00000000..f35ef875 --- /dev/null +++ b/act/runner/testdata/set-env-step-env-override/push.yml @@ -0,0 +1,24 @@ +on: push +jobs: + _: + runs-on: ubuntu-latest + env: + MY_ENV: test + steps: + - run: exit 1 + if: env.MY_ENV != 'test' + - run: | + runs: + using: composite + steps: + - run: exit 1 + shell: bash + if: env.MY_ENV != 'val' + - run: echo "MY_ENV=returnedenv" > $GITHUB_ENV + shell: bash + shell: cp {0} action.yml + - uses: ./ + env: + MY_ENV: val + - run: exit 1 + if: env.MY_ENV != 'returnedenv' \ No newline at end of file From bc654738806f8a67a8458c3bf4e754126dcc9a62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 03:05:59 +0000 Subject: [PATCH 52/73] build(deps): bump megalinter/megalinter from 6.18.0 to 6.19.0 (#1610) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.18.0 to 6.19.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.18.0...v6.19.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 38514270..1f5327d9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.4.0 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.18.0 + - uses: megalinter/megalinter/flavors/go@v6.19.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f81e2d684349ff246cb827ccdff7eeb30d7f317d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Feb 2023 12:33:42 +0000 Subject: [PATCH 53/73] build(deps): bump github.com/docker/cli from 23.0.0-rc.1+incompatible to 23.0.0+incompatible (#1611) * build(deps): bump github.com/docker/cli Bumps [github.com/docker/cli](https://github.com/docker/cli) from 23.0.0-rc.1+incompatible to 23.0.0+incompatible. - [Release notes](https://github.com/docker/cli/releases) - [Commits](https://github.com/docker/cli/compare/v23.0.0-rc.1...v23.0.0) --- updated-dependencies: - dependency-name: github.com/docker/cli dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * update-test * update test --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: ChristopherHX --- act/container/docker_cli_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/act/container/docker_cli_test.go b/act/container/docker_cli_test.go index cdd91f6a..a6445be6 100644 --- a/act/container/docker_cli_test.go +++ b/act/container/docker_cli_test.go @@ -663,8 +663,8 @@ func TestRunFlagsParseShmSize(t *testing.T) { func TestParseRestartPolicy(t *testing.T) { invalids := map[string]string{ - "always:2:3": "invalid restart policy format", - "on-failure:invalid": "maximum retry count must be an integer", + "always:2:3": "invalid restart policy format: maximum retry count must be an integer", + "on-failure:invalid": "invalid restart policy format: maximum retry count must be an integer", } valids := map[string]container.RestartPolicy{ "": {}, From e3667a54feea653333816e7071992436c714e002 Mon Sep 17 00:00:00 2001 From: Aidan Date: Wed, 8 Feb 2023 09:14:43 -0800 Subject: [PATCH 54/73] Pass dockerfile to build executor (#1606) This allows testing actions with non standard dockerfile names Signed-off-by: Aidan Jensen --- act/container/container_types.go | 1 + act/container/docker_build.go | 3 ++- act/runner/action.go | 3 ++- .../testdata/actions-environment-and-context-tests/push.yml | 2 ++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/act/container/container_types.go b/act/container/container_types.go index c83ec35b..6b3fc860 100644 --- a/act/container/container_types.go +++ b/act/container/container_types.go @@ -55,6 +55,7 @@ type Container interface { // NewDockerBuildExecutorInput the input for the NewDockerBuildExecutor function type NewDockerBuildExecutorInput struct { ContextDir string + Dockerfile string Container Container ImageTag string Platform string diff --git a/act/container/docker_build.go b/act/container/docker_build.go index 0c87cfdf..72150234 100644 --- a/act/container/docker_build.go +++ b/act/container/docker_build.go @@ -45,12 +45,13 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor { Remove: true, Platform: input.Platform, AuthConfigs: LoadDockerAuthConfigs(ctx), + Dockerfile: input.Dockerfile, } var buildContext io.ReadCloser if input.Container != nil { buildContext, err = input.Container.GetContainerArchive(ctx, input.ContextDir+"/.") } else { - buildContext, err = createBuildContext(ctx, input.ContextDir, "Dockerfile") + buildContext, err = createBuildContext(ctx, input.ContextDir, input.Dockerfile) } if err != nil { return err diff --git a/act/runner/action.go b/act/runner/action.go index 1da76959..c476ad9f 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -231,7 +231,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based image = fmt.Sprintf("%s-dockeraction:%s", regexp.MustCompile("[^a-zA-Z0-9]").ReplaceAllString(actionName, "-"), "latest") image = fmt.Sprintf("act-%s", strings.TrimLeft(image, "-")) image = strings.ToLower(image) - contextDir, _ := filepath.Split(filepath.Join(basedir, action.Runs.Image)) + contextDir, fileName := filepath.Split(filepath.Join(basedir, action.Runs.Image)) anyArchExists, err := container.ImageExistsLocally(ctx, image, "any") if err != nil { @@ -261,6 +261,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based } prepImage = container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ ContextDir: contextDir, + Dockerfile: fileName, ImageTag: image, Container: actionContainer, Platform: rc.Config.ContainerArchitecture, diff --git a/act/runner/testdata/actions-environment-and-context-tests/push.yml b/act/runner/testdata/actions-environment-and-context-tests/push.yml index db3c3413..1d799d57 100644 --- a/act/runner/testdata/actions-environment-and-context-tests/push.yml +++ b/act/runner/testdata/actions-environment-and-context-tests/push.yml @@ -11,3 +11,5 @@ jobs: - uses: './actions-environment-and-context-tests/docker' - uses: 'nektos/act-test-actions/js@main' - uses: 'nektos/act-test-actions/docker@main' + - uses: 'nektos/act-test-actions/docker-file@main' + - uses: 'nektos/act-test-actions/docker-relative-context/action@main' From 41da84cbb5c134108fe8bd5305ce8ddb8ec0a977 Mon Sep 17 00:00:00 2001 From: sitiom Date: Tue, 14 Feb 2023 01:59:19 +0800 Subject: [PATCH 55/73] ci: add Winget Releaser workflow (#1623) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ad8665d..69b5aecf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -56,3 +56,12 @@ jobs: ref: context.ref, sha: mainRef.object.sha, }); + winget: + needs: release + runs-on: windows-latest # Action can only run on Windows + steps: + - uses: vedantmgoyal2009/winget-releaser@v2 + with: + identifier: nektos.act + installers-regex: '_Windows_\w+\.zip$' + token: ${{ secrets.WINGET_TOKEN }} From 1b88ccb8030573c751ee45ed72ff51c2d05495ff Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 16 Feb 2023 23:16:46 +0800 Subject: [PATCH 56/73] fix: don't override env (#1629) --- act/runner/run_context.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index bfd090ce..24acfe11 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -186,9 +186,11 @@ func (rc *RunContext) startHostEnvironment() common.Executor { } } for _, env := range os.Environ() { - i := strings.Index(env, "=") - if i > 0 { - rc.Env[env[0:i]] = env[i+1:] + if k, v, ok := strings.Cut(env, "="); ok { + // don't override + if _, ok := rc.Env[k]; !ok { + rc.Env[k] = v + } } } From 32b8839b683491d5222ba0671771fa2034947ac2 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 16 Feb 2023 23:34:51 +0800 Subject: [PATCH 57/73] chore: use new style octal (#1630) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/artifacts/server.go | 5 +++-- act/common/git/git.go | 6 +++--- act/common/git/git_test.go | 2 +- act/container/file_collector.go | 2 +- act/container/file_collector_test.go | 2 +- act/container/host_environment.go | 8 ++++---- act/runner/action.go | 4 +++- act/runner/run_context.go | 14 +++++++------- act/runner/step.go | 8 ++++---- act/runner/step_run.go | 3 ++- act/runner/step_run_test.go | 7 ++++--- cmd/notices.go | 4 ++-- 12 files changed, 35 insertions(+), 30 deletions(-) diff --git a/act/artifacts/server.go b/act/artifacts/server.go index 7c3df56c..d0c7a6aa 100644 --- a/act/artifacts/server.go +++ b/act/artifacts/server.go @@ -14,6 +14,7 @@ import ( "time" "github.com/julienschmidt/httprouter" + "github.com/nektos/act/pkg/common" ) @@ -65,14 +66,14 @@ func (fwfs readWriteFSImpl) OpenWritable(name string) (WritableFile, error) { if err := os.MkdirAll(filepath.Dir(name), os.ModePerm); err != nil { return nil, err } - return os.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + return os.OpenFile(name, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o644) } func (fwfs readWriteFSImpl) OpenAppendable(name string) (WritableFile, error) { if err := os.MkdirAll(filepath.Dir(name), os.ModePerm); err != nil { return nil, err } - file, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0644) + file, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0o644) if err != nil { return nil, err diff --git a/act/common/git/git.go b/act/common/git/git.go index 74df3c0f..954c2cc4 100644 --- a/act/common/git/git.go +++ b/act/common/git/git.go @@ -11,8 +11,6 @@ import ( "strings" "sync" - "github.com/nektos/act/pkg/common" - "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" @@ -20,6 +18,8 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/mattn/go-isatty" log "github.com/sirupsen/logrus" + + "github.com/nektos/act/pkg/common" ) var ( @@ -260,7 +260,7 @@ func CloneIfRequired(ctx context.Context, refName plumbing.ReferenceName, input return nil, err } - if err = os.Chmod(input.Dir, 0755); err != nil { + if err = os.Chmod(input.Dir, 0o755); err != nil { return nil, err } } diff --git a/act/common/git/git_test.go b/act/common/git/git_test.go index 5d3e5886..6ad66b67 100644 --- a/act/common/git/git_test.go +++ b/act/common/git/git_test.go @@ -167,7 +167,7 @@ func TestGitFindRef(t *testing.T) { name := name t.Run(name, func(t *testing.T) { dir := filepath.Join(basedir, name) - require.NoError(t, os.MkdirAll(dir, 0755)) + require.NoError(t, os.MkdirAll(dir, 0o755)) require.NoError(t, gitCmd("-C", dir, "init", "--initial-branch=master")) require.NoError(t, cleanGitHooks(dir)) tt.Prepare(t, dir) diff --git a/act/container/file_collector.go b/act/container/file_collector.go index a4143edc..b4be0e88 100644 --- a/act/container/file_collector.go +++ b/act/container/file_collector.go @@ -65,7 +65,7 @@ type copyCollector struct { func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { fdestpath := filepath.Join(cc.DstDir, fpath) - if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil { + if err := os.MkdirAll(filepath.Dir(fdestpath), 0o777); err != nil { return err } if f == nil { diff --git a/act/container/file_collector_test.go b/act/container/file_collector_test.go index 86b80034..241fd34b 100644 --- a/act/container/file_collector_test.go +++ b/act/container/file_collector_test.go @@ -76,7 +76,7 @@ func (mfs *memoryFs) Readlink(path string) (string, error) { func TestIgnoredTrackedfile(t *testing.T) { fs := memfs.New() - _ = fs.MkdirAll("mygitrepo/.git", 0777) + _ = fs.MkdirAll("mygitrepo/.git", 0o777) dotgit, _ := fs.Chroot("mygitrepo/.git") worktree, _ := fs.Chroot("mygitrepo") repo, _ := git.Init(filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()), worktree) diff --git a/act/container/host_environment.go b/act/container/host_environment.go index ff21b0ad..5912db8a 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -5,6 +5,7 @@ import ( "bufio" "bytes" "context" + "errors" "fmt" "io" "io/fs" @@ -15,14 +16,13 @@ import ( "strings" "time" - "errors" - "github.com/go-git/go-billy/v5/helper/polyfill" "github.com/go-git/go-billy/v5/osfs" "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "golang.org/x/term" + "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/lookpath" - "golang.org/x/term" ) type HostEnvironment struct { @@ -50,7 +50,7 @@ func (e *HostEnvironment) Close() common.Executor { func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor { return func(ctx context.Context) error { for _, f := range files { - if err := os.MkdirAll(filepath.Dir(filepath.Join(destPath, f.Name)), 0777); err != nil { + if err := os.MkdirAll(filepath.Dir(filepath.Join(destPath, f.Name)), 0o777); err != nil { return err } if err := os.WriteFile(filepath.Join(destPath, f.Name), []byte(f.Body), fs.FileMode(f.Mode)); err != nil { diff --git a/act/runner/action.go b/act/runner/action.go index c476ad9f..4d860a22 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/kballard/go-shellquote" + "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/model" @@ -30,6 +31,7 @@ type actionStep interface { type readAction func(ctx context.Context, step *model.Step, actionDir string, actionPath string, readFile actionYamlReader, writeFile fileWriter) (*model.Action, error) type actionYamlReader func(filename string) (io.Reader, io.Closer, error) + type fileWriter func(filename string, data []byte, perm fs.FileMode) error type runAction func(step actionStep, actionDir string, remoteAction *remoteAction) common.Executor @@ -61,7 +63,7 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir string, act if b, err = trampoline.ReadFile("res/trampoline.js"); err != nil { return nil, err } - err2 := writeFile(filepath.Join(actionDir, actionPath, "trampoline.js"), b, 0400) + err2 := writeFile(filepath.Join(actionDir, actionPath, "trampoline.js"), b, 0o400) if err2 != nil { return nil, err2 } diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 24acfe11..e24a236c 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -156,15 +156,15 @@ func (rc *RunContext) startHostEnvironment() common.Executor { _, _ = rand.Read(randBytes) miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) actPath := filepath.Join(miscpath, "act") - if err := os.MkdirAll(actPath, 0777); err != nil { + if err := os.MkdirAll(actPath, 0o777); err != nil { return err } path := filepath.Join(miscpath, "hostexecutor") - if err := os.MkdirAll(path, 0777); err != nil { + if err := os.MkdirAll(path, 0o777); err != nil { return err } runnerTmp := filepath.Join(miscpath, "tmp") - if err := os.MkdirAll(runnerTmp, 0777); err != nil { + if err := os.MkdirAll(runnerTmp, 0o777); err != nil { return err } toolCache := filepath.Join(cacheDir, "tool_cache") @@ -197,11 +197,11 @@ func (rc *RunContext) startHostEnvironment() common.Executor { return common.NewPipelineExecutor( rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ Name: "workflow/event.json", - Mode: 0644, + Mode: 0o644, Body: rc.EventJSON, }, &container.FileEntry{ Name: "workflow/envs.txt", - Mode: 0666, + Mode: 0o666, Body: "", }), )(ctx) @@ -280,11 +280,11 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.JobContainer.Start(false), rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ Name: "workflow/event.json", - Mode: 0644, + Mode: 0o644, Body: rc.EventJSON, }, &container.FileEntry{ Name: "workflow/envs.txt", - Mode: 0666, + Mode: 0o666, Body: "", }), )(ctx) diff --git a/act/runner/step.go b/act/runner/step.go index d66827a9..8a7ecc64 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -111,16 +111,16 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo (*step.getEnv())["GITHUB_ENV"] = path.Join(actPath, envFileCommand) _ = rc.JobContainer.Copy(actPath, &container.FileEntry{ Name: outputFileCommand, - Mode: 0666, + Mode: 0o666, }, &container.FileEntry{ Name: stateFileCommand, - Mode: 0666, + Mode: 0o666, }, &container.FileEntry{ Name: pathFileCommand, - Mode: 0666, + Mode: 0o666, }, &container.FileEntry{ Name: envFileCommand, - Mode: 0666, + Mode: 0o666, })(ctx) err = executor(ctx) diff --git a/act/runner/step_run.go b/act/runner/step_run.go index f833fc1a..ca77d569 100644 --- a/act/runner/step_run.go +++ b/act/runner/step_run.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/kballard/go-shellquote" + "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/model" @@ -72,7 +73,7 @@ func (sr *stepRun) setupShellCommandExecutor() common.Executor { rc := sr.getRunContext() return rc.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{ Name: scriptName, - Mode: 0755, + Mode: 0o755, Body: script, })(ctx) } diff --git a/act/runner/step_run_test.go b/act/runner/step_run_test.go index 4ca2eb97..fc5e6595 100644 --- a/act/runner/step_run_test.go +++ b/act/runner/step_run_test.go @@ -6,17 +6,18 @@ import ( "io" "testing" - "github.com/nektos/act/pkg/container" - "github.com/nektos/act/pkg/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + + "github.com/nektos/act/pkg/container" + "github.com/nektos/act/pkg/model" ) func TestStepRun(t *testing.T) { cm := &containerMock{} fileEntry := &container.FileEntry{ Name: "workflow/1.sh", - Mode: 0755, + Mode: 0o755, Body: "\ncmd\n", } diff --git a/cmd/notices.go b/cmd/notices.go index ffa84312..bd03aa3e 100644 --- a/cmd/notices.go +++ b/cmd/notices.go @@ -126,7 +126,7 @@ func loadNoticesEtag() string { func saveNoticesEtag(etag string) { p := etagPath() - err := os.WriteFile(p, []byte(strings.TrimSuffix(etag, "\n")), 0600) + err := os.WriteFile(p, []byte(strings.TrimSuffix(etag, "\n")), 0o600) if err != nil { log.Debugf("Unable to save etag to %s: %e", p, err) } @@ -143,7 +143,7 @@ func etagPath() string { } } dir := filepath.Join(xdgCache, "act") - if err := os.MkdirAll(dir, 0777); err != nil { + if err := os.MkdirAll(dir, 0o777); err != nil { log.Fatal(err) } return filepath.Join(dir, ".notices.etag") From de558842bb0ffe505cd6c966ed398e77bbb66dca Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Thu, 16 Feb 2023 17:11:26 +0100 Subject: [PATCH 58/73] chore: Remove obsolete Container.UpdateFromPath (#1631) * chore: Remove obsolete Container.UpdateFromPath * remove unused import --- act/container/container_types.go | 1 - act/container/docker_run.go | 30 ------------------------------ act/container/host_environment.go | 27 --------------------------- act/runner/container_mock_test.go | 5 ----- 4 files changed, 63 deletions(-) diff --git a/act/container/container_types.go b/act/container/container_types.go index 6b3fc860..cba2ebf7 100644 --- a/act/container/container_types.go +++ b/act/container/container_types.go @@ -46,7 +46,6 @@ type Container interface { Exec(command []string, env map[string]string, user, workdir string) common.Executor UpdateFromEnv(srcPath string, env *map[string]string) common.Executor UpdateFromImageEnv(env *map[string]string) common.Executor - UpdateFromPath(env *map[string]string) common.Executor Remove() common.Executor Close() common.Executor ReplaceLogWriter(io.Writer, io.Writer) (io.Writer, io.Writer) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 4ef68602..0ba562d0 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -4,7 +4,6 @@ package container import ( "archive/tar" - "bufio" "bytes" "context" "errors" @@ -152,10 +151,6 @@ func (cr *containerReference) UpdateFromImageEnv(env *map[string]string) common. return cr.extractFromImageEnv(env).IfNot(common.Dryrun) } -func (cr *containerReference) UpdateFromPath(env *map[string]string) common.Executor { - return cr.extractPath(env).IfNot(common.Dryrun) -} - func (cr *containerReference) Exec(command []string, env map[string]string, user, workdir string) common.Executor { return common.NewPipelineExecutor( common.NewInfoExecutor("%sdocker exec cmd=[%s] user=%s workdir=%s", logPrefix, strings.Join(command, " "), user, workdir), @@ -492,31 +487,6 @@ func (cr *containerReference) extractFromImageEnv(env *map[string]string) common } } -func (cr *containerReference) extractPath(env *map[string]string) common.Executor { - localEnv := *env - return func(ctx context.Context) error { - pathTar, _, err := cr.cli.CopyFromContainer(ctx, cr.id, localEnv["GITHUB_PATH"]) - if err != nil { - return fmt.Errorf("failed to copy from container: %w", err) - } - defer pathTar.Close() - - reader := tar.NewReader(pathTar) - _, err = reader.Next() - if err != nil && err != io.EOF { - return fmt.Errorf("failed to read tar archive: %w", err) - } - s := bufio.NewScanner(reader) - for s.Scan() { - line := s.Text() - localEnv["PATH"] = fmt.Sprintf("%s:%s", line, localEnv["PATH"]) - } - - env = &localEnv - return nil - } -} - func (cr *containerReference) exec(cmd []string, env map[string]string, user, workdir string) common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) diff --git a/act/container/host_environment.go b/act/container/host_environment.go index 5912db8a..5d8c7dcd 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -2,7 +2,6 @@ package container import ( "archive/tar" - "bufio" "bytes" "context" "errors" @@ -344,32 +343,6 @@ func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) return parseEnvFile(e, srcPath, env) } -func (e *HostEnvironment) UpdateFromPath(env *map[string]string) common.Executor { - localEnv := *env - return func(ctx context.Context) error { - pathTar, err := e.GetContainerArchive(ctx, localEnv["GITHUB_PATH"]) - if err != nil { - return err - } - defer pathTar.Close() - - reader := tar.NewReader(pathTar) - _, err = reader.Next() - if err != nil && err != io.EOF { - return err - } - s := bufio.NewScanner(reader) - for s.Scan() { - line := s.Text() - pathSep := string(filepath.ListSeparator) - localEnv[e.GetPathVariableName()] = fmt.Sprintf("%s%s%s", line, pathSep, localEnv[e.GetPathVariableName()]) - } - - env = &localEnv - return nil - } -} - func (e *HostEnvironment) Remove() common.Executor { return func(ctx context.Context) error { if e.CleanUp != nil { diff --git a/act/runner/container_mock_test.go b/act/runner/container_mock_test.go index 19f89039..04d6261b 100644 --- a/act/runner/container_mock_test.go +++ b/act/runner/container_mock_test.go @@ -50,11 +50,6 @@ func (cm *containerMock) UpdateFromImageEnv(env *map[string]string) common.Execu return args.Get(0).(func(context.Context) error) } -func (cm *containerMock) UpdateFromPath(env *map[string]string) common.Executor { - args := cm.Called(env) - return args.Get(0).(func(context.Context) error) -} - func (cm *containerMock) Copy(destPath string, files ...*container.FileEntry) common.Executor { args := cm.Called(destPath, files) return args.Get(0).(func(context.Context) error) From bfe9d9f671186816ed655535b7bcbfdb5a861e25 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Thu, 16 Feb 2023 11:41:59 -0500 Subject: [PATCH 59/73] fix: tolerate workflow that needs a missing job (#1595) (#1619) Change planner functions to return errors This enables createStages to return `unable to build dependency graph` Fix PlanEvent to properly report errors relating to events/workflows --- act/artifacts/server_test.go | 15 ++- act/model/planner.go | 71 +++++++++--- act/model/workflow_test.go | 3 +- act/runner/reusable_workflow.go | 5 +- act/runner/runner_test.go | 115 ++++++++++++++++++-- act/runner/testdata/issue-1595/missing.yml | 16 +++ act/runner/testdata/issue-1595/no-event.yml | 8 ++ act/runner/testdata/issue-1595/no-first.yml | 10 ++ cmd/root.go | 43 ++++++-- 9 files changed, 238 insertions(+), 48 deletions(-) create mode 100644 act/runner/testdata/issue-1595/missing.yml create mode 100644 act/runner/testdata/issue-1595/no-event.yml create mode 100644 act/runner/testdata/issue-1595/no-first.yml diff --git a/act/artifacts/server_test.go b/act/artifacts/server_test.go index 259c9542..943820ca 100644 --- a/act/artifacts/server_test.go +++ b/act/artifacts/server_test.go @@ -297,13 +297,16 @@ func runTestJobFile(ctx context.Context, t *testing.T, tjfi TestJobFileInfo) { planner, err := model.NewWorkflowPlanner(fullWorkflowPath, true) assert.Nil(t, err, fullWorkflowPath) - plan := planner.PlanEvent(tjfi.eventName) - - err = runner.NewPlanExecutor(plan)(ctx) - if tjfi.errorMessage == "" { - assert.Nil(t, err, fullWorkflowPath) + plan, err := planner.PlanEvent(tjfi.eventName) + if err == nil { + err = runner.NewPlanExecutor(plan)(ctx) + if tjfi.errorMessage == "" { + assert.Nil(t, err, fullWorkflowPath) + } else { + assert.Error(t, err, tjfi.errorMessage) + } } else { - assert.Error(t, err, tjfi.errorMessage) + assert.Nil(t, plan) } fmt.Println("::endgroup::") diff --git a/act/model/planner.go b/act/model/planner.go index 73e3488a..1769b732 100644 --- a/act/model/planner.go +++ b/act/model/planner.go @@ -15,9 +15,9 @@ import ( // WorkflowPlanner contains methods for creating plans type WorkflowPlanner interface { - PlanEvent(eventName string) *Plan - PlanJob(jobName string) *Plan - PlanAll() *Plan + PlanEvent(eventName string) (*Plan, error) + PlanJob(jobName string) (*Plan, error) + PlanAll() (*Plan, error) GetEvents() []string } @@ -169,47 +169,76 @@ type workflowPlanner struct { } // PlanEvent builds a new list of runs to execute in parallel for an event name -func (wp *workflowPlanner) PlanEvent(eventName string) *Plan { +func (wp *workflowPlanner) PlanEvent(eventName string) (*Plan, error) { plan := new(Plan) if len(wp.workflows) == 0 { - log.Debugf("no events found for workflow: %s", eventName) + log.Debug("no workflows found by planner") + return plan, nil } + var lastErr error for _, w := range wp.workflows { - for _, e := range w.On() { + events := w.On() + if len(events) == 0 { + log.Debugf("no events found for workflow: %s", w.File) + continue + } + + for _, e := range events { if e == eventName { - plan.mergeStages(createStages(w, w.GetJobIDs()...)) + stages, err := createStages(w, w.GetJobIDs()...) + if err != nil { + log.Warn(err) + lastErr = err + } else { + plan.mergeStages(stages) + } } } } - return plan + return plan, lastErr } // PlanJob builds a new run to execute in parallel for a job name -func (wp *workflowPlanner) PlanJob(jobName string) *Plan { +func (wp *workflowPlanner) PlanJob(jobName string) (*Plan, error) { plan := new(Plan) if len(wp.workflows) == 0 { log.Debugf("no jobs found for workflow: %s", jobName) } + var lastErr error for _, w := range wp.workflows { - plan.mergeStages(createStages(w, jobName)) + stages, err := createStages(w, jobName) + if err != nil { + log.Warn(err) + lastErr = err + } else { + plan.mergeStages(stages) + } } - return plan + return plan, lastErr } // PlanAll builds a new run to execute in parallel all -func (wp *workflowPlanner) PlanAll() *Plan { +func (wp *workflowPlanner) PlanAll() (*Plan, error) { plan := new(Plan) if len(wp.workflows) == 0 { - log.Debugf("no jobs found for loaded workflows") + log.Debug("no workflows found by planner") + return plan, nil } + var lastErr error for _, w := range wp.workflows { - plan.mergeStages(createStages(w, w.GetJobIDs()...)) + stages, err := createStages(w, w.GetJobIDs()...) + if err != nil { + log.Warn(err) + lastErr = err + } else { + plan.mergeStages(stages) + } } - return plan + return plan, lastErr } // GetEvents gets all the events in the workflows file @@ -282,7 +311,7 @@ func (p *Plan) mergeStages(stages []*Stage) { p.Stages = newStages } -func createStages(w *Workflow, jobIDs ...string) []*Stage { +func createStages(w *Workflow, jobIDs ...string) ([]*Stage, error) { // first, build a list of all the necessary jobs to run, and their dependencies jobDependencies := make(map[string][]string) for len(jobIDs) > 0 { @@ -299,6 +328,8 @@ func createStages(w *Workflow, jobIDs ...string) []*Stage { jobIDs = newJobIDs } + var err error + // next, build an execution graph stages := make([]*Stage, 0) for len(jobDependencies) > 0 { @@ -314,12 +345,16 @@ func createStages(w *Workflow, jobIDs ...string) []*Stage { } } if len(stage.Runs) == 0 { - log.Fatalf("Unable to build dependency graph!") + return nil, fmt.Errorf("unable to build dependency graph for %s (%s)", w.Name, w.File) } stages = append(stages, stage) } - return stages + if len(stages) == 0 && err != nil { + return nil, err + } + + return stages, nil } // return true iff all strings in srcList exist in at least one of the stages diff --git a/act/model/workflow_test.go b/act/model/workflow_test.go index d978f163..a7892338 100644 --- a/act/model/workflow_test.go +++ b/act/model/workflow_test.go @@ -241,7 +241,8 @@ func TestReadWorkflow_Strategy(t *testing.T) { w, err := NewWorkflowPlanner("testdata/strategy/push.yml", true) assert.NoError(t, err) - p := w.PlanJob("strategy-only-max-parallel") + p, err := w.PlanJob("strategy-only-max-parallel") + assert.NoError(t, err) assert.Equal(t, len(p.Stages), 1) assert.Equal(t, len(p.Stages[0].Runs), 1) diff --git a/act/runner/reusable_workflow.go b/act/runner/reusable_workflow.go index b080b4db..a5687f93 100644 --- a/act/runner/reusable_workflow.go +++ b/act/runner/reusable_workflow.go @@ -74,7 +74,10 @@ func newReusableWorkflowExecutor(rc *RunContext, directory string, workflow stri return err } - plan := planner.PlanEvent("workflow_call") + plan, err := planner.PlanEvent("workflow_call") + if err != nil { + return err + } runner, err := NewReusableWorkflowRunner(rc) if err != nil { diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 0a83537e..fb51be97 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -49,12 +49,99 @@ func init() { secrets = map[string]string{} } +func TestNoWorkflowsFoundByPlanner(t *testing.T) { + planner, err := model.NewWorkflowPlanner("res", true) + assert.NoError(t, err) + + out := log.StandardLogger().Out + var buf bytes.Buffer + log.SetOutput(&buf) + log.SetLevel(log.DebugLevel) + plan, err := planner.PlanEvent("pull_request") + assert.NotNil(t, plan) + assert.NoError(t, err) + assert.Contains(t, buf.String(), "no workflows found by planner") + buf.Reset() + plan, err = planner.PlanAll() + assert.NotNil(t, plan) + assert.NoError(t, err) + assert.Contains(t, buf.String(), "no workflows found by planner") + log.SetOutput(out) +} + +func TestGraphMissingEvent(t *testing.T) { + planner, err := model.NewWorkflowPlanner("testdata/issue-1595/no-event.yml", true) + assert.NoError(t, err) + + out := log.StandardLogger().Out + var buf bytes.Buffer + log.SetOutput(&buf) + log.SetLevel(log.DebugLevel) + + plan, err := planner.PlanEvent("push") + assert.NoError(t, err) + assert.NotNil(t, plan) + assert.Equal(t, 0, len(plan.Stages)) + + assert.Contains(t, buf.String(), "no events found for workflow: no-event.yml") + log.SetOutput(out) +} + +func TestGraphMissingFirst(t *testing.T) { + planner, err := model.NewWorkflowPlanner("testdata/issue-1595/no-first.yml", true) + assert.NoError(t, err) + + plan, err := planner.PlanEvent("push") + assert.EqualError(t, err, "unable to build dependency graph for no first (no-first.yml)") + assert.NotNil(t, plan) + assert.Equal(t, 0, len(plan.Stages)) +} + +func TestGraphWithMissing(t *testing.T) { + planner, err := model.NewWorkflowPlanner("testdata/issue-1595/missing.yml", true) + assert.NoError(t, err) + + out := log.StandardLogger().Out + var buf bytes.Buffer + log.SetOutput(&buf) + log.SetLevel(log.DebugLevel) + + plan, err := planner.PlanEvent("push") + assert.NotNil(t, plan) + assert.Equal(t, 0, len(plan.Stages)) + assert.EqualError(t, err, "unable to build dependency graph for missing (missing.yml)") + assert.Contains(t, buf.String(), "unable to build dependency graph for missing (missing.yml)") + log.SetOutput(out) +} + +func TestGraphWithSomeMissing(t *testing.T) { + log.SetLevel(log.DebugLevel) + + planner, err := model.NewWorkflowPlanner("testdata/issue-1595/", true) + assert.NoError(t, err) + + out := log.StandardLogger().Out + var buf bytes.Buffer + log.SetOutput(&buf) + log.SetLevel(log.DebugLevel) + + plan, err := planner.PlanAll() + assert.Error(t, err, "unable to build dependency graph for no first (no-first.yml)") + assert.NotNil(t, plan) + assert.Equal(t, 1, len(plan.Stages)) + assert.Contains(t, buf.String(), "unable to build dependency graph for missing (missing.yml)") + assert.Contains(t, buf.String(), "unable to build dependency graph for no first (no-first.yml)") + log.SetOutput(out) +} + func TestGraphEvent(t *testing.T) { planner, err := model.NewWorkflowPlanner("testdata/basic", true) - assert.Nil(t, err) + assert.NoError(t, err) - plan := planner.PlanEvent("push") - assert.Nil(t, err) + plan, err := planner.PlanEvent("push") + assert.NoError(t, err) + assert.NotNil(t, plan) + assert.NotNil(t, plan.Stages) assert.Equal(t, len(plan.Stages), 3, "stages") assert.Equal(t, len(plan.Stages[0].Runs), 1, "stage0.runs") assert.Equal(t, len(plan.Stages[1].Runs), 1, "stage1.runs") @@ -63,8 +150,10 @@ func TestGraphEvent(t *testing.T) { assert.Equal(t, plan.Stages[1].Runs[0].JobID, "build", "jobid") assert.Equal(t, plan.Stages[2].Runs[0].JobID, "test", "jobid") - plan = planner.PlanEvent("release") - assert.Equal(t, len(plan.Stages), 0, "stages") + plan, err = planner.PlanEvent("release") + assert.NoError(t, err) + assert.NotNil(t, plan) + assert.Equal(t, 0, len(plan.Stages)) } type TestJobFileInfo struct { @@ -105,13 +194,15 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config planner, err := model.NewWorkflowPlanner(fullWorkflowPath, true) assert.Nil(t, err, fullWorkflowPath) - plan := planner.PlanEvent(j.eventName) - - err = runner.NewPlanExecutor(plan)(ctx) - if j.errorMessage == "" { - assert.Nil(t, err, fullWorkflowPath) - } else { - assert.Error(t, err, j.errorMessage) + plan, err := planner.PlanEvent(j.eventName) + assert.True(t, (err == nil) != (plan == nil), "PlanEvent should return either a plan or an error") + if err == nil && plan != nil { + err = runner.NewPlanExecutor(plan)(ctx) + if j.errorMessage == "" { + assert.Nil(t, err, fullWorkflowPath) + } else { + assert.Error(t, err, j.errorMessage) + } } fmt.Println("::endgroup::") diff --git a/act/runner/testdata/issue-1595/missing.yml b/act/runner/testdata/issue-1595/missing.yml new file mode 100644 index 00000000..3b4adf48 --- /dev/null +++ b/act/runner/testdata/issue-1595/missing.yml @@ -0,0 +1,16 @@ +name: missing +on: push + +jobs: + second: + runs-on: ubuntu-latest + needs: first + steps: + - run: echo How did you get here? + shell: bash + + standalone: + runs-on: ubuntu-latest + steps: + - run: echo Hello world + shell: bash diff --git a/act/runner/testdata/issue-1595/no-event.yml b/act/runner/testdata/issue-1595/no-event.yml new file mode 100644 index 00000000..2140a0bd --- /dev/null +++ b/act/runner/testdata/issue-1595/no-event.yml @@ -0,0 +1,8 @@ +name: no event + +jobs: + stuck: + runs-on: ubuntu-latest + steps: + - run: echo How did you get here? + shell: bash diff --git a/act/runner/testdata/issue-1595/no-first.yml b/act/runner/testdata/issue-1595/no-first.yml new file mode 100644 index 00000000..48d4b55d --- /dev/null +++ b/act/runner/testdata/issue-1595/no-first.yml @@ -0,0 +1,10 @@ +name: no first +on: push + +jobs: + second: + runs-on: ubuntu-latest + needs: first + steps: + - run: echo How did you get here? + shell: bash diff --git a/cmd/root.go b/cmd/root.go index 23f411c9..3619c7bf 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -354,7 +354,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str var filterPlan *model.Plan // Determine the event name to be filtered - var filterEventName string = "" + var filterEventName string if len(args) > 0 { log.Debugf("Using first passed in arguments event for filtering: %s", args[0]) @@ -366,23 +366,35 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str filterEventName = events[0] } + var plannerErr error if jobID != "" { log.Debugf("Preparing plan with a job: %s", jobID) - filterPlan = planner.PlanJob(jobID) + filterPlan, plannerErr = planner.PlanJob(jobID) } else if filterEventName != "" { log.Debugf("Preparing plan for a event: %s", filterEventName) - filterPlan = planner.PlanEvent(filterEventName) + filterPlan, plannerErr = planner.PlanEvent(filterEventName) } else { log.Debugf("Preparing plan with all jobs") - filterPlan = planner.PlanAll() + filterPlan, plannerErr = planner.PlanAll() + } + if filterPlan == nil && plannerErr != nil { + return plannerErr } if list { - return printList(filterPlan) + err = printList(filterPlan) + if err != nil { + return err + } + return plannerErr } if graph { - return drawGraph(filterPlan) + err = drawGraph(filterPlan) + if err != nil { + return err + } + return plannerErr } // plan with triggered jobs @@ -410,10 +422,13 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str // build the plan for this run if jobID != "" { log.Debugf("Planning job: %s", jobID) - plan = planner.PlanJob(jobID) + plan, plannerErr = planner.PlanJob(jobID) } else { log.Debugf("Planning jobs for event: %s", eventName) - plan = planner.PlanEvent(eventName) + plan, plannerErr = planner.PlanEvent(eventName) + } + if plan == nil && plannerErr != nil { + return plannerErr } // check to see if the main branch was defined @@ -501,14 +516,22 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str if watch, err := cmd.Flags().GetBool("watch"); err != nil { return err } else if watch { - return watchAndRun(ctx, r.NewPlanExecutor(plan)) + err = watchAndRun(ctx, r.NewPlanExecutor(plan)) + if err != nil { + return err + } + return plannerErr } executor := r.NewPlanExecutor(plan).Finally(func(ctx context.Context) error { cancel() return nil }) - return executor(ctx) + err = executor(ctx) + if err != nil { + return err + } + return plannerErr } } From c28143e1f900902717d6efcb1b99941a37467d42 Mon Sep 17 00:00:00 2001 From: R Date: Thu, 23 Feb 2023 16:24:44 +0100 Subject: [PATCH 60/73] fix: add GITHUB_STEP_SUMMARY (#1607) --- act/runner/step.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/act/runner/step.go b/act/runner/step.go index 8a7ecc64..7cc355f4 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -101,14 +101,22 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo // Prepare and clean Runner File Commands actPath := rc.JobContainer.GetActPath() + outputFileCommand := path.Join("workflow", "outputcmd.txt") - stateFileCommand := path.Join("workflow", "statecmd.txt") - pathFileCommand := path.Join("workflow", "pathcmd.txt") - envFileCommand := path.Join("workflow", "envs.txt") (*step.getEnv())["GITHUB_OUTPUT"] = path.Join(actPath, outputFileCommand) + + stateFileCommand := path.Join("workflow", "statecmd.txt") (*step.getEnv())["GITHUB_STATE"] = path.Join(actPath, stateFileCommand) + + pathFileCommand := path.Join("workflow", "pathcmd.txt") (*step.getEnv())["GITHUB_PATH"] = path.Join(actPath, pathFileCommand) + + envFileCommand := path.Join("workflow", "envs.txt") (*step.getEnv())["GITHUB_ENV"] = path.Join(actPath, envFileCommand) + + summaryFileCommand := path.Join("workflow", "SUMMARY.md") + (*step.getEnv())["GITHUB_STEP_SUMMARY"] = path.Join(actPath, summaryFileCommand) + _ = rc.JobContainer.Copy(actPath, &container.FileEntry{ Name: outputFileCommand, Mode: 0o666, @@ -120,6 +128,9 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo Mode: 0o666, }, &container.FileEntry{ Name: envFileCommand, + Mode: 0666, + }, &container.FileEntry{ + Name: summaryFileCommand, Mode: 0o666, })(ctx) From 75baa9dc3b10e8ee599b9d4a9dd23b7ac136c2ce Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Thu, 23 Feb 2023 18:21:08 +0000 Subject: [PATCH 61/73] feat: workflowpattern package (#1618) * feat: workflowpattern package * nolint:gocyclo --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/workflowpattern/trace_writer.go | 18 + act/workflowpattern/workflow_pattern.go | 196 +++++++++ act/workflowpattern/workflow_pattern_test.go | 414 +++++++++++++++++++ 3 files changed, 628 insertions(+) create mode 100644 act/workflowpattern/trace_writer.go create mode 100644 act/workflowpattern/workflow_pattern.go create mode 100644 act/workflowpattern/workflow_pattern_test.go diff --git a/act/workflowpattern/trace_writer.go b/act/workflowpattern/trace_writer.go new file mode 100644 index 00000000..d5d990f6 --- /dev/null +++ b/act/workflowpattern/trace_writer.go @@ -0,0 +1,18 @@ +package workflowpattern + +import "fmt" + +type TraceWriter interface { + Info(string, ...interface{}) +} + +type EmptyTraceWriter struct{} + +func (*EmptyTraceWriter) Info(string, ...interface{}) { +} + +type StdOutTraceWriter struct{} + +func (*StdOutTraceWriter) Info(format string, args ...interface{}) { + fmt.Printf(format+"\n", args...) +} diff --git a/act/workflowpattern/workflow_pattern.go b/act/workflowpattern/workflow_pattern.go new file mode 100644 index 00000000..cc03e405 --- /dev/null +++ b/act/workflowpattern/workflow_pattern.go @@ -0,0 +1,196 @@ +package workflowpattern + +import ( + "fmt" + "regexp" + "strings" +) + +type WorkflowPattern struct { + Pattern string + Negative bool + Regex *regexp.Regexp +} + +func CompilePattern(rawpattern string) (*WorkflowPattern, error) { + negative := false + pattern := rawpattern + if strings.HasPrefix(rawpattern, "!") { + negative = true + pattern = rawpattern[1:] + } + rpattern, err := PatternToRegex(pattern) + if err != nil { + return nil, err + } + regex, err := regexp.Compile(rpattern) + if err != nil { + return nil, err + } + return &WorkflowPattern{ + Pattern: pattern, + Negative: negative, + Regex: regex, + }, nil +} + +//nolint:gocyclo +func PatternToRegex(pattern string) (string, error) { + var rpattern strings.Builder + rpattern.WriteString("^") + pos := 0 + errors := map[int]string{} + for pos < len(pattern) { + switch pattern[pos] { + case '*': + if pos+1 < len(pattern) && pattern[pos+1] == '*' { + if pos+2 < len(pattern) && pattern[pos+2] == '/' { + rpattern.WriteString("(.+/)?") + pos += 3 + } else { + rpattern.WriteString(".*") + pos += 2 + } + } else { + rpattern.WriteString("[^/]*") + pos++ + } + case '+', '?': + if pos > 0 { + rpattern.WriteByte(pattern[pos]) + } else { + rpattern.WriteString(regexp.QuoteMeta(string([]byte{pattern[pos]}))) + } + pos++ + case '[': + rpattern.WriteByte(pattern[pos]) + pos++ + if pos < len(pattern) && pattern[pos] == ']' { + errors[pos] = "Unexpected empty brackets '[]'" + pos++ + break + } + validChar := func(a, b, test byte) bool { + return test >= a && test <= b + } + startPos := pos + for pos < len(pattern) && pattern[pos] != ']' { + switch pattern[pos] { + case '-': + if pos <= startPos || pos+1 >= len(pattern) { + errors[pos] = "Invalid range" + pos++ + break + } + validRange := func(a, b byte) bool { + return validChar(a, b, pattern[pos-1]) && validChar(a, b, pattern[pos+1]) && pattern[pos-1] <= pattern[pos+1] + } + if !validRange('A', 'z') && !validRange('0', '9') { + errors[pos] = "Ranges can only include a-z, A-Z, A-z, and 0-9" + pos++ + break + } + rpattern.WriteString(pattern[pos : pos+2]) + pos += 2 + default: + if !validChar('A', 'z', pattern[pos]) && !validChar('0', '9', pattern[pos]) { + errors[pos] = "Ranges can only include a-z, A-Z and 0-9" + pos++ + break + } + rpattern.WriteString(regexp.QuoteMeta(string([]byte{pattern[pos]}))) + pos++ + } + } + if pos >= len(pattern) || pattern[pos] != ']' { + errors[pos] = "Missing closing bracket ']' after '['" + pos++ + } + rpattern.WriteString("]") + pos++ + case '\\': + if pos+1 >= len(pattern) { + errors[pos] = "Missing symbol after \\" + pos++ + break + } + rpattern.WriteString(regexp.QuoteMeta(string([]byte{pattern[pos+1]}))) + pos += 2 + default: + rpattern.WriteString(regexp.QuoteMeta(string([]byte{pattern[pos]}))) + pos++ + } + } + if len(errors) > 0 { + var errorMessage strings.Builder + for position, err := range errors { + if errorMessage.Len() > 0 { + errorMessage.WriteString(", ") + } + errorMessage.WriteString(fmt.Sprintf("Position: %d Error: %s", position, err)) + } + return "", fmt.Errorf("invalid Pattern '%s': %s", pattern, errorMessage.String()) + } + rpattern.WriteString("$") + return rpattern.String(), nil +} + +func CompilePatterns(patterns ...string) ([]*WorkflowPattern, error) { + ret := []*WorkflowPattern{} + for _, pattern := range patterns { + cp, err := CompilePattern(pattern) + if err != nil { + return nil, err + } + ret = append(ret, cp) + } + return ret, nil +} + +// returns true if the workflow should be skipped paths/branches +func Skip(sequence []*WorkflowPattern, input []string, traceWriter TraceWriter) bool { + if len(sequence) == 0 { + return false + } + for _, file := range input { + matched := false + for _, item := range sequence { + if item.Regex.MatchString(file) { + pattern := item.Pattern + if item.Negative { + matched = false + traceWriter.Info("%s excluded by pattern %s", file, pattern) + } else { + matched = true + traceWriter.Info("%s included by pattern %s", file, pattern) + } + } + } + if matched { + return false + } + } + return true +} + +// returns true if the workflow should be skipped paths-ignore/branches-ignore +func Filter(sequence []*WorkflowPattern, input []string, traceWriter TraceWriter) bool { + if len(sequence) == 0 { + return false + } + for _, file := range input { + matched := false + for _, item := range sequence { + if item.Regex.MatchString(file) == !item.Negative { + pattern := item.Pattern + traceWriter.Info("%s ignored by pattern %s", file, pattern) + matched = true + break + } + } + if !matched { + return false + } + } + return true +} diff --git a/act/workflowpattern/workflow_pattern_test.go b/act/workflowpattern/workflow_pattern_test.go new file mode 100644 index 00000000..a62d529b --- /dev/null +++ b/act/workflowpattern/workflow_pattern_test.go @@ -0,0 +1,414 @@ +package workflowpattern + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMatchPattern(t *testing.T) { + kases := []struct { + inputs []string + patterns []string + skipResult bool + filterResult bool + }{ + { + patterns: []string{"*"}, + inputs: []string{"path/with/slash"}, + skipResult: true, + filterResult: false, + }, + { + patterns: []string{"path/a", "path/b", "path/c"}, + inputs: []string{"meta", "path/b", "otherfile"}, + skipResult: false, + filterResult: false, + }, + { + patterns: []string{"path/a", "path/b", "path/c"}, + inputs: []string{"path/b"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"path/a", "path/b", "path/c"}, + inputs: []string{"path/c", "path/b"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"path/a", "path/b", "path/c"}, + inputs: []string{"path/c", "path/b", "path/a"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"path/a", "path/b", "path/c"}, + inputs: []string{"path/c", "path/b", "path/d", "path/a"}, + skipResult: false, + filterResult: false, + }, + { + patterns: []string{}, + inputs: []string{}, + skipResult: false, + filterResult: false, + }, + { + patterns: []string{"\\!file"}, + inputs: []string{"!file"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"escape\\\\backslash"}, + inputs: []string{"escape\\backslash"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{".yml"}, + inputs: []string{"fyml"}, + skipResult: true, + filterResult: false, + }, + // https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#patterns-to-match-branches-and-tags + { + patterns: []string{"feature/*"}, + inputs: []string{"feature/my-branch"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"feature/*"}, + inputs: []string{"feature/your-branch"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"feature/**"}, + inputs: []string{"feature/beta-a/my-branch"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"feature/**"}, + inputs: []string{"feature/beta-a/my-branch"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"feature/**"}, + inputs: []string{"feature/mona/the/octocat"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"main", "releases/mona-the-octocat"}, + inputs: []string{"main"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"main", "releases/mona-the-octocat"}, + inputs: []string{"releases/mona-the-octocat"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*"}, + inputs: []string{"main"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*"}, + inputs: []string{"releases"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**"}, + inputs: []string{"all/the/branches"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**"}, + inputs: []string{"every/tag"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*feature"}, + inputs: []string{"mona-feature"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*feature"}, + inputs: []string{"feature"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*feature"}, + inputs: []string{"ver-10-feature"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"v2*"}, + inputs: []string{"v2"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"v2*"}, + inputs: []string{"v2.0"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"v2*"}, + inputs: []string{"v2.9"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"v[12].[0-9]+.[0-9]+"}, + inputs: []string{"v1.10.1"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"v[12].[0-9]+.[0-9]+"}, + inputs: []string{"v2.0.0"}, + skipResult: false, + filterResult: true, + }, + // https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#patterns-to-match-file-paths + { + patterns: []string{"*"}, + inputs: []string{"README.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*"}, + inputs: []string{"server.rb"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*.jsx?"}, + inputs: []string{"page.js"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*.jsx?"}, + inputs: []string{"page.jsx"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**"}, + inputs: []string{"all/the/files.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*.js"}, + inputs: []string{"app.js"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*.js"}, + inputs: []string{"index.js"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**.js"}, + inputs: []string{"index.js"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**.js"}, + inputs: []string{"js/index.js"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**.js"}, + inputs: []string{"src/js/app.js"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"docs/*"}, + inputs: []string{"docs/README.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"docs/*"}, + inputs: []string{"docs/file.txt"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"docs/**"}, + inputs: []string{"docs/README.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"docs/**"}, + inputs: []string{"docs/mona/octocat.txt"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"docs/**/*.md"}, + inputs: []string{"docs/README.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"docs/**/*.md"}, + inputs: []string{"docs/mona/hello-world.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"docs/**/*.md"}, + inputs: []string{"docs/a/markdown/file.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/docs/**"}, + inputs: []string{"docs/hello.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/docs/**"}, + inputs: []string{"dir/docs/my-file.txt"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/docs/**"}, + inputs: []string{"space/docs/plan/space.doc"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/README.md"}, + inputs: []string{"README.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/README.md"}, + inputs: []string{"js/README.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/*src/**"}, + inputs: []string{"a/src/app.js"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/*src/**"}, + inputs: []string{"my-src/code/js/app.js"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/*-post.md"}, + inputs: []string{"my-post.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/*-post.md"}, + inputs: []string{"path/their-post.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/migrate-*.sql"}, + inputs: []string{"migrate-10909.sql"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/migrate-*.sql"}, + inputs: []string{"db/migrate-v1.0.sql"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"**/migrate-*.sql"}, + inputs: []string{"db/sept/migrate-v1.sql"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*.md", "!README.md"}, + inputs: []string{"hello.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*.md", "!README.md"}, + inputs: []string{"README.md"}, + skipResult: true, + filterResult: true, + }, + { + patterns: []string{"*.md", "!README.md"}, + inputs: []string{"docs/hello.md"}, + skipResult: true, + filterResult: true, + }, + { + patterns: []string{"*.md", "!README.md", "README*"}, + inputs: []string{"hello.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*.md", "!README.md", "README*"}, + inputs: []string{"README.md"}, + skipResult: false, + filterResult: true, + }, + { + patterns: []string{"*.md", "!README.md", "README*"}, + inputs: []string{"README.doc"}, + skipResult: false, + filterResult: true, + }, + } + + for _, kase := range kases { + t.Run(strings.Join(kase.patterns, ","), func(t *testing.T) { + patterns, err := CompilePatterns(kase.patterns...) + assert.NoError(t, err) + + assert.EqualValues(t, kase.skipResult, Skip(patterns, kase.inputs, &StdOutTraceWriter{}), "skipResult") + assert.EqualValues(t, kase.filterResult, Filter(patterns, kase.inputs, &StdOutTraceWriter{}), "filterResult") + }) + } +} From 606fd4bde138fd91001d49d1edb58530e4aa593e Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Thu, 23 Feb 2023 22:16:07 +0000 Subject: [PATCH 62/73] fix: crash malformed composite action (#1616) * fix: crash malformed composite action * Add remote composite action test --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/action.go | 10 +++++-- act/runner/action_composite.go | 4 +++ act/runner/runner_test.go | 2 ++ .../push.yml | 29 +++++++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 act/runner/testdata/no-panic-on-invalid-composite-action/push.yml diff --git a/act/runner/action.go b/act/runner/action.go index 4d860a22..94d9cfc1 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -503,7 +503,10 @@ func runPreStep(step actionStep) common.Executor { step.getCompositeRunContext(ctx) } - return step.getCompositeSteps().pre(ctx) + if steps := step.getCompositeSteps(); steps != nil && steps.pre != nil { + return steps.pre(ctx) + } + return fmt.Errorf("missing steps in composite action") default: return nil @@ -592,7 +595,10 @@ func runPostStep(step actionStep) common.Executor { return err } - return step.getCompositeSteps().post(ctx) + if steps := step.getCompositeSteps(); steps != nil && steps.post != nil { + return steps.post(ctx) + } + return fmt.Errorf("missing steps in composite action") default: return nil diff --git a/act/runner/action_composite.go b/act/runner/action_composite.go index 0b6fbe55..b6ef58c8 100644 --- a/act/runner/action_composite.go +++ b/act/runner/action_composite.go @@ -86,6 +86,10 @@ func execAsComposite(step actionStep) common.Executor { steps := step.getCompositeSteps() + if steps == nil || steps.main == nil { + return fmt.Errorf("missing steps in composite action") + } + ctx = WithCompositeLogger(ctx, &compositeRC.Masks) err := steps.main(ctx) diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index fb51be97..b0ce4f20 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -299,6 +299,7 @@ func TestRunEvent(t *testing.T) { {workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets}, {workdir, "set-env-step-env-override", "push", "", platforms, secrets}, {workdir, "set-env-new-env-file-per-step", "push", "", platforms, secrets}, + {workdir, "no-panic-on-invalid-composite-action", "push", "jobs failed due to invalid action", platforms, secrets}, } for _, table := range tables { @@ -401,6 +402,7 @@ func TestRunEventHostEnvironment(t *testing.T) { {workdir, "do-not-leak-step-env-in-composite", "push", "", platforms, secrets}, {workdir, "set-env-step-env-override", "push", "", platforms, secrets}, {workdir, "set-env-new-env-file-per-step", "push", "", platforms, secrets}, + {workdir, "no-panic-on-invalid-composite-action", "push", "jobs failed due to invalid action", platforms, secrets}, }...) } diff --git a/act/runner/testdata/no-panic-on-invalid-composite-action/push.yml b/act/runner/testdata/no-panic-on-invalid-composite-action/push.yml new file mode 100644 index 00000000..6b9e4ae6 --- /dev/null +++ b/act/runner/testdata/no-panic-on-invalid-composite-action/push.yml @@ -0,0 +1,29 @@ +on: push +jobs: + local-invalid-step: + runs-on: ubuntu-latest + steps: + - run: | + runs: + using: composite + steps: + - name: Foo + - uses: Foo/Bar + shell: cp {0} action.yml + - uses: ./ + local-missing-steps: + runs-on: ubuntu-latest + steps: + - run: | + runs: + using: composite + shell: cp {0} action.yml + - uses: ./ + remote-invalid-step: + runs-on: ubuntu-latest + steps: + - uses: nektos/act-test-actions/invalid-composite-action/invalid-step@main + remote-missing-steps: + runs-on: ubuntu-latest + steps: + - uses: nektos/act-test-actions/invalid-composite-action/missing-steps@main \ No newline at end of file From 943f34732719afab115d4d158c3adfadd895165c Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Thu, 23 Feb 2023 23:34:47 +0100 Subject: [PATCH 63/73] fix: update output handling for reusable workflows (#1521) * fix: map job output for reusable workflows This fixes the job outputs for reusable workflows. There is a required indirection. Before this we took the outputs from all jobs which is not what users express with the workflow outputs. * fix: remove double evaluation --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/exprparser/interpreter.go | 6 ++++++ act/model/workflow.go | 4 ++++ act/runner/expression.go | 20 +++++++++++++++++++ act/runner/job_executor.go | 5 +++-- .../workflows/local-reusable-workflow.yml | 4 ++-- 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/act/exprparser/interpreter.go b/act/exprparser/interpreter.go index 6b276fd1..ea912713 100644 --- a/act/exprparser/interpreter.go +++ b/act/exprparser/interpreter.go @@ -15,6 +15,7 @@ type EvaluationEnvironment struct { Github *model.GithubContext Env map[string]string Job *model.JobContext + Jobs *map[string]*model.WorkflowCallResult Steps map[string]*model.StepResult Runner map[string]interface{} Secrets map[string]string @@ -155,6 +156,11 @@ func (impl *interperterImpl) evaluateVariable(variableNode *actionlint.VariableN return impl.env.Env, nil case "job": return impl.env.Job, nil + case "jobs": + if impl.env.Jobs == nil { + return nil, fmt.Errorf("Unavailable context: jobs") + } + return impl.env.Jobs, nil case "steps": return impl.env.Steps, nil case "runner": diff --git a/act/model/workflow.go b/act/model/workflow.go index 3da7a133..d7e2922b 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -117,6 +117,10 @@ type WorkflowCall struct { Outputs map[string]WorkflowCallOutput `yaml:"outputs"` } +type WorkflowCallResult struct { + Outputs map[string]string +} + func (w *Workflow) WorkflowCallConfig() *WorkflowCall { if w.RawOn.Kind != yaml.MappingNode { return nil diff --git a/act/runner/expression.go b/act/runner/expression.go index a8d506ea..ca40e95c 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -25,6 +25,8 @@ func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEval } func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map[string]string) ExpressionEvaluator { + var workflowCallResult map[string]*model.WorkflowCallResult + // todo: cleanup EvaluationEnvironment creation using := make(map[string]exprparser.Needs) strategy := make(map[string]interface{}) @@ -44,6 +46,23 @@ func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map Result: jobs[needs].Result, } } + + // only setup jobs context in case of workflow_call + // and existing expression evaluator (this means, jobs are at + // least ready to run) + if rc.caller != nil && rc.ExprEval != nil { + workflowCallResult = map[string]*model.WorkflowCallResult{} + + for jobName, job := range jobs { + result := model.WorkflowCallResult{ + Outputs: map[string]string{}, + } + for k, v := range job.Outputs { + result.Outputs[k] = v + } + workflowCallResult[jobName] = &result + } + } } ghc := rc.getGithubContext(ctx) @@ -53,6 +72,7 @@ func (rc *RunContext) NewExpressionEvaluatorWithEnv(ctx context.Context, env map Github: ghc, Env: env, Job: rc.getJobContext(), + Jobs: &workflowCallResult, // todo: should be unavailable // but required to interpolate/evaluate the step outputs on the job Steps: rc.getStepsContext(), diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index 88c227fe..2f98ada4 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -162,8 +162,9 @@ func setJobOutputs(ctx context.Context, rc *RunContext) { callerOutputs := make(map[string]string) ee := rc.NewExpressionEvaluator(ctx) - for k, v := range rc.Run.Job().Outputs { - callerOutputs[k] = ee.Interpolate(ctx, v) + + for k, v := range rc.Run.Workflow.WorkflowCallConfig().Outputs { + callerOutputs[k] = ee.Interpolate(ctx, ee.Interpolate(ctx, v.Value)) } rc.caller.runContext.Run.Job().Outputs = callerOutputs diff --git a/act/runner/testdata/.github/workflows/local-reusable-workflow.yml b/act/runner/testdata/.github/workflows/local-reusable-workflow.yml index a52fdcf3..d32dc5b8 100644 --- a/act/runner/testdata/.github/workflows/local-reusable-workflow.yml +++ b/act/runner/testdata/.github/workflows/local-reusable-workflow.yml @@ -27,7 +27,7 @@ on: outputs: output: description: "A workflow output" - value: ${{ jobs.reusable_workflow_job.outputs.output }} + value: ${{ jobs.reusable_workflow_job.outputs.job-output }} jobs: reusable_workflow_job: @@ -79,4 +79,4 @@ jobs: echo "value=${{ inputs.string_required }}" >> $GITHUB_OUTPUT outputs: - output: ${{ steps.output_test.outputs.value }} + job-output: ${{ steps.output_test.outputs.value }} From 33b4484fcd2296aa4b8e2f5271a9487e0cce278d Mon Sep 17 00:00:00 2001 From: Alex Savchuk Date: Mon, 27 Feb 2023 22:10:31 +0300 Subject: [PATCH 64/73] fix: github.job property is empty, GITHUB_JOB should be job id (#1646) * fix: github.job property is empty, GITHUB_JOB should be job id fix: github.job property is empty #1621 fix: GITHUB_JOB should be the id not the name #1473 * fix linter problem. --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 3 ++- act/runner/run_context_test.go | 3 +++ act/runner/step_test.go | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index e24a236c..dd082fd7 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -567,6 +567,7 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext EventName: rc.Config.EventName, Action: rc.CurrentStep, Token: rc.Config.Token, + Job: rc.Run.JobID, ActionPath: rc.ActionPath, RepositoryOwner: rc.Config.Env["GITHUB_REPOSITORY_OWNER"], RetentionDays: rc.Config.Env["GITHUB_RETENTION_DAYS"], @@ -693,7 +694,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon env["GITHUB_REF_NAME"] = github.RefName env["GITHUB_REF_TYPE"] = github.RefType env["GITHUB_TOKEN"] = github.Token - env["GITHUB_JOB"] = rc.JobName + env["GITHUB_JOB"] = github.Job env["GITHUB_REPOSITORY_OWNER"] = github.RepositoryOwner env["GITHUB_RETENTION_DAYS"] = github.RetentionDays env["RUNNER_PERFLOG"] = github.RunnerPerflog diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index de8f177a..e7059461 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -144,6 +144,7 @@ func TestRunContext_EvalBool(t *testing.T) { // Check github context {in: "github.actor == 'nektos/act'", out: true}, {in: "github.actor == 'unknown'", out: false}, + {in: "github.job == 'job1'", out: true}, // The special ACT flag {in: "${{ env.ACT }}", out: true}, {in: "${{ !env.ACT }}", out: false}, @@ -364,6 +365,7 @@ func TestGetGitHubContext(t *testing.T) { StepResults: map[string]*model.StepResult{}, OutputMappings: map[MappableOutput]MappableOutput{}, } + rc.Run.JobID = "job1" ghc := rc.getGithubContext(context.Background()) @@ -392,6 +394,7 @@ func TestGetGitHubContext(t *testing.T) { assert.Equal(t, ghc.RepositoryOwner, owner) assert.Equal(t, ghc.RunnerPerflog, "/dev/null") assert.Equal(t, ghc.Token, rc.Config.Secrets["GITHUB_TOKEN"]) + assert.Equal(t, ghc.Job, "job1") } func TestGetGithubContextRef(t *testing.T) { diff --git a/act/runner/step_test.go b/act/runner/step_test.go index 86e5acc4..4fc77652 100644 --- a/act/runner/step_test.go +++ b/act/runner/step_test.go @@ -175,7 +175,7 @@ func TestSetupEnv(t *testing.T) { "GITHUB_EVENT_PATH": "/var/run/act/workflow/event.json", "GITHUB_GRAPHQL_URL": "https:///api/graphql", "GITHUB_HEAD_REF": "", - "GITHUB_JOB": "", + "GITHUB_JOB": "1", "GITHUB_RETENTION_DAYS": "0", "GITHUB_RUN_ID": "runId", "GITHUB_RUN_NUMBER": "1", From 1f1adbac3d0e161d28aa504f91dbeee3cbef6f53 Mon Sep 17 00:00:00 2001 From: Alex Savchuk Date: Fri, 3 Mar 2023 11:16:33 +0300 Subject: [PATCH 65/73] fix: compare properties of Invalid types (#1645) * fix: compare properties of Invalid types fix: compare properties of Invalid types #1643 * fix linter problem * Fix review comment --- act/exprparser/interpreter.go | 10 +++++++++- act/exprparser/interpreter_test.go | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/act/exprparser/interpreter.go b/act/exprparser/interpreter.go index ea912713..ef3e8e11 100644 --- a/act/exprparser/interpreter.go +++ b/act/exprparser/interpreter.go @@ -372,8 +372,16 @@ func (impl *interperterImpl) compareValues(leftValue reflect.Value, rightValue r return impl.compareNumber(leftValue.Float(), rightValue.Float(), kind) + case reflect.Invalid: + if rightValue.Kind() == reflect.Invalid { + return true, nil + } + + // not possible situation - params are converted to the same type in code above + return nil, fmt.Errorf("Compare params of Invalid type: left: %+v, right: %+v", leftValue.Kind(), rightValue.Kind()) + default: - return nil, fmt.Errorf("TODO: evaluateCompare not implemented! left: %+v, right: %+v", leftValue.Kind(), rightValue.Kind()) + return nil, fmt.Errorf("Compare not implemented for types: left: %+v, right: %+v", leftValue.Kind(), rightValue.Kind()) } } diff --git a/act/exprparser/interpreter_test.go b/act/exprparser/interpreter_test.go index d6f58a7c..01eb25f4 100644 --- a/act/exprparser/interpreter_test.go +++ b/act/exprparser/interpreter_test.go @@ -69,6 +69,11 @@ func TestOperators(t *testing.T) { {`true || false`, true, "or", ""}, {`fromJSON('{}') && true`, true, "and-boolean-object", ""}, {`fromJSON('{}') || false`, make(map[string]interface{}), "or-boolean-object", ""}, + {"github.event.commits[0].author.username != github.event.commits[1].author.username", true, "property-comparison1", ""}, + {"github.event.commits[0].author.username1 != github.event.commits[1].author.username", true, "property-comparison2", ""}, + {"github.event.commits[0].author.username != github.event.commits[1].author.username1", true, "property-comparison3", ""}, + {"github.event.commits[0].author.username1 != github.event.commits[1].author.username2", true, "property-comparison4", ""}, + {"secrets != env", nil, "property-comparison5", "Compare not implemented for types: left: map, right: map"}, } env := &EvaluationEnvironment{ From d8ea1eb23659f104adaa85917e5460634c7fe77b Mon Sep 17 00:00:00 2001 From: Jason Song Date: Fri, 3 Mar 2023 16:38:33 +0800 Subject: [PATCH 66/73] fix: safe file name (#1651) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/action.go | 4 ++-- act/runner/reusable_workflow.go | 3 +-- act/runner/step_action_remote.go | 24 +++++++++++++++++----- act/runner/step_action_remote_test.go | 29 +++++++++++++++++++++++---- 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/act/runner/action.go b/act/runner/action.go index 94d9cfc1..f871653b 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -472,7 +472,7 @@ func runPreStep(step actionStep) common.Executor { var actionPath string if _, ok := step.(*stepActionRemote); ok { actionPath = newRemoteAction(stepModel.Uses).Path - actionDir = fmt.Sprintf("%s/%s", rc.ActionCacheDir(), strings.ReplaceAll(stepModel.Uses, "/", "-")) + actionDir = fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(stepModel.Uses)) } else { actionDir = filepath.Join(rc.Config.Workdir, stepModel.Uses) actionPath = "" @@ -563,7 +563,7 @@ func runPostStep(step actionStep) common.Executor { var actionPath string if _, ok := step.(*stepActionRemote); ok { actionPath = newRemoteAction(stepModel.Uses).Path - actionDir = fmt.Sprintf("%s/%s", rc.ActionCacheDir(), strings.ReplaceAll(stepModel.Uses, "/", "-")) + actionDir = fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(stepModel.Uses)) } else { actionDir = filepath.Join(rc.Config.Workdir, stepModel.Uses) actionPath = "" diff --git a/act/runner/reusable_workflow.go b/act/runner/reusable_workflow.go index a5687f93..1ffa22b7 100644 --- a/act/runner/reusable_workflow.go +++ b/act/runner/reusable_workflow.go @@ -8,7 +8,6 @@ import ( "os" "path" "regexp" - "strings" "sync" "github.com/nektos/act/pkg/common" @@ -29,7 +28,7 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor { } remoteReusableWorkflow.URL = rc.Config.GitHubInstance - workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), strings.ReplaceAll(uses, "/", "-")) + workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(uses)) return common.NewPipelineExecutor( newMutexExecutor(cloneIfRequired(rc, *remoteReusableWorkflow, workflowDir)), diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index 17834866..029ed5c4 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -11,11 +11,11 @@ import ( "regexp" "strings" + gogit "github.com/go-git/go-git/v5" + "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/common/git" "github.com/nektos/act/pkg/model" - - gogit "github.com/go-git/go-git/v5" ) type stepActionRemote struct { @@ -62,7 +62,7 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { } } - actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), strings.ReplaceAll(sar.Step.Uses, "/", "-")) + actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses)) gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{ URL: sar.remoteAction.CloneURL(), Ref: sar.remoteAction.Ref, @@ -122,7 +122,7 @@ func (sar *stepActionRemote) main() common.Executor { return sar.RunContext.JobContainer.CopyDir(copyToPath, sar.RunContext.Config.Workdir+string(filepath.Separator)+".", sar.RunContext.Config.UseGitIgnore)(ctx) } - actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), strings.ReplaceAll(sar.Step.Uses, "/", "-")) + actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses)) return sar.runAction(sar, actionDir, sar.remoteAction)(ctx) }), @@ -181,7 +181,7 @@ func (sar *stepActionRemote) getActionModel() *model.Action { func (sar *stepActionRemote) getCompositeRunContext(ctx context.Context) *RunContext { if sar.compositeRunContext == nil { - actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), strings.ReplaceAll(sar.Step.Uses, "/", "-")) + actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses)) actionLocation := path.Join(actionDir, sar.remoteAction.Path) _, containerActionDir := getContainerActionPaths(sar.getStepModel(), actionLocation, sar.RunContext) @@ -243,3 +243,17 @@ func newRemoteAction(action string) *remoteAction { URL: "github.com", } } + +func safeFilename(s string) string { + return strings.NewReplacer( + `<`, "-", + `>`, "-", + `:`, "-", + `"`, "-", + `/`, "-", + `\`, "-", + `|`, "-", + `?`, "-", + `*`, "-", + ).Replace(s) +} diff --git a/act/runner/step_action_remote_test.go b/act/runner/step_action_remote_test.go index 23d65545..dfc49d27 100644 --- a/act/runner/step_action_remote_test.go +++ b/act/runner/step_action_remote_test.go @@ -8,13 +8,13 @@ import ( "strings" "testing" - "github.com/nektos/act/pkg/common" - "github.com/nektos/act/pkg/common/git" - "github.com/nektos/act/pkg/model" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "gopkg.in/yaml.v3" + + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/common/git" + "github.com/nektos/act/pkg/model" ) type stepActionRemoteMocks struct { @@ -615,3 +615,24 @@ func TestStepActionRemotePost(t *testing.T) { }) } } + +func Test_safeFilename(t *testing.T) { + tests := []struct { + s string + want string + }{ + { + s: "https://test.com/test/", + want: "https---test.com-test-", + }, + { + s: `<>:"/\|?*`, + want: "---------", + }, + } + for _, tt := range tests { + t.Run(tt.s, func(t *testing.T) { + assert.Equalf(t, tt.want, safeFilename(tt.s), "safeFilename(%v)", tt.s) + }) + } +} From 5b7fb3fb223fa664bdc4c9a22b65a5dccbd3de8c Mon Sep 17 00:00:00 2001 From: Tony Soloveyv <33671815+Tony-Sol@users.noreply.github.com> Date: Fri, 3 Mar 2023 17:39:02 +0300 Subject: [PATCH 67/73] Improve XDG Spec supporting (#1656) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- cmd/root.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 3619c7bf..e5c04791 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/AlecAivazis/survey/v2" + "github.com/adrg/xdg" "github.com/andreaskoch/go-fswatch" "github.com/joho/godotenv" "github.com/mitchellh/go-homedir" @@ -98,18 +99,21 @@ func configLocations() []string { log.Fatal(err) } + configFileName := ".actrc" + // reference: https://specifications.freedesktop.org/basedir-spec/latest/ar01s03.html var actrcXdg string - if xdg, ok := os.LookupEnv("XDG_CONFIG_HOME"); ok && xdg != "" { - actrcXdg = filepath.Join(xdg, ".actrc") - } else { - actrcXdg = filepath.Join(home, ".config", ".actrc") + for _, fileName := range []string{"act/actrc", configFileName} { + if foundConfig, err := xdg.SearchConfigFile(fileName); foundConfig != "" && err == nil { + actrcXdg = foundConfig + break + } } return []string{ - filepath.Join(home, ".actrc"), + filepath.Join(home, configFileName), actrcXdg, - filepath.Join(".", ".actrc"), + filepath.Join(".", configFileName), } } From 554522b044b70272837b766fda7a870ded9587e5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Mar 2023 03:30:59 +0000 Subject: [PATCH 68/73] build(deps): bump megalinter/megalinter from 6.19.0 to 6.20.0 (#1665) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 6.19.0 to 6.20.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v6.19.0...v6.20.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1f5327d9..69365dd5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,7 +22,7 @@ jobs: - uses: golangci/golangci-lint-action@v3.4.0 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.19.0 + - uses: megalinter/megalinter/flavors/go@v6.20.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 073d2055475ed77aaf41c559e936cab7966ef808 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 8 Mar 2023 15:41:25 +0100 Subject: [PATCH 69/73] fix: crash if the id tool fails to run in the container (1660) --- act/container/docker_run.go | 2 +- .../testdata/remote-action-js-node-user/push.yml | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 0ba562d0..62390f6e 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -583,7 +583,7 @@ func (cr *containerReference) tryReadID(opt string, cbk func(id int)) common.Exe } exp := regexp.MustCompile(`\d+\n`) found := exp.FindString(sid) - id, err := strconv.ParseInt(found[:len(found)-1], 10, 32) + id, err := strconv.ParseInt(strings.TrimSpace(found), 10, 32) if err != nil { return nil } diff --git a/act/runner/testdata/remote-action-js-node-user/push.yml b/act/runner/testdata/remote-action-js-node-user/push.yml index cede7b0a..8bf45da4 100644 --- a/act/runner/testdata/remote-action-js-node-user/push.yml +++ b/act/runner/testdata/remote-action-js-node-user/push.yml @@ -8,6 +8,21 @@ jobs: image: node:16-buster-slim options: --user node steps: + - name: check permissions of env files + id: test + run: | + echo "USER: $(id -un) expected: node" + [[ "$(id -un)" = "node" ]] + echo "TEST=Value" >> $GITHUB_OUTPUT + shell: bash + + - name: check if file command worked + if: steps.test.outputs.test != 'Value' + run: | + echo "steps.test.outputs.test=${{ steps.test.outputs.test || 'missing value!' }}" + exit 1 + shell: bash + - uses: actions/hello-world-javascript-action@v1 with: who-to-greet: 'Mona the Octocat' From b2784e31049dfecaa566aee834ccf6ad13716957 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Wed, 8 Mar 2023 15:57:49 +0100 Subject: [PATCH 70/73] test: Enshure ForcePull config doesn't break docker actions (1661) --- act/runner/runner_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index b0ce4f20..60a81937 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -442,6 +442,30 @@ func TestDryrunEvent(t *testing.T) { } } +func TestDockerActionForcePullForceRebuild(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + ctx := context.Background() + + config := &Config{ + ForcePull: true, + ForceRebuild: true, + } + + tables := []TestJobFileInfo{ + {workdir, "local-action-dockerfile", "push", "", platforms, secrets}, + {workdir, "local-action-via-composite-dockerfile", "push", "", platforms, secrets}, + } + + for _, table := range tables { + t.Run(table.workflowPath, func(t *testing.T) { + table.runTest(ctx, t, config) + }) + } +} + func TestRunDifferentArchitecture(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") From e29fd4cac713d34f53cab66422561a049e22602e Mon Sep 17 00:00:00 2001 From: Jason Song Date: Wed, 8 Mar 2023 23:13:11 +0800 Subject: [PATCH 71/73] fix: return err in walk (#1667) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/exprparser/functions.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/act/exprparser/functions.go b/act/exprparser/functions.go index 047a0e3c..83b2a080 100644 --- a/act/exprparser/functions.go +++ b/act/exprparser/functions.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "github.com/nektos/act/pkg/model" "github.com/rhysd/actionlint" ) @@ -202,6 +203,9 @@ func (impl *interperterImpl) hashFiles(paths ...reflect.Value) (string, error) { var files []string if err := filepath.Walk(impl.config.WorkingDir, func(path string, fi fs.FileInfo, err error) error { + if err != nil { + return err + } sansPrefix := strings.TrimPrefix(path, impl.config.WorkingDir+string(filepath.Separator)) parts := strings.Split(sansPrefix, string(filepath.Separator)) if fi.IsDir() || !matcher.Match(parts, fi.IsDir()) { From 8d098545de46de86042a7fd7ca183d5c66ed01a1 Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Thu, 9 Mar 2023 21:03:13 +0100 Subject: [PATCH 72/73] fix: correct ref and ref_name (#1672) * fix: correct ref and ref_name The ref in the GitHub context is always full qualified (e.g. refs/heads/branch, refs/tags/v1). The ref_name is the ref with the strippep prefix. In case of pull_requests, this is the merge commit ref (e.g. refs/pull/123/merge -> 123/merge). * test: update test data --- act/model/github_context.go | 5 ++++- act/model/github_context_test.go | 20 +++++++++++++++----- act/runner/run_context_test.go | 2 +- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/act/model/github_context.go b/act/model/github_context.go index 9ce8d08a..e4c31fcc 100644 --- a/act/model/github_context.go +++ b/act/model/github_context.go @@ -103,7 +103,7 @@ func (ghc *GithubContext) SetRef(ctx context.Context, defaultBranch string, repo case "deployment", "deployment_status": ghc.Ref = asString(nestedMapLookup(ghc.Event, "deployment", "ref")) case "release": - ghc.Ref = asString(nestedMapLookup(ghc.Event, "release", "tag_name")) + ghc.Ref = fmt.Sprintf("refs/tags/%s", asString(nestedMapLookup(ghc.Event, "release", "tag_name"))) case "push", "create", "workflow_dispatch": ghc.Ref = asString(ghc.Event["ref"]) default: @@ -183,6 +183,9 @@ func (ghc *GithubContext) SetRefTypeAndName() { } else if strings.HasPrefix(ghc.Ref, "refs/heads/") { refType = "branch" refName = ghc.Ref[len("refs/heads/"):] + } else if strings.HasPrefix(ghc.Ref, "refs/pull/") { + refType = "" + refName = ghc.Ref[len("refs/pull/"):] } if ghc.RefType == "" { diff --git a/act/model/github_context_test.go b/act/model/github_context_test.go index 29bb546a..ed08e231 100644 --- a/act/model/github_context_test.go +++ b/act/model/github_context_test.go @@ -29,18 +29,21 @@ func TestSetRef(t *testing.T) { eventName string event map[string]interface{} ref string + refName string }{ { eventName: "pull_request_target", event: map[string]interface{}{}, ref: "refs/heads/master", + refName: "master", }, { eventName: "pull_request", event: map[string]interface{}{ "number": 1234., }, - ref: "refs/pull/1234/merge", + ref: "refs/pull/1234/merge", + refName: "1234/merge", }, { eventName: "deployment", @@ -49,7 +52,8 @@ func TestSetRef(t *testing.T) { "ref": "refs/heads/somebranch", }, }, - ref: "refs/heads/somebranch", + ref: "refs/heads/somebranch", + refName: "somebranch", }, { eventName: "release", @@ -58,14 +62,16 @@ func TestSetRef(t *testing.T) { "tag_name": "v1.0.0", }, }, - ref: "v1.0.0", + ref: "refs/tags/v1.0.0", + refName: "v1.0.0", }, { eventName: "push", event: map[string]interface{}{ "ref": "refs/heads/somebranch", }, - ref: "refs/heads/somebranch", + ref: "refs/heads/somebranch", + refName: "somebranch", }, { eventName: "unknown", @@ -74,12 +80,14 @@ func TestSetRef(t *testing.T) { "default_branch": "main", }, }, - ref: "refs/heads/main", + ref: "refs/heads/main", + refName: "main", }, { eventName: "no-event", event: map[string]interface{}{}, ref: "refs/heads/master", + refName: "master", }, } @@ -92,8 +100,10 @@ func TestSetRef(t *testing.T) { } ghc.SetRef(context.Background(), "main", "/some/dir") + ghc.SetRefTypeAndName() assert.Equal(t, table.ref, ghc.Ref) + assert.Equal(t, table.refName, ghc.RefName) }) } diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index e7059461..3e26a022 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -413,7 +413,7 @@ func TestGetGithubContextRef(t *testing.T) { {event: "pull_request_target", json: `{"pull_request":{"base":{"ref": "main"}}}`, ref: "refs/heads/main"}, {event: "deployment", json: `{"deployment": {"ref": "tag-name"}}`, ref: "tag-name"}, {event: "deployment_status", json: `{"deployment": {"ref": "tag-name"}}`, ref: "tag-name"}, - {event: "release", json: `{"release": {"tag_name": "tag-name"}}`, ref: "tag-name"}, + {event: "release", json: `{"release": {"tag_name": "tag-name"}}`, ref: "refs/tags/tag-name"}, } for _, data := range table { From 6f6aad9a9b48c282c761024a1ef88947bf48f45a Mon Sep 17 00:00:00 2001 From: Shubham Agrawal Date: Tue, 14 Mar 2023 19:37:31 +0530 Subject: [PATCH 73/73] Support for docker steps in host environment (#1674) * Support for docker steps in host environment * removed workdir changes --- act/runner/action.go | 7 +++++-- act/runner/run_context.go | 8 ++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/act/runner/action.go b/act/runner/action.go index f871653b..a534170d 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -356,7 +356,10 @@ func newStepContainer(ctx context.Context, step step, image string, cmd []string envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp")) binds, mounts := rc.GetBindsAndMounts() - + networkMode := fmt.Sprintf("container:%s", rc.jobContainerName()) + if rc.IsHostEnv(ctx) { + networkMode = "default" + } stepContainer := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, @@ -367,7 +370,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd []string Name: createContainerName(rc.jobContainerName(), stepModel.ID), Env: envList, Mounts: mounts, - NetworkMode: fmt.Sprintf("container:%s", rc.jobContainerName()), + NetworkMode: networkMode, Binds: binds, Stdout: logWriter, Stderr: logWriter, diff --git a/act/runner/run_context.go b/act/runner/run_context.go index dd082fd7..27bbe878 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -384,14 +384,18 @@ func (rc *RunContext) interpolateOutputs() common.Executor { func (rc *RunContext) startContainer() common.Executor { return func(ctx context.Context) error { - image := rc.platformImage(ctx) - if strings.EqualFold(image, "-self-hosted") { + if rc.IsHostEnv(ctx) { return rc.startHostEnvironment()(ctx) } return rc.startJobContainer()(ctx) } } +func (rc *RunContext) IsHostEnv(ctx context.Context) bool { + image := rc.platformImage(ctx) + return strings.EqualFold(image, "-self-hosted") +} + func (rc *RunContext) stopContainer() common.Executor { return rc.stopJobContainer() }