From 82d61eaf05ae117a161565146809b90e686f3584 Mon Sep 17 00:00:00 2001 From: Jon Jensen Date: Fri, 13 Oct 2023 14:01:04 -0600 Subject: [PATCH 01/38] Fix float formatting (#2018) Format floats the same way as actions/runner (precision 15, remove trailing zeroes) See: https://github.com/actions/runner/blob/67d70803a95fca2fc86d89231acbc319f9a9be2a/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/NumberToken.cs#L34 --- act/exprparser/functions_test.go | 1 + act/exprparser/interpreter.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/act/exprparser/functions_test.go b/act/exprparser/functions_test.go index 3c6392c1..ea51a2bc 100644 --- a/act/exprparser/functions_test.go +++ b/act/exprparser/functions_test.go @@ -230,6 +230,7 @@ func TestFunctionFormat(t *testing.T) { {"format('{0', '{1}', 'World')", nil, "Unclosed brackets. The following format string is invalid: '{0'", "format-invalid-format-string"}, {"format('{2}', '{1}', 'World')", "", "The following format string references more arguments than were supplied: '{2}'", "format-invalid-replacement-reference"}, {"format('{2147483648}')", "", "The following format string is invalid: '{2147483648}'", "format-invalid-replacement-reference"}, + {"format('{0} {1} {2} {3}', 1.0, 1.1, 1234567890.0, 12345678901234567890.0)", "1 1.1 1234567890 1.23456789012346E+19", nil, "format-floats"}, } env := &EvaluationEnvironment{ diff --git a/act/exprparser/interpreter.go b/act/exprparser/interpreter.go index ff4d0a20..ce3aca38 100644 --- a/act/exprparser/interpreter.go +++ b/act/exprparser/interpreter.go @@ -447,7 +447,7 @@ func (impl *interperterImpl) coerceToString(value reflect.Value) reflect.Value { } else if math.IsInf(value.Float(), -1) { return reflect.ValueOf("-Infinity") } - return reflect.ValueOf(fmt.Sprint(value)) + return reflect.ValueOf(fmt.Sprintf("%.15G", value.Float())) case reflect.Slice: return reflect.ValueOf("Array") From 2de6a8e3aa1ea334f14eb205f16349c0798d34bb Mon Sep 17 00:00:00 2001 From: Sam Foo Date: Thu, 19 Oct 2023 02:24:52 -0700 Subject: [PATCH 02/38] Add support for service containers (#1949) * Support services (#42) Removed createSimpleContainerName and AutoRemove flag Co-authored-by: Lunny Xiao Co-authored-by: Jason Song Reviewed-on: https://gitea.com/gitea/act/pulls/42 Reviewed-by: Jason Song Co-authored-by: Zettat123 Co-committed-by: Zettat123 * Support services options (#45) Reviewed-on: https://gitea.com/gitea/act/pulls/45 Reviewed-by: Lunny Xiao Co-authored-by: Zettat123 Co-committed-by: Zettat123 * Support intepolation for `env` of `services` (#47) Reviewed-on: https://gitea.com/gitea/act/pulls/47 Reviewed-by: Lunny Xiao Co-authored-by: Zettat123 Co-committed-by: Zettat123 * Support services `credentials` (#51) If a service's image is from a container registry requires authentication, `act_runner` will need `credentials` to pull the image, see [documentation](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idservicesservice_idcredentials). Currently, `act_runner` incorrectly uses the `credentials` of `containers` to pull services' images and the `credentials` of services won't be used, see the related code: https://gitea.com/gitea/act/src/commit/ba7ef95f06fe237175a1495979479eb185260135/pkg/runner/run_context.go#L228-L269 Co-authored-by: Jason Song Reviewed-on: https://gitea.com/gitea/act/pulls/51 Reviewed-by: Jason Song Reviewed-by: Lunny Xiao Co-authored-by: Zettat123 Co-committed-by: Zettat123 * Add ContainerMaxLifetime and ContainerNetworkMode options from: https://gitea.com/gitea/act/commit/1d92791718bcb426f2a95f90dc1baf0045b89eb2 * Fix container network issue (#56) Follow: https://gitea.com/gitea/act_runner/pulls/184 Close https://gitea.com/gitea/act_runner/issues/177 - `act` create new networks only if the value of `NeedCreateNetwork` is true, and remove these networks at last. `NeedCreateNetwork` is passed by `act_runner`. 'NeedCreateNetwork' is true only if `container.network` in the configuration file of the `act_runner` is empty. - In the `docker create` phase, specify the network to which containers will connect. Because, if not specify , container will connect to `bridge` network which is created automatically by Docker. - If the network is user defined network ( the value of `container.network` is empty or ``. Because, the network created by `act` is also user defined network.), will also specify alias by `--network-alias`. The alias of service is ``. So we can be access service container by `:` in the steps of job. - Won't try to `docker network connect ` network after `docker start` any more. - Because on the one hand, `docker network connect` applies only to user defined networks, if try to `docker network connect host ` will return error. - On the other hand, we just specify network in the stage of `docker create`, the same effect can be achieved. - Won't try to remove containers and networks berfore the stage of `docker start`, because the name of these containers and netwoks won't be repeat. Co-authored-by: Jason Song Reviewed-on: https://gitea.com/gitea/act/pulls/56 Reviewed-by: Jason Song Co-authored-by: sillyguodong Co-committed-by: sillyguodong * Check volumes (#60) This PR adds a `ValidVolumes` config. Users can specify the volumes (including bind mounts) that can be mounted to containers by this config. Options related to volumes: - [jobs..container.volumes](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idcontainervolumes) - [jobs..services..volumes](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idservicesservice_idvolumes) In addition, volumes specified by `options` will also be checked. Currently, the following default volumes (see https://gitea.com/gitea/act/src/commit/f78a6206d3204d14f6b1ad97c1771385236d52e3/pkg/runner/run_context.go#L116-L166) will be added to `ValidVolumes`: - `act-toolcache` - `` and `-env` - `/var/run/docker.sock` (We need to add a new configuration to control whether the docker daemon can be mounted) Co-authored-by: Jason Song Reviewed-on: https://gitea.com/gitea/act/pulls/60 Reviewed-by: Jason Song Co-authored-by: Zettat123 Co-committed-by: Zettat123 * Remove ContainerMaxLifetime; fix lint * Remove unused ValidVolumes * Remove ConnectToNetwork * Add docker stubs * Close docker clients to prevent file descriptor leaks * Fix the error when removing network in self-hosted mode (#69) Fixes https://gitea.com/gitea/act_runner/issues/255 Reviewed-on: https://gitea.com/gitea/act/pulls/69 Co-authored-by: Zettat123 Co-committed-by: Zettat123 * Move service container and network cleanup to rc.cleanUpJobContainer * Add --network flag; default to host if not using service containers or set explicitly * Correctly close executor to prevent fd leak * Revert to tail instead of full path * fix network duplication * backport networkingConfig for aliaes * don't hardcode netMode host * Convert services test to table driven tests * Add failing tests for services * Expose service container ports onto the host * Set container network mode in artifacts server test to host mode * Log container network mode when creating/starting a container * fix: Correctly handle ContainerNetworkMode * fix: missing service container network * Always remove service containers Although we usually keep containers running if the workflow errored (unless `--rm` is given) in order to facilitate debugging and we have a flag (`--reuse`) to always keep containers running in order to speed up repeated `act` invocations, I believe that these should only apply to job containers and not service containers, because changing the network settings on a service container requires re-creating it anyway. * Remove networks only if no active endpoints exist * Ensure job containers are stopped before starting a new job * fix: go build -tags WITHOUT_DOCKER --------- Co-authored-by: Zettat123 Co-authored-by: Lunny Xiao Co-authored-by: Jason Song Co-authored-by: sillyguodong Co-authored-by: ChristopherHX Co-authored-by: ZauberNerd --- act/container/container_types.go | 38 +-- act/container/docker_network.go | 79 ++++++ act/container/docker_run.go | 50 ++-- act/container/docker_run_test.go | 1 + act/container/docker_stub.go | 12 + act/runner/job_executor.go | 10 +- act/runner/run_context.go | 233 ++++++++++++++++-- act/runner/runner.go | 81 +++--- act/runner/runner_test.go | 5 + .../testdata/services-host-network/push.yml | 14 ++ .../testdata/services-with-container/push.yml | 16 ++ act/runner/testdata/services/push.yaml | 26 ++ cmd/input.go | 1 + cmd/root.go | 3 + 14 files changed, 469 insertions(+), 100 deletions(-) create mode 100644 act/container/docker_network.go create mode 100644 act/runner/testdata/services-host-network/push.yml create mode 100644 act/runner/testdata/services-with-container/push.yml create mode 100644 act/runner/testdata/services/push.yaml diff --git a/act/container/container_types.go b/act/container/container_types.go index 767beb52..37d293a3 100644 --- a/act/container/container_types.go +++ b/act/container/container_types.go @@ -4,28 +4,32 @@ import ( "context" "io" + "github.com/docker/go-connections/nat" "github.com/nektos/act/pkg/common" ) // NewContainerInput the input for the New function type NewContainerInput struct { - Image string - Username string - Password string - Entrypoint []string - Cmd []string - WorkingDir string - Env []string - Binds []string - Mounts map[string]string - Name string - Stdout io.Writer - Stderr io.Writer - NetworkMode string - Privileged bool - UsernsMode string - Platform string - Options string + Image string + Username string + Password string + Entrypoint []string + Cmd []string + WorkingDir string + Env []string + Binds []string + Mounts map[string]string + Name string + Stdout io.Writer + Stderr io.Writer + NetworkMode string + Privileged bool + UsernsMode string + Platform string + Options string + NetworkAliases []string + ExposedPorts nat.PortSet + PortBindings nat.PortMap } // FileEntry is a file to copy to a container diff --git a/act/container/docker_network.go b/act/container/docker_network.go new file mode 100644 index 00000000..8a7528ab --- /dev/null +++ b/act/container/docker_network.go @@ -0,0 +1,79 @@ +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) + +package container + +import ( + "context" + + "github.com/docker/docker/api/types" + "github.com/nektos/act/pkg/common" +) + +func NewDockerNetworkCreateExecutor(name string) common.Executor { + return func(ctx context.Context) error { + cli, err := GetDockerClient(ctx) + if err != nil { + return err + } + defer cli.Close() + + // Only create the network if it doesn't exist + networks, err := cli.NetworkList(ctx, types.NetworkListOptions{}) + if err != nil { + return err + } + common.Logger(ctx).Debugf("%v", networks) + for _, network := range networks { + if network.Name == name { + common.Logger(ctx).Debugf("Network %v exists", name) + return nil + } + } + + _, err = cli.NetworkCreate(ctx, name, types.NetworkCreate{ + Driver: "bridge", + Scope: "local", + }) + if err != nil { + return err + } + + return nil + } +} + +func NewDockerNetworkRemoveExecutor(name string) common.Executor { + return func(ctx context.Context) error { + cli, err := GetDockerClient(ctx) + if err != nil { + return err + } + defer cli.Close() + + // Make shure that all network of the specified name are removed + // cli.NetworkRemove refuses to remove a network if there are duplicates + networks, err := cli.NetworkList(ctx, types.NetworkListOptions{}) + if err != nil { + return err + } + common.Logger(ctx).Debugf("%v", networks) + for _, network := range networks { + if network.Name == name { + result, err := cli.NetworkInspect(ctx, network.ID, types.NetworkInspectOptions{}) + if err != nil { + return err + } + + if len(result.Containers) == 0 { + if err = cli.NetworkRemove(ctx, network.ID); err != nil { + common.Logger(ctx).Debugf("%v", err) + } + } else { + common.Logger(ctx).Debugf("Refusing to remove network %v because it still has active endpoints", name) + } + } + } + + return err + } +} diff --git a/act/container/docker_run.go b/act/container/docker_run.go index cf58aee9..d24c4ac3 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -29,6 +29,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" specs "github.com/opencontainers/image-spec/specs-go/v1" @@ -66,7 +67,7 @@ func supportsContainerImagePlatform(ctx context.Context, cli client.APIClient) b func (cr *containerReference) Create(capAdd []string, capDrop []string) common.Executor { return common. - NewInfoExecutor("%sdocker create image=%s platform=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Platform, cr.input.Entrypoint, cr.input.Cmd). + NewInfoExecutor("%sdocker create image=%s platform=%s entrypoint=%+q cmd=%+q network=%+q", logPrefix, cr.input.Image, cr.input.Platform, cr.input.Entrypoint, cr.input.Cmd, cr.input.NetworkMode). Then( common.NewPipelineExecutor( cr.connect(), @@ -78,7 +79,7 @@ func (cr *containerReference) Create(capAdd []string, capDrop []string) common.E func (cr *containerReference) Start(attach bool) common.Executor { return common. - NewInfoExecutor("%sdocker run image=%s platform=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Platform, cr.input.Entrypoint, cr.input.Cmd). + NewInfoExecutor("%sdocker run image=%s platform=%s entrypoint=%+q cmd=%+q network=%+q", logPrefix, cr.input.Image, cr.input.Platform, cr.input.Entrypoint, cr.input.Cmd, cr.input.NetworkMode). Then( common.NewPipelineExecutor( cr.connect(), @@ -346,8 +347,8 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config } if len(copts.netMode.Value()) == 0 { - if err = copts.netMode.Set("host"); err != nil { - return nil, nil, fmt.Errorf("Cannot parse networkmode=host. This is an internal error and should not happen: '%w'", err) + if err = copts.netMode.Set(cr.input.NetworkMode); err != nil { + return nil, nil, fmt.Errorf("Cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err) } } @@ -391,10 +392,11 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E input := cr.input config := &container.Config{ - Image: input.Image, - WorkingDir: input.WorkingDir, - Env: input.Env, - Tty: isTerminal, + Image: input.Image, + WorkingDir: input.WorkingDir, + Env: input.Env, + ExposedPorts: input.ExposedPorts, + Tty: isTerminal, } logger.Debugf("Common container.Config ==> %+v", config) @@ -430,13 +432,14 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E } hostConfig := &container.HostConfig{ - CapAdd: capAdd, - CapDrop: capDrop, - Binds: input.Binds, - Mounts: mounts, - NetworkMode: container.NetworkMode(input.NetworkMode), - Privileged: input.Privileged, - UsernsMode: container.UsernsMode(input.UsernsMode), + CapAdd: capAdd, + CapDrop: capDrop, + Binds: input.Binds, + Mounts: mounts, + NetworkMode: container.NetworkMode(input.NetworkMode), + Privileged: input.Privileged, + UsernsMode: container.UsernsMode(input.UsernsMode), + PortBindings: input.PortBindings, } logger.Debugf("Common container.HostConfig ==> %+v", hostConfig) @@ -445,7 +448,22 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E return err } - resp, err := cr.cli.ContainerCreate(ctx, config, hostConfig, nil, platSpecs, input.Name) + var networkingConfig *network.NetworkingConfig + logger.Debugf("input.NetworkAliases ==> %v", input.NetworkAliases) + if hostConfig.NetworkMode.IsUserDefined() && len(input.NetworkAliases) > 0 { + endpointConfig := &network.EndpointSettings{ + Aliases: input.NetworkAliases, + } + networkingConfig = &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + input.NetworkMode: endpointConfig, + }, + } + } else { + logger.Debugf("not a use defined config??") + } + + resp, err := cr.cli.ContainerCreate(ctx, config, hostConfig, networkingConfig, platSpecs, input.Name) if err != nil { return fmt.Errorf("failed to create container: '%w'", err) } diff --git a/act/container/docker_run_test.go b/act/container/docker_run_test.go index 8309df6c..96bda592 100644 --- a/act/container/docker_run_test.go +++ b/act/container/docker_run_test.go @@ -19,6 +19,7 @@ func TestDocker(t *testing.T) { ctx := context.Background() client, err := GetDockerClient(ctx) assert.NoError(t, err) + defer client.Close() dockerBuild := NewDockerBuildExecutor(NewDockerBuildExecutorInput{ ContextDir: "testdata", diff --git a/act/container/docker_stub.go b/act/container/docker_stub.go index b28c90de..36f530ee 100644 --- a/act/container/docker_stub.go +++ b/act/container/docker_stub.go @@ -55,3 +55,15 @@ func NewDockerVolumeRemoveExecutor(volume string, force bool) common.Executor { return nil } } + +func NewDockerNetworkCreateExecutor(name string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func NewDockerNetworkRemoveExecutor(name string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index 3f2e41e2..148c9ff5 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -19,6 +19,7 @@ type jobInfo interface { result(result string) } +//nolint:contextcheck,gocyclo func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executor { steps := make([]common.Executor, 0) preSteps := make([]common.Executor, 0) @@ -87,7 +88,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo postExec := useStepLogger(rc, stepModel, stepStagePost, step.post()) if postExecutor != nil { - // run the post exector in reverse order + // run the post executor in reverse order postExecutor = postExec.Finally(postExecutor) } else { postExecutor = postExec @@ -101,7 +102,12 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo // always allow 1 min for stopping and removing the runner, even if we were cancelled ctx, cancel := context.WithTimeout(common.WithLogger(context.Background(), common.Logger(ctx)), time.Minute) defer cancel() - err = info.stopContainer()(ctx) //nolint:contextcheck + + logger := common.Logger(ctx) + logger.Infof("Cleaning up container for job %s", rc.JobName) + if err = info.stopContainer()(ctx); err != nil { + logger.Errorf("Error while stop job container: %v", err) + } } setJobResult(ctx, info, rc, jobError == nil) setJobOutputs(ctx, rc) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 2c55e8a8..70d2f8f4 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -17,12 +17,12 @@ import ( "runtime" "strings" - "github.com/opencontainers/selinux/go-selinux" - + "github.com/docker/go-connections/nat" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/exprparser" "github.com/nektos/act/pkg/model" + "github.com/opencontainers/selinux/go-selinux" ) // RunContext contains info about current job @@ -40,6 +40,7 @@ type RunContext struct { IntraActionState map[string]map[string]string ExprEval ExpressionEvaluator JobContainer container.ExecutionsEnvironment + ServiceContainers []container.ExecutionsEnvironment OutputMappings map[MappableOutput]MappableOutput JobName string ActionPath string @@ -87,6 +88,18 @@ func (rc *RunContext) jobContainerName() string { return createContainerName("act", rc.String()) } +// networkName return the name of the network which will be created by `act` automatically for job, +// only create network if using a service container +func (rc *RunContext) networkName() (string, bool) { + if len(rc.Run.Job().Services) > 0 { + return fmt.Sprintf("%s-%s-network", rc.jobContainerName(), rc.Run.JobID), true + } + if rc.Config.ContainerNetworkMode == "" { + return "host", false + } + return string(rc.Config.ContainerNetworkMode), false +} + func getDockerDaemonSocketMountPath(daemonPath string) string { if protoIndex := strings.Index(daemonPath, "://"); protoIndex != -1 { scheme := daemonPath[:protoIndex] @@ -226,6 +239,7 @@ func (rc *RunContext) startHostEnvironment() common.Executor { } } +//nolint:gocyclo func (rc *RunContext) startJobContainer() common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) @@ -259,41 +273,126 @@ func (rc *RunContext) startJobContainer() common.Executor { ext := container.LinuxContainerEnvironmentExtensions{} binds, mounts := rc.GetBindsAndMounts() + // specify the network to which the container will connect when `docker create` stage. (like execute command line: docker create --network ) + // if using service containers, will create a new network for the containers. + // and it will be removed after at last. + networkName, createAndDeleteNetwork := rc.networkName() + + // add service containers + for serviceID, spec := range rc.Run.Job().Services { + // interpolate env + interpolatedEnvs := make(map[string]string, len(spec.Env)) + for k, v := range spec.Env { + interpolatedEnvs[k] = rc.ExprEval.Interpolate(ctx, v) + } + envs := make([]string, 0, len(interpolatedEnvs)) + for k, v := range interpolatedEnvs { + envs = append(envs, fmt.Sprintf("%s=%s", k, v)) + } + username, password, err = rc.handleServiceCredentials(ctx, spec.Credentials) + if err != nil { + return fmt.Errorf("failed to handle service %s credentials: %w", serviceID, err) + } + serviceBinds, serviceMounts := rc.GetServiceBindsAndMounts(spec.Volumes) + + exposedPorts, portBindings, err := nat.ParsePortSpecs(spec.Ports) + if err != nil { + return fmt.Errorf("failed to parse service %s ports: %w", serviceID, err) + } + + serviceContainerName := createContainerName(rc.jobContainerName(), serviceID) + c := container.NewContainer(&container.NewContainerInput{ + Name: serviceContainerName, + WorkingDir: ext.ToContainerPath(rc.Config.Workdir), + Image: spec.Image, + Username: username, + Password: password, + Env: envs, + Mounts: serviceMounts, + Binds: serviceBinds, + Stdout: logWriter, + Stderr: logWriter, + Privileged: rc.Config.Privileged, + UsernsMode: rc.Config.UsernsMode, + Platform: rc.Config.ContainerArchitecture, + Options: spec.Options, + NetworkMode: networkName, + NetworkAliases: []string{serviceID}, + ExposedPorts: exposedPorts, + PortBindings: portBindings, + }) + rc.ServiceContainers = append(rc.ServiceContainers, c) + } + rc.cleanUpJobContainer = func(ctx context.Context) error { - if rc.JobContainer != nil && !rc.Config.ReuseContainers { - return rc.JobContainer.Remove(). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + reuseJobContainer := func(ctx context.Context) bool { + return rc.Config.ReuseContainers + } + + if rc.JobContainer != nil { + return rc.JobContainer.Remove().IfNot(reuseJobContainer). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)).IfNot(reuseJobContainer). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false)).IfNot(reuseJobContainer). + Then(func(ctx context.Context) error { + if len(rc.ServiceContainers) > 0 { + logger.Infof("Cleaning up services for job %s", rc.JobName) + if err := rc.stopServiceContainers()(ctx); err != nil { + logger.Errorf("Error while cleaning services: %v", err) + } + if createAndDeleteNetwork { + // clean network if it has been created by act + // if using service containers + // it means that the network to which containers are connecting is created by `act_runner`, + // so, we should remove the network at last. + logger.Infof("Cleaning up network for job %s, and network name is: %s", rc.JobName, networkName) + if err := container.NewDockerNetworkRemoveExecutor(networkName)(ctx); err != nil { + logger.Errorf("Error while cleaning network: %v", err) + } + } + } + return nil + })(ctx) } return nil } + jobContainerNetwork := rc.Config.ContainerNetworkMode.NetworkName() + if rc.containerImage(ctx) != "" { + jobContainerNetwork = networkName + } else if jobContainerNetwork == "" { + jobContainerNetwork = "host" + } + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ - Cmd: nil, - Entrypoint: []string{"tail", "-f", "/dev/null"}, - WorkingDir: ext.ToContainerPath(rc.Config.Workdir), - Image: image, - Username: username, - Password: password, - Name: name, - Env: envList, - Mounts: mounts, - NetworkMode: "host", - Binds: binds, - Stdout: logWriter, - Stderr: logWriter, - Privileged: rc.Config.Privileged, - UsernsMode: rc.Config.UsernsMode, - Platform: rc.Config.ContainerArchitecture, - Options: rc.options(ctx), + Cmd: nil, + Entrypoint: []string{"tail", "-f", "/dev/null"}, + WorkingDir: ext.ToContainerPath(rc.Config.Workdir), + Image: image, + Username: username, + Password: password, + Name: name, + Env: envList, + Mounts: mounts, + NetworkMode: jobContainerNetwork, + NetworkAliases: []string{rc.Name}, + Binds: binds, + Stdout: logWriter, + Stderr: logWriter, + Privileged: rc.Config.Privileged, + UsernsMode: rc.Config.UsernsMode, + Platform: rc.Config.ContainerArchitecture, + Options: rc.options(ctx), }) if rc.JobContainer == nil { return errors.New("Failed to create job container") } return common.NewPipelineExecutor( + rc.pullServicesImages(rc.Config.ForcePull), rc.JobContainer.Pull(rc.Config.ForcePull), rc.stopJobContainer(), + container.NewDockerNetworkCreateExecutor(networkName).IfBool(createAndDeleteNetwork), + rc.startServiceContainers(networkName), rc.JobContainer.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop), rc.JobContainer.Start(false), rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ @@ -369,16 +468,50 @@ func (rc *RunContext) UpdateExtraPath(ctx context.Context, githubEnvPath string) return nil } -// stopJobContainer removes the job container (if it exists) and its volume (if it exists) if !rc.Config.ReuseContainers +// stopJobContainer removes the job container (if it exists) and its volume (if it exists) func (rc *RunContext) stopJobContainer() common.Executor { return func(ctx context.Context) error { - if rc.cleanUpJobContainer != nil && !rc.Config.ReuseContainers { + if rc.cleanUpJobContainer != nil { return rc.cleanUpJobContainer(ctx) } return nil } } +func (rc *RunContext) pullServicesImages(forcePull bool) common.Executor { + return func(ctx context.Context) error { + execs := []common.Executor{} + for _, c := range rc.ServiceContainers { + execs = append(execs, c.Pull(forcePull)) + } + return common.NewParallelExecutor(len(execs), execs...)(ctx) + } +} + +func (rc *RunContext) startServiceContainers(_ string) common.Executor { + return func(ctx context.Context) error { + execs := []common.Executor{} + for _, c := range rc.ServiceContainers { + execs = append(execs, common.NewPipelineExecutor( + c.Pull(false), + c.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop), + c.Start(false), + )) + } + return common.NewParallelExecutor(len(execs), execs...)(ctx) + } +} + +func (rc *RunContext) stopServiceContainers() common.Executor { + return func(ctx context.Context) error { + execs := []common.Executor{} + for _, c := range rc.ServiceContainers { + execs = append(execs, c.Remove().Finally(c.Close())) + } + return common.NewParallelExecutor(len(execs), execs...)(ctx) + } +} + // Prepare the mounts and binds for the worker // ActionCacheDir is for rc @@ -853,3 +986,53 @@ func (rc *RunContext) handleCredentials(ctx context.Context) (string, string, er return username, password, nil } + +func (rc *RunContext) handleServiceCredentials(ctx context.Context, creds map[string]string) (username, password string, err error) { + if creds == nil { + return + } + if len(creds) != 2 { + err = fmt.Errorf("invalid property count for key 'credentials:'") + return + } + + ee := rc.NewExpressionEvaluator(ctx) + if username = ee.Interpolate(ctx, creds["username"]); username == "" { + err = fmt.Errorf("failed to interpolate credentials.username") + return + } + + if password = ee.Interpolate(ctx, creds["password"]); password == "" { + err = fmt.Errorf("failed to interpolate credentials.password") + return + } + + return +} + +// GetServiceBindsAndMounts returns the binds and mounts for the service container, resolving paths as appopriate +func (rc *RunContext) GetServiceBindsAndMounts(svcVolumes []string) ([]string, map[string]string) { + if rc.Config.ContainerDaemonSocket == "" { + rc.Config.ContainerDaemonSocket = "/var/run/docker.sock" + } + binds := []string{} + if rc.Config.ContainerDaemonSocket != "-" { + daemonPath := getDockerDaemonSocketMountPath(rc.Config.ContainerDaemonSocket) + binds = append(binds, fmt.Sprintf("%s:%s", daemonPath, "/var/run/docker.sock")) + } + + mounts := map[string]string{} + + for _, v := range svcVolumes { + if !strings.Contains(v, ":") || filepath.IsAbs(v) { + // Bind anonymous volume or host file. + binds = append(binds, v) + } else { + // Mount existing volume. + paths := strings.SplitN(v, ":", 2) + mounts[paths[0]] = paths[1] + } + } + + return binds, mounts +} diff --git a/act/runner/runner.go b/act/runner/runner.go index 09c17319..e1d646eb 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -7,10 +7,10 @@ import ( "os" "runtime" - log "github.com/sirupsen/logrus" - + docker_container "github.com/docker/docker/api/types/container" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/model" + log "github.com/sirupsen/logrus" ) // Runner provides capabilities to run GitHub actions @@ -20,44 +20,45 @@ type Runner interface { // Config contains the config for a new runner type Config struct { - Actor string // the user that triggered the event - Workdir string // path to working directory - ActionCacheDir string // path used for caching action contents - BindWorkdir bool // bind the workdir to the job container - EventName string // name of event to run - EventPath string // path to JSON file to use for event.json in containers - DefaultBranch string // name of the main branch for this repository - ReuseContainers bool // reuse containers to maintain state - ForcePull bool // force pulling of the image, even if already present - ForceRebuild bool // force rebuilding local docker image action - LogOutput bool // log the output from docker run - JSONLogger bool // use json or text logger - LogPrefixJobID bool // switches from the full job name to the job id - Env map[string]string // env for containers - Inputs map[string]string // manually passed action inputs - Secrets map[string]string // list of secrets - Vars map[string]string // list of vars - Token string // GitHub token - InsecureSecrets bool // switch hiding output when printing to terminal - Platforms map[string]string // list of platforms - Privileged bool // use privileged mode - UsernsMode string // user namespace to use - ContainerArchitecture string // Desired OS/architecture platform for running containers - ContainerDaemonSocket string // Path to Docker daemon socket - ContainerOptions string // Options for the job container - UseGitIgnore bool // controls if paths in .gitignore should not be copied into container, default true - GitHubInstance string // GitHub instance to use, default "github.com" - ContainerCapAdd []string // list of kernel capabilities to add to the containers - ContainerCapDrop []string // list of kernel capabilities to remove from the containers - AutoRemove bool // controls if the container is automatically removed upon workflow completion - ArtifactServerPath string // the path where the artifact server stores uploads - ArtifactServerAddr string // the address the artifact server binds to - ArtifactServerPort string // the port the artifact server binds to - NoSkipCheckout bool // do not skip actions/checkout - RemoteName string // remote name in local git repo config - ReplaceGheActionWithGithubCom []string // Use actions from GitHub Enterprise instance to GitHub - ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. - Matrix map[string]map[string]bool // Matrix config to run + Actor string // the user that triggered the event + Workdir string // path to working directory + ActionCacheDir string // path used for caching action contents + BindWorkdir bool // bind the workdir to the job container + EventName string // name of event to run + EventPath string // path to JSON file to use for event.json in containers + DefaultBranch string // name of the main branch for this repository + ReuseContainers bool // reuse containers to maintain state + ForcePull bool // force pulling of the image, even if already present + ForceRebuild bool // force rebuilding local docker image action + LogOutput bool // log the output from docker run + JSONLogger bool // use json or text logger + LogPrefixJobID bool // switches from the full job name to the job id + Env map[string]string // env for containers + Inputs map[string]string // manually passed action inputs + Secrets map[string]string // list of secrets + Vars map[string]string // list of vars + Token string // GitHub token + InsecureSecrets bool // switch hiding output when printing to terminal + Platforms map[string]string // list of platforms + Privileged bool // use privileged mode + UsernsMode string // user namespace to use + ContainerArchitecture string // Desired OS/architecture platform for running containers + ContainerDaemonSocket string // Path to Docker daemon socket + ContainerOptions string // Options for the job container + UseGitIgnore bool // controls if paths in .gitignore should not be copied into container, default true + GitHubInstance string // GitHub instance to use, default "github.com" + ContainerCapAdd []string // list of kernel capabilities to add to the containers + ContainerCapDrop []string // list of kernel capabilities to remove from the containers + AutoRemove bool // controls if the container is automatically removed upon workflow completion + ArtifactServerPath string // the path where the artifact server stores uploads + ArtifactServerAddr string // the address the artifact server binds to + ArtifactServerPort string // the port the artifact server binds to + NoSkipCheckout bool // do not skip actions/checkout + RemoteName string // remote name in local git repo config + ReplaceGheActionWithGithubCom []string // Use actions from GitHub Enterprise instance to GitHub + ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. + Matrix map[string]map[string]bool // Matrix config to run + ContainerNetworkMode docker_container.NetworkMode // the network mode of job containers (the value of --network) } type caller struct { diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index 8d0ee771..96738a88 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -302,6 +302,11 @@ func TestRunEvent(t *testing.T) { {workdir, "set-env-step-env-override", "push", "", platforms, secrets}, {workdir, "set-env-new-env-file-per-step", "push", "", platforms, secrets}, {workdir, "no-panic-on-invalid-composite-action", "push", "jobs failed due to invalid action", platforms, secrets}, + + // services + {workdir, "services", "push", "", platforms, secrets}, + {workdir, "services-host-network", "push", "", platforms, secrets}, + {workdir, "services-with-container", "push", "", platforms, secrets}, } for _, table := range tables { diff --git a/act/runner/testdata/services-host-network/push.yml b/act/runner/testdata/services-host-network/push.yml new file mode 100644 index 00000000..8d0eb294 --- /dev/null +++ b/act/runner/testdata/services-host-network/push.yml @@ -0,0 +1,14 @@ +name: services-host-network +on: push +jobs: + services-host-network: + runs-on: ubuntu-latest + services: + nginx: + image: "nginx:latest" + ports: + - "8080:80" + steps: + - run: apt-get -qq update && apt-get -yqq install --no-install-recommends curl net-tools + - run: netstat -tlpen + - run: curl -v http://localhost:8080 diff --git a/act/runner/testdata/services-with-container/push.yml b/act/runner/testdata/services-with-container/push.yml new file mode 100644 index 00000000..b37e5dcd --- /dev/null +++ b/act/runner/testdata/services-with-container/push.yml @@ -0,0 +1,16 @@ +name: services-with-containers +on: push +jobs: + services-with-containers: + runs-on: ubuntu-latest + # https://docs.github.com/en/actions/using-containerized-services/about-service-containers#running-jobs-in-a-container + container: + image: "ubuntu:latest" + services: + nginx: + image: "nginx:latest" + ports: + - "8080:80" + steps: + - run: apt-get -qq update && apt-get -yqq install --no-install-recommends curl + - run: curl -v http://nginx:80 diff --git a/act/runner/testdata/services/push.yaml b/act/runner/testdata/services/push.yaml new file mode 100644 index 00000000..f6ca7bc4 --- /dev/null +++ b/act/runner/testdata/services/push.yaml @@ -0,0 +1,26 @@ +name: services +on: push +jobs: + services: + name: Reproduction of failing Services interpolation + runs-on: ubuntu-latest + services: + postgres: + image: postgres:12 + env: + POSTGRES_USER: runner + POSTGRES_PASSWORD: mysecretdbpass + POSTGRES_DB: mydb + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: + - name: Echo the Postgres service ID / Network / Ports + run: | + echo "id: ${{ job.services.postgres.id }}" + echo "network: ${{ job.services.postgres.network }}" + echo "ports: ${{ job.services.postgres.ports }}" diff --git a/cmd/input.go b/cmd/input.go index 1ae27930..f2f8edcd 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -56,6 +56,7 @@ type Input struct { matrix []string actionCachePath string logPrefixJobID bool + networkName string } func (i *Input) resolve(path string) string { diff --git a/cmd/root.go b/cmd/root.go index c076bf17..6b84cd76 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -14,6 +14,7 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/adrg/xdg" "github.com/andreaskoch/go-fswatch" + docker_container "github.com/docker/docker/api/types/container" "github.com/joho/godotenv" gitignore "github.com/sabhiram/go-gitignore" log "github.com/sirupsen/logrus" @@ -96,6 +97,7 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().StringVarP(&input.cacheServerAddr, "cache-server-addr", "", common.GetOutboundIP().String(), "Defines the address to which the cache server binds.") rootCmd.PersistentFlags().Uint16VarP(&input.cacheServerPort, "cache-server-port", "", 0, "Defines the port where the artifact server listens. 0 means a randomly available port.") rootCmd.PersistentFlags().StringVarP(&input.actionCachePath, "action-cache-path", "", filepath.Join(CacheHomeDir, "act"), "Defines the path where the actions get cached and host workspaces created.") + rootCmd.PersistentFlags().StringVarP(&input.networkName, "network", "", "host", "Sets a docker network name. Defaults to host.") rootCmd.SetArgs(args()) if err := rootCmd.Execute(); err != nil { @@ -612,6 +614,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str ReplaceGheActionWithGithubCom: input.replaceGheActionWithGithubCom, ReplaceGheActionTokenWithGithubCom: input.replaceGheActionTokenWithGithubCom, Matrix: matrixes, + ContainerNetworkMode: docker_container.NetworkMode(input.networkName), } r, err := runner.New(config) if err != nil { From 1a0ca2c6aee4aeffca96dfbd7921b276dcb2a91b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 03:16:31 +0000 Subject: [PATCH 03/38] build(deps): bump megalinter/megalinter from 7.4.0 to 7.5.0 (#2070) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 7.4.0 to 7.5.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v7.4.0...v7.5.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5c72a93b..cae6dc3a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -26,7 +26,7 @@ jobs: with: version: v1.53 only-new-issues: true - - uses: megalinter/megalinter/flavors/go@v7.4.0 + - uses: megalinter/megalinter/flavors/go@v7.5.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 0fe2aa1cd9d7576be2dee7cd886401b039c003f2 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Sat, 4 Nov 2023 22:10:53 +0800 Subject: [PATCH 04/38] fix: panic (#2071) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/docker_run.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index d24c4ac3..f5a898ee 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -16,15 +16,7 @@ import ( "strconv" "strings" - "github.com/go-git/go-billy/v5/helper/polyfill" - "github.com/go-git/go-billy/v5/osfs" - "github.com/go-git/go-git/v5/plumbing/format/gitignore" - "github.com/joho/godotenv" - - "github.com/imdario/mergo" - "github.com/kballard/go-shellquote" - "github.com/spf13/pflag" - + "github.com/Masterminds/semver" "github.com/docker/cli/cli/connhelper" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -32,9 +24,14 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" + "github.com/go-git/go-billy/v5/helper/polyfill" + "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "github.com/imdario/mergo" + "github.com/joho/godotenv" + "github.com/kballard/go-shellquote" specs "github.com/opencontainers/image-spec/specs-go/v1" - - "github.com/Masterminds/semver" + "github.com/spf13/pflag" "golang.org/x/term" "github.com/nektos/act/pkg/common" @@ -484,11 +481,17 @@ func (cr *containerReference) extractFromImageEnv(env *map[string]string) common inspect, _, err := cr.cli.ImageInspectWithRaw(ctx, cr.input.Image) if err != nil { logger.Error(err) + return fmt.Errorf("inspect image: %w", err) + } + + if inspect.Config == nil { + return nil } imageEnv, err := godotenv.Unmarshal(strings.Join(inspect.Config.Env, "\n")) if err != nil { logger.Error(err) + return fmt.Errorf("unmarshal image env: %w", err) } for k, v := range imageEnv { From 078c67a6ee678090d6488e51481fbbd598b72b64 Mon Sep 17 00:00:00 2001 From: Andreas Taylor Date: Sun, 12 Nov 2023 11:21:41 -0600 Subject: [PATCH 05/38] Use unique name for reusable workflow (#2015) Co-authored-by: ChristopherHX --- act/runner/run_context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 70d2f8f4..a69abc6e 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -64,7 +64,7 @@ func (rc *RunContext) String() string { if rc.caller != nil { // prefix the reusable workflow with the caller job // this is required to create unique container names - name = fmt.Sprintf("%s/%s", rc.caller.runContext.Run.JobID, name) + name = fmt.Sprintf("%s/%s", rc.caller.runContext.Name, name) } return name } From 16d6000e9a73472fd2985764c5d21b229109e31c Mon Sep 17 00:00:00 2001 From: Jon Jensen Date: Sun, 12 Nov 2023 12:40:06 -0500 Subject: [PATCH 06/38] Support array expressions in runs-on (#2088) * Support array expressions in runs-on * Simplify appproach to use EvaluateYamlNode, fix case-sensitivity bug --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 36 ++++++++++++++------------ act/runner/run_context_test.go | 47 ++++++++++++++++++++++++++++++++++ cmd/platforms.go | 2 +- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index a69abc6e..faf64e11 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -626,14 +626,7 @@ func (rc *RunContext) containerImage(ctx context.Context) string { } func (rc *RunContext) runsOnImage(ctx context.Context) string { - job := rc.Run.Job() - - if job.RunsOn() == nil { - common.Logger(ctx).Errorf("'runs-on' key not defined in %s", rc.String()) - } - - for _, runnerLabel := range job.RunsOn() { - platformName := rc.ExprEval.Interpolate(ctx, runnerLabel) + for _, platformName := range rc.runsOnPlatformNames(ctx) { image := rc.Config.Platforms[strings.ToLower(platformName)] if image != "" { return image @@ -643,6 +636,22 @@ func (rc *RunContext) runsOnImage(ctx context.Context) string { return "" } +func (rc *RunContext) runsOnPlatformNames(ctx context.Context) []string { + job := rc.Run.Job() + + if job.RunsOn() == nil { + common.Logger(ctx).Errorf("'runs-on' key not defined in %s", rc.String()) + return []string{} + } + + if err := rc.ExprEval.EvaluateYamlNode(ctx, &job.RawRunsOn); err != nil { + common.Logger(ctx).Errorf("Error while evaluating runs-on: %v", err) + return []string{} + } + + return job.RunsOn() +} + func (rc *RunContext) platformImage(ctx context.Context) string { if containerImage := rc.containerImage(ctx); containerImage != "" { return containerImage @@ -684,12 +693,7 @@ func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) { img := rc.platformImage(ctx) if img == "" { - if job.RunsOn() == nil { - l.Errorf("'runs-on' key not defined in %s", rc.String()) - } - - for _, runnerLabel := range job.RunsOn() { - platformName := rc.ExprEval.Interpolate(ctx, runnerLabel) + for _, platformName := range rc.runsOnPlatformNames(ctx) { l.Infof("\U0001F6A7 Skipping unsupported platform -- Try running with `-P %+v=...`", platformName) } return false, nil @@ -923,9 +927,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon setActionRuntimeVars(rc, env) } - job := rc.Run.Job() - for _, runnerLabel := range job.RunsOn() { - platformName := rc.ExprEval.Interpolate(ctx, runnerLabel) + for _, platformName := range rc.runsOnPlatformNames(ctx) { if platformName != "" { if platformName == "ubuntu-latest" { // hardcode current ubuntu-latest since we have no way to check that 'on the fly' diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index 3e26a022..8ab643ec 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -470,6 +470,53 @@ func createJob(t *testing.T, input string, result string) *model.Job { return job } +func TestRunContextRunsOnPlatformNames(t *testing.T) { + log.SetLevel(log.DebugLevel) + assertObject := assert.New(t) + + rc := createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: ubuntu-latest`, ""), + }) + assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: ${{ 'ubuntu-latest' }}`, ""), + }) + assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: [self-hosted, my-runner]`, ""), + }) + assertObject.Equal([]string{"self-hosted", "my-runner"}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: [self-hosted, "${{ 'my-runner' }}"]`, ""), + }) + assertObject.Equal([]string{"self-hosted", "my-runner"}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: ${{ fromJSON('["ubuntu-latest"]') }}`, ""), + }) + assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(context.Background())) + + // test missing / invalid runs-on + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `name: something`, ""), + }) + assertObject.Equal([]string{}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: + mapping: value`, ""), + }) + assertObject.Equal([]string{}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: ${{ invalid expression }}`, ""), + }) + assertObject.Equal([]string{}, rc.runsOnPlatformNames(context.Background())) +} + func TestRunContextIsEnabled(t *testing.T) { log.SetLevel(log.DebugLevel) assertObject := assert.New(t) diff --git a/cmd/platforms.go b/cmd/platforms.go index 9d7e97a8..45724d75 100644 --- a/cmd/platforms.go +++ b/cmd/platforms.go @@ -15,7 +15,7 @@ func (i *Input) newPlatforms() map[string]string { for _, p := range i.platforms { pParts := strings.Split(p, "=") if len(pParts) == 2 { - platforms[pParts[0]] = pParts[1] + platforms[strings.ToLower(pParts[0])] = pParts[1] } } return platforms From d2b2ed2d663a3db59e9ae018c03b197b3652e263 Mon Sep 17 00:00:00 2001 From: Jon Jensen Date: Sun, 12 Nov 2023 12:52:08 -0500 Subject: [PATCH 07/38] Don't set GITHUB_TOKEN (#2089) This needs to be explicitly in the `env` to be consistent with GitHub Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 1 - act/runner/step_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index faf64e11..a5493f63 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -911,7 +911,6 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon env["GITHUB_REF"] = github.Ref env["GITHUB_REF_NAME"] = github.RefName env["GITHUB_REF_TYPE"] = github.RefType - env["GITHUB_TOKEN"] = github.Token env["GITHUB_JOB"] = github.Job env["GITHUB_REPOSITORY_OWNER"] = github.RepositoryOwner env["GITHUB_RETENTION_DAYS"] = github.RetentionDays diff --git a/act/runner/step_test.go b/act/runner/step_test.go index d08a1297..cdb870e2 100644 --- a/act/runner/step_test.go +++ b/act/runner/step_test.go @@ -182,7 +182,6 @@ func TestSetupEnv(t *testing.T) { "GITHUB_RUN_ID": "runId", "GITHUB_RUN_NUMBER": "1", "GITHUB_SERVER_URL": "https://", - "GITHUB_TOKEN": "", "GITHUB_WORKFLOW": "", "INPUT_STEP_WITH": "with-value", "RC_KEY": "rcvalue", From 78a021bc1440bbb3d0ea85f2b2cbdbf0f600cc1c Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sun, 12 Nov 2023 19:09:25 +0100 Subject: [PATCH 08/38] fix: (#2075) network-scoped alias is supported only for containers in user defined networks Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/docker_run.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index f5a898ee..5a8afa22 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -447,7 +447,9 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E var networkingConfig *network.NetworkingConfig logger.Debugf("input.NetworkAliases ==> %v", input.NetworkAliases) - if hostConfig.NetworkMode.IsUserDefined() && len(input.NetworkAliases) > 0 { + n := hostConfig.NetworkMode + // TODO: use IsUserDefined() once it's windows implementation matches the unix one + if !n.IsDefault() && !n.IsBridge() && !n.IsHost() && !n.IsNone() && !n.IsContainer() && len(input.NetworkAliases) > 0 { endpointConfig := &network.EndpointSettings{ Aliases: input.NetworkAliases, } From bfde430c64223d0de8e4caae932d31217bfc353d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Brauer?= Date: Sun, 12 Nov 2023 19:30:21 +0100 Subject: [PATCH 09/38] Evaluate all service values (#2054) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index a5493f63..00e67f5d 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -293,9 +293,18 @@ func (rc *RunContext) startJobContainer() common.Executor { if err != nil { return fmt.Errorf("failed to handle service %s credentials: %w", serviceID, err) } - serviceBinds, serviceMounts := rc.GetServiceBindsAndMounts(spec.Volumes) - exposedPorts, portBindings, err := nat.ParsePortSpecs(spec.Ports) + interpolatedVolumes := make([]string, 0, len(spec.Volumes)) + for _, volume := range spec.Volumes { + interpolatedVolumes = append(interpolatedVolumes, rc.ExprEval.Interpolate(ctx, volume)) + } + serviceBinds, serviceMounts := rc.GetServiceBindsAndMounts(interpolatedVolumes) + + interpolatedPorts := make([]string, 0, len(spec.Ports)) + for _, port := range spec.Ports { + interpolatedPorts = append(interpolatedPorts, rc.ExprEval.Interpolate(ctx, port)) + } + exposedPorts, portBindings, err := nat.ParsePortSpecs(interpolatedPorts) if err != nil { return fmt.Errorf("failed to parse service %s ports: %w", serviceID, err) } @@ -304,7 +313,7 @@ func (rc *RunContext) startJobContainer() common.Executor { c := container.NewContainer(&container.NewContainerInput{ Name: serviceContainerName, WorkingDir: ext.ToContainerPath(rc.Config.Workdir), - Image: spec.Image, + Image: rc.ExprEval.Interpolate(ctx, spec.Image), Username: username, Password: password, Env: envs, @@ -315,7 +324,7 @@ func (rc *RunContext) startJobContainer() common.Executor { Privileged: rc.Config.Privileged, UsernsMode: rc.Config.UsernsMode, Platform: rc.Config.ContainerArchitecture, - Options: spec.Options, + Options: rc.ExprEval.Interpolate(ctx, spec.Options), NetworkMode: networkName, NetworkAliases: []string{serviceID}, ExposedPorts: exposedPorts, From 0956a75aca2325e5ee7e13de51e2051e6194afc7 Mon Sep 17 00:00:00 2001 From: raffis Date: Sun, 12 Nov 2023 20:46:38 +0100 Subject: [PATCH 10/38] feat: support runs-on labels and group (#2062) Signed-off-by: Raffael Sahli Co-authored-by: ChristopherHX --- act/model/workflow.go | 28 ++++++++++++++++++++++++++-- act/model/workflow_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/act/model/workflow.go b/act/model/workflow.go index 1573c76f..9860aad4 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -275,15 +275,39 @@ func (j *Job) Needs() []string { // RunsOn list for Job func (j *Job) RunsOn() []string { switch j.RawRunsOn.Kind { + case yaml.MappingNode: + var val struct { + Group string + Labels yaml.Node + } + + if !decodeNode(j.RawRunsOn, &val) { + return nil + } + + labels := nodeAsStringSlice(val.Labels) + + if val.Group != "" { + labels = append(labels, val.Group) + } + + return labels + default: + return nodeAsStringSlice(j.RawRunsOn) + } +} + +func nodeAsStringSlice(node yaml.Node) []string { + switch node.Kind { case yaml.ScalarNode: var val string - if !decodeNode(j.RawRunsOn, &val) { + if !decodeNode(node, &val) { return nil } return []string{val} case yaml.SequenceNode: var val []string - if !decodeNode(j.RawRunsOn, &val) { + if !decodeNode(node, &val) { return nil } return val diff --git a/act/model/workflow_test.go b/act/model/workflow_test.go index 292c0bff..bc12f8ee 100644 --- a/act/model/workflow_test.go +++ b/act/model/workflow_test.go @@ -71,6 +71,41 @@ jobs: assert.Contains(t, workflow.On(), "pull_request") } +func TestReadWorkflow_RunsOnLabels(t *testing.T) { + yaml := ` +name: local-action-docker-url + +jobs: + test: + container: nginx:latest + runs-on: + labels: ubuntu-latest + steps: + - uses: ./actions/docker-url` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + assert.Equal(t, workflow.Jobs["test"].RunsOn(), []string{"ubuntu-latest"}) +} + +func TestReadWorkflow_RunsOnLabelsWithGroup(t *testing.T) { + yaml := ` +name: local-action-docker-url + +jobs: + test: + container: nginx:latest + runs-on: + labels: [ubuntu-latest] + group: linux + steps: + - uses: ./actions/docker-url` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + assert.Equal(t, workflow.Jobs["test"].RunsOn(), []string{"ubuntu-latest", "linux"}) +} + func TestReadWorkflow_StringContainer(t *testing.T) { yaml := ` name: local-action-docker-url From 21fcd3b3dda87a66ec5f14e8059dc965f8f92a7e Mon Sep 17 00:00:00 2001 From: Jon Jensen Date: Sun, 12 Nov 2023 15:01:32 -0500 Subject: [PATCH 11/38] Evaluate if condition when calling a reusable workflow (#2087) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: ChristopherHX --- act/runner/run_context.go | 6 ++++-- act/runner/run_context_test.go | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 00e67f5d..5afa6f74 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -691,8 +691,6 @@ func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) { if jobType == model.JobTypeInvalid { return false, jobTypeErr - } else if jobType != model.JobTypeDefault { - return true, nil } if !runJob { @@ -700,6 +698,10 @@ func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) { return false, nil } + if jobType != model.JobTypeDefault { + return true, nil + } + img := rc.platformImage(ctx) if img == "" { for _, platformName := range rc.runsOnPlatformNames(ctx) { diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index 8ab643ec..c9d3830d 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -619,6 +619,17 @@ if: always()`, ""), }) rc.Run.JobID = "job2" assertObject.True(rc.isEnabled(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `uses: ./.github/workflows/reusable.yml`, ""), + }) + assertObject.True(rc.isEnabled(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `uses: ./.github/workflows/reusable.yml +if: false`, ""), + }) + assertObject.False(rc.isEnabled(context.Background())) } func TestRunContextGetEnv(t *testing.T) { From c26768fe90609065e50bd60696912d59a01c962c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 02:53:47 +0000 Subject: [PATCH 12/38] build(deps): bump actions/github-script from 6 to 7 (#2097) Bumps [actions/github-script](https://github.com/actions/github-script) from 6 to 7. - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/github-script dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c40b07e7..fc295c74 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,7 +43,7 @@ jobs: apiKey: ${{ secrets.CHOCO_APIKEY }} push: true - name: GitHub CLI extension - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} script: | From 773e52f02e33b2cd812c02916490b9b9ff884fd0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 03:19:49 +0000 Subject: [PATCH 13/38] build(deps): bump megalinter/megalinter from 7.5.0 to 7.6.0 (#2098) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 7.5.0 to 7.6.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v7.5.0...v7.6.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index cae6dc3a..2e9c7b12 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -26,7 +26,7 @@ jobs: with: version: v1.53 only-new-issues: true - - uses: megalinter/megalinter/flavors/go@v7.5.0 + - uses: megalinter/megalinter/flavors/go@v7.6.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f5fa27a3586a72c0ee5df8dcda1c4d22e60ca1c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 02:53:36 +0000 Subject: [PATCH 14/38] build(deps): bump megalinter/megalinter from 7.6.0 to 7.7.0 (#2119) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 7.6.0 to 7.7.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v7.6.0...v7.7.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2e9c7b12..1a81f6e6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -26,7 +26,7 @@ jobs: with: version: v1.53 only-new-issues: true - - uses: megalinter/megalinter/flavors/go@v7.6.0 + - uses: megalinter/megalinter/flavors/go@v7.7.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 24ebffd3ff5d66c6b92690c378e51bee66bd1d53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 03:07:56 +0000 Subject: [PATCH 15/38] build(deps): bump actions/setup-go from 4 to 5 (#2118) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 4 to 5. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 8 ++++---- .github/workflows/promote.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 1a81f6e6..5ee2b947 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true @@ -43,7 +43,7 @@ jobs: fetch-depth: 2 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true @@ -78,7 +78,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 2 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true @@ -93,7 +93,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index d6cd1eab..66a2396f 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -15,7 +15,7 @@ jobs: ref: master token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} - uses: fregante/setup-git-user@v2 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc295c74..d72173d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true From 98afefd09208465f7ea8f14d3e8d47adc44ff27e Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sat, 16 Dec 2023 23:46:17 +0100 Subject: [PATCH 16/38] fix: IsHost is defined as false on windows (#2093) * fix: IsHost is defined as false on windows * Update docker_run.go * Update docker_run.go --- act/container/docker_run.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 5a8afa22..18328845 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -448,8 +448,8 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E var networkingConfig *network.NetworkingConfig logger.Debugf("input.NetworkAliases ==> %v", input.NetworkAliases) n := hostConfig.NetworkMode - // TODO: use IsUserDefined() once it's windows implementation matches the unix one - if !n.IsDefault() && !n.IsBridge() && !n.IsHost() && !n.IsNone() && !n.IsContainer() && len(input.NetworkAliases) > 0 { + // IsUserDefined and IsHost are broken on windows + if n.IsUserDefined() && n != "host" && len(input.NetworkAliases) > 0 { endpointConfig := &network.EndpointSettings{ Aliases: input.NetworkAliases, } @@ -458,8 +458,6 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E input.NetworkMode: endpointConfig, }, } - } else { - logger.Debugf("not a use defined config??") } resp, err := cr.cli.ContainerCreate(ctx, config, hostConfig, networkingConfig, platSpecs, input.Name) From 182e1be4694dfe0eaa6a5d41455bcb310f4116cd Mon Sep 17 00:00:00 2001 From: Jon Jensen Date: Sat, 16 Dec 2023 15:04:54 -0800 Subject: [PATCH 17/38] Fix noisy `runs-on` error logging (#2102) Move the logging back up a level to fix a minor logging issue introduced in #2088 `RunContext`s for composite actions have dummy/blank `Job`s with no `runs-on`, meaning their calls to `withGithubEnv` would result in an inaccurate log message complaining that `'runs-on' key not defined in ...` Co-authored-by: Jason Song Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 5afa6f74..937b4803 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -635,6 +635,10 @@ func (rc *RunContext) containerImage(ctx context.Context) string { } func (rc *RunContext) runsOnImage(ctx context.Context) string { + if rc.Run.Job().RunsOn() == nil { + common.Logger(ctx).Errorf("'runs-on' key not defined in %s", rc.String()) + } + for _, platformName := range rc.runsOnPlatformNames(ctx) { image := rc.Config.Platforms[strings.ToLower(platformName)] if image != "" { @@ -649,7 +653,6 @@ func (rc *RunContext) runsOnPlatformNames(ctx context.Context) []string { job := rc.Run.Job() if job.RunsOn() == nil { - common.Logger(ctx).Errorf("'runs-on' key not defined in %s", rc.String()) return []string{} } From f36f685aabd4c71df84e4675d0a68648e6d91189 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Dec 2023 23:47:32 +0000 Subject: [PATCH 18/38] build(deps): bump actions/checkout from 3 to 4 (#1998) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 8 ++++---- .github/workflows/promote.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5ee2b947..25daad18 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,7 +15,7 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: actions/setup-go@v5 @@ -38,7 +38,7 @@ jobs: name: test-linux runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 - name: Set up QEMU @@ -75,7 +75,7 @@ jobs: name: test-${{matrix.os}} runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 - uses: actions/setup-go@v5 @@ -92,7 +92,7 @@ jobs: name: snapshot runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version-file: go.mod diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 66a2396f..536afe88 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -9,7 +9,7 @@ jobs: name: promote runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: master diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d72173d7..afdf3f52 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ jobs: name: release runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: actions/setup-go@v5 From 360fd52355654198c7f579544f42c7c2961c078c Mon Sep 17 00:00:00 2001 From: raffis Date: Sun, 17 Dec 2023 09:59:13 +0100 Subject: [PATCH 19/38] feat: support config env expansion (#2063) Signed-off-by: Raffael Sahli Co-authored-by: ChristopherHX Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- cmd/root.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index 6b84cd76..fe7a130b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -267,7 +267,8 @@ func readArgsFile(file string, split bool) []string { }() scanner := bufio.NewScanner(f) for scanner.Scan() { - arg := strings.TrimSpace(scanner.Text()) + arg := os.ExpandEnv(strings.TrimSpace(scanner.Text())) + if strings.HasPrefix(arg, "-") && split { args = append(args, regexp.MustCompile(`\s`).Split(arg, 2)...) } else if !split { From c16757adf1717fcb6d59d55876e12445e48c43ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 17 Dec 2023 09:14:56 +0000 Subject: [PATCH 20/38] build(deps): bump actions/stale from 8 to 9 (#2120) Bumps [actions/stale](https://github.com/actions/stale) from 8 to 9. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v8...v9) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 88e815c1..87948a2e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -8,7 +8,7 @@ jobs: name: Stale runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'Issue is stale and will be closed in 14 days unless there is new activity' From 4485482c1f28de77822e7c18b43b874489ccc89d Mon Sep 17 00:00:00 2001 From: Leonardo Taccari Date: Mon, 8 Jan 2024 20:26:03 +0100 Subject: [PATCH 21/38] Add support for NetBSD (#2023) NetBSD can run Docker CLI and then use Docker on some remote machine via DOCKER_HOST. (This can be probably extended to all other Unix-es capable of running just Docker CLI.) Co-authored-by: ChristopherHX --- act/container/docker_auth.go | 2 +- act/container/docker_build.go | 2 +- act/container/docker_cli.go | 2 +- act/container/docker_images.go | 2 +- act/container/docker_logger.go | 2 +- act/container/docker_network.go | 2 +- act/container/docker_pull.go | 2 +- act/container/docker_run.go | 2 +- act/container/docker_stub.go | 2 +- act/container/docker_volume.go | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/act/container/docker_auth.go b/act/container/docker_auth.go index 9c263f55..e8dfb7d5 100644 --- a/act/container/docker_auth.go +++ b/act/container/docker_auth.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_build.go b/act/container/docker_build.go index fdc04d8d..5f56ec70 100644 --- a/act/container/docker_build.go +++ b/act/container/docker_build.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_cli.go b/act/container/docker_cli.go index a1481c36..82d32466 100644 --- a/act/container/docker_cli.go +++ b/act/container/docker_cli.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) // This file is exact copy of https://github.com/docker/cli/blob/9ac8584acfd501c3f4da0e845e3a40ed15c85041/cli/command/container/opts.go // appended with license information. diff --git a/act/container/docker_images.go b/act/container/docker_images.go index 22772307..50ec68da 100644 --- a/act/container/docker_images.go +++ b/act/container/docker_images.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_logger.go b/act/container/docker_logger.go index f2c21e6c..b9eb503e 100644 --- a/act/container/docker_logger.go +++ b/act/container/docker_logger.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_network.go b/act/container/docker_network.go index 8a7528ab..6c2676f9 100644 --- a/act/container/docker_network.go +++ b/act/container/docker_network.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_pull.go b/act/container/docker_pull.go index ad75958f..7c94c104 100644 --- a/act/container/docker_pull.go +++ b/act/container/docker_pull.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_run.go b/act/container/docker_run.go index 18328845..dcb2df5b 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_stub.go b/act/container/docker_stub.go index 36f530ee..d5c5ef12 100644 --- a/act/container/docker_stub.go +++ b/act/container/docker_stub.go @@ -1,4 +1,4 @@ -//go:build WITHOUT_DOCKER || !(linux || darwin || windows) +//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd) package container diff --git a/act/container/docker_volume.go b/act/container/docker_volume.go index 0bb2cd7c..f99c5845 100644 --- a/act/container/docker_volume.go +++ b/act/container/docker_volume.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container From 57a16ab0f7ff96c6f9489e0f36c0f6fff9508d05 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sat, 20 Jan 2024 00:49:35 +0100 Subject: [PATCH 22/38] feat: cli option to enable the new action cache (#1954) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Enable the new action cache * fix * fix: CopyTarStream (Docker) * suppress panic in test * add a cli option for opt in * fixups * add package * fix * rc.Config nil in test??? * add feature flag * patch * Fix respect --action-cache-path Co-authored-by: Björn Brauer * add remote reusable workflow to ActionCache * fixup --------- Co-authored-by: Björn Brauer Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/container/docker_run.go | 20 +++++++++++++- act/model/planner.go | 46 +++++++++++++++++++++++++++----- act/runner/action.go | 22 ++++++++++++++- act/runner/reusable_workflow.go | 40 +++++++++++++++++++++++++++ act/runner/runner.go | 1 + act/runner/step.go | 13 +++++++++ act/runner/step_action_local.go | 35 ++++++++++++++++++------ act/runner/step_action_remote.go | 43 +++++++++++++++++++++++++++++ cmd/input.go | 1 + cmd/root.go | 6 +++++ 10 files changed, 211 insertions(+), 16 deletions(-) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index dcb2df5b..dff2ac61 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -671,10 +671,28 @@ func (cr *containerReference) waitForCommand(ctx context.Context, isTerminal boo } func (cr *containerReference) CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error { - err := cr.cli.CopyToContainer(ctx, cr.id, destPath, tarStream, types.CopyToContainerOptions{}) + // Mkdir + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + _ = tw.WriteHeader(&tar.Header{ + Name: destPath, + Mode: 777, + Typeflag: tar.TypeDir, + }) + tw.Close() + err := cr.cli.CopyToContainer(ctx, cr.id, "/", buf, types.CopyToContainerOptions{}) + if err != nil { + return fmt.Errorf("failed to mkdir to copy content to container: %w", err) + } + // Copy Content + err = cr.cli.CopyToContainer(ctx, cr.id, destPath, tarStream, types.CopyToContainerOptions{}) if err != nil { return fmt.Errorf("failed to copy content to container: %w", err) } + // If this fails, then folders have wrong permissions on non root container + if cr.UID != 0 || cr.GID != 0 { + _ = cr.Exec([]string{"chown", "-R", fmt.Sprintf("%d:%d", cr.UID, cr.GID), destPath}, nil, "0", "")(ctx) + } return nil } diff --git a/act/model/planner.go b/act/model/planner.go index 089d67d9..4d23c082 100644 --- a/act/model/planner.go +++ b/act/model/planner.go @@ -148,12 +148,10 @@ func NewWorkflowPlanner(path string, noWorkflowRecurse bool) (WorkflowPlanner, e workflow.Name = wf.workflowDirEntry.Name() } - jobNameRegex := regexp.MustCompile(`^([[:alpha:]_][[:alnum:]_\-]*)$`) - for k := range workflow.Jobs { - if ok := jobNameRegex.MatchString(k); !ok { - _ = f.Close() - return nil, fmt.Errorf("workflow is not valid. '%s': Job name '%s' is invalid. Names must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'", workflow.Name, k) - } + err = validateJobName(workflow) + if err != nil { + _ = f.Close() + return nil, err } wp.workflows = append(wp.workflows, workflow) @@ -164,6 +162,42 @@ func NewWorkflowPlanner(path string, noWorkflowRecurse bool) (WorkflowPlanner, e return wp, nil } +func NewSingleWorkflowPlanner(name string, f io.Reader) (WorkflowPlanner, error) { + wp := new(workflowPlanner) + + log.Debugf("Reading workflow %s", name) + workflow, err := ReadWorkflow(f) + if err != nil { + if err == io.EOF { + return nil, fmt.Errorf("unable to read workflow '%s': file is empty: %w", name, err) + } + return nil, fmt.Errorf("workflow is not valid. '%s': %w", name, err) + } + workflow.File = name + if workflow.Name == "" { + workflow.Name = name + } + + err = validateJobName(workflow) + if err != nil { + return nil, err + } + + wp.workflows = append(wp.workflows, workflow) + + return wp, nil +} + +func validateJobName(workflow *Workflow) error { + jobNameRegex := regexp.MustCompile(`^([[:alpha:]_][[:alnum:]_\-]*)$`) + for k := range workflow.Jobs { + if ok := jobNameRegex.MatchString(k); !ok { + return fmt.Errorf("workflow is not valid. '%s': Job name '%s' is invalid. Names must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'", workflow.Name, k) + } + } + return nil +} + type workflowPlanner struct { workflows []*Workflow } diff --git a/act/runner/action.go b/act/runner/action.go index a8b8912d..0af6c653 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -44,7 +44,7 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir string, act reader, closer, err := readFile("action.yml") if os.IsNotExist(err) { reader, closer, err = readFile("action.yaml") - if err != nil { + if os.IsNotExist(err) { if _, closer, err2 := readFile("Dockerfile"); err2 == nil { closer.Close() action := &model.Action{ @@ -91,6 +91,8 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir string, act } } return nil, err + } else if err != nil { + return nil, err } } else if err != nil { return nil, err @@ -110,6 +112,17 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir string if stepModel.Type() != model.StepTypeUsesActionRemote { return nil } + + if rc.Config != nil && rc.Config.ActionCache != nil { + raction := step.(*stepActionRemote) + ta, err := rc.Config.ActionCache.GetTarArchive(ctx, raction.cacheDir, raction.resolvedSha, "") + if err != nil { + return err + } + defer ta.Close() + return rc.JobContainer.CopyTarStream(ctx, containerActionDir, ta) + } + if err := removeGitIgnore(ctx, actionDir); err != nil { return err } @@ -265,6 +278,13 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based return err } defer buildContext.Close() + } else if rc.Config.ActionCache != nil { + rstep := step.(*stepActionRemote) + buildContext, err = rc.Config.ActionCache.GetTarArchive(ctx, rstep.cacheDir, rstep.resolvedSha, contextDir) + if err != nil { + return err + } + defer buildContext.Close() } prepImage = container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ ContextDir: contextDir, diff --git a/act/runner/reusable_workflow.go b/act/runner/reusable_workflow.go index 67e04037..b5e3d5ba 100644 --- a/act/runner/reusable_workflow.go +++ b/act/runner/reusable_workflow.go @@ -1,6 +1,7 @@ package runner import ( + "archive/tar" "context" "errors" "fmt" @@ -33,12 +34,51 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor { filename := fmt.Sprintf("%s/%s@%s", remoteReusableWorkflow.Org, remoteReusableWorkflow.Repo, remoteReusableWorkflow.Ref) workflowDir := fmt.Sprintf("%s/%s", rc.ActionCacheDir(), safeFilename(filename)) + if rc.Config.ActionCache != nil { + return newActionCacheReusableWorkflowExecutor(rc, filename, remoteReusableWorkflow) + } + return common.NewPipelineExecutor( newMutexExecutor(cloneIfRequired(rc, *remoteReusableWorkflow, workflowDir)), newReusableWorkflowExecutor(rc, workflowDir, fmt.Sprintf("./.github/workflows/%s", remoteReusableWorkflow.Filename)), ) } +func newActionCacheReusableWorkflowExecutor(rc *RunContext, filename string, remoteReusableWorkflow *remoteReusableWorkflow) common.Executor { + return func(ctx context.Context) error { + ghctx := rc.getGithubContext(ctx) + remoteReusableWorkflow.URL = ghctx.ServerURL + sha, err := rc.Config.ActionCache.Fetch(ctx, filename, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, ghctx.Token) + if err != nil { + return err + } + archive, err := rc.Config.ActionCache.GetTarArchive(ctx, filename, sha, fmt.Sprintf(".github/workflows/%s", remoteReusableWorkflow.Filename)) + if err != nil { + return err + } + defer archive.Close() + treader := tar.NewReader(archive) + if _, err = treader.Next(); err != nil { + return err + } + planner, err := model.NewSingleWorkflowPlanner(remoteReusableWorkflow.Filename, treader) + if err != nil { + return err + } + plan, err := planner.PlanEvent("workflow_call") + if err != nil { + return err + } + + runner, err := NewReusableWorkflowRunner(rc) + if err != nil { + return err + } + + return runner.NewPlanExecutor(plan)(ctx) + } +} + var ( executorLock sync.Mutex ) diff --git a/act/runner/runner.go b/act/runner/runner.go index e1d646eb..5a7b1ad1 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -59,6 +59,7 @@ type Config struct { ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. Matrix map[string]map[string]bool // Matrix config to run ContainerNetworkMode docker_container.NetworkMode // the network mode of job containers (the value of --network) + ActionCache ActionCache // Use a custom ActionCache Implementation } type caller struct { diff --git a/act/runner/step.go b/act/runner/step.go index ffb2efbc..c67b5b04 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -34,6 +34,9 @@ const ( stepStagePost ) +// Controls how many symlinks are resolved for local and remote Actions +const maxSymlinkDepth = 10 + func (s stepStage) String() string { switch s { case stepStagePre: @@ -307,3 +310,13 @@ func mergeIntoMapCaseInsensitive(target map[string]string, maps ...map[string]st } } } + +func symlinkJoin(filename, sym, parent string) (string, error) { + dir := path.Dir(filename) + dest := path.Join(dir, sym) + prefix := path.Clean(parent) + "/" + if strings.HasPrefix(dest, prefix) || prefix == "./" { + return dest, nil + } + return "", fmt.Errorf("symlink tries to access file '%s' outside of '%s'", strings.ReplaceAll(dest, "'", "''"), strings.ReplaceAll(parent, "'", "''")) +} diff --git a/act/runner/step_action_local.go b/act/runner/step_action_local.go index a745e686..f8daf5cb 100644 --- a/act/runner/step_action_local.go +++ b/act/runner/step_action_local.go @@ -3,7 +3,10 @@ package runner import ( "archive/tar" "context" + "errors" + "fmt" "io" + "io/fs" "os" "path" "path/filepath" @@ -42,15 +45,31 @@ func (sal *stepActionLocal) main() common.Executor { localReader := func(ctx context.Context) actionYamlReader { _, cpath := getContainerActionPaths(sal.Step, path.Join(actionDir, ""), sal.RunContext) return func(filename string) (io.Reader, io.Closer, error) { - tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, path.Join(cpath, filename)) - if err != nil { - return nil, nil, os.ErrNotExist + spath := path.Join(cpath, filename) + for i := 0; i < maxSymlinkDepth; i++ { + tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, spath) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil, err + } else if err != nil { + return nil, nil, fs.ErrNotExist + } + treader := tar.NewReader(tars) + header, err := treader.Next() + if errors.Is(err, io.EOF) { + return nil, nil, os.ErrNotExist + } else if err != nil { + return nil, nil, err + } + if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink { + spath, err = symlinkJoin(spath, header.Linkname, cpath) + if err != nil { + return nil, nil, err + } + } else { + return treader, tars, nil + } } - treader := tar.NewReader(tars) - if _, err := treader.Next(); err != nil { - return nil, nil, os.ErrNotExist - } - return treader, tars, nil + return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath) } } diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index e23dcf99..4019388f 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -1,6 +1,7 @@ package runner import ( + "archive/tar" "context" "errors" "fmt" @@ -28,6 +29,8 @@ type stepActionRemote struct { action *model.Action env map[string]string remoteAction *remoteAction + cacheDir string + resolvedSha string } var ( @@ -60,6 +63,46 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { github.Token = sar.RunContext.Config.ReplaceGheActionTokenWithGithubCom } } + if sar.RunContext.Config.ActionCache != nil { + cache := sar.RunContext.Config.ActionCache + + var err error + sar.cacheDir = fmt.Sprintf("%s/%s", sar.remoteAction.Org, sar.remoteAction.Repo) + sar.resolvedSha, err = cache.Fetch(ctx, sar.cacheDir, sar.remoteAction.URL+"/"+sar.cacheDir, sar.remoteAction.Ref, github.Token) + if err != nil { + return err + } + + remoteReader := func(ctx context.Context) actionYamlReader { + return func(filename string) (io.Reader, io.Closer, error) { + spath := filename + for i := 0; i < maxSymlinkDepth; i++ { + tars, err := cache.GetTarArchive(ctx, sar.cacheDir, sar.resolvedSha, spath) + if err != nil { + return nil, nil, os.ErrNotExist + } + treader := tar.NewReader(tars) + header, err := treader.Next() + if err != nil { + return nil, nil, os.ErrNotExist + } + if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink { + spath, err = symlinkJoin(spath, header.Linkname, ".") + if err != nil { + return nil, nil, err + } + } else { + return treader, tars, nil + } + } + return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath) + } + } + + actionModel, err := sar.readAction(ctx, sar.Step, sar.resolvedSha, sar.remoteAction.Path, remoteReader(ctx), os.WriteFile) + sar.action = actionModel + return err + } actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses)) gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{ diff --git a/cmd/input.go b/cmd/input.go index f2f8edcd..a6d70dd5 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -57,6 +57,7 @@ type Input struct { actionCachePath string logPrefixJobID bool networkName string + useNewActionCache bool } func (i *Input) resolve(path string) string { diff --git a/cmd/root.go b/cmd/root.go index fe7a130b..0494b861 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -98,6 +98,7 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().Uint16VarP(&input.cacheServerPort, "cache-server-port", "", 0, "Defines the port where the artifact server listens. 0 means a randomly available port.") rootCmd.PersistentFlags().StringVarP(&input.actionCachePath, "action-cache-path", "", filepath.Join(CacheHomeDir, "act"), "Defines the path where the actions get cached and host workspaces created.") rootCmd.PersistentFlags().StringVarP(&input.networkName, "network", "", "host", "Sets a docker network name. Defaults to host.") + rootCmd.PersistentFlags().BoolVarP(&input.useNewActionCache, "use-new-action-cache", "", false, "Enable using the new Action Cache for storing Actions locally") rootCmd.SetArgs(args()) if err := rootCmd.Execute(); err != nil { @@ -617,6 +618,11 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str Matrix: matrixes, ContainerNetworkMode: docker_container.NetworkMode(input.networkName), } + if input.useNewActionCache { + config.ActionCache = &runner.GoGitActionCache{ + Path: config.ActionCacheDir, + } + } r, err := runner.New(config) if err != nil { return err From 141b614c3680fa9ba28ec0d4bfbeb595623aab5b Mon Sep 17 00:00:00 2001 From: TKaxv_7S <954067342@qq.com> Date: Sat, 20 Jan 2024 08:20:15 +0800 Subject: [PATCH 23/38] feat: support offline mode (#2128) * Add: Actions Offline Mode * Add: Actions Offline Mode --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/common/git/git.go | 26 ++++++++++++++++---------- act/runner/reusable_workflow.go | 9 +++++---- act/runner/runner.go | 1 + act/runner/step_action_remote.go | 9 +++++---- cmd/input.go | 1 + cmd/root.go | 4 +++- 6 files changed, 31 insertions(+), 19 deletions(-) diff --git a/act/common/git/git.go b/act/common/git/git.go index bf771558..0a819b95 100644 --- a/act/common/git/git.go +++ b/act/common/git/git.go @@ -221,10 +221,11 @@ func findGitSlug(url string, githubInstance string) (string, string, error) { // NewGitCloneExecutorInput the input for the NewGitCloneExecutor type NewGitCloneExecutorInput struct { - URL string - Ref string - Dir string - Token string + URL string + Ref string + Dir string + Token string + OfflineMode bool } // CloneIfRequired ... @@ -302,12 +303,16 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor { return err } + isOfflineMode := input.OfflineMode + // fetch latest changes fetchOptions, pullOptions := gitOptions(input.Token) - err = r.Fetch(&fetchOptions) - if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { - return err + if !isOfflineMode { + err = r.Fetch(&fetchOptions) + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return err + } } var hash *plumbing.Hash @@ -367,9 +372,10 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor { return err } } - - if err = w.Pull(&pullOptions); err != nil && err != git.NoErrAlreadyUpToDate { - logger.Debugf("Unable to pull %s: %v", refName, err) + if !isOfflineMode { + if err = w.Pull(&pullOptions); err != nil && err != git.NoErrAlreadyUpToDate { + logger.Debugf("Unable to pull %s: %v", refName, err) + } } logger.Debugf("Cloned %s to %s", input.URL, input.Dir) diff --git a/act/runner/reusable_workflow.go b/act/runner/reusable_workflow.go index b5e3d5ba..7ce68e92 100644 --- a/act/runner/reusable_workflow.go +++ b/act/runner/reusable_workflow.go @@ -102,10 +102,11 @@ func cloneIfRequired(rc *RunContext, remoteReusableWorkflow remoteReusableWorkfl func(ctx context.Context) error { remoteReusableWorkflow.URL = rc.getGithubContext(ctx).ServerURL return git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{ - URL: remoteReusableWorkflow.CloneURL(), - Ref: remoteReusableWorkflow.Ref, - Dir: targetDirectory, - Token: rc.Config.Token, + URL: remoteReusableWorkflow.CloneURL(), + Ref: remoteReusableWorkflow.Ref, + Dir: targetDirectory, + Token: rc.Config.Token, + OfflineMode: rc.Config.ActionOfflineMode, })(ctx) }, nil, diff --git a/act/runner/runner.go b/act/runner/runner.go index 5a7b1ad1..dd8afe54 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -23,6 +23,7 @@ type Config struct { Actor string // the user that triggered the event Workdir string // path to working directory ActionCacheDir string // path used for caching action contents + ActionOfflineMode bool // when offline, use caching action contents BindWorkdir bool // bind the workdir to the job container EventName string // name of event to run EventPath string // path to JSON file to use for event.json in containers diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index 4019388f..5c8a8f2f 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -106,10 +106,11 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses)) gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{ - URL: sar.remoteAction.CloneURL(), - Ref: sar.remoteAction.Ref, - Dir: actionDir, - Token: github.Token, + URL: sar.remoteAction.CloneURL(), + Ref: sar.remoteAction.Ref, + Dir: actionDir, + Token: github.Token, + OfflineMode: sar.RunContext.Config.ActionOfflineMode, }) var ntErr common.Executor if err := gitClone(ctx); err != nil { diff --git a/cmd/input.go b/cmd/input.go index a6d70dd5..36af6d86 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -55,6 +55,7 @@ type Input struct { replaceGheActionTokenWithGithubCom string matrix []string actionCachePath string + actionOfflineMode bool logPrefixJobID bool networkName string useNewActionCache bool diff --git a/cmd/root.go b/cmd/root.go index 0494b861..e595e529 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -97,6 +97,7 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().StringVarP(&input.cacheServerAddr, "cache-server-addr", "", common.GetOutboundIP().String(), "Defines the address to which the cache server binds.") rootCmd.PersistentFlags().Uint16VarP(&input.cacheServerPort, "cache-server-port", "", 0, "Defines the port where the artifact server listens. 0 means a randomly available port.") rootCmd.PersistentFlags().StringVarP(&input.actionCachePath, "action-cache-path", "", filepath.Join(CacheHomeDir, "act"), "Defines the path where the actions get cached and host workspaces created.") + rootCmd.PersistentFlags().BoolVarP(&input.actionOfflineMode, "action-offline-mode", "", false, "If action contents exists, it will not be fetch and pull again. If turn on this,will turn off force pull") rootCmd.PersistentFlags().StringVarP(&input.networkName, "network", "", "host", "Sets a docker network name. Defaults to host.") rootCmd.PersistentFlags().BoolVarP(&input.useNewActionCache, "use-new-action-cache", "", false, "Enable using the new Action Cache for storing Actions locally") rootCmd.SetArgs(args()) @@ -582,11 +583,12 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str EventName: eventName, EventPath: input.EventPath(), DefaultBranch: defaultbranch, - ForcePull: input.forcePull, + ForcePull: !input.actionOfflineMode && input.forcePull, ForceRebuild: input.forceRebuild, ReuseContainers: input.reuseContainers, Workdir: input.Workdir(), ActionCacheDir: input.actionCachePath, + ActionOfflineMode: input.actionOfflineMode, BindWorkdir: input.bindWorkdir, LogOutput: !input.noOutput, JSONLogger: input.jsonLogger, From ec23c70ab32912b13075ef9dcecb31c3c2c0dd19 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 00:32:38 +0000 Subject: [PATCH 24/38] build(deps): bump actions/upload-artifact from 3 to 4 (#2133) Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3 to 4. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Casey Lee Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 25daad18..e8751526 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -111,67 +111,67 @@ jobs: args: release --snapshot --clean - name: Capture x86_64 (64-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-amd64 path: dist/act_linux_amd64_v1/act - name: Capture i386 (32-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-i386 path: dist/act_linux_386/act - name: Capture arm64 (64-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-arm64 path: dist/act_linux_arm64/act - name: Capture armv6 (32-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-armv6 path: dist/act_linux_arm_6/act - name: Capture armv7 (32-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-armv7 path: dist/act_linux_arm_7/act - name: Capture x86_64 (64-bit) Windows binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-windows-amd64 path: dist/act_windows_amd64_v1/act.exe - name: Capture i386 (32-bit) Windows binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-windows-i386 path: dist/act_windows_386/act.exe - name: Capture arm64 (64-bit) Windows binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-windows-arm64 path: dist/act_windows_arm64/act.exe - name: Capture armv7 (32-bit) Windows binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-windows-armv7 path: dist/act_windows_arm_7/act.exe - name: Capture x86_64 (64-bit) MacOS binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-macos-amd64 path: dist/act_darwin_amd64_v1/act - name: Capture arm64 (64-bit) MacOS binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-macos-arm64 path: dist/act_darwin_arm64/act From a25c37e83c7bf9e26d3ba0ff1cbed165ecb38db1 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Sat, 20 Jan 2024 14:11:50 +0200 Subject: [PATCH 25/38] fix: match cache `restore-keys` in creation reverse order (#2153) * Match cache restore-keys in creation reverse order * Match full prefix when selecting cache --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/artifactcache/handler.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/act/artifactcache/handler.go b/act/artifactcache/handler.go index b6600b6b..3178260b 100644 --- a/act/artifactcache/handler.go +++ b/act/artifactcache/handler.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "strconv" "strings" "sync/atomic" @@ -386,7 +387,12 @@ func (h *Handler) findCache(db *bolthold.Store, keys []string, version string) ( for _, prefix := range keys[1:] { found := false - if err := db.ForEach(bolthold.Where("Key").Ge(prefix).And("Version").Eq(version).SortBy("Key"), func(v *Cache) error { + prefixPattern := fmt.Sprintf("^%s", regexp.QuoteMeta(prefix)) + re, err := regexp.Compile(prefixPattern) + if err != nil { + continue + } + if err := db.ForEach(bolthold.Where("Key").RegExp(re).And("Version").Eq(version).SortBy("CreatedAt").Reverse(), func(v *Cache) error { if !strings.HasPrefix(v.Key, prefix) { return stop } From 853c5a22a465de67816ce49b08ff26c72549ebf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=96=E6=A2=81?= <418094911@qq.com> Date: Sat, 20 Jan 2024 22:07:36 +0800 Subject: [PATCH 26/38] WorkflowDispatchConfig supports multiple yaml node kinds (#2123) * WorkflowDispatchConfig supports ScalarNode and SequenceNode yaml node kinds * Avoid using log.Fatal * package slices is not in golang 1.20 --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/model/workflow.go | 46 +++++++++++----- act/model/workflow_test.go | 104 +++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 14 deletions(-) diff --git a/act/model/workflow.go b/act/model/workflow.go index 9860aad4..11864540 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -79,22 +79,40 @@ type WorkflowDispatch struct { } func (w *Workflow) WorkflowDispatchConfig() *WorkflowDispatch { - if w.RawOn.Kind != yaml.MappingNode { + switch w.RawOn.Kind { + case yaml.ScalarNode: + var val string + if !decodeNode(w.RawOn, &val) { + return nil + } + if val == "workflow_dispatch" { + return &WorkflowDispatch{} + } + case yaml.SequenceNode: + var val []string + if !decodeNode(w.RawOn, &val) { + return nil + } + for _, v := range val { + if v == "workflow_dispatch" { + return &WorkflowDispatch{} + } + } + case yaml.MappingNode: + var val map[string]yaml.Node + if !decodeNode(w.RawOn, &val) { + return nil + } + + n, found := val["workflow_dispatch"] + var workflowDispatch WorkflowDispatch + if found && decodeNode(n, &workflowDispatch) { + return &workflowDispatch + } + default: return nil } - - var val map[string]yaml.Node - if !decodeNode(w.RawOn, &val) { - return nil - } - - var config WorkflowDispatch - node := val["workflow_dispatch"] - if !decodeNode(node, &config) { - return nil - } - - return &config + return nil } type WorkflowCallInput struct { diff --git a/act/model/workflow_test.go b/act/model/workflow_test.go index bc12f8ee..88c2096a 100644 --- a/act/model/workflow_test.go +++ b/act/model/workflow_test.go @@ -417,3 +417,107 @@ func TestStep_ShellCommand(t *testing.T) { }) } } + +func TestReadWorkflow_WorkflowDispatchConfig(t *testing.T) { + yaml := ` + name: local-action-docker-url + ` + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch := workflow.WorkflowDispatchConfig() + assert.Nil(t, workflowDispatch) + + yaml = ` + name: local-action-docker-url + on: push + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.Nil(t, workflowDispatch) + + yaml = ` + name: local-action-docker-url + on: workflow_dispatch + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.NotNil(t, workflowDispatch) + assert.Nil(t, workflowDispatch.Inputs) + + yaml = ` + name: local-action-docker-url + on: [push, pull_request] + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.Nil(t, workflowDispatch) + + yaml = ` + name: local-action-docker-url + on: [push, workflow_dispatch] + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.NotNil(t, workflowDispatch) + assert.Nil(t, workflowDispatch.Inputs) + + yaml = ` + name: local-action-docker-url + on: + - push + - workflow_dispatch + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.NotNil(t, workflowDispatch) + assert.Nil(t, workflowDispatch.Inputs) + + yaml = ` + name: local-action-docker-url + on: + push: + pull_request: + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.Nil(t, workflowDispatch) + + yaml = ` + name: local-action-docker-url + on: + push: + pull_request: + workflow_dispatch: + inputs: + logLevel: + description: 'Log level' + required: true + default: 'warning' + type: choice + options: + - info + - warning + - debug + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.NotNil(t, workflowDispatch) + assert.Equal(t, WorkflowDispatchInput{ + Default: "warning", + Description: "Log level", + Options: []string{ + "info", + "warning", + "debug", + }, + Required: true, + Type: "choice", + }, workflowDispatch.Inputs["logLevel"]) +} From 2103956b4fed0d858ef0c8667e941a8170582789 Mon Sep 17 00:00:00 2001 From: Milo Moisson Date: Sun, 21 Jan 2024 21:16:06 +0100 Subject: [PATCH 27/38] fix: write default config in XDG config dir to avoid cluttering the HOME directory by default (#2140) Co-authored-by: Casey Lee Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- cmd/root.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/root.go b/cmd/root.go index e595e529..319fdd54 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -107,6 +107,7 @@ func Execute(ctx context.Context, version string) { } } +// Return locations where Act's config can be found in order : XDG spec, .actrc in HOME directory, .actrc in invocation directory func configLocations() []string { configFileName := ".actrc" @@ -120,8 +121,8 @@ func configLocations() []string { } return []string{ - filepath.Join(UserHomeDir, configFileName), actrcXdg, + filepath.Join(UserHomeDir, configFileName), filepath.Join(".", configFileName), } } @@ -557,6 +558,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str } } if !cfgFound && len(cfgLocations) > 0 { + // The first config location refers to the XDG spec one if err := defaultImageSurvey(cfgLocations[0]); err != nil { log.Fatal(err) } From 724a00e9c1b3f51d29ee79499e7297f15e11e8bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 02:22:08 +0000 Subject: [PATCH 28/38] build(deps): bump megalinter/megalinter from 7.7.0 to 7.8.0 (#2164) Bumps [megalinter/megalinter](https://github.com/megalinter/megalinter) from 7.7.0 to 7.8.0. - [Release notes](https://github.com/megalinter/megalinter/releases) - [Changelog](https://github.com/oxsecurity/megalinter/blob/main/CHANGELOG.md) - [Commits](https://github.com/megalinter/megalinter/compare/v7.7.0...v7.8.0) --- updated-dependencies: - dependency-name: megalinter/megalinter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e8751526..b6dac11e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -26,7 +26,7 @@ jobs: with: version: v1.53 only-new-issues: true - - uses: megalinter/megalinter/flavors/go@v7.7.0 + - uses: megalinter/megalinter/flavors/go@v7.8.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c358c37324a4abc8b52c8d2ffbc7a78b873e682f Mon Sep 17 00:00:00 2001 From: Matthew Date: Tue, 23 Jan 2024 18:44:48 -0800 Subject: [PATCH 29/38] Add containerd's normalized architectures to archMapper (#2168) --- act/container/docker_run.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index dff2ac61..bf220e67 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -238,8 +238,10 @@ func RunnerArch(ctx context.Context) string { archMapper := map[string]string{ "x86_64": "X64", + "amd64": "X64", "386": "X86", "aarch64": "ARM64", + "arm64": "ARM64", } if arch, ok := archMapper[info.Architecture]; ok { return arch From df3d88caa1388d7646f97b2bca583b973ca9bb7b Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Wed, 24 Jan 2024 11:06:31 +0800 Subject: [PATCH 30/38] refactor(cmd/root): simplify `parseEnvs` (#2162) Prior to this commit, `parseEnvs` accept two parameters: 1. env []string 2. envs map[string]string `parseEnvs` then do a `nil` check for `env`. However, we never pass a `nil` `env` to `parseEnvs` in `newRunCommand`. This commit simplify the `parseEnvs` function to accept just one `env []string` parameter and return the result as `map[string]string` instead. Signed-off-by: Eng Zer Jun --- cmd/root.go | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 319fdd54..e506699f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -297,19 +297,17 @@ func cleanup(inputs *Input) func(*cobra.Command, []string) { } } -func parseEnvs(env []string, envs map[string]string) bool { - if env != nil { - for _, envVar := range env { - e := strings.SplitN(envVar, `=`, 2) - if len(e) == 2 { - envs[e[0]] = e[1] - } else { - envs[e[0]] = "" - } +func parseEnvs(env []string) map[string]string { + envs := make(map[string]string, len(env)) + for _, envVar := range env { + e := strings.SplitN(envVar, `=`, 2) + if len(e) == 2 { + envs[e[0]] = e[1] + } else { + envs[e[0]] = "" } - return true } - return false + return envs } func readYamlFile(file string) (map[string]string, error) { @@ -415,13 +413,11 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str } log.Debugf("Loading environment from %s", input.Envfile()) - envs := make(map[string]string) - _ = parseEnvs(input.envs, envs) + envs := parseEnvs(input.envs) _ = readEnvs(input.Envfile(), envs) log.Debugf("Loading action inputs from %s", input.Inputfile()) - inputs := make(map[string]string) - _ = parseEnvs(input.inputs, inputs) + inputs := parseEnvs(input.inputs) _ = readEnvs(input.Inputfile(), inputs) log.Debugf("Loading secrets from %s", input.Secretfile()) From 5f0f24d596c983d25e9182a63943b284b7c833f3 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sun, 28 Jan 2024 17:37:19 +0100 Subject: [PATCH 31/38] fix: improve action not found error (#2171) --- act/runner/action.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/act/runner/action.go b/act/runner/action.go index 0af6c653..42742c4d 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -3,6 +3,7 @@ package runner import ( "context" "embed" + "errors" "fmt" "io" "io/fs" @@ -41,11 +42,24 @@ var trampoline embed.FS func readActionImpl(ctx context.Context, step *model.Step, actionDir string, actionPath string, readFile actionYamlReader, writeFile fileWriter) (*model.Action, error) { logger := common.Logger(ctx) + allErrors := []error{} + addError := func(fileName string, err error) { + if err != nil { + allErrors = append(allErrors, fmt.Errorf("failed to read '%s' from action '%s' with path '%s' of step %w", fileName, step.String(), actionPath, err)) + } else { + // One successful read, clear error state + allErrors = nil + } + } reader, closer, err := readFile("action.yml") + addError("action.yml", err) if os.IsNotExist(err) { reader, closer, err = readFile("action.yaml") + addError("action.yaml", err) if os.IsNotExist(err) { - if _, closer, err2 := readFile("Dockerfile"); err2 == nil { + _, closer, err := readFile("Dockerfile") + addError("Dockerfile", err) + if err == nil { closer.Close() action := &model.Action{ Name: "(Synthetic)", @@ -90,12 +104,10 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir string, act return action, nil } } - return nil, err - } else if err != nil { - return nil, err } - } else if err != nil { - return nil, err + } + if allErrors != nil { + return nil, errors.Join(allErrors...) } defer closer.Close() From c5cae8a5683ec0ae1e46d411a9a4cb8e7dbf5fc8 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sun, 28 Jan 2024 17:49:47 +0100 Subject: [PATCH 32/38] fix: subpath actions via new artifact cache (#2170) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/action.go | 29 +++++++++++++++-------------- act/runner/step_action_remote.go | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/act/runner/action.go b/act/runner/action.go index 42742c4d..9b14f1a4 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -125,20 +125,6 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir string return nil } - if rc.Config != nil && rc.Config.ActionCache != nil { - raction := step.(*stepActionRemote) - ta, err := rc.Config.ActionCache.GetTarArchive(ctx, raction.cacheDir, raction.resolvedSha, "") - if err != nil { - return err - } - defer ta.Close() - return rc.JobContainer.CopyTarStream(ctx, containerActionDir, ta) - } - - if err := removeGitIgnore(ctx, actionDir); err != nil { - return err - } - var containerActionDirCopy string containerActionDirCopy = strings.TrimSuffix(containerActionDir, actionPath) logger.Debug(containerActionDirCopy) @@ -146,6 +132,21 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir string if !strings.HasSuffix(containerActionDirCopy, `/`) { containerActionDirCopy += `/` } + + if rc.Config != nil && rc.Config.ActionCache != nil { + raction := step.(*stepActionRemote) + ta, err := rc.Config.ActionCache.GetTarArchive(ctx, raction.cacheDir, raction.resolvedSha, "") + if err != nil { + return err + } + defer ta.Close() + return rc.JobContainer.CopyTarStream(ctx, containerActionDirCopy, ta) + } + + if err := removeGitIgnore(ctx, actionDir); err != nil { + return err + } + return rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", rc.Config.UseGitIgnore)(ctx) } diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index 5c8a8f2f..7d0caa84 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -75,7 +75,7 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { remoteReader := func(ctx context.Context) actionYamlReader { return func(filename string) (io.Reader, io.Closer, error) { - spath := filename + spath := path.Join(sar.remoteAction.Path, filename) for i := 0; i < maxSymlinkDepth; i++ { tars, err := cache.GetTarArchive(ctx, sar.cacheDir, sar.resolvedSha, spath) if err != nil { From ce4598b0af3b732865ca6e25609bb08b93d0f4a3 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Sun, 28 Jan 2024 18:02:15 +0100 Subject: [PATCH 33/38] fix: improve new-action-cache fetch failure error (#2172) - include repoURL and repoRef in error - map NoErrAlreadyUptodate to `couldn't find remote ref` for branchOrtag fetch request Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/action_cache.go | 4 ++++ act/runner/step_action_remote.go | 6 ++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/act/runner/action_cache.go b/act/runner/action_cache.go index 1173206e..da4e651c 100644 --- a/act/runner/action_cache.go +++ b/act/runner/action_cache.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "encoding/hex" "errors" + "fmt" "io" "io/fs" "path" @@ -86,6 +87,9 @@ func (c GoGitActionCache) Fetch(ctx context.Context, cacheDir, url, ref, token s Auth: auth, Force: true, }); err != nil { + if tagOrSha && errors.Is(err, git.NoErrAlreadyUpToDate) { + return "", fmt.Errorf("couldn't find remote ref \"%s\"", ref) + } return "", err } if tagOrSha { diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index 7d0caa84..b34c6d9c 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -68,9 +68,11 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { var err error sar.cacheDir = fmt.Sprintf("%s/%s", sar.remoteAction.Org, sar.remoteAction.Repo) - sar.resolvedSha, err = cache.Fetch(ctx, sar.cacheDir, sar.remoteAction.URL+"/"+sar.cacheDir, sar.remoteAction.Ref, github.Token) + repoURL := sar.remoteAction.URL + "/" + sar.cacheDir + repoRef := sar.remoteAction.Ref + sar.resolvedSha, err = cache.Fetch(ctx, sar.cacheDir, repoURL, repoRef, github.Token) if err != nil { - return err + return fmt.Errorf("failed to fetch \"%s\" version \"%s\": %w", repoURL, repoRef, err) } remoteReader := func(ctx context.Context) actionYamlReader { From 79e54641f4843fba6aa7b68b96d09b46dc35db40 Mon Sep 17 00:00:00 2001 From: Josh Soref <2119212+jsoref@users.noreply.github.com> Date: Sun, 28 Jan 2024 14:21:21 -0500 Subject: [PATCH 34/38] fix: improve warning about remote not found (#2169) Signed-off-by: Josh Soref <2119212+jsoref@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/model/github_context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/act/model/github_context.go b/act/model/github_context.go index 5ed3d962..71221a59 100644 --- a/act/model/github_context.go +++ b/act/model/github_context.go @@ -168,7 +168,7 @@ func (ghc *GithubContext) SetRepositoryAndOwner(ctx context.Context, githubInsta if ghc.Repository == "" { repo, err := git.FindGithubRepo(ctx, repoPath, githubInstance, remoteName) if err != nil { - common.Logger(ctx).Warningf("unable to get git repo: %v", err) + common.Logger(ctx).Warningf("unable to get git repo (githubInstance: %v; remoteName: %v, repoPath: %v): %v", githubInstance, remoteName, repoPath, err) return } ghc.Repository = repo From bc545cd76fa0ef8f4871c7af137bedc3e08f3518 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 02:34:33 +0000 Subject: [PATCH 35/38] build(deps): bump codecov/codecov-action from 3.1.4 to 3.1.5 (#2175) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3.1.4 to 3.1.5. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3.1.4...v3.1.5) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b6dac11e..0a8c0349 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -61,7 +61,7 @@ jobs: - name: Run act from cli run: go run main.go -P ubuntu-latest=node:16-buster-slim -C ./pkg/runner/testdata/ -W ./basic/push.yml - name: Upload Codecov report - uses: codecov/codecov-action@v3.1.4 + uses: codecov/codecov-action@v3.1.5 with: files: coverage.txt fail_ci_if_error: true # optional (default = false) From 5e0a9ffa532d5bc428c709f757075d8c4a2abc87 Mon Sep 17 00:00:00 2001 From: ChristopherHX Date: Tue, 30 Jan 2024 01:46:45 +0100 Subject: [PATCH 36/38] refactor: filecollector into new package (#2174) * refactor: filecollector into new package * Add test for symlinks * add test fix bug of GetContainerArchive * add test data --- act/container/docker_run.go | 9 +-- act/container/host_environment.go | 21 +++--- act/container/host_environment_test.go | 67 +++++++++++++++++++ act/container/testdata/scratch/test.txt | 1 + .../file_collector.go | 34 +++++----- .../file_collector_test.go | 63 +++++++++++++++-- 6 files changed, 160 insertions(+), 35 deletions(-) create mode 100644 act/container/testdata/scratch/test.txt rename act/{container => filecollector}/file_collector.go (84%) rename act/{container => filecollector}/file_collector_test.go (62%) diff --git a/act/container/docker_run.go b/act/container/docker_run.go index bf220e67..c61301b5 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -35,6 +35,7 @@ import ( "golang.org/x/term" "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/filecollector" ) // NewContainer creates a reference to a container @@ -735,12 +736,12 @@ func (cr *containerReference) copyDir(dstPath string, srcPath string, useGitIgno ignorer = gitignore.NewMatcher(ps) } - fc := &fileCollector{ - Fs: &defaultFs{}, + fc := &filecollector.FileCollector{ + Fs: &filecollector.DefaultFs{}, Ignorer: ignorer, SrcPath: srcPath, SrcPrefix: srcPrefix, - Handler: &tarCollector{ + Handler: &filecollector.TarCollector{ TarWriter: tw, UID: cr.UID, GID: cr.GID, @@ -748,7 +749,7 @@ func (cr *containerReference) copyDir(dstPath string, srcPath string, useGitIgno }, } - err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{})) if err != nil { return err } diff --git a/act/container/host_environment.go b/act/container/host_environment.go index 91dae4c5..8130414d 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -21,6 +21,7 @@ import ( "golang.org/x/term" "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/filecollector" "github.com/nektos/act/pkg/lookpath" ) @@ -65,7 +66,7 @@ func (e *HostEnvironment) CopyTarStream(ctx context.Context, destPath string, ta return err } tr := tar.NewReader(tarStream) - cp := ©Collector{ + cp := &filecollector.CopyCollector{ DstDir: destPath, } for { @@ -104,16 +105,16 @@ func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore ignorer = gitignore.NewMatcher(ps) } - fc := &fileCollector{ - Fs: &defaultFs{}, + fc := &filecollector.FileCollector{ + Fs: &filecollector.DefaultFs{}, Ignorer: ignorer, SrcPath: srcPath, SrcPrefix: srcPrefix, - Handler: ©Collector{ + Handler: &filecollector.CopyCollector{ DstDir: destPath, }, } - return filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + return filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{})) } } @@ -126,21 +127,21 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin if err != nil { return nil, err } - tc := &tarCollector{ + tc := &filecollector.TarCollector{ TarWriter: tw, } if fi.IsDir() { - srcPrefix := filepath.Dir(srcPath) + srcPrefix := srcPath if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { srcPrefix += string(filepath.Separator) } - fc := &fileCollector{ - Fs: &defaultFs{}, + fc := &filecollector.FileCollector{ + Fs: &filecollector.DefaultFs{}, SrcPath: srcPath, SrcPrefix: srcPrefix, Handler: tc, } - err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{})) if err != nil { return nil, err } diff --git a/act/container/host_environment_test.go b/act/container/host_environment_test.go index 67787d95..2614a2f8 100644 --- a/act/container/host_environment_test.go +++ b/act/container/host_environment_test.go @@ -1,4 +1,71 @@ package container +import ( + "archive/tar" + "context" + "io" + "os" + "path" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + // Type assert HostEnvironment implements ExecutionsEnvironment var _ ExecutionsEnvironment = &HostEnvironment{} + +func TestCopyDir(t *testing.T) { + dir, err := os.MkdirTemp("", "test-host-env-*") + assert.NoError(t, err) + defer os.RemoveAll(dir) + ctx := context.Background() + e := &HostEnvironment{ + Path: filepath.Join(dir, "path"), + TmpDir: filepath.Join(dir, "tmp"), + ToolCache: filepath.Join(dir, "tool_cache"), + ActPath: filepath.Join(dir, "act_path"), + StdOut: os.Stdout, + Workdir: path.Join("testdata", "scratch"), + } + _ = os.MkdirAll(e.Path, 0700) + _ = os.MkdirAll(e.TmpDir, 0700) + _ = os.MkdirAll(e.ToolCache, 0700) + _ = os.MkdirAll(e.ActPath, 0700) + err = e.CopyDir(e.Workdir, e.Path, true)(ctx) + assert.NoError(t, err) +} + +func TestGetContainerArchive(t *testing.T) { + dir, err := os.MkdirTemp("", "test-host-env-*") + assert.NoError(t, err) + defer os.RemoveAll(dir) + ctx := context.Background() + e := &HostEnvironment{ + Path: filepath.Join(dir, "path"), + TmpDir: filepath.Join(dir, "tmp"), + ToolCache: filepath.Join(dir, "tool_cache"), + ActPath: filepath.Join(dir, "act_path"), + StdOut: os.Stdout, + Workdir: path.Join("testdata", "scratch"), + } + _ = os.MkdirAll(e.Path, 0700) + _ = os.MkdirAll(e.TmpDir, 0700) + _ = os.MkdirAll(e.ToolCache, 0700) + _ = os.MkdirAll(e.ActPath, 0700) + expectedContent := []byte("sdde/7sh") + err = os.WriteFile(filepath.Join(e.Path, "action.yml"), expectedContent, 0600) + assert.NoError(t, err) + archive, err := e.GetContainerArchive(ctx, e.Path) + assert.NoError(t, err) + defer archive.Close() + reader := tar.NewReader(archive) + h, err := reader.Next() + assert.NoError(t, err) + assert.Equal(t, "action.yml", h.Name) + content, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, expectedContent, content) + _, err = reader.Next() + assert.ErrorIs(t, err, io.EOF) +} diff --git a/act/container/testdata/scratch/test.txt b/act/container/testdata/scratch/test.txt new file mode 100644 index 00000000..e7cbb71a --- /dev/null +++ b/act/container/testdata/scratch/test.txt @@ -0,0 +1 @@ +testfile \ No newline at end of file diff --git a/act/container/file_collector.go b/act/filecollector/file_collector.go similarity index 84% rename from act/container/file_collector.go rename to act/filecollector/file_collector.go index b4be0e88..8547bb7c 100644 --- a/act/container/file_collector.go +++ b/act/filecollector/file_collector.go @@ -1,4 +1,4 @@ -package container +package filecollector import ( "archive/tar" @@ -17,18 +17,18 @@ import ( "github.com/go-git/go-git/v5/plumbing/format/index" ) -type fileCollectorHandler interface { +type Handler interface { WriteFile(path string, fi fs.FileInfo, linkName string, f io.Reader) error } -type tarCollector struct { +type TarCollector struct { TarWriter *tar.Writer UID int GID int DstDir string } -func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { +func (tc TarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { // create a new dir/file header header, err := tar.FileInfoHeader(fi, linkName) if err != nil { @@ -59,11 +59,11 @@ func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, return nil } -type copyCollector struct { +type CopyCollector struct { DstDir string } -func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { +func (cc *CopyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { fdestpath := filepath.Join(cc.DstDir, fpath) if err := os.MkdirAll(filepath.Dir(fdestpath), 0o777); err != nil { return err @@ -82,29 +82,29 @@ func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string return nil } -type fileCollector struct { +type FileCollector struct { Ignorer gitignore.Matcher SrcPath string SrcPrefix string - Fs fileCollectorFs - Handler fileCollectorHandler + Fs Fs + Handler Handler } -type fileCollectorFs interface { +type Fs interface { Walk(root string, fn filepath.WalkFunc) error OpenGitIndex(path string) (*index.Index, error) Open(path string) (io.ReadCloser, error) Readlink(path string) (string, error) } -type defaultFs struct { +type DefaultFs struct { } -func (*defaultFs) Walk(root string, fn filepath.WalkFunc) error { +func (*DefaultFs) Walk(root string, fn filepath.WalkFunc) error { return filepath.Walk(root, fn) } -func (*defaultFs) OpenGitIndex(path string) (*index.Index, error) { +func (*DefaultFs) OpenGitIndex(path string) (*index.Index, error) { r, err := git.PlainOpen(path) if err != nil { return nil, err @@ -116,16 +116,16 @@ func (*defaultFs) OpenGitIndex(path string) (*index.Index, error) { return i, nil } -func (*defaultFs) Open(path string) (io.ReadCloser, error) { +func (*DefaultFs) Open(path string) (io.ReadCloser, error) { return os.Open(path) } -func (*defaultFs) Readlink(path string) (string, error) { +func (*DefaultFs) Readlink(path string) (string, error) { return os.Readlink(path) } //nolint:gocyclo -func (fc *fileCollector) collectFiles(ctx context.Context, submodulePath []string) filepath.WalkFunc { +func (fc *FileCollector) CollectFiles(ctx context.Context, submodulePath []string) filepath.WalkFunc { i, _ := fc.Fs.OpenGitIndex(path.Join(fc.SrcPath, path.Join(submodulePath...))) return func(file string, fi os.FileInfo, err error) error { if err != nil { @@ -166,7 +166,7 @@ func (fc *fileCollector) collectFiles(ctx context.Context, submodulePath []strin } } if err == nil && entry.Mode == filemode.Submodule { - err = fc.Fs.Walk(file, fc.collectFiles(ctx, split)) + err = fc.Fs.Walk(file, fc.CollectFiles(ctx, split)) if err != nil { return err } diff --git a/act/container/file_collector_test.go b/act/filecollector/file_collector_test.go similarity index 62% rename from act/container/file_collector_test.go rename to act/filecollector/file_collector_test.go index 241fd34b..60a8d4dd 100644 --- a/act/container/file_collector_test.go +++ b/act/filecollector/file_collector_test.go @@ -1,4 +1,4 @@ -package container +package filecollector import ( "archive/tar" @@ -95,16 +95,16 @@ func TestIgnoredTrackedfile(t *testing.T) { tw := tar.NewWriter(tmpTar) ps, _ := gitignore.ReadPatterns(worktree, []string{}) ignorer := gitignore.NewMatcher(ps) - fc := &fileCollector{ + fc := &FileCollector{ Fs: &memoryFs{Filesystem: fs}, Ignorer: ignorer, SrcPath: "mygitrepo", SrcPrefix: "mygitrepo" + string(filepath.Separator), - Handler: &tarCollector{ + Handler: &TarCollector{ TarWriter: tw, }, } - err := fc.Fs.Walk("mygitrepo", fc.collectFiles(context.Background(), []string{})) + err := fc.Fs.Walk("mygitrepo", fc.CollectFiles(context.Background(), []string{})) assert.NoError(t, err, "successfully collect files") tw.Close() _, _ = tmpTar.Seek(0, io.SeekStart) @@ -115,3 +115,58 @@ func TestIgnoredTrackedfile(t *testing.T) { _, err = tr.Next() assert.ErrorIs(t, err, io.EOF, "tar must only contain one element") } + +func TestSymlinks(t *testing.T) { + fs := memfs.New() + _ = fs.MkdirAll("mygitrepo/.git", 0o777) + dotgit, _ := fs.Chroot("mygitrepo/.git") + worktree, _ := fs.Chroot("mygitrepo") + repo, _ := git.Init(filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()), worktree) + // This file shouldn't be in the tar + f, err := worktree.Create(".env") + assert.NoError(t, err) + _, err = f.Write([]byte("test=val1\n")) + assert.NoError(t, err) + f.Close() + err = worktree.Symlink(".env", "test.env") + assert.NoError(t, err) + + w, err := repo.Worktree() + assert.NoError(t, err) + + // .gitignore is in the tar after adding it to the index + _, err = w.Add(".env") + assert.NoError(t, err) + _, err = w.Add("test.env") + assert.NoError(t, err) + + tmpTar, _ := fs.Create("temp.tar") + tw := tar.NewWriter(tmpTar) + ps, _ := gitignore.ReadPatterns(worktree, []string{}) + ignorer := gitignore.NewMatcher(ps) + fc := &FileCollector{ + Fs: &memoryFs{Filesystem: fs}, + Ignorer: ignorer, + SrcPath: "mygitrepo", + SrcPrefix: "mygitrepo" + string(filepath.Separator), + Handler: &TarCollector{ + TarWriter: tw, + }, + } + err = fc.Fs.Walk("mygitrepo", fc.CollectFiles(context.Background(), []string{})) + assert.NoError(t, err, "successfully collect files") + tw.Close() + _, _ = tmpTar.Seek(0, io.SeekStart) + tr := tar.NewReader(tmpTar) + h, err := tr.Next() + files := map[string]tar.Header{} + for err == nil { + files[h.Name] = *h + h, err = tr.Next() + } + + assert.Equal(t, ".env", files[".env"].Name) + assert.Equal(t, "test.env", files["test.env"].Name) + assert.Equal(t, ".env", files["test.env"].Linkname) + assert.ErrorIs(t, err, io.EOF, "tar must be read cleanly to EOF") +} From f99698eb769b506d24935f5bc904adfb3e8807d1 Mon Sep 17 00:00:00 2001 From: Markus Wolf Date: Tue, 30 Jan 2024 23:43:52 +0100 Subject: [PATCH 37/38] fix: use correct path to toolcache (#1494) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The toolcache on GitHub Actions need to be in /opt/hostedtoolcache. This is the case for all environment variables set by act, but it's not the case for the volume mounted into the container. Co-authored-by: Björn Brauer Co-authored-by: ChristopherHX Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- act/runner/run_context.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index 937b4803..8027f791 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -135,7 +135,7 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { ext := container.LinuxContainerEnvironmentExtensions{} mounts := map[string]string{ - "act-toolcache": "/toolcache", + "act-toolcache": "/opt/hostedtoolcache", name + "-env": ext.GetActPath(), } From 0c6fe20cfb1abcc36c11110f1c2c813357eae777 Mon Sep 17 00:00:00 2001 From: Milo Moisson Date: Thu, 1 Feb 2024 22:57:16 +0100 Subject: [PATCH 38/38] feat: correctly use the xdg library, which has the side effect to fix the config survey (#2195) --- cmd/root.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index e506699f..349e6ac4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -107,24 +107,22 @@ func Execute(ctx context.Context, version string) { } } -// Return locations where Act's config can be found in order : XDG spec, .actrc in HOME directory, .actrc in invocation directory +// Return locations where Act's config can be found in order: XDG spec, .actrc in HOME directory, .actrc in invocation directory func configLocations() []string { configFileName := ".actrc" - // reference: https://specifications.freedesktop.org/basedir-spec/latest/ar01s03.html - var actrcXdg string - for _, fileName := range []string{"act/actrc", configFileName} { - if foundConfig, err := xdg.SearchConfigFile(fileName); foundConfig != "" && err == nil { - actrcXdg = foundConfig - break - } + homePath := filepath.Join(UserHomeDir, configFileName) + invocationPath := filepath.Join(".", configFileName) + + // Though named xdg, adrg's lib support macOS and Windows config paths as well + // It also takes cares of creating the parent folder so we don't need to bother later + specPath, err := xdg.ConfigFile("act/actrc") + if err != nil { + specPath = homePath } - return []string{ - actrcXdg, - filepath.Join(UserHomeDir, configFileName), - filepath.Join(".", configFileName), - } + // This order should be enforced since the survey part relies on it + return []string{specPath, homePath, invocationPath} } var commonSocketPaths = []string{ @@ -554,7 +552,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str } } if !cfgFound && len(cfgLocations) > 0 { - // The first config location refers to the XDG spec one + // The first config location refers to the global config folder one if err := defaultImageSurvey(cfgLocations[0]); err != nil { log.Fatal(err) }