diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 5c72a93b..0a8c0349 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -15,10 +15,10 @@ jobs: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true @@ -26,7 +26,7 @@ jobs: with: version: v1.53 only-new-issues: true - - uses: megalinter/megalinter/flavors/go@v7.4.0 + - uses: megalinter/megalinter/flavors/go@v7.8.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -38,12 +38,12 @@ jobs: name: test-linux runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true @@ -61,7 +61,7 @@ jobs: - name: Run act from cli run: go run main.go -P ubuntu-latest=node:16-buster-slim -C ./pkg/runner/testdata/ -W ./basic/push.yml - name: Upload Codecov report - uses: codecov/codecov-action@v3.1.4 + uses: codecov/codecov-action@v3.1.5 with: files: coverage.txt fail_ci_if_error: true # optional (default = false) @@ -75,10 +75,10 @@ jobs: name: test-${{matrix.os}} runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 2 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true @@ -92,8 +92,8 @@ jobs: name: snapshot runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true @@ -111,67 +111,67 @@ jobs: args: release --snapshot --clean - name: Capture x86_64 (64-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-amd64 path: dist/act_linux_amd64_v1/act - name: Capture i386 (32-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-i386 path: dist/act_linux_386/act - name: Capture arm64 (64-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-arm64 path: dist/act_linux_arm64/act - name: Capture armv6 (32-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-armv6 path: dist/act_linux_arm_6/act - name: Capture armv7 (32-bit) Linux binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-linux-armv7 path: dist/act_linux_arm_7/act - name: Capture x86_64 (64-bit) Windows binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-windows-amd64 path: dist/act_windows_amd64_v1/act.exe - name: Capture i386 (32-bit) Windows binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-windows-i386 path: dist/act_windows_386/act.exe - name: Capture arm64 (64-bit) Windows binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-windows-arm64 path: dist/act_windows_arm64/act.exe - name: Capture armv7 (32-bit) Windows binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-windows-armv7 path: dist/act_windows_arm_7/act.exe - name: Capture x86_64 (64-bit) MacOS binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-macos-amd64 path: dist/act_darwin_amd64_v1/act - name: Capture arm64 (64-bit) MacOS binary if: ${{ !env.ACT }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: act-macos-arm64 path: dist/act_darwin_arm64/act diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index d6cd1eab..536afe88 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -9,13 +9,13 @@ jobs: name: promote runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: master token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} - uses: fregante/setup-git-user@v2 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c40b07e7..afdf3f52 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,10 +9,10 @@ jobs: name: release runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version-file: go.mod check-latest: true @@ -43,7 +43,7 @@ jobs: apiKey: ${{ secrets.CHOCO_APIKEY }} push: true - name: GitHub CLI extension - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: github-token: ${{ secrets.GORELEASER_GITHUB_TOKEN }} script: | diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 88e815c1..87948a2e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -8,7 +8,7 @@ jobs: name: Stale runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-issue-message: 'Issue is stale and will be closed in 14 days unless there is new activity' diff --git a/act/artifactcache/handler.go b/act/artifactcache/handler.go index b6600b6b..3178260b 100644 --- a/act/artifactcache/handler.go +++ b/act/artifactcache/handler.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "strconv" "strings" "sync/atomic" @@ -386,7 +387,12 @@ func (h *Handler) findCache(db *bolthold.Store, keys []string, version string) ( for _, prefix := range keys[1:] { found := false - if err := db.ForEach(bolthold.Where("Key").Ge(prefix).And("Version").Eq(version).SortBy("Key"), func(v *Cache) error { + prefixPattern := fmt.Sprintf("^%s", regexp.QuoteMeta(prefix)) + re, err := regexp.Compile(prefixPattern) + if err != nil { + continue + } + if err := db.ForEach(bolthold.Where("Key").RegExp(re).And("Version").Eq(version).SortBy("CreatedAt").Reverse(), func(v *Cache) error { if !strings.HasPrefix(v.Key, prefix) { return stop } diff --git a/act/common/git/git.go b/act/common/git/git.go index bf771558..0a819b95 100644 --- a/act/common/git/git.go +++ b/act/common/git/git.go @@ -221,10 +221,11 @@ func findGitSlug(url string, githubInstance string) (string, string, error) { // NewGitCloneExecutorInput the input for the NewGitCloneExecutor type NewGitCloneExecutorInput struct { - URL string - Ref string - Dir string - Token string + URL string + Ref string + Dir string + Token string + OfflineMode bool } // CloneIfRequired ... @@ -302,12 +303,16 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor { return err } + isOfflineMode := input.OfflineMode + // fetch latest changes fetchOptions, pullOptions := gitOptions(input.Token) - err = r.Fetch(&fetchOptions) - if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { - return err + if !isOfflineMode { + err = r.Fetch(&fetchOptions) + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return err + } } var hash *plumbing.Hash @@ -367,9 +372,10 @@ func NewGitCloneExecutor(input NewGitCloneExecutorInput) common.Executor { return err } } - - if err = w.Pull(&pullOptions); err != nil && err != git.NoErrAlreadyUpToDate { - logger.Debugf("Unable to pull %s: %v", refName, err) + if !isOfflineMode { + if err = w.Pull(&pullOptions); err != nil && err != git.NoErrAlreadyUpToDate { + logger.Debugf("Unable to pull %s: %v", refName, err) + } } logger.Debugf("Cloned %s to %s", input.URL, input.Dir) diff --git a/act/container/container_types.go b/act/container/container_types.go index ab70f983..511db9e9 100644 --- a/act/container/container_types.go +++ b/act/container/container_types.go @@ -4,34 +4,37 @@ import ( "context" "io" + "github.com/docker/go-connections/nat" "github.com/nektos/act/pkg/common" ) // NewContainerInput the input for the New function type NewContainerInput struct { - Image string - Username string - Password string - Entrypoint []string - Cmd []string - WorkingDir string - Env []string - Binds []string - Mounts map[string]string - Name string - Stdout io.Writer - Stderr io.Writer - NetworkMode string - Privileged bool - UsernsMode string - Platform string - Options string + Image string + Username string + Password string + Entrypoint []string + Cmd []string + WorkingDir string + Env []string + Binds []string + Mounts map[string]string + Name string + Stdout io.Writer + Stderr io.Writer + NetworkMode string + Privileged bool + UsernsMode string + Platform string + Options string + NetworkAliases []string + ExposedPorts nat.PortSet + PortBindings nat.PortMap // Gitea specific AutoRemove bool - NetworkAliases []string - ValidVolumes []string + ValidVolumes []string } // FileEntry is a file to copy to a container diff --git a/act/container/docker_auth.go b/act/container/docker_auth.go index 9c263f55..e8dfb7d5 100644 --- a/act/container/docker_auth.go +++ b/act/container/docker_auth.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_build.go b/act/container/docker_build.go index fdc04d8d..5f56ec70 100644 --- a/act/container/docker_build.go +++ b/act/container/docker_build.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_cli.go b/act/container/docker_cli.go index a1481c36..82d32466 100644 --- a/act/container/docker_cli.go +++ b/act/container/docker_cli.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) // This file is exact copy of https://github.com/docker/cli/blob/9ac8584acfd501c3f4da0e845e3a40ed15c85041/cli/command/container/opts.go // appended with license information. diff --git a/act/container/docker_images.go b/act/container/docker_images.go index 22772307..50ec68da 100644 --- a/act/container/docker_images.go +++ b/act/container/docker_images.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_logger.go b/act/container/docker_logger.go index f2c21e6c..b9eb503e 100644 --- a/act/container/docker_logger.go +++ b/act/container/docker_logger.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_network.go b/act/container/docker_network.go index d3a0046f..6c2676f9 100644 --- a/act/container/docker_network.go +++ b/act/container/docker_network.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container @@ -15,6 +15,20 @@ func NewDockerNetworkCreateExecutor(name string) common.Executor { if err != nil { return err } + defer cli.Close() + + // Only create the network if it doesn't exist + networks, err := cli.NetworkList(ctx, types.NetworkListOptions{}) + if err != nil { + return err + } + common.Logger(ctx).Debugf("%v", networks) + for _, network := range networks { + if network.Name == name { + common.Logger(ctx).Debugf("Network %v exists", name) + return nil + } + } _, err = cli.NetworkCreate(ctx, name, types.NetworkCreate{ Driver: "bridge", @@ -34,7 +48,32 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor { if err != nil { return err } + defer cli.Close() - return cli.NetworkRemove(ctx, name) + // Make shure that all network of the specified name are removed + // cli.NetworkRemove refuses to remove a network if there are duplicates + networks, err := cli.NetworkList(ctx, types.NetworkListOptions{}) + if err != nil { + return err + } + common.Logger(ctx).Debugf("%v", networks) + for _, network := range networks { + if network.Name == name { + result, err := cli.NetworkInspect(ctx, network.ID, types.NetworkInspectOptions{}) + if err != nil { + return err + } + + if len(result.Containers) == 0 { + if err = cli.NetworkRemove(ctx, network.ID); err != nil { + common.Logger(ctx).Debugf("%v", err) + } + } else { + common.Logger(ctx).Debugf("Refusing to remove network %v because it still has active endpoints", name) + } + } + } + + return err } } diff --git a/act/container/docker_pull.go b/act/container/docker_pull.go index ad75958f..7c94c104 100644 --- a/act/container/docker_pull.go +++ b/act/container/docker_pull.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/docker_run.go b/act/container/docker_run.go index fe13f290..191b424f 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container @@ -38,6 +38,7 @@ import ( "golang.org/x/term" "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/filecollector" ) // NewContainer creates a reference to a container @@ -86,7 +87,7 @@ func supportsContainerImagePlatform(ctx context.Context, cli client.APIClient) b func (cr *containerReference) Create(capAdd []string, capDrop []string) common.Executor { return common. - NewInfoExecutor("%sdocker create image=%s platform=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Platform, cr.input.Entrypoint, cr.input.Cmd). + NewInfoExecutor("%sdocker create image=%s platform=%s entrypoint=%+q cmd=%+q network=%+q", logPrefix, cr.input.Image, cr.input.Platform, cr.input.Entrypoint, cr.input.Cmd, cr.input.NetworkMode). Then( common.NewPipelineExecutor( cr.connect(), @@ -98,7 +99,7 @@ func (cr *containerReference) Create(capAdd []string, capDrop []string) common.E func (cr *containerReference) Start(attach bool) common.Executor { return common. - NewInfoExecutor("%sdocker run image=%s platform=%s entrypoint=%+q cmd=%+q", logPrefix, cr.input.Image, cr.input.Platform, cr.input.Entrypoint, cr.input.Cmd). + NewInfoExecutor("%sdocker run image=%s platform=%s entrypoint=%+q cmd=%+q network=%+q", logPrefix, cr.input.Image, cr.input.Platform, cr.input.Entrypoint, cr.input.Cmd, cr.input.NetworkMode). Then( common.NewPipelineExecutor( cr.connect(), @@ -260,8 +261,10 @@ func RunnerArch(ctx context.Context) string { archMapper := map[string]string{ "x86_64": "X64", + "amd64": "X64", "386": "X86", "aarch64": "ARM64", + "arm64": "ARM64", } if arch, ok := archMapper[info.Architecture]; ok { return arch @@ -370,8 +373,8 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config // So comment out the following code. // if len(copts.netMode.Value()) == 0 { - // if err = copts.netMode.Set("host"); err != nil { - // return nil, nil, fmt.Errorf("Cannot parse networkmode=host. This is an internal error and should not happen: '%w'", err) + // if err = copts.netMode.Set(cr.input.NetworkMode); err != nil { + // return nil, nil, fmt.Errorf("Cannot parse networkmode=%s. This is an internal error and should not happen: '%w'", cr.input.NetworkMode, err) // } // } @@ -426,10 +429,11 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E input := cr.input config := &container.Config{ - Image: input.Image, - WorkingDir: input.WorkingDir, - Env: input.Env, - Tty: isTerminal, + Image: input.Image, + WorkingDir: input.WorkingDir, + Env: input.Env, + ExposedPorts: input.ExposedPorts, + Tty: isTerminal, } logger.Debugf("Common container.Config ==> %+v", config) @@ -465,14 +469,15 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E } hostConfig := &container.HostConfig{ - CapAdd: capAdd, - CapDrop: capDrop, - Binds: input.Binds, - Mounts: mounts, - NetworkMode: container.NetworkMode(input.NetworkMode), - Privileged: input.Privileged, - UsernsMode: container.UsernsMode(input.UsernsMode), - AutoRemove: input.AutoRemove, + CapAdd: capAdd, + CapDrop: capDrop, + Binds: input.Binds, + Mounts: mounts, + NetworkMode: container.NetworkMode(input.NetworkMode), + Privileged: input.Privileged, + UsernsMode: container.UsernsMode(input.UsernsMode), + PortBindings: input.PortBindings, + AutoRemove: input.AutoRemove, } logger.Debugf("Common container.HostConfig ==> %+v", hostConfig) @@ -487,7 +492,10 @@ func (cr *containerReference) create(capAdd []string, capDrop []string) common.E // For Gitea // network-scoped alias is supported only for containers in user defined networks var networkingConfig *network.NetworkingConfig - if hostConfig.NetworkMode.IsUserDefined() && len(input.NetworkAliases) > 0 { + logger.Debugf("input.NetworkAliases ==> %v", input.NetworkAliases) + n := hostConfig.NetworkMode + // IsUserDefined and IsHost are broken on windows + if n.IsUserDefined() && n != "host" && len(input.NetworkAliases) > 0 { endpointConfig := &network.EndpointSettings{ Aliases: input.NetworkAliases, } @@ -709,10 +717,28 @@ func (cr *containerReference) waitForCommand(ctx context.Context, isTerminal boo } func (cr *containerReference) CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error { - err := cr.cli.CopyToContainer(ctx, cr.id, destPath, tarStream, types.CopyToContainerOptions{}) + // Mkdir + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + _ = tw.WriteHeader(&tar.Header{ + Name: destPath, + Mode: 777, + Typeflag: tar.TypeDir, + }) + tw.Close() + err := cr.cli.CopyToContainer(ctx, cr.id, "/", buf, types.CopyToContainerOptions{}) + if err != nil { + return fmt.Errorf("failed to mkdir to copy content to container: %w", err) + } + // Copy Content + err = cr.cli.CopyToContainer(ctx, cr.id, destPath, tarStream, types.CopyToContainerOptions{}) if err != nil { return fmt.Errorf("failed to copy content to container: %w", err) } + // If this fails, then folders have wrong permissions on non root container + if cr.UID != 0 || cr.GID != 0 { + _ = cr.Exec([]string{"chown", "-R", fmt.Sprintf("%d:%d", cr.UID, cr.GID), destPath}, nil, "0", "")(ctx) + } return nil } @@ -753,12 +779,12 @@ func (cr *containerReference) copyDir(dstPath string, srcPath string, useGitIgno ignorer = gitignore.NewMatcher(ps) } - fc := &fileCollector{ - Fs: &defaultFs{}, + fc := &filecollector.FileCollector{ + Fs: &filecollector.DefaultFs{}, Ignorer: ignorer, SrcPath: srcPath, SrcPrefix: srcPrefix, - Handler: &tarCollector{ + Handler: &filecollector.TarCollector{ TarWriter: tw, UID: cr.UID, GID: cr.GID, @@ -766,7 +792,7 @@ func (cr *containerReference) copyDir(dstPath string, srcPath string, useGitIgno }, } - err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{})) if err != nil { return err } diff --git a/act/container/docker_run_test.go b/act/container/docker_run_test.go index 2a2007af..9e85fb71 100644 --- a/act/container/docker_run_test.go +++ b/act/container/docker_run_test.go @@ -23,6 +23,7 @@ func TestDocker(t *testing.T) { ctx := context.Background() client, err := GetDockerClient(ctx) assert.NoError(t, err) + defer client.Close() dockerBuild := NewDockerBuildExecutor(NewDockerBuildExecutorInput{ ContextDir: "testdata", diff --git a/act/container/docker_stub.go b/act/container/docker_stub.go index 36f530ee..d5c5ef12 100644 --- a/act/container/docker_stub.go +++ b/act/container/docker_stub.go @@ -1,4 +1,4 @@ -//go:build WITHOUT_DOCKER || !(linux || darwin || windows) +//go:build WITHOUT_DOCKER || !(linux || darwin || windows || netbsd) package container diff --git a/act/container/docker_volume.go b/act/container/docker_volume.go index 0bb2cd7c..f99c5845 100644 --- a/act/container/docker_volume.go +++ b/act/container/docker_volume.go @@ -1,4 +1,4 @@ -//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows)) +//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) package container diff --git a/act/container/host_environment.go b/act/container/host_environment.go index a131f814..bdda16cd 100644 --- a/act/container/host_environment.go +++ b/act/container/host_environment.go @@ -21,6 +21,7 @@ import ( "golang.org/x/term" "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/filecollector" "github.com/nektos/act/pkg/lookpath" ) @@ -71,7 +72,7 @@ func (e *HostEnvironment) CopyTarStream(ctx context.Context, destPath string, ta return err } tr := tar.NewReader(tarStream) - cp := ©Collector{ + cp := &filecollector.CopyCollector{ DstDir: destPath, } for { @@ -110,16 +111,16 @@ func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore ignorer = gitignore.NewMatcher(ps) } - fc := &fileCollector{ - Fs: &defaultFs{}, + fc := &filecollector.FileCollector{ + Fs: &filecollector.DefaultFs{}, Ignorer: ignorer, SrcPath: srcPath, SrcPrefix: srcPrefix, - Handler: ©Collector{ + Handler: &filecollector.CopyCollector{ DstDir: destPath, }, } - return filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + return filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{})) } } @@ -132,21 +133,21 @@ func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath strin if err != nil { return nil, err } - tc := &tarCollector{ + tc := &filecollector.TarCollector{ TarWriter: tw, } if fi.IsDir() { - srcPrefix := filepath.Dir(srcPath) + srcPrefix := srcPath if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { srcPrefix += string(filepath.Separator) } - fc := &fileCollector{ - Fs: &defaultFs{}, + fc := &filecollector.FileCollector{ + Fs: &filecollector.DefaultFs{}, SrcPath: srcPath, SrcPrefix: srcPrefix, Handler: tc, } - err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + err = filepath.Walk(srcPath, fc.CollectFiles(ctx, []string{})) if err != nil { return nil, err } diff --git a/act/container/host_environment_test.go b/act/container/host_environment_test.go index 67787d95..2614a2f8 100644 --- a/act/container/host_environment_test.go +++ b/act/container/host_environment_test.go @@ -1,4 +1,71 @@ package container +import ( + "archive/tar" + "context" + "io" + "os" + "path" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + // Type assert HostEnvironment implements ExecutionsEnvironment var _ ExecutionsEnvironment = &HostEnvironment{} + +func TestCopyDir(t *testing.T) { + dir, err := os.MkdirTemp("", "test-host-env-*") + assert.NoError(t, err) + defer os.RemoveAll(dir) + ctx := context.Background() + e := &HostEnvironment{ + Path: filepath.Join(dir, "path"), + TmpDir: filepath.Join(dir, "tmp"), + ToolCache: filepath.Join(dir, "tool_cache"), + ActPath: filepath.Join(dir, "act_path"), + StdOut: os.Stdout, + Workdir: path.Join("testdata", "scratch"), + } + _ = os.MkdirAll(e.Path, 0700) + _ = os.MkdirAll(e.TmpDir, 0700) + _ = os.MkdirAll(e.ToolCache, 0700) + _ = os.MkdirAll(e.ActPath, 0700) + err = e.CopyDir(e.Workdir, e.Path, true)(ctx) + assert.NoError(t, err) +} + +func TestGetContainerArchive(t *testing.T) { + dir, err := os.MkdirTemp("", "test-host-env-*") + assert.NoError(t, err) + defer os.RemoveAll(dir) + ctx := context.Background() + e := &HostEnvironment{ + Path: filepath.Join(dir, "path"), + TmpDir: filepath.Join(dir, "tmp"), + ToolCache: filepath.Join(dir, "tool_cache"), + ActPath: filepath.Join(dir, "act_path"), + StdOut: os.Stdout, + Workdir: path.Join("testdata", "scratch"), + } + _ = os.MkdirAll(e.Path, 0700) + _ = os.MkdirAll(e.TmpDir, 0700) + _ = os.MkdirAll(e.ToolCache, 0700) + _ = os.MkdirAll(e.ActPath, 0700) + expectedContent := []byte("sdde/7sh") + err = os.WriteFile(filepath.Join(e.Path, "action.yml"), expectedContent, 0600) + assert.NoError(t, err) + archive, err := e.GetContainerArchive(ctx, e.Path) + assert.NoError(t, err) + defer archive.Close() + reader := tar.NewReader(archive) + h, err := reader.Next() + assert.NoError(t, err) + assert.Equal(t, "action.yml", h.Name) + content, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Equal(t, expectedContent, content) + _, err = reader.Next() + assert.ErrorIs(t, err, io.EOF) +} diff --git a/act/container/testdata/scratch/test.txt b/act/container/testdata/scratch/test.txt new file mode 100644 index 00000000..e7cbb71a --- /dev/null +++ b/act/container/testdata/scratch/test.txt @@ -0,0 +1 @@ +testfile \ No newline at end of file diff --git a/act/exprparser/functions_test.go b/act/exprparser/functions_test.go index 3c6392c1..ea51a2bc 100644 --- a/act/exprparser/functions_test.go +++ b/act/exprparser/functions_test.go @@ -230,6 +230,7 @@ func TestFunctionFormat(t *testing.T) { {"format('{0', '{1}', 'World')", nil, "Unclosed brackets. The following format string is invalid: '{0'", "format-invalid-format-string"}, {"format('{2}', '{1}', 'World')", "", "The following format string references more arguments than were supplied: '{2}'", "format-invalid-replacement-reference"}, {"format('{2147483648}')", "", "The following format string is invalid: '{2147483648}'", "format-invalid-replacement-reference"}, + {"format('{0} {1} {2} {3}', 1.0, 1.1, 1234567890.0, 12345678901234567890.0)", "1 1.1 1234567890 1.23456789012346E+19", nil, "format-floats"}, } env := &EvaluationEnvironment{ diff --git a/act/exprparser/interpreter.go b/act/exprparser/interpreter.go index d23db889..6c661b7b 100644 --- a/act/exprparser/interpreter.go +++ b/act/exprparser/interpreter.go @@ -449,7 +449,7 @@ func (impl *interperterImpl) coerceToString(value reflect.Value) reflect.Value { } else if math.IsInf(value.Float(), -1) { return reflect.ValueOf("-Infinity") } - return reflect.ValueOf(fmt.Sprint(value)) + return reflect.ValueOf(fmt.Sprintf("%.15G", value.Float())) case reflect.Slice: return reflect.ValueOf("Array") diff --git a/act/container/file_collector.go b/act/filecollector/file_collector.go similarity index 84% rename from act/container/file_collector.go rename to act/filecollector/file_collector.go index b4be0e88..8547bb7c 100644 --- a/act/container/file_collector.go +++ b/act/filecollector/file_collector.go @@ -1,4 +1,4 @@ -package container +package filecollector import ( "archive/tar" @@ -17,18 +17,18 @@ import ( "github.com/go-git/go-git/v5/plumbing/format/index" ) -type fileCollectorHandler interface { +type Handler interface { WriteFile(path string, fi fs.FileInfo, linkName string, f io.Reader) error } -type tarCollector struct { +type TarCollector struct { TarWriter *tar.Writer UID int GID int DstDir string } -func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { +func (tc TarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { // create a new dir/file header header, err := tar.FileInfoHeader(fi, linkName) if err != nil { @@ -59,11 +59,11 @@ func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, return nil } -type copyCollector struct { +type CopyCollector struct { DstDir string } -func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { +func (cc *CopyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { fdestpath := filepath.Join(cc.DstDir, fpath) if err := os.MkdirAll(filepath.Dir(fdestpath), 0o777); err != nil { return err @@ -82,29 +82,29 @@ func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string return nil } -type fileCollector struct { +type FileCollector struct { Ignorer gitignore.Matcher SrcPath string SrcPrefix string - Fs fileCollectorFs - Handler fileCollectorHandler + Fs Fs + Handler Handler } -type fileCollectorFs interface { +type Fs interface { Walk(root string, fn filepath.WalkFunc) error OpenGitIndex(path string) (*index.Index, error) Open(path string) (io.ReadCloser, error) Readlink(path string) (string, error) } -type defaultFs struct { +type DefaultFs struct { } -func (*defaultFs) Walk(root string, fn filepath.WalkFunc) error { +func (*DefaultFs) Walk(root string, fn filepath.WalkFunc) error { return filepath.Walk(root, fn) } -func (*defaultFs) OpenGitIndex(path string) (*index.Index, error) { +func (*DefaultFs) OpenGitIndex(path string) (*index.Index, error) { r, err := git.PlainOpen(path) if err != nil { return nil, err @@ -116,16 +116,16 @@ func (*defaultFs) OpenGitIndex(path string) (*index.Index, error) { return i, nil } -func (*defaultFs) Open(path string) (io.ReadCloser, error) { +func (*DefaultFs) Open(path string) (io.ReadCloser, error) { return os.Open(path) } -func (*defaultFs) Readlink(path string) (string, error) { +func (*DefaultFs) Readlink(path string) (string, error) { return os.Readlink(path) } //nolint:gocyclo -func (fc *fileCollector) collectFiles(ctx context.Context, submodulePath []string) filepath.WalkFunc { +func (fc *FileCollector) CollectFiles(ctx context.Context, submodulePath []string) filepath.WalkFunc { i, _ := fc.Fs.OpenGitIndex(path.Join(fc.SrcPath, path.Join(submodulePath...))) return func(file string, fi os.FileInfo, err error) error { if err != nil { @@ -166,7 +166,7 @@ func (fc *fileCollector) collectFiles(ctx context.Context, submodulePath []strin } } if err == nil && entry.Mode == filemode.Submodule { - err = fc.Fs.Walk(file, fc.collectFiles(ctx, split)) + err = fc.Fs.Walk(file, fc.CollectFiles(ctx, split)) if err != nil { return err } diff --git a/act/container/file_collector_test.go b/act/filecollector/file_collector_test.go similarity index 62% rename from act/container/file_collector_test.go rename to act/filecollector/file_collector_test.go index 241fd34b..60a8d4dd 100644 --- a/act/container/file_collector_test.go +++ b/act/filecollector/file_collector_test.go @@ -1,4 +1,4 @@ -package container +package filecollector import ( "archive/tar" @@ -95,16 +95,16 @@ func TestIgnoredTrackedfile(t *testing.T) { tw := tar.NewWriter(tmpTar) ps, _ := gitignore.ReadPatterns(worktree, []string{}) ignorer := gitignore.NewMatcher(ps) - fc := &fileCollector{ + fc := &FileCollector{ Fs: &memoryFs{Filesystem: fs}, Ignorer: ignorer, SrcPath: "mygitrepo", SrcPrefix: "mygitrepo" + string(filepath.Separator), - Handler: &tarCollector{ + Handler: &TarCollector{ TarWriter: tw, }, } - err := fc.Fs.Walk("mygitrepo", fc.collectFiles(context.Background(), []string{})) + err := fc.Fs.Walk("mygitrepo", fc.CollectFiles(context.Background(), []string{})) assert.NoError(t, err, "successfully collect files") tw.Close() _, _ = tmpTar.Seek(0, io.SeekStart) @@ -115,3 +115,58 @@ func TestIgnoredTrackedfile(t *testing.T) { _, err = tr.Next() assert.ErrorIs(t, err, io.EOF, "tar must only contain one element") } + +func TestSymlinks(t *testing.T) { + fs := memfs.New() + _ = fs.MkdirAll("mygitrepo/.git", 0o777) + dotgit, _ := fs.Chroot("mygitrepo/.git") + worktree, _ := fs.Chroot("mygitrepo") + repo, _ := git.Init(filesystem.NewStorage(dotgit, cache.NewObjectLRUDefault()), worktree) + // This file shouldn't be in the tar + f, err := worktree.Create(".env") + assert.NoError(t, err) + _, err = f.Write([]byte("test=val1\n")) + assert.NoError(t, err) + f.Close() + err = worktree.Symlink(".env", "test.env") + assert.NoError(t, err) + + w, err := repo.Worktree() + assert.NoError(t, err) + + // .gitignore is in the tar after adding it to the index + _, err = w.Add(".env") + assert.NoError(t, err) + _, err = w.Add("test.env") + assert.NoError(t, err) + + tmpTar, _ := fs.Create("temp.tar") + tw := tar.NewWriter(tmpTar) + ps, _ := gitignore.ReadPatterns(worktree, []string{}) + ignorer := gitignore.NewMatcher(ps) + fc := &FileCollector{ + Fs: &memoryFs{Filesystem: fs}, + Ignorer: ignorer, + SrcPath: "mygitrepo", + SrcPrefix: "mygitrepo" + string(filepath.Separator), + Handler: &TarCollector{ + TarWriter: tw, + }, + } + err = fc.Fs.Walk("mygitrepo", fc.CollectFiles(context.Background(), []string{})) + assert.NoError(t, err, "successfully collect files") + tw.Close() + _, _ = tmpTar.Seek(0, io.SeekStart) + tr := tar.NewReader(tmpTar) + h, err := tr.Next() + files := map[string]tar.Header{} + for err == nil { + files[h.Name] = *h + h, err = tr.Next() + } + + assert.Equal(t, ".env", files[".env"].Name) + assert.Equal(t, "test.env", files["test.env"].Name) + assert.Equal(t, ".env", files["test.env"].Linkname) + assert.ErrorIs(t, err, io.EOF, "tar must be read cleanly to EOF") +} diff --git a/act/model/github_context.go b/act/model/github_context.go index 5ed3d962..71221a59 100644 --- a/act/model/github_context.go +++ b/act/model/github_context.go @@ -168,7 +168,7 @@ func (ghc *GithubContext) SetRepositoryAndOwner(ctx context.Context, githubInsta if ghc.Repository == "" { repo, err := git.FindGithubRepo(ctx, repoPath, githubInstance, remoteName) if err != nil { - common.Logger(ctx).Warningf("unable to get git repo: %v", err) + common.Logger(ctx).Warningf("unable to get git repo (githubInstance: %v; remoteName: %v, repoPath: %v): %v", githubInstance, remoteName, repoPath, err) return } ghc.Repository = repo diff --git a/act/model/planner.go b/act/model/planner.go index 1b34d153..2d681456 100644 --- a/act/model/planner.go +++ b/act/model/planner.go @@ -148,12 +148,10 @@ func NewWorkflowPlanner(path string, noWorkflowRecurse bool) (WorkflowPlanner, e workflow.Name = wf.workflowDirEntry.Name() } - jobNameRegex := regexp.MustCompile(`^([[:alpha:]_][[:alnum:]_\-]*)$`) - for k := range workflow.Jobs { - if ok := jobNameRegex.MatchString(k); !ok { - _ = f.Close() - return nil, fmt.Errorf("workflow is not valid. '%s': Job name '%s' is invalid. Names must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'", workflow.Name, k) - } + err = validateJobName(workflow) + if err != nil { + _ = f.Close() + return nil, err } wp.workflows = append(wp.workflows, workflow) @@ -171,6 +169,42 @@ func CombineWorkflowPlanner(workflows ...*Workflow) WorkflowPlanner { } } +func NewSingleWorkflowPlanner(name string, f io.Reader) (WorkflowPlanner, error) { + wp := new(workflowPlanner) + + log.Debugf("Reading workflow %s", name) + workflow, err := ReadWorkflow(f) + if err != nil { + if err == io.EOF { + return nil, fmt.Errorf("unable to read workflow '%s': file is empty: %w", name, err) + } + return nil, fmt.Errorf("workflow is not valid. '%s': %w", name, err) + } + workflow.File = name + if workflow.Name == "" { + workflow.Name = name + } + + err = validateJobName(workflow) + if err != nil { + return nil, err + } + + wp.workflows = append(wp.workflows, workflow) + + return wp, nil +} + +func validateJobName(workflow *Workflow) error { + jobNameRegex := regexp.MustCompile(`^([[:alpha:]_][[:alnum:]_\-]*)$`) + for k := range workflow.Jobs { + if ok := jobNameRegex.MatchString(k); !ok { + return fmt.Errorf("workflow is not valid. '%s': Job name '%s' is invalid. Names must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'", workflow.Name, k) + } + } + return nil +} + type workflowPlanner struct { workflows []*Workflow } diff --git a/act/model/workflow.go b/act/model/workflow.go index a54004bf..a8d167e2 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -103,22 +103,40 @@ type WorkflowDispatch struct { } func (w *Workflow) WorkflowDispatchConfig() *WorkflowDispatch { - if w.RawOn.Kind != yaml.MappingNode { + switch w.RawOn.Kind { + case yaml.ScalarNode: + var val string + if !decodeNode(w.RawOn, &val) { + return nil + } + if val == "workflow_dispatch" { + return &WorkflowDispatch{} + } + case yaml.SequenceNode: + var val []string + if !decodeNode(w.RawOn, &val) { + return nil + } + for _, v := range val { + if v == "workflow_dispatch" { + return &WorkflowDispatch{} + } + } + case yaml.MappingNode: + var val map[string]yaml.Node + if !decodeNode(w.RawOn, &val) { + return nil + } + + n, found := val["workflow_dispatch"] + var workflowDispatch WorkflowDispatch + if found && decodeNode(n, &workflowDispatch) { + return &workflowDispatch + } + default: return nil } - - var val map[string]yaml.Node - if !decodeNode(w.RawOn, &val) { - return nil - } - - var config WorkflowDispatch - node := val["workflow_dispatch"] - if !decodeNode(node, &config) { - return nil - } - - return &config + return nil } type WorkflowCallInput struct { @@ -299,15 +317,39 @@ func (j *Job) Needs() []string { // RunsOn list for Job func (j *Job) RunsOn() []string { switch j.RawRunsOn.Kind { + case yaml.MappingNode: + var val struct { + Group string + Labels yaml.Node + } + + if !decodeNode(j.RawRunsOn, &val) { + return nil + } + + labels := nodeAsStringSlice(val.Labels) + + if val.Group != "" { + labels = append(labels, val.Group) + } + + return labels + default: + return nodeAsStringSlice(j.RawRunsOn) + } +} + +func nodeAsStringSlice(node yaml.Node) []string { + switch node.Kind { case yaml.ScalarNode: var val string - if !decodeNode(j.RawRunsOn, &val) { + if !decodeNode(node, &val) { return nil } return []string{val} case yaml.SequenceNode: var val []string - if !decodeNode(j.RawRunsOn, &val) { + if !decodeNode(node, &val) { return nil } return val diff --git a/act/model/workflow_test.go b/act/model/workflow_test.go index 8b336629..9f219ee7 100644 --- a/act/model/workflow_test.go +++ b/act/model/workflow_test.go @@ -153,6 +153,41 @@ jobs: assert.Contains(t, workflow.On(), "pull_request") } +func TestReadWorkflow_RunsOnLabels(t *testing.T) { + yaml := ` +name: local-action-docker-url + +jobs: + test: + container: nginx:latest + runs-on: + labels: ubuntu-latest + steps: + - uses: ./actions/docker-url` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + assert.Equal(t, workflow.Jobs["test"].RunsOn(), []string{"ubuntu-latest"}) +} + +func TestReadWorkflow_RunsOnLabelsWithGroup(t *testing.T) { + yaml := ` +name: local-action-docker-url + +jobs: + test: + container: nginx:latest + runs-on: + labels: [ubuntu-latest] + group: linux + steps: + - uses: ./actions/docker-url` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + assert.Equal(t, workflow.Jobs["test"].RunsOn(), []string{"ubuntu-latest", "linux"}) +} + func TestReadWorkflow_StringContainer(t *testing.T) { yaml := ` name: local-action-docker-url @@ -464,3 +499,107 @@ func TestStep_ShellCommand(t *testing.T) { }) } } + +func TestReadWorkflow_WorkflowDispatchConfig(t *testing.T) { + yaml := ` + name: local-action-docker-url + ` + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch := workflow.WorkflowDispatchConfig() + assert.Nil(t, workflowDispatch) + + yaml = ` + name: local-action-docker-url + on: push + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.Nil(t, workflowDispatch) + + yaml = ` + name: local-action-docker-url + on: workflow_dispatch + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.NotNil(t, workflowDispatch) + assert.Nil(t, workflowDispatch.Inputs) + + yaml = ` + name: local-action-docker-url + on: [push, pull_request] + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.Nil(t, workflowDispatch) + + yaml = ` + name: local-action-docker-url + on: [push, workflow_dispatch] + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.NotNil(t, workflowDispatch) + assert.Nil(t, workflowDispatch.Inputs) + + yaml = ` + name: local-action-docker-url + on: + - push + - workflow_dispatch + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.NotNil(t, workflowDispatch) + assert.Nil(t, workflowDispatch.Inputs) + + yaml = ` + name: local-action-docker-url + on: + push: + pull_request: + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.Nil(t, workflowDispatch) + + yaml = ` + name: local-action-docker-url + on: + push: + pull_request: + workflow_dispatch: + inputs: + logLevel: + description: 'Log level' + required: true + default: 'warning' + type: choice + options: + - info + - warning + - debug + ` + workflow, err = ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + workflowDispatch = workflow.WorkflowDispatchConfig() + assert.NotNil(t, workflowDispatch) + assert.Equal(t, WorkflowDispatchInput{ + Default: "warning", + Description: "Log level", + Options: []string{ + "info", + "warning", + "debug", + }, + Required: true, + Type: "choice", + }, workflowDispatch.Inputs["logLevel"]) +} diff --git a/act/runner/action.go b/act/runner/action.go index f769eeeb..416e5e40 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -3,6 +3,7 @@ package runner import ( "context" "embed" + "errors" "fmt" "io" "io/fs" @@ -41,11 +42,24 @@ var trampoline embed.FS func readActionImpl(ctx context.Context, step *model.Step, actionDir string, actionPath string, readFile actionYamlReader, writeFile fileWriter) (*model.Action, error) { logger := common.Logger(ctx) + allErrors := []error{} + addError := func(fileName string, err error) { + if err != nil { + allErrors = append(allErrors, fmt.Errorf("failed to read '%s' from action '%s' with path '%s' of step %w", fileName, step.String(), actionPath, err)) + } else { + // One successful read, clear error state + allErrors = nil + } + } reader, closer, err := readFile("action.yml") + addError("action.yml", err) if os.IsNotExist(err) { reader, closer, err = readFile("action.yaml") - if err != nil { - if _, closer, err2 := readFile("Dockerfile"); err2 == nil { + addError("action.yaml", err) + if os.IsNotExist(err) { + _, closer, err := readFile("Dockerfile") + addError("Dockerfile", err) + if err == nil { closer.Close() action := &model.Action{ Name: "(Synthetic)", @@ -90,10 +104,10 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir string, act return action, nil } } - return nil, err } - } else if err != nil { - return nil, err + } + if allErrors != nil { + return nil, errors.Join(allErrors...) } defer closer.Close() @@ -110,9 +124,6 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir string if stepModel.Type() != model.StepTypeUsesActionRemote { return nil } - if err := removeGitIgnore(ctx, actionDir); err != nil { - return err - } var containerActionDirCopy string containerActionDirCopy = strings.TrimSuffix(containerActionDir, actionPath) @@ -121,6 +132,21 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir string if !strings.HasSuffix(containerActionDirCopy, `/`) { containerActionDirCopy += `/` } + + if rc.Config != nil && rc.Config.ActionCache != nil { + raction := step.(*stepActionRemote) + ta, err := rc.Config.ActionCache.GetTarArchive(ctx, raction.cacheDir, raction.resolvedSha, "") + if err != nil { + return err + } + defer ta.Close() + return rc.JobContainer.CopyTarStream(ctx, containerActionDirCopy, ta) + } + + if err := removeGitIgnore(ctx, actionDir); err != nil { + return err + } + return rc.JobContainer.CopyDir(containerActionDirCopy, actionDir+"/", rc.Config.UseGitIgnore)(ctx) } @@ -281,6 +307,13 @@ func execAsDocker(ctx context.Context, step actionStep, actionName string, based return err } defer buildContext.Close() + } else if rc.Config.ActionCache != nil { + rstep := step.(*stepActionRemote) + buildContext, err = rc.Config.ActionCache.GetTarArchive(ctx, rstep.cacheDir, rstep.resolvedSha, contextDir) + if err != nil { + return err + } + defer buildContext.Close() } prepImage = container.NewDockerBuildExecutor(container.NewDockerBuildExecutorInput{ ContextDir: contextDir, diff --git a/act/runner/action_cache.go b/act/runner/action_cache.go index 1173206e..da4e651c 100644 --- a/act/runner/action_cache.go +++ b/act/runner/action_cache.go @@ -6,6 +6,7 @@ import ( "crypto/rand" "encoding/hex" "errors" + "fmt" "io" "io/fs" "path" @@ -86,6 +87,9 @@ func (c GoGitActionCache) Fetch(ctx context.Context, cacheDir, url, ref, token s Auth: auth, Force: true, }); err != nil { + if tagOrSha && errors.Is(err, git.NoErrAlreadyUpToDate) { + return "", fmt.Errorf("couldn't find remote ref \"%s\"", ref) + } return "", err } if tagOrSha { diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index 67708c09..0a866835 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -20,6 +20,7 @@ type jobInfo interface { result(result string) } +//nolint:contextcheck,gocyclo func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executor { steps := make([]common.Executor, 0) preSteps := make([]common.Executor, 0) @@ -101,7 +102,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo postExec := useStepLogger(rc, stepModel, stepStagePost, step.post()) if postExecutor != nil { - // run the post exector in reverse order + // run the post executor in reverse order postExecutor = postExec.Finally(postExecutor) } else { postExecutor = postExec @@ -117,22 +118,19 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo defer cancel() logger := common.Logger(ctx) - logger.Infof("Cleaning up services for job %s", rc.JobName) - if err := rc.stopServiceContainers()(ctx); err != nil { - logger.Errorf("Error while cleaning services: %v", err) - } - logger.Infof("Cleaning up container for job %s", rc.JobName) if err = info.stopContainer()(ctx); err != nil { logger.Errorf("Error while stop job container: %v", err) } + if !rc.IsHostEnv(ctx) && rc.Config.ContainerNetworkMode == "" { // clean network in docker mode only // if the value of `ContainerNetworkMode` is empty string, // it means that the network to which containers are connecting is created by `act_runner`, // so, we should remove the network at last. - logger.Infof("Cleaning up network for job %s, and network name is: %s", rc.JobName, rc.networkName()) - if err := container.NewDockerNetworkRemoveExecutor(rc.networkName())(ctx); err != nil { + networkName, _ := rc.networkName() + logger.Infof("Cleaning up network for job %s, and network name is: %s", rc.JobName, networkName) + if err := container.NewDockerNetworkRemoveExecutor(networkName)(ctx); err != nil { logger.Errorf("Error while cleaning network: %v", err) } } diff --git a/act/runner/reusable_workflow.go b/act/runner/reusable_workflow.go index 721c2a2a..717913f4 100644 --- a/act/runner/reusable_workflow.go +++ b/act/runner/reusable_workflow.go @@ -1,6 +1,7 @@ package runner import ( + "archive/tar" "context" "errors" "fmt" @@ -67,12 +68,51 @@ func newRemoteReusableWorkflowExecutor(rc *RunContext) common.Executor { // FIXME: if the reusable workflow is from a private repository, we need to provide a token to access the repository. token := "" + if rc.Config.ActionCache != nil { + return newActionCacheReusableWorkflowExecutor(rc, filename, remoteReusableWorkflow) + } + return common.NewPipelineExecutor( newMutexExecutor(cloneIfRequired(rc, *remoteReusableWorkflow, workflowDir, token)), newReusableWorkflowExecutor(rc, workflowDir, remoteReusableWorkflow.FilePath()), ) } +func newActionCacheReusableWorkflowExecutor(rc *RunContext, filename string, remoteReusableWorkflow *remoteReusableWorkflow) common.Executor { + return func(ctx context.Context) error { + ghctx := rc.getGithubContext(ctx) + remoteReusableWorkflow.URL = ghctx.ServerURL + sha, err := rc.Config.ActionCache.Fetch(ctx, filename, remoteReusableWorkflow.CloneURL(), remoteReusableWorkflow.Ref, ghctx.Token) + if err != nil { + return err + } + archive, err := rc.Config.ActionCache.GetTarArchive(ctx, filename, sha, fmt.Sprintf(".github/workflows/%s", remoteReusableWorkflow.Filename)) + if err != nil { + return err + } + defer archive.Close() + treader := tar.NewReader(archive) + if _, err = treader.Next(); err != nil { + return err + } + planner, err := model.NewSingleWorkflowPlanner(remoteReusableWorkflow.Filename, treader) + if err != nil { + return err + } + plan, err := planner.PlanEvent("workflow_call") + if err != nil { + return err + } + + runner, err := NewReusableWorkflowRunner(rc) + if err != nil { + return err + } + + return runner.NewPlanExecutor(plan)(ctx) + } +} + var ( executorLock sync.Mutex ) @@ -99,10 +139,11 @@ func cloneIfRequired(rc *RunContext, remoteReusableWorkflow remoteReusableWorkfl // 2. Gitea has already full URL with rc.Config.GitHubInstance when calling newRemoteReusableWorkflowWithPlat // remoteReusableWorkflow.URL = rc.getGithubContext(ctx).ServerURL return git.NewGitCloneExecutor(git.NewGitCloneExecutorInput{ - URL: remoteReusableWorkflow.CloneURL(), - Ref: remoteReusableWorkflow.Ref, - Dir: targetDirectory, - Token: token, + URL: remoteReusableWorkflow.CloneURL(), + Ref: remoteReusableWorkflow.Ref, + Dir: targetDirectory, + Token: token, + OfflineMode: rc.Config.ActionOfflineMode, })(ctx) }, nil, diff --git a/act/runner/run_context.go b/act/runner/run_context.go index d412106d..59a01615 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -16,14 +16,13 @@ import ( "regexp" "runtime" "strings" - "time" - - "github.com/opencontainers/selinux/go-selinux" + "github.com/docker/go-connections/nat" "github.com/nektos/act/pkg/common" "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/exprparser" "github.com/nektos/act/pkg/model" + "github.com/opencontainers/selinux/go-selinux" ) // RunContext contains info about current job @@ -65,7 +64,7 @@ func (rc *RunContext) String() string { if rc.caller != nil { // prefix the reusable workflow with the caller job // this is required to create unique container names - name = fmt.Sprintf("%s/%s", rc.caller.runContext.Run.JobID, name) + name = fmt.Sprintf("%s/%s", rc.caller.runContext.Name, name) } return name } @@ -95,9 +94,15 @@ func (rc *RunContext) jobContainerName() string { } // networkName return the name of the network which will be created by `act` automatically for job, -// only create network if `rc.Config.ContainerNetworkMode` is empty string. -func (rc *RunContext) networkName() string { - return fmt.Sprintf("%s-network", rc.jobContainerName()) +// only create network if using a service container +func (rc *RunContext) networkName() (string, bool) { + if len(rc.Run.Job().Services) > 0 { + return fmt.Sprintf("%s-%s-network", rc.jobContainerName(), rc.Run.JobID), true + } + if rc.Config.ContainerNetworkMode == "" { + return "host", false + } + return string(rc.Config.ContainerNetworkMode), false } func getDockerDaemonSocketMountPath(daemonPath string) string { @@ -135,7 +140,7 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { ext := container.LinuxContainerEnvironmentExtensions{} mounts := map[string]string{ - "act-toolcache": "/toolcache", + "act-toolcache": "/opt/hostedtoolcache", name + "-env": ext.GetActPath(), } @@ -247,6 +252,7 @@ func (rc *RunContext) startHostEnvironment() common.Executor { } } +//nolint:gocyclo func (rc *RunContext) startJobContainer() common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) @@ -285,14 +291,15 @@ func (rc *RunContext) startJobContainer() common.Executor { // specify the network to which the container will connect when `docker create` stage. (like execute command line: docker create --network ) networkName := string(rc.Config.ContainerNetworkMode) + var createAndDeleteNetwork bool if networkName == "" { // if networkName is empty string, will create a new network for the containers. // and it will be removed after at last. - networkName = rc.networkName() + networkName, createAndDeleteNetwork = rc.networkName() } // add service containers - for serviceId, spec := range rc.Run.Job().Services { + for serviceID, spec := range rc.Run.Job().Services { // interpolate env interpolatedEnvs := make(map[string]string, len(spec.Env)) for k, v := range spec.Env { @@ -302,21 +309,36 @@ func (rc *RunContext) startJobContainer() common.Executor { for k, v := range interpolatedEnvs { envs = append(envs, fmt.Sprintf("%s=%s", k, v)) } - // interpolate cmd interpolatedCmd := make([]string, 0, len(spec.Cmd)) for _, v := range spec.Cmd { interpolatedCmd = append(interpolatedCmd, rc.ExprEval.Interpolate(ctx, v)) } - username, password, err := rc.handleServiceCredentials(ctx, spec.Credentials) + + username, password, err = rc.handleServiceCredentials(ctx, spec.Credentials) if err != nil { - return fmt.Errorf("failed to handle service %s credentials: %w", serviceId, err) + return fmt.Errorf("failed to handle service %s credentials: %w", serviceID, err) } - serviceBinds, serviceMounts := rc.GetServiceBindsAndMounts(spec.Volumes) - serviceContainerName := createSimpleContainerName(rc.jobContainerName(), serviceId) + + interpolatedVolumes := make([]string, 0, len(spec.Volumes)) + for _, volume := range spec.Volumes { + interpolatedVolumes = append(interpolatedVolumes, rc.ExprEval.Interpolate(ctx, volume)) + } + serviceBinds, serviceMounts := rc.GetServiceBindsAndMounts(interpolatedVolumes) + + interpolatedPorts := make([]string, 0, len(spec.Ports)) + for _, port := range spec.Ports { + interpolatedPorts = append(interpolatedPorts, rc.ExprEval.Interpolate(ctx, port)) + } + exposedPorts, portBindings, err := nat.ParsePortSpecs(interpolatedPorts) + if err != nil { + return fmt.Errorf("failed to parse service %s ports: %w", serviceID, err) + } + + serviceContainerName := createContainerName(rc.jobContainerName(), serviceID) c := container.NewContainer(&container.NewContainerInput{ Name: serviceContainerName, WorkingDir: ext.ToContainerPath(rc.Config.Workdir), - Image: spec.Image, + Image: rc.ExprEval.Interpolate(ctx, spec.Image), Username: username, Password: password, Cmd: interpolatedCmd, @@ -329,26 +351,58 @@ func (rc *RunContext) startJobContainer() common.Executor { UsernsMode: rc.Config.UsernsMode, Platform: rc.Config.ContainerArchitecture, AutoRemove: rc.Config.AutoRemove, - Options: spec.Options, + Options: rc.ExprEval.Interpolate(ctx, spec.Options), NetworkMode: networkName, - NetworkAliases: []string{serviceId}, + NetworkAliases: []string{serviceID}, + ExposedPorts: exposedPorts, + PortBindings: portBindings, ValidVolumes: rc.Config.ValidVolumes, }) rc.ServiceContainers = append(rc.ServiceContainers, c) } rc.cleanUpJobContainer = func(ctx context.Context) error { - if rc.JobContainer != nil && !rc.Config.ReuseContainers { - return rc.JobContainer.Remove(). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + reuseJobContainer := func(ctx context.Context) bool { + return rc.Config.ReuseContainers + } + + if rc.JobContainer != nil { + return rc.JobContainer.Remove().IfNot(reuseJobContainer). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)).IfNot(reuseJobContainer). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false)).IfNot(reuseJobContainer). + Then(func(ctx context.Context) error { + if len(rc.ServiceContainers) > 0 { + logger.Infof("Cleaning up services for job %s", rc.JobName) + if err := rc.stopServiceContainers()(ctx); err != nil { + logger.Errorf("Error while cleaning services: %v", err) + } + if createAndDeleteNetwork { + // clean network if it has been created by act + // if using service containers + // it means that the network to which containers are connecting is created by `act_runner`, + // so, we should remove the network at last. + logger.Infof("Cleaning up network for job %s, and network name is: %s", rc.JobName, networkName) + if err := container.NewDockerNetworkRemoveExecutor(networkName)(ctx); err != nil { + logger.Errorf("Error while cleaning network: %v", err) + } + } + } + return nil + })(ctx) } return nil } + jobContainerNetwork := rc.Config.ContainerNetworkMode.NetworkName() + if rc.containerImage(ctx) != "" { + jobContainerNetwork = networkName + } else if jobContainerNetwork == "" { + jobContainerNetwork = "host" + } + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, - Entrypoint: []string{"/bin/sleep", fmt.Sprint(rc.Config.ContainerMaxLifetime.Round(time.Second).Seconds())}, + Entrypoint: []string{"tail", "-f", "/dev/null"}, WorkingDir: ext.ToContainerPath(rc.Config.Workdir), Image: image, Username: username, @@ -356,7 +410,7 @@ func (rc *RunContext) startJobContainer() common.Executor { Name: name, Env: envList, Mounts: mounts, - NetworkMode: networkName, + NetworkMode: jobContainerNetwork, NetworkAliases: []string{rc.Name}, Binds: binds, Stdout: logWriter, @@ -375,6 +429,7 @@ func (rc *RunContext) startJobContainer() common.Executor { return common.NewPipelineExecutor( rc.pullServicesImages(rc.Config.ForcePull), rc.JobContainer.Pull(rc.Config.ForcePull), + rc.stopJobContainer(), container.NewDockerNetworkCreateExecutor(networkName).IfBool(!rc.IsHostEnv(ctx) && rc.Config.ContainerNetworkMode == ""), // if the value of `ContainerNetworkMode` is empty string, then will create a new network for containers. rc.startServiceContainers(networkName), rc.JobContainer.Create(rc.Config.ContainerCapAdd, rc.Config.ContainerCapDrop), @@ -452,10 +507,10 @@ func (rc *RunContext) UpdateExtraPath(ctx context.Context, githubEnvPath string) return nil } -// stopJobContainer removes the job container (if it exists) and its volume (if it exists) if !rc.Config.ReuseContainers +// stopJobContainer removes the job container (if it exists) and its volume (if it exists) func (rc *RunContext) stopJobContainer() common.Executor { return func(ctx context.Context) error { - if rc.cleanUpJobContainer != nil && !rc.Config.ReuseContainers { + if rc.cleanUpJobContainer != nil { return rc.cleanUpJobContainer(ctx) } return nil @@ -472,7 +527,7 @@ func (rc *RunContext) pullServicesImages(forcePull bool) common.Executor { } } -func (rc *RunContext) startServiceContainers(networkName string) common.Executor { +func (rc *RunContext) startServiceContainers(_ string) common.Executor { return func(ctx context.Context) error { execs := []common.Executor{} for _, c := range rc.ServiceContainers { @@ -490,7 +545,7 @@ func (rc *RunContext) stopServiceContainers() common.Executor { return func(ctx context.Context) error { execs := []common.Executor{} for _, c := range rc.ServiceContainers { - execs = append(execs, c.Remove()) + execs = append(execs, c.Remove().Finally(c.Close())) } return common.NewParallelExecutor(len(execs), execs...)(ctx) } @@ -610,13 +665,11 @@ func (rc *RunContext) containerImage(ctx context.Context) string { } func (rc *RunContext) runsOnImage(ctx context.Context) string { - job := rc.Run.Job() - - if job.RunsOn() == nil { + if rc.Run.Job().RunsOn() == nil { common.Logger(ctx).Errorf("'runs-on' key not defined in %s", rc.String()) } - runsOn := job.RunsOn() + runsOn := rc.Run.Job().RunsOn() for i, v := range runsOn { runsOn[i] = rc.ExprEval.Interpolate(ctx, v) } @@ -627,8 +680,8 @@ func (rc *RunContext) runsOnImage(ctx context.Context) string { } } - for _, runnerLabel := range runsOn { - image := rc.Config.Platforms[strings.ToLower(runnerLabel)] + for _, platformName := range rc.runsOnPlatformNames(ctx) { + image := rc.Config.Platforms[strings.ToLower(platformName)] if image != "" { return image } @@ -637,6 +690,21 @@ func (rc *RunContext) runsOnImage(ctx context.Context) string { return "" } +func (rc *RunContext) runsOnPlatformNames(ctx context.Context) []string { + job := rc.Run.Job() + + if job.RunsOn() == nil { + return []string{} + } + + if err := rc.ExprEval.EvaluateYamlNode(ctx, &job.RawRunsOn); err != nil { + common.Logger(ctx).Errorf("Error while evaluating runs-on: %v", err) + return []string{} + } + + return job.RunsOn() +} + func (rc *RunContext) platformImage(ctx context.Context) string { if containerImage := rc.containerImage(ctx); containerImage != "" { return containerImage @@ -667,8 +735,6 @@ func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) { if jobType == model.JobTypeInvalid { return false, jobTypeErr - } else if jobType != model.JobTypeDefault { - return true, nil } if !runJob { @@ -676,14 +742,13 @@ func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) { return false, nil } + if jobType != model.JobTypeDefault { + return true, nil + } + img := rc.platformImage(ctx) if img == "" { - if job.RunsOn() == nil { - l.Errorf("'runs-on' key not defined in %s", rc.String()) - } - - for _, runnerLabel := range job.RunsOn() { - platformName := rc.ExprEval.Interpolate(ctx, runnerLabel) + for _, platformName := range rc.runsOnPlatformNames(ctx) { l.Infof("\U0001F6A7 Skipping unsupported platform -- Try running with `-P %+v=...`", platformName) } return false, nil @@ -960,7 +1025,6 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon env["GITHUB_REF"] = github.Ref env["GITHUB_REF_NAME"] = github.RefName env["GITHUB_REF_TYPE"] = github.RefType - env["GITHUB_TOKEN"] = github.Token env["GITHUB_JOB"] = github.Job env["GITHUB_REPOSITORY_OWNER"] = github.RepositoryOwner env["GITHUB_RETENTION_DAYS"] = github.RetentionDays @@ -987,9 +1051,7 @@ func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubCon setActionRuntimeVars(rc, env) } - job := rc.Run.Job() - for _, runnerLabel := range job.RunsOn() { - platformName := rc.ExprEval.Interpolate(ctx, runnerLabel) + for _, platformName := range rc.runsOnPlatformNames(ctx) { if platformName != "" { if platformName == "ubuntu-latest" { // hardcode current ubuntu-latest since we have no way to check that 'on the fly' diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index 86ad44ed..af0cd4ee 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -470,6 +470,53 @@ func createJob(t *testing.T, input string, result string) *model.Job { return job } +func TestRunContextRunsOnPlatformNames(t *testing.T) { + log.SetLevel(log.DebugLevel) + assertObject := assert.New(t) + + rc := createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: ubuntu-latest`, ""), + }) + assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: ${{ 'ubuntu-latest' }}`, ""), + }) + assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: [self-hosted, my-runner]`, ""), + }) + assertObject.Equal([]string{"self-hosted", "my-runner"}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: [self-hosted, "${{ 'my-runner' }}"]`, ""), + }) + assertObject.Equal([]string{"self-hosted", "my-runner"}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: ${{ fromJSON('["ubuntu-latest"]') }}`, ""), + }) + assertObject.Equal([]string{"ubuntu-latest"}, rc.runsOnPlatformNames(context.Background())) + + // test missing / invalid runs-on + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `name: something`, ""), + }) + assertObject.Equal([]string{}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: + mapping: value`, ""), + }) + assertObject.Equal([]string{}, rc.runsOnPlatformNames(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `runs-on: ${{ invalid expression }}`, ""), + }) + assertObject.Equal([]string{}, rc.runsOnPlatformNames(context.Background())) +} + func TestRunContextIsEnabled(t *testing.T) { log.SetLevel(log.DebugLevel) assertObject := assert.New(t) @@ -572,6 +619,17 @@ if: always()`, ""), }) rc.Run.JobID = "job2" assertObject.True(rc.isEnabled(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `uses: ./.github/workflows/reusable.yml`, ""), + }) + assertObject.True(rc.isEnabled(context.Background())) + + rc = createIfTestRunContext(map[string]*model.Job{ + "job1": createJob(t, `uses: ./.github/workflows/reusable.yml +if: false`, ""), + }) + assertObject.False(rc.isEnabled(context.Background())) } func TestRunContextGetEnv(t *testing.T) { diff --git a/act/runner/runner.go b/act/runner/runner.go index e9f00e28..015c3648 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -22,50 +22,52 @@ type Runner interface { // Config contains the config for a new runner type Config struct { - Actor string // the user that triggered the event - Workdir string // path to working directory - ActionCacheDir string // path used for caching action contents - BindWorkdir bool // bind the workdir to the job container - EventName string // name of event to run - EventPath string // path to JSON file to use for event.json in containers - DefaultBranch string // name of the main branch for this repository - ReuseContainers bool // reuse containers to maintain state - ForcePull bool // force pulling of the image, even if already present - ForceRebuild bool // force rebuilding local docker image action - LogOutput bool // log the output from docker run - JSONLogger bool // use json or text logger - LogPrefixJobID bool // switches from the full job name to the job id - Env map[string]string // env for containers - Inputs map[string]string // manually passed action inputs - Secrets map[string]string // list of secrets - Vars map[string]string // list of vars - Token string // GitHub token - InsecureSecrets bool // switch hiding output when printing to terminal - Platforms map[string]string // list of platforms - Privileged bool // use privileged mode - UsernsMode string // user namespace to use - ContainerArchitecture string // Desired OS/architecture platform for running containers - ContainerDaemonSocket string // Path to Docker daemon socket - ContainerOptions string // Options for the job container - UseGitIgnore bool // controls if paths in .gitignore should not be copied into container, default true - GitHubInstance string // GitHub instance to use, default "github.com" - ContainerCapAdd []string // list of kernel capabilities to add to the containers - ContainerCapDrop []string // list of kernel capabilities to remove from the containers - AutoRemove bool // controls if the container is automatically removed upon workflow completion - ArtifactServerPath string // the path where the artifact server stores uploads - ArtifactServerAddr string // the address the artifact server binds to - ArtifactServerPort string // the port the artifact server binds to - NoSkipCheckout bool // do not skip actions/checkout - RemoteName string // remote name in local git repo config - ReplaceGheActionWithGithubCom []string // Use actions from GitHub Enterprise instance to GitHub - ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. - Matrix map[string]map[string]bool // Matrix config to run + Actor string // the user that triggered the event + Workdir string // path to working directory + ActionCacheDir string // path used for caching action contents + ActionOfflineMode bool // when offline, use caching action contents + BindWorkdir bool // bind the workdir to the job container + EventName string // name of event to run + EventPath string // path to JSON file to use for event.json in containers + DefaultBranch string // name of the main branch for this repository + ReuseContainers bool // reuse containers to maintain state + ForcePull bool // force pulling of the image, even if already present + ForceRebuild bool // force rebuilding local docker image action + LogOutput bool // log the output from docker run + JSONLogger bool // use json or text logger + LogPrefixJobID bool // switches from the full job name to the job id + Env map[string]string // env for containers + Inputs map[string]string // manually passed action inputs + Secrets map[string]string // list of secrets + Vars map[string]string // list of vars + Token string // GitHub token + InsecureSecrets bool // switch hiding output when printing to terminal + Platforms map[string]string // list of platforms + Privileged bool // use privileged mode + UsernsMode string // user namespace to use + ContainerArchitecture string // Desired OS/architecture platform for running containers + ContainerDaemonSocket string // Path to Docker daemon socket + ContainerOptions string // Options for the job container + UseGitIgnore bool // controls if paths in .gitignore should not be copied into container, default true + GitHubInstance string // GitHub instance to use, default "github.com" + ContainerCapAdd []string // list of kernel capabilities to add to the containers + ContainerCapDrop []string // list of kernel capabilities to remove from the containers + AutoRemove bool // controls if the container is automatically removed upon workflow completion + ArtifactServerPath string // the path where the artifact server stores uploads + ArtifactServerAddr string // the address the artifact server binds to + ArtifactServerPort string // the port the artifact server binds to + NoSkipCheckout bool // do not skip actions/checkout + RemoteName string // remote name in local git repo config + ReplaceGheActionWithGithubCom []string // Use actions from GitHub Enterprise instance to GitHub + ReplaceGheActionTokenWithGithubCom string // Token of private action repo on GitHub. + Matrix map[string]map[string]bool // Matrix config to run + ContainerNetworkMode docker_container.NetworkMode // the network mode of job containers (the value of --network) + ActionCache ActionCache // Use a custom ActionCache Implementation PresetGitHubContext *model.GithubContext // the preset github context, overrides some fields like DefaultBranch, Env, Secrets etc. EventJSON string // the content of JSON file to use for event.json in containers, overrides EventPath ContainerNamePrefix string // the prefix of container name ContainerMaxLifetime time.Duration // the max lifetime of job containers - ContainerNetworkMode docker_container.NetworkMode // the network mode of job containers (the value of --network) DefaultActionInstance string // the default actions web site PlatformPicker func(labels []string) string // platform picker, it will take precedence over Platforms if isn't nil JobLoggerLevel *log.Level // the level of job logger diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index d1cdab29..70e9634d 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -302,6 +302,11 @@ func TestRunEvent(t *testing.T) { {workdir, "set-env-step-env-override", "push", "", platforms, secrets}, {workdir, "set-env-new-env-file-per-step", "push", "", platforms, secrets}, {workdir, "no-panic-on-invalid-composite-action", "push", "jobs failed due to invalid action", platforms, secrets}, + + // services + {workdir, "services", "push", "", platforms, secrets}, + {workdir, "services-host-network", "push", "", platforms, secrets}, + {workdir, "services-with-container", "push", "", platforms, secrets}, } for _, table := range tables { diff --git a/act/runner/step.go b/act/runner/step.go index ffb2efbc..c67b5b04 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -34,6 +34,9 @@ const ( stepStagePost ) +// Controls how many symlinks are resolved for local and remote Actions +const maxSymlinkDepth = 10 + func (s stepStage) String() string { switch s { case stepStagePre: @@ -307,3 +310,13 @@ func mergeIntoMapCaseInsensitive(target map[string]string, maps ...map[string]st } } } + +func symlinkJoin(filename, sym, parent string) (string, error) { + dir := path.Dir(filename) + dest := path.Join(dir, sym) + prefix := path.Clean(parent) + "/" + if strings.HasPrefix(dest, prefix) || prefix == "./" { + return dest, nil + } + return "", fmt.Errorf("symlink tries to access file '%s' outside of '%s'", strings.ReplaceAll(dest, "'", "''"), strings.ReplaceAll(parent, "'", "''")) +} diff --git a/act/runner/step_action_local.go b/act/runner/step_action_local.go index a745e686..f8daf5cb 100644 --- a/act/runner/step_action_local.go +++ b/act/runner/step_action_local.go @@ -3,7 +3,10 @@ package runner import ( "archive/tar" "context" + "errors" + "fmt" "io" + "io/fs" "os" "path" "path/filepath" @@ -42,15 +45,31 @@ func (sal *stepActionLocal) main() common.Executor { localReader := func(ctx context.Context) actionYamlReader { _, cpath := getContainerActionPaths(sal.Step, path.Join(actionDir, ""), sal.RunContext) return func(filename string) (io.Reader, io.Closer, error) { - tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, path.Join(cpath, filename)) - if err != nil { - return nil, nil, os.ErrNotExist + spath := path.Join(cpath, filename) + for i := 0; i < maxSymlinkDepth; i++ { + tars, err := sal.RunContext.JobContainer.GetContainerArchive(ctx, spath) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil, err + } else if err != nil { + return nil, nil, fs.ErrNotExist + } + treader := tar.NewReader(tars) + header, err := treader.Next() + if errors.Is(err, io.EOF) { + return nil, nil, os.ErrNotExist + } else if err != nil { + return nil, nil, err + } + if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink { + spath, err = symlinkJoin(spath, header.Linkname, cpath) + if err != nil { + return nil, nil, err + } + } else { + return treader, tars, nil + } } - treader := tar.NewReader(tars) - if _, err := treader.Next(); err != nil { - return nil, nil, os.ErrNotExist - } - return treader, tars, nil + return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath) } } diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index 4aca8f53..3243188c 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -1,6 +1,7 @@ package runner import ( + "archive/tar" "context" "errors" "fmt" @@ -28,6 +29,8 @@ type stepActionRemote struct { action *model.Action env map[string]string remoteAction *remoteAction + cacheDir string + resolvedSha string } var stepActionRemoteNewCloneExecutor = git.NewGitCloneExecutor @@ -62,6 +65,48 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { github.Token = sar.RunContext.Config.ReplaceGheActionTokenWithGithubCom } } + if sar.RunContext.Config.ActionCache != nil { + cache := sar.RunContext.Config.ActionCache + + var err error + sar.cacheDir = fmt.Sprintf("%s/%s", sar.remoteAction.Org, sar.remoteAction.Repo) + repoURL := sar.remoteAction.URL + "/" + sar.cacheDir + repoRef := sar.remoteAction.Ref + sar.resolvedSha, err = cache.Fetch(ctx, sar.cacheDir, repoURL, repoRef, github.Token) + if err != nil { + return fmt.Errorf("failed to fetch \"%s\" version \"%s\": %w", repoURL, repoRef, err) + } + + remoteReader := func(ctx context.Context) actionYamlReader { + return func(filename string) (io.Reader, io.Closer, error) { + spath := path.Join(sar.remoteAction.Path, filename) + for i := 0; i < maxSymlinkDepth; i++ { + tars, err := cache.GetTarArchive(ctx, sar.cacheDir, sar.resolvedSha, spath) + if err != nil { + return nil, nil, os.ErrNotExist + } + treader := tar.NewReader(tars) + header, err := treader.Next() + if err != nil { + return nil, nil, os.ErrNotExist + } + if header.FileInfo().Mode()&os.ModeSymlink == os.ModeSymlink { + spath, err = symlinkJoin(spath, header.Linkname, ".") + if err != nil { + return nil, nil, err + } + } else { + return treader, tars, nil + } + } + return nil, nil, fmt.Errorf("max depth %d of symlinks exceeded while reading %s", maxSymlinkDepth, spath) + } + } + + actionModel, err := sar.readAction(ctx, sar.Step, sar.resolvedSha, sar.remoteAction.Path, remoteReader(ctx), os.WriteFile) + sar.action = actionModel + return err + } actionDir := fmt.Sprintf("%s/%s", sar.RunContext.ActionCacheDir(), safeFilename(sar.Step.Uses)) gitClone := stepActionRemoteNewCloneExecutor(git.NewGitCloneExecutorInput{ @@ -75,6 +120,7 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor { For GitHub, they are the same, always github.com. But for Gitea, tasks triggered by a.com can clone actions from b.com. */ + OfflineMode: sar.RunContext.Config.ActionOfflineMode, }) var ntErr common.Executor if err := gitClone(ctx); err != nil { diff --git a/act/runner/step_test.go b/act/runner/step_test.go index d08a1297..cdb870e2 100644 --- a/act/runner/step_test.go +++ b/act/runner/step_test.go @@ -182,7 +182,6 @@ func TestSetupEnv(t *testing.T) { "GITHUB_RUN_ID": "runId", "GITHUB_RUN_NUMBER": "1", "GITHUB_SERVER_URL": "https://", - "GITHUB_TOKEN": "", "GITHUB_WORKFLOW": "", "INPUT_STEP_WITH": "with-value", "RC_KEY": "rcvalue", diff --git a/act/runner/testdata/services-host-network/push.yml b/act/runner/testdata/services-host-network/push.yml new file mode 100644 index 00000000..8d0eb294 --- /dev/null +++ b/act/runner/testdata/services-host-network/push.yml @@ -0,0 +1,14 @@ +name: services-host-network +on: push +jobs: + services-host-network: + runs-on: ubuntu-latest + services: + nginx: + image: "nginx:latest" + ports: + - "8080:80" + steps: + - run: apt-get -qq update && apt-get -yqq install --no-install-recommends curl net-tools + - run: netstat -tlpen + - run: curl -v http://localhost:8080 diff --git a/act/runner/testdata/services-with-container/push.yml b/act/runner/testdata/services-with-container/push.yml new file mode 100644 index 00000000..b37e5dcd --- /dev/null +++ b/act/runner/testdata/services-with-container/push.yml @@ -0,0 +1,16 @@ +name: services-with-containers +on: push +jobs: + services-with-containers: + runs-on: ubuntu-latest + # https://docs.github.com/en/actions/using-containerized-services/about-service-containers#running-jobs-in-a-container + container: + image: "ubuntu:latest" + services: + nginx: + image: "nginx:latest" + ports: + - "8080:80" + steps: + - run: apt-get -qq update && apt-get -yqq install --no-install-recommends curl + - run: curl -v http://nginx:80 diff --git a/cmd/input.go b/cmd/input.go index 1ae27930..36af6d86 100644 --- a/cmd/input.go +++ b/cmd/input.go @@ -55,7 +55,10 @@ type Input struct { replaceGheActionTokenWithGithubCom string matrix []string actionCachePath string + actionOfflineMode bool logPrefixJobID bool + networkName string + useNewActionCache bool } func (i *Input) resolve(path string) string { diff --git a/cmd/platforms.go b/cmd/platforms.go index 9d7e97a8..45724d75 100644 --- a/cmd/platforms.go +++ b/cmd/platforms.go @@ -15,7 +15,7 @@ func (i *Input) newPlatforms() map[string]string { for _, p := range i.platforms { pParts := strings.Split(p, "=") if len(pParts) == 2 { - platforms[pParts[0]] = pParts[1] + platforms[strings.ToLower(pParts[0])] = pParts[1] } } return platforms diff --git a/cmd/root.go b/cmd/root.go index c076bf17..349e6ac4 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -14,6 +14,7 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/adrg/xdg" "github.com/andreaskoch/go-fswatch" + docker_container "github.com/docker/docker/api/types/container" "github.com/joho/godotenv" gitignore "github.com/sabhiram/go-gitignore" log "github.com/sirupsen/logrus" @@ -96,6 +97,9 @@ func Execute(ctx context.Context, version string) { rootCmd.PersistentFlags().StringVarP(&input.cacheServerAddr, "cache-server-addr", "", common.GetOutboundIP().String(), "Defines the address to which the cache server binds.") rootCmd.PersistentFlags().Uint16VarP(&input.cacheServerPort, "cache-server-port", "", 0, "Defines the port where the artifact server listens. 0 means a randomly available port.") rootCmd.PersistentFlags().StringVarP(&input.actionCachePath, "action-cache-path", "", filepath.Join(CacheHomeDir, "act"), "Defines the path where the actions get cached and host workspaces created.") + rootCmd.PersistentFlags().BoolVarP(&input.actionOfflineMode, "action-offline-mode", "", false, "If action contents exists, it will not be fetch and pull again. If turn on this,will turn off force pull") + rootCmd.PersistentFlags().StringVarP(&input.networkName, "network", "", "host", "Sets a docker network name. Defaults to host.") + rootCmd.PersistentFlags().BoolVarP(&input.useNewActionCache, "use-new-action-cache", "", false, "Enable using the new Action Cache for storing Actions locally") rootCmd.SetArgs(args()) if err := rootCmd.Execute(); err != nil { @@ -103,23 +107,22 @@ func Execute(ctx context.Context, version string) { } } +// Return locations where Act's config can be found in order: XDG spec, .actrc in HOME directory, .actrc in invocation directory func configLocations() []string { configFileName := ".actrc" - // reference: https://specifications.freedesktop.org/basedir-spec/latest/ar01s03.html - var actrcXdg string - for _, fileName := range []string{"act/actrc", configFileName} { - if foundConfig, err := xdg.SearchConfigFile(fileName); foundConfig != "" && err == nil { - actrcXdg = foundConfig - break - } + homePath := filepath.Join(UserHomeDir, configFileName) + invocationPath := filepath.Join(".", configFileName) + + // Though named xdg, adrg's lib support macOS and Windows config paths as well + // It also takes cares of creating the parent folder so we don't need to bother later + specPath, err := xdg.ConfigFile("act/actrc") + if err != nil { + specPath = homePath } - return []string{ - filepath.Join(UserHomeDir, configFileName), - actrcXdg, - filepath.Join(".", configFileName), - } + // This order should be enforced since the survey part relies on it + return []string{specPath, homePath, invocationPath} } var commonSocketPaths = []string{ @@ -265,7 +268,8 @@ func readArgsFile(file string, split bool) []string { }() scanner := bufio.NewScanner(f) for scanner.Scan() { - arg := strings.TrimSpace(scanner.Text()) + arg := os.ExpandEnv(strings.TrimSpace(scanner.Text())) + if strings.HasPrefix(arg, "-") && split { args = append(args, regexp.MustCompile(`\s`).Split(arg, 2)...) } else if !split { @@ -291,19 +295,17 @@ func cleanup(inputs *Input) func(*cobra.Command, []string) { } } -func parseEnvs(env []string, envs map[string]string) bool { - if env != nil { - for _, envVar := range env { - e := strings.SplitN(envVar, `=`, 2) - if len(e) == 2 { - envs[e[0]] = e[1] - } else { - envs[e[0]] = "" - } +func parseEnvs(env []string) map[string]string { + envs := make(map[string]string, len(env)) + for _, envVar := range env { + e := strings.SplitN(envVar, `=`, 2) + if len(e) == 2 { + envs[e[0]] = e[1] + } else { + envs[e[0]] = "" } - return true } - return false + return envs } func readYamlFile(file string) (map[string]string, error) { @@ -409,13 +411,11 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str } log.Debugf("Loading environment from %s", input.Envfile()) - envs := make(map[string]string) - _ = parseEnvs(input.envs, envs) + envs := parseEnvs(input.envs) _ = readEnvs(input.Envfile(), envs) log.Debugf("Loading action inputs from %s", input.Inputfile()) - inputs := make(map[string]string) - _ = parseEnvs(input.inputs, inputs) + inputs := parseEnvs(input.inputs) _ = readEnvs(input.Inputfile(), inputs) log.Debugf("Loading secrets from %s", input.Secretfile()) @@ -552,6 +552,7 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str } } if !cfgFound && len(cfgLocations) > 0 { + // The first config location refers to the global config folder one if err := defaultImageSurvey(cfgLocations[0]); err != nil { log.Fatal(err) } @@ -578,11 +579,12 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str EventName: eventName, EventPath: input.EventPath(), DefaultBranch: defaultbranch, - ForcePull: input.forcePull, + ForcePull: !input.actionOfflineMode && input.forcePull, ForceRebuild: input.forceRebuild, ReuseContainers: input.reuseContainers, Workdir: input.Workdir(), ActionCacheDir: input.actionCachePath, + ActionOfflineMode: input.actionOfflineMode, BindWorkdir: input.bindWorkdir, LogOutput: !input.noOutput, JSONLogger: input.jsonLogger, @@ -612,6 +614,12 @@ func newRunCommand(ctx context.Context, input *Input) func(*cobra.Command, []str ReplaceGheActionWithGithubCom: input.replaceGheActionWithGithubCom, ReplaceGheActionTokenWithGithubCom: input.replaceGheActionTokenWithGithubCom, Matrix: matrixes, + ContainerNetworkMode: docker_container.NetworkMode(input.networkName), + } + if input.useNewActionCache { + config.ActionCache = &runner.GoGitActionCache{ + Path: config.ActionCacheDir, + } } r, err := runner.New(config) if err != nil {