diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 80d04e93..ffd9b7ec 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,10 +19,10 @@ jobs: with: go-version: ${{ env.GO_VERSION }} check-latest: true - - uses: golangci/golangci-lint-action@v3.3.0 + - uses: golangci/golangci-lint-action@v3.3.1 with: version: v1.47.2 - - uses: megalinter/megalinter/flavors/go@v6.13.0 + - uses: megalinter/megalinter/flavors/go@v6.15.0 env: DEFAULT_BRANCH: master GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -50,13 +50,32 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- - - run: go test -v -cover -coverprofile=coverage.txt -covermode=atomic ./... + - run: go test -v -cover -coverprofile=coverage.txt -covermode=atomic -timeout 15m ./... - name: Upload Codecov report uses: codecov/codecov-action@v3.1.1 with: files: coverage.txt fail_ci_if_error: true # optional (default = false) + test-host: + strategy: + matrix: + os: + - windows-latest + - macos-latest + name: test-${{matrix.os}} + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + - uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + - run: go test -v -run ^TestRunEventHostEnvironment$ ./... + # TODO merge coverage with test-linux + snapshot: name: snapshot runs-on: ubuntu-latest diff --git a/act/container/docker_logger.go b/act/container/docker_logger.go index 5c85785c..b6b2f150 100644 --- a/act/container/docker_logger.go +++ b/act/container/docker_logger.go @@ -22,59 +22,6 @@ type dockerMessage struct { const logPrefix = " \U0001F433 " -/* -func logDockerOutput(ctx context.Context, dockerResponse io.Reader) { - logger := common.Logger(ctx) - if entry, ok := logger.(*logrus.Entry); ok { - w := entry.Writer() - _, err := stdcopy.StdCopy(w, w, dockerResponse) - if err != nil { - logrus.Error(err) - } - } else if lgr, ok := logger.(*logrus.Logger); ok { - w := lgr.Writer() - _, err := stdcopy.StdCopy(w, w, dockerResponse) - if err != nil { - logrus.Error(err) - } - } else { - logrus.Errorf("Unable to get writer from logger (type=%T)", logger) - } -} -*/ - -/* -func streamDockerOutput(ctx context.Context, dockerResponse io.Reader) { - /* - out := os.Stdout - go func() { - <-ctx.Done() - //fmt.Println() - }() - - _, err := io.Copy(out, dockerResponse) - if err != nil { - logrus.Error(err) - } - * / - - logger := common.Logger(ctx) - reader := bufio.NewReader(dockerResponse) - - for { - if ctx.Err() != nil { - break - } - line, _, err := reader.ReadLine() - if err == io.EOF { - break - } - logger.Debugf("%s\n", line) - } - -} -*/ - func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser, isError bool) error { if dockerResponse == nil { return nil diff --git a/act/container/docker_run.go b/act/container/docker_run.go index add0e2e3..351cf32b 100644 --- a/act/container/docker_run.go +++ b/act/container/docker_run.go @@ -86,7 +86,7 @@ type Container interface { } // NewContainer creates a reference to a container -func NewContainer(input *NewContainerInput) Container { +func NewContainer(input *NewContainerInput) ExecutionsEnvironment { cr := new(containerReference) cr.input = input return cr @@ -235,6 +235,7 @@ type containerReference struct { input *NewContainerInput UID int GID int + LinuxContainerEnvironmentExtensions } func GetDockerClient(ctx context.Context) (cli client.APIClient, err error) { diff --git a/act/container/docker_run_test.go b/act/container/docker_run_test.go index 2e12fcfc..bc3ab4bd 100644 --- a/act/container/docker_run_test.go +++ b/act/container/docker_run_test.go @@ -163,3 +163,6 @@ func TestDockerExecFailure(t *testing.T) { conn.AssertExpectations(t) client.AssertExpectations(t) } + +// Type assert containerReference implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &containerReference{} diff --git a/act/container/executions_environment.go b/act/container/executions_environment.go new file mode 100644 index 00000000..1c21f943 --- /dev/null +++ b/act/container/executions_environment.go @@ -0,0 +1,13 @@ +package container + +import "context" + +type ExecutionsEnvironment interface { + Container + ToContainerPath(string) string + GetActPath() string + GetPathVariableName() string + DefaultPathVariable() string + JoinPathVariable(...string) string + GetRunnerContext(ctx context.Context) map[string]interface{} +} diff --git a/act/container/file_collector.go b/act/container/file_collector.go index 164bfe7d..a4143edc 100644 --- a/act/container/file_collector.go +++ b/act/container/file_collector.go @@ -59,6 +59,29 @@ func (tc tarCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, return nil } +type copyCollector struct { + DstDir string +} + +func (cc *copyCollector) WriteFile(fpath string, fi fs.FileInfo, linkName string, f io.Reader) error { + fdestpath := filepath.Join(cc.DstDir, fpath) + if err := os.MkdirAll(filepath.Dir(fdestpath), 0777); err != nil { + return err + } + if f == nil { + return os.Symlink(linkName, fdestpath) + } + df, err := os.OpenFile(fdestpath, os.O_CREATE|os.O_WRONLY, fi.Mode()) + if err != nil { + return err + } + defer df.Close() + if _, err := io.Copy(df, f); err != nil { + return err + } + return nil +} + type fileCollector struct { Ignorer gitignore.Matcher SrcPath string diff --git a/act/container/host_environment.go b/act/container/host_environment.go new file mode 100644 index 00000000..b404e86d --- /dev/null +++ b/act/container/host_environment.go @@ -0,0 +1,470 @@ +package container + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "errors" + + "github.com/go-git/go-billy/v5/helper/polyfill" + "github.com/go-git/go-billy/v5/osfs" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/lookpath" + "golang.org/x/term" +) + +type HostEnvironment struct { + Path string + TmpDir string + ToolCache string + Workdir string + ActPath string + CleanUp func() + StdOut io.Writer +} + +func (e *HostEnvironment) Create(capAdd []string, capDrop []string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Close() common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Copy(destPath string, files ...*FileEntry) common.Executor { + return func(ctx context.Context) error { + for _, f := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(destPath, f.Name)), 0777); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(destPath, f.Name), []byte(f.Body), fs.FileMode(f.Mode)); err != nil { + return err + } + } + return nil + } +} + +func (e *HostEnvironment) CopyDir(destPath string, srcPath string, useGitIgnore bool) common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + logger.Debugf("Stripping prefix:%s src:%s", srcPrefix, srcPath) + var ignorer gitignore.Matcher + if useGitIgnore { + ps, err := gitignore.ReadPatterns(polyfill.New(osfs.New(srcPath)), nil) + if err != nil { + logger.Debugf("Error loading .gitignore: %v", err) + } + + ignorer = gitignore.NewMatcher(ps) + } + fc := &fileCollector{ + Fs: &defaultFs{}, + Ignorer: ignorer, + SrcPath: srcPath, + SrcPrefix: srcPrefix, + Handler: ©Collector{ + DstDir: destPath, + }, + } + return filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + } +} + +func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + defer tw.Close() + srcPath = filepath.Clean(srcPath) + fi, err := os.Lstat(srcPath) + if err != nil { + return nil, err + } + tc := &tarCollector{ + TarWriter: tw, + } + if fi.IsDir() { + srcPrefix := filepath.Dir(srcPath) + if !strings.HasSuffix(srcPrefix, string(filepath.Separator)) { + srcPrefix += string(filepath.Separator) + } + fc := &fileCollector{ + Fs: &defaultFs{}, + SrcPath: srcPath, + SrcPrefix: srcPrefix, + Handler: tc, + } + err = filepath.Walk(srcPath, fc.collectFiles(ctx, []string{})) + if err != nil { + return nil, err + } + } else { + var f io.ReadCloser + var linkname string + if fi.Mode()&fs.ModeSymlink != 0 { + linkname, err = os.Readlink(srcPath) + if err != nil { + return nil, err + } + } else { + f, err = os.Open(srcPath) + if err != nil { + return nil, err + } + defer f.Close() + } + err := tc.WriteFile(fi.Name(), fi, linkname, f) + if err != nil { + return nil, err + } + } + return io.NopCloser(buf), nil +} + +func (e *HostEnvironment) Pull(forcePull bool) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func (e *HostEnvironment) Start(attach bool) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +type ptyWriter struct { + Out io.Writer + AutoStop bool + dirtyLine bool +} + +func (w *ptyWriter) Write(buf []byte) (int, error) { + if w.AutoStop && len(buf) > 0 && buf[len(buf)-1] == 4 { + n, err := w.Out.Write(buf[:len(buf)-1]) + if err != nil { + return n, err + } + if w.dirtyLine || len(buf) > 1 && buf[len(buf)-2] != '\n' { + _, _ = w.Out.Write([]byte("\n")) + return n, io.EOF + } + return n, io.EOF + } + w.dirtyLine = strings.LastIndex(string(buf), "\n") < len(buf)-1 + return w.Out.Write(buf) +} + +type localEnv struct { + env map[string]string +} + +func (l *localEnv) Getenv(name string) string { + if runtime.GOOS == "windows" { + for k, v := range l.env { + if strings.EqualFold(name, k) { + return v + } + } + return "" + } + return l.env[name] +} + +func lookupPathHost(cmd string, env map[string]string, writer io.Writer) (string, error) { + f, err := lookpath.LookPath2(cmd, &localEnv{env: env}) + if err != nil { + err := "Cannot find: " + fmt.Sprint(cmd) + " in PATH" + if _, _err := writer.Write([]byte(err + "\n")); _err != nil { + return "", fmt.Errorf("%v: %w", err, _err) + } + return "", errors.New(err) + } + return f, nil +} + +func setupPty(cmd *exec.Cmd, cmdline string) (*os.File, *os.File, error) { + ppty, tty, err := openPty() + if err != nil { + return nil, nil, err + } + if term.IsTerminal(int(tty.Fd())) { + _, err := term.MakeRaw(int(tty.Fd())) + if err != nil { + ppty.Close() + tty.Close() + return nil, nil, err + } + } + cmd.Stdin = tty + cmd.Stdout = tty + cmd.Stderr = tty + cmd.SysProcAttr = getSysProcAttr(cmdline, true) + return ppty, tty, nil +} + +func writeKeepAlive(ppty io.Writer) { + c := 1 + var err error + for c == 1 && err == nil { + c, err = ppty.Write([]byte{4}) + <-time.After(time.Second) + } +} + +func copyPtyOutput(writer io.Writer, ppty io.Reader, finishLog context.CancelFunc) { + defer func() { + finishLog() + }() + if _, err := io.Copy(writer, ppty); err != nil { + return + } +} + +func (e *HostEnvironment) UpdateFromImageEnv(env *map[string]string) common.Executor { + return func(ctx context.Context) error { + return nil + } +} + +func getEnvListFromMap(env map[string]string) []string { + envList := make([]string, 0) + for k, v := range env { + envList = append(envList, fmt.Sprintf("%s=%s", k, v)) + } + return envList +} + +func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline string, env map[string]string, user, workdir string) error { + envList := getEnvListFromMap(env) + var wd string + if workdir != "" { + if filepath.IsAbs(workdir) { + wd = workdir + } else { + wd = filepath.Join(e.Path, workdir) + } + } else { + wd = e.Path + } + f, err := lookupPathHost(command[0], env, e.StdOut) + if err != nil { + return err + } + cmd := exec.CommandContext(ctx, f) + cmd.Path = f + cmd.Args = command + cmd.Stdin = nil + cmd.Stdout = e.StdOut + cmd.Env = envList + cmd.Stderr = e.StdOut + cmd.Dir = wd + cmd.SysProcAttr = getSysProcAttr(cmdline, false) + var ppty *os.File + var tty *os.File + defer func() { + if ppty != nil { + ppty.Close() + } + if tty != nil { + tty.Close() + } + }() + if true /* allocate Terminal */ { + var err error + ppty, tty, err = setupPty(cmd, cmdline) + if err != nil { + common.Logger(ctx).Debugf("Failed to setup Pty %v\n", err.Error()) + } + } + writer := &ptyWriter{Out: e.StdOut} + logctx, finishLog := context.WithCancel(context.Background()) + if ppty != nil { + go copyPtyOutput(writer, ppty, finishLog) + } else { + finishLog() + } + if ppty != nil { + go writeKeepAlive(ppty) + } + err = cmd.Run() + if err != nil { + return err + } + if tty != nil { + writer.AutoStop = true + if _, err := tty.Write([]byte("\x04")); err != nil { + common.Logger(ctx).Debug("Failed to write EOT") + } + } + <-logctx.Done() + + if ppty != nil { + ppty.Close() + ppty = nil + } + return err +} + +func (e *HostEnvironment) Exec(command []string /*cmdline string, */, env map[string]string, user, workdir string) common.Executor { + return func(ctx context.Context) error { + if err := e.exec(ctx, command, "" /*cmdline*/, env, user, workdir); err != nil { + select { + case <-ctx.Done(): + return fmt.Errorf("this step has been cancelled: %w", err) + default: + return err + } + } + return nil + } +} + +func (e *HostEnvironment) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor { + localEnv := *env + return func(ctx context.Context) error { + envTar, err := e.GetContainerArchive(ctx, srcPath) + if err != nil { + return nil + } + defer envTar.Close() + reader := tar.NewReader(envTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + for s.Scan() { + line := s.Text() + singleLineEnv := strings.Index(line, "=") + multiLineEnv := strings.Index(line, "<<") + if singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv) { + localEnv[line[:singleLineEnv]] = line[singleLineEnv+1:] + } else if multiLineEnv != -1 { + multiLineEnvContent := "" + multiLineEnvDelimiter := line[multiLineEnv+2:] + delimiterFound := false + for s.Scan() { + content := s.Text() + if content == multiLineEnvDelimiter { + delimiterFound = true + break + } + if multiLineEnvContent != "" { + multiLineEnvContent += "\n" + } + multiLineEnvContent += content + } + if !delimiterFound { + return fmt.Errorf("invalid format delimiter '%v' not found before end of file", multiLineEnvDelimiter) + } + localEnv[line[:multiLineEnv]] = multiLineEnvContent + } else { + return fmt.Errorf("invalid format '%v', expected a line with '=' or '<<'", line) + } + } + env = &localEnv + return nil + } +} + +func (e *HostEnvironment) UpdateFromPath(env *map[string]string) common.Executor { + localEnv := *env + return func(ctx context.Context) error { + pathTar, err := e.GetContainerArchive(ctx, localEnv["GITHUB_PATH"]) + if err != nil { + return err + } + defer pathTar.Close() + + reader := tar.NewReader(pathTar) + _, err = reader.Next() + if err != nil && err != io.EOF { + return err + } + s := bufio.NewScanner(reader) + for s.Scan() { + line := s.Text() + pathSep := string(filepath.ListSeparator) + localEnv[e.GetPathVariableName()] = fmt.Sprintf("%s%s%s", line, pathSep, localEnv[e.GetPathVariableName()]) + } + + env = &localEnv + return nil + } +} + +func (e *HostEnvironment) Remove() common.Executor { + return func(ctx context.Context) error { + if e.CleanUp != nil { + e.CleanUp() + } + return os.RemoveAll(e.Path) + } +} + +func (e *HostEnvironment) ToContainerPath(path string) string { + if bp, err := filepath.Rel(e.Workdir, path); err != nil { + return filepath.Join(e.Path, bp) + } else if filepath.Clean(e.Workdir) == filepath.Clean(path) { + return e.Path + } + return path +} + +func (e *HostEnvironment) GetActPath() string { + return e.ActPath +} + +func (*HostEnvironment) GetPathVariableName() string { + if runtime.GOOS == "plan9" { + return "path" + } else if runtime.GOOS == "windows" { + return "Path" // Actually we need a case insensitive map + } + return "PATH" +} + +func (e *HostEnvironment) DefaultPathVariable() string { + v, _ := os.LookupEnv(e.GetPathVariableName()) + return v +} + +func (*HostEnvironment) JoinPathVariable(paths ...string) string { + return strings.Join(paths, string(filepath.ListSeparator)) +} + +func (e *HostEnvironment) GetRunnerContext(ctx context.Context) map[string]interface{} { + return map[string]interface{}{ + "os": runtime.GOOS, + "arch": runtime.GOARCH, + "temp": e.TmpDir, + "tool_cache": e.ToolCache, + } +} + +func (e *HostEnvironment) ReplaceLogWriter(stdout io.Writer, stderr io.Writer) (io.Writer, io.Writer) { + org := e.StdOut + e.StdOut = stdout + return org, org +} diff --git a/act/container/host_environment_test.go b/act/container/host_environment_test.go new file mode 100644 index 00000000..67787d95 --- /dev/null +++ b/act/container/host_environment_test.go @@ -0,0 +1,4 @@ +package container + +// Type assert HostEnvironment implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &HostEnvironment{} diff --git a/act/container/linux_container_environment_extensions.go b/act/container/linux_container_environment_extensions.go new file mode 100644 index 00000000..c369055d --- /dev/null +++ b/act/container/linux_container_environment_extensions.go @@ -0,0 +1,73 @@ +package container + +import ( + "context" + "path/filepath" + "regexp" + "runtime" + "strings" + + log "github.com/sirupsen/logrus" +) + +type LinuxContainerEnvironmentExtensions struct { +} + +// Resolves the equivalent host path inside the container +// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject +// For use in docker volumes and binds +func (*LinuxContainerEnvironmentExtensions) ToContainerPath(path string) string { + if runtime.GOOS == "windows" && strings.Contains(path, "/") { + log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.") + return "" + } + + abspath, err := filepath.Abs(path) + if err != nil { + log.Error(err) + return "" + } + + // Test if the path is a windows path + windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`) + windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath) + + // Return as-is if no match + if windowsPathComponents == nil { + return abspath + } + + // Convert to WSL2-compatible path if it is a windows path + // NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows + // based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work. + driveLetter := strings.ToLower(windowsPathComponents[1]) + translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`) + // Should make something like /mnt/c/Users/person/My Folder/MyActProject + result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`) + return result +} + +func (*LinuxContainerEnvironmentExtensions) GetActPath() string { + return "/var/run/act" +} + +func (*LinuxContainerEnvironmentExtensions) GetPathVariableName() string { + return "PATH" +} + +func (*LinuxContainerEnvironmentExtensions) DefaultPathVariable() string { + return "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +} + +func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) string { + return strings.Join(paths, ":") +} + +func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]interface{} { + return map[string]interface{}{ + "os": "Linux", + "arch": RunnerArch(ctx), + "temp": "/tmp", + "tool_cache": "/opt/hostedtoolcache", + } +} diff --git a/act/container/linux_container_environment_extensions_test.go b/act/container/linux_container_environment_extensions_test.go new file mode 100644 index 00000000..38111714 --- /dev/null +++ b/act/container/linux_container_environment_extensions_test.go @@ -0,0 +1,71 @@ +package container + +import ( + "fmt" + "os" + "runtime" + "strings" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func TestContainerPath(t *testing.T) { + type containerPathJob struct { + destinationPath string + sourcePath string + workDir string + } + + linuxcontainerext := &LinuxContainerEnvironmentExtensions{} + + if runtime.GOOS == "windows" { + cwd, err := os.Getwd() + if err != nil { + log.Error(err) + } + + rootDrive := os.Getenv("SystemDrive") + rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") + for _, v := range []containerPathJob{ + {"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""}, + {"/mnt/f/work/dir", `F:\work\dir`, ""}, + {"/mnt/c/windows/to/unix", "windows\\to\\unix", fmt.Sprintf("%s\\", rootDrive)}, + {fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)}, + } { + if v.workDir != "" { + if err := os.Chdir(v.workDir); err != nil { + log.Error(err) + t.Fail() + } + } + + assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath)) + } + + if err := os.Chdir(cwd); err != nil { + log.Error(err) + } + } else { + cwd, err := os.Getwd() + if err != nil { + log.Error(err) + } + for _, v := range []containerPathJob{ + {"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""}, + {"/home/act", `/home/act/`, ""}, + {cwd, ".", ""}, + } { + assert.Equal(t, v.destinationPath, linuxcontainerext.ToContainerPath(v.sourcePath)) + } + } +} + +type typeAssertMockContainer struct { + Container + LinuxContainerEnvironmentExtensions +} + +// Type assert Container + LinuxContainerEnvironmentExtensions implements ExecutionsEnvironment +var _ ExecutionsEnvironment = &typeAssertMockContainer{} diff --git a/act/container/util.go b/act/container/util.go new file mode 100644 index 00000000..eb7f46c6 --- /dev/null +++ b/act/container/util.go @@ -0,0 +1,26 @@ +//go:build (!windows && !plan9 && !openbsd) || (!windows && !plan9 && !mips64) + +package container + +import ( + "os" + "syscall" + + "github.com/creack/pty" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + if tty { + return &syscall.SysProcAttr{ + Setsid: true, + Setctty: true, + } + } + return &syscall.SysProcAttr{ + Setpgid: true, + } +} + +func openPty() (*os.File, *os.File, error) { + return pty.Open() +} diff --git a/act/container/util_openbsd_mips64.go b/act/container/util_openbsd_mips64.go new file mode 100644 index 00000000..b991d694 --- /dev/null +++ b/act/container/util_openbsd_mips64.go @@ -0,0 +1,17 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + Setpgid: true, + } +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/act/container/util_plan9.go b/act/container/util_plan9.go new file mode 100644 index 00000000..cd64b4ab --- /dev/null +++ b/act/container/util_plan9.go @@ -0,0 +1,17 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{ + Rfork: syscall.RFNOTEG, + } +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/act/container/util_windows.go b/act/container/util_windows.go new file mode 100644 index 00000000..0a94c83a --- /dev/null +++ b/act/container/util_windows.go @@ -0,0 +1,15 @@ +package container + +import ( + "errors" + "os" + "syscall" +) + +func getSysProcAttr(cmdLine string, tty bool) *syscall.SysProcAttr { + return &syscall.SysProcAttr{CmdLine: cmdLine, CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP} +} + +func openPty() (*os.File, *os.File, error) { + return nil, nil, errors.New("Unsupported") +} diff --git a/act/exprparser/functions.go b/act/exprparser/functions.go index 37a38db5..047a0e3c 100644 --- a/act/exprparser/functions.go +++ b/act/exprparser/functions.go @@ -6,12 +6,14 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "os" "path/filepath" "reflect" "strconv" "strings" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/nektos/act/pkg/model" "github.com/rhysd/actionlint" ) @@ -178,25 +180,37 @@ func (impl *interperterImpl) fromJSON(value reflect.Value) (interface{}, error) } func (impl *interperterImpl) hashFiles(paths ...reflect.Value) (string, error) { - var filepaths []string + var ps []gitignore.Pattern + const cwdPrefix = "." + string(filepath.Separator) + const excludeCwdPrefix = "!" + cwdPrefix for _, path := range paths { if path.Kind() == reflect.String { - filepaths = append(filepaths, path.String()) + cleanPath := path.String() + if strings.HasPrefix(cleanPath, cwdPrefix) { + cleanPath = cleanPath[len(cwdPrefix):] + } else if strings.HasPrefix(cleanPath, excludeCwdPrefix) { + cleanPath = "!" + cleanPath[len(excludeCwdPrefix):] + } + ps = append(ps, gitignore.ParsePattern(cleanPath, nil)) } else { return "", fmt.Errorf("Non-string path passed to hashFiles") } } + matcher := gitignore.NewMatcher(ps) + var files []string - - for i := range filepaths { - newFiles, err := filepath.Glob(filepath.Join(impl.config.WorkingDir, filepaths[i])) - if err != nil { - return "", fmt.Errorf("Unable to glob.Glob: %v", err) + if err := filepath.Walk(impl.config.WorkingDir, func(path string, fi fs.FileInfo, err error) error { + sansPrefix := strings.TrimPrefix(path, impl.config.WorkingDir+string(filepath.Separator)) + parts := strings.Split(sansPrefix, string(filepath.Separator)) + if fi.IsDir() || !matcher.Match(parts, fi.IsDir()) { + return nil } - - files = append(files, newFiles...) + files = append(files, path) + return nil + }); err != nil { + return "", fmt.Errorf("Unable to filepath.Walk: %v", err) } if len(files) == 0 { diff --git a/act/exprparser/functions_test.go b/act/exprparser/functions_test.go index 009e9113..3c6392c1 100644 --- a/act/exprparser/functions_test.go +++ b/act/exprparser/functions_test.go @@ -188,7 +188,11 @@ func TestFunctionHashFiles(t *testing.T) { {"hashFiles('**/non-extant-files') }}", "", "hash-non-existing-file"}, {"hashFiles('**/non-extant-files', '**/more-non-extant-files') }}", "", "hash-multiple-non-existing-files"}, {"hashFiles('./for-hashing-1.txt') }}", "66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18", "hash-single-file"}, - {"hashFiles('./for-hashing-*') }}", "8e5935e7e13368cd9688fe8f48a0955293676a021562582c7e848dafe13fb046", "hash-multiple-files"}, + {"hashFiles('./for-hashing-*.txt') }}", "8e5935e7e13368cd9688fe8f48a0955293676a021562582c7e848dafe13fb046", "hash-multiple-files"}, + {"hashFiles('./for-hashing-*.txt', '!./for-hashing-2.txt') }}", "66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18", "hash-negative-pattern"}, + {"hashFiles('./for-hashing-**') }}", "c418ba693753c84115ced0da77f876cddc662b9054f4b129b90f822597ee2f94", "hash-multiple-files-and-directories"}, + {"hashFiles('./for-hashing-3/**') }}", "6f5696b546a7a9d6d42a449dc9a56bef244aaa826601ef27466168846139d2c2", "hash-nested-directories"}, + {"hashFiles('./for-hashing-3/**/nested-data.txt') }}", "8ecadfb49f7f978d0a9f3a957e9c8da6cc9ab871f5203b5d9f9d1dc87d8af18c", "hash-nested-directories-2"}, } env := &EvaluationEnvironment{} diff --git a/act/exprparser/testdata/for-hashing-3/data.txt b/act/exprparser/testdata/for-hashing-3/data.txt new file mode 100644 index 00000000..5ac7bf9b --- /dev/null +++ b/act/exprparser/testdata/for-hashing-3/data.txt @@ -0,0 +1 @@ +Knock knock! diff --git a/act/exprparser/testdata/for-hashing-3/nested/nested-data.txt b/act/exprparser/testdata/for-hashing-3/nested/nested-data.txt new file mode 100644 index 00000000..ebe288b2 --- /dev/null +++ b/act/exprparser/testdata/for-hashing-3/nested/nested-data.txt @@ -0,0 +1 @@ +Anybody home? diff --git a/act/lookpath/LICENSE b/act/lookpath/LICENSE new file mode 100644 index 00000000..83403eff --- /dev/null +++ b/act/lookpath/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/act/lookpath/env.go b/act/lookpath/env.go new file mode 100644 index 00000000..dc376e7b --- /dev/null +++ b/act/lookpath/env.go @@ -0,0 +1,18 @@ +package lookpath + +import "os" + +type Env interface { + Getenv(name string) string +} + +type defaultEnv struct { +} + +func (*defaultEnv) Getenv(name string) string { + return os.Getenv(name) +} + +func LookPath(file string) (string, error) { + return LookPath2(file, &defaultEnv{}) +} diff --git a/act/lookpath/error.go b/act/lookpath/error.go new file mode 100644 index 00000000..0e3a3735 --- /dev/null +++ b/act/lookpath/error.go @@ -0,0 +1,10 @@ +package lookpath + +type Error struct { + Name string + Err error +} + +func (e *Error) Error() string { + return e.Err.Error() +} diff --git a/act/lookpath/lp_js.go b/act/lookpath/lp_js.go new file mode 100644 index 00000000..a967b861 --- /dev/null +++ b/act/lookpath/lp_js.go @@ -0,0 +1,23 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build js && wasm + +package lookpath + +import ( + "errors" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $PATH") + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // Wasm can not execute processes, so act as if there are no executables at all. + return "", &Error{file, ErrNotFound} +} diff --git a/act/lookpath/lp_plan9.go b/act/lookpath/lp_plan9.go new file mode 100644 index 00000000..a201b715 --- /dev/null +++ b/act/lookpath/lp_plan9.go @@ -0,0 +1,56 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $path") + +func findExecutable(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if m := d.Mode(); !m.IsDir() && m&0111 != 0 { + return nil + } + return fs.ErrPermission +} + +// LookPath searches for an executable named file in the +// directories named by the path environment variable. +// If file begins with "/", "#", "./", or "../", it is tried +// directly and the path is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // skip the path lookup for these prefixes + skip := []string{"/", "#", "./", "../"} + + for _, p := range skip { + if strings.HasPrefix(file, p) { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + } + + path := lenv.Getenv("path") + for _, dir := range filepath.SplitList(path) { + path := filepath.Join(dir, file) + if err := findExecutable(path); err == nil { + return path, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/act/lookpath/lp_unix.go b/act/lookpath/lp_unix.go new file mode 100644 index 00000000..233e21f9 --- /dev/null +++ b/act/lookpath/lp_unix.go @@ -0,0 +1,59 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in $PATH") + +func findExecutable(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if m := d.Mode(); !m.IsDir() && m&0111 != 0 { + return nil + } + return fs.ErrPermission +} + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + // NOTE(rsc): I wish we could use the Plan 9 behavior here + // (only bypass the path if file begins with / or ./ or ../) + // but that would not match all the Unix shells. + + if strings.Contains(file, "/") { + err := findExecutable(file) + if err == nil { + return file, nil + } + return "", &Error{file, err} + } + path := lenv.Getenv("PATH") + for _, dir := range filepath.SplitList(path) { + if dir == "" { + // Unix shell semantics: path element "" means "." + dir = "." + } + path := filepath.Join(dir, file) + if err := findExecutable(path); err == nil { + return path, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/act/lookpath/lp_windows.go b/act/lookpath/lp_windows.go new file mode 100644 index 00000000..48204a79 --- /dev/null +++ b/act/lookpath/lp_windows.go @@ -0,0 +1,94 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lookpath + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// ErrNotFound is the error resulting if a path search failed to find an executable file. +var ErrNotFound = errors.New("executable file not found in %PATH%") + +func chkStat(file string) error { + d, err := os.Stat(file) + if err != nil { + return err + } + if d.IsDir() { + return fs.ErrPermission + } + return nil +} + +func hasExt(file string) bool { + i := strings.LastIndex(file, ".") + if i < 0 { + return false + } + return strings.LastIndexAny(file, `:\/`) < i +} + +func findExecutable(file string, exts []string) (string, error) { + if len(exts) == 0 { + return file, chkStat(file) + } + if hasExt(file) { + if chkStat(file) == nil { + return file, nil + } + } + for _, e := range exts { + if f := file + e; chkStat(f) == nil { + return f, nil + } + } + return "", fs.ErrNotExist +} + +// LookPath searches for an executable named file in the +// directories named by the PATH environment variable. +// If file contains a slash, it is tried directly and the PATH is not consulted. +// LookPath also uses PATHEXT environment variable to match +// a suitable candidate. +// The result may be an absolute path or a path relative to the current directory. +func LookPath2(file string, lenv Env) (string, error) { + var exts []string + x := lenv.Getenv(`PATHEXT`) + if x != "" { + for _, e := range strings.Split(strings.ToLower(x), `;`) { + if e == "" { + continue + } + if e[0] != '.' { + e = "." + e + } + exts = append(exts, e) + } + } else { + exts = []string{".com", ".exe", ".bat", ".cmd"} + } + + if strings.ContainsAny(file, `:\/`) { + if f, err := findExecutable(file, exts); err == nil { + return f, nil + } else { + return "", &Error{file, err} + } + } + if f, err := findExecutable(filepath.Join(".", file), exts); err == nil { + return f, nil + } + path := lenv.Getenv("path") + for _, dir := range filepath.SplitList(path) { + if f, err := findExecutable(filepath.Join(dir, file), exts); err == nil { + return f, nil + } + } + return "", &Error{file, ErrNotFound} +} diff --git a/act/model/workflow.go b/act/model/workflow.go index 0a6183aa..f1a2cb37 100644 --- a/act/model/workflow.go +++ b/act/model/workflow.go @@ -380,6 +380,42 @@ func commonKeysMatch2(a map[string]interface{}, b map[string]interface{}, m map[ return true } +// JobType describes what type of job we are about to run +type JobType int + +const ( + // StepTypeRun is all steps that have a `run` attribute + JobTypeDefault JobType = iota + + // StepTypeReusableWorkflowLocal is all steps that have a `uses` that is a local workflow in the .github/workflows directory + JobTypeReusableWorkflowLocal + + // JobTypeReusableWorkflowRemote is all steps that have a `uses` that references a workflow file in a github repo + JobTypeReusableWorkflowRemote +) + +func (j JobType) String() string { + switch j { + case JobTypeDefault: + return "default" + case JobTypeReusableWorkflowLocal: + return "local-reusable-workflow" + case JobTypeReusableWorkflowRemote: + return "remote-reusable-workflow" + } + return "unknown" +} + +// Type returns the type of the job +func (j *Job) Type() JobType { + if strings.HasPrefix(j.Uses, "./.github/workflows") && (strings.HasSuffix(j.Uses, ".yml") || strings.HasSuffix(j.Uses, ".yaml")) { + return JobTypeReusableWorkflowLocal + } else if !strings.HasPrefix(j.Uses, "./") && strings.Contains(j.Uses, ".github/workflows") && (strings.Contains(j.Uses, ".yml@") || strings.Contains(j.Uses, ".yaml@")) { + return JobTypeReusableWorkflowRemote + } + return JobTypeDefault +} + // ContainerSpec is the specification of the container to use for the job type ContainerSpec struct { Image string `yaml:"image"` @@ -460,7 +496,7 @@ func (s *Step) ShellCommand() string { case "python": shellCommand = "python {0}" case "sh": - shellCommand = "sh -e -c {0}" + shellCommand = "sh -e {0}" case "cmd": shellCommand = "%ComSpec% /D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"" case "powershell": @@ -487,6 +523,12 @@ const ( // StepTypeUsesActionRemote is all steps that have a `uses` that is a reference to a github repo StepTypeUsesActionRemote + // StepTypeReusableWorkflowLocal is all steps that have a `uses` that is a local workflow in the .github/workflows directory + StepTypeReusableWorkflowLocal + + // StepTypeReusableWorkflowRemote is all steps that have a `uses` that references a workflow file in a github repo + StepTypeReusableWorkflowRemote + // StepTypeInvalid is for steps that have invalid step action StepTypeInvalid ) @@ -503,6 +545,10 @@ func (s StepType) String() string { return "remote-action" case StepTypeUsesDockerURL: return "docker" + case StepTypeReusableWorkflowLocal: + return "local-reusable-workflow" + case StepTypeReusableWorkflowRemote: + return "remote-reusable-workflow" } return "unknown" } @@ -520,6 +566,10 @@ func (s *Step) Type() StepType { return StepTypeRun } else if strings.HasPrefix(s.Uses, "docker://") { return StepTypeUsesDockerURL + } else if strings.HasPrefix(s.Uses, "./.github/workflows") && (strings.HasSuffix(s.Uses, ".yml") || strings.HasSuffix(s.Uses, ".yaml")) { + return StepTypeReusableWorkflowLocal + } else if !strings.HasPrefix(s.Uses, "./") && strings.Contains(s.Uses, ".github/workflows") && (strings.Contains(s.Uses, ".yml@") || strings.Contains(s.Uses, ".yaml@")) { + return StepTypeReusableWorkflowRemote } else if strings.HasPrefix(s.Uses, "./") { return StepTypeUsesActionLocal } diff --git a/act/model/workflow_test.go b/act/model/workflow_test.go index 6d3b307f..d978f163 100644 --- a/act/model/workflow_test.go +++ b/act/model/workflow_test.go @@ -138,6 +138,31 @@ jobs: }) } +func TestReadWorkflow_JobTypes(t *testing.T) { + yaml := ` +name: invalid job definition + +jobs: + default-job: + runs-on: ubuntu-latest + steps: + - run: echo + remote-reusable-workflow: + runs-on: ubuntu-latest + uses: remote/repo/.github/workflows/workflow.yml@main + local-reusable-workflow: + runs-on: ubuntu-latest + uses: ./.github/workflows/workflow.yml +` + + workflow, err := ReadWorkflow(strings.NewReader(yaml)) + assert.NoError(t, err, "read workflow should succeed") + assert.Len(t, workflow.Jobs, 3) + assert.Equal(t, workflow.Jobs["default-job"].Type(), JobTypeDefault) + assert.Equal(t, workflow.Jobs["remote-reusable-workflow"].Type(), JobTypeReusableWorkflowRemote) + assert.Equal(t, workflow.Jobs["local-reusable-workflow"].Type(), JobTypeReusableWorkflowLocal) +} + func TestReadWorkflow_StepsTypes(t *testing.T) { yaml := ` name: invalid step definition diff --git a/act/runner/action.go b/act/runner/action.go index bae766e4..74a6560a 100644 --- a/act/runner/action.go +++ b/act/runner/action.go @@ -352,7 +352,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd []string stepContainer := container.NewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: rc.JobContainer.ToContainerPath(rc.Config.Workdir), Image: image, Username: rc.Config.Secrets["DOCKER_USERNAME"], Password: rc.Config.Secrets["DOCKER_PASSWORD"], @@ -397,11 +397,11 @@ func getContainerActionPaths(step *model.Step, actionDir string, rc *RunContext) containerActionDir := "." if step.Type() != model.StepTypeUsesActionRemote { actionName = getOsSafeRelativePath(actionDir, rc.Config.Workdir) - containerActionDir = rc.Config.ContainerWorkdir() + "/" + actionName + containerActionDir = rc.JobContainer.ToContainerPath(rc.Config.Workdir) + "/" + actionName actionName = "./" + actionName } else if step.Type() == model.StepTypeUsesActionRemote { actionName = getOsSafeRelativePath(actionDir, rc.ActionCacheDir()) - containerActionDir = ActPath + "/actions/" + actionName + containerActionDir = rc.JobContainer.GetActPath() + "/actions/" + actionName } if actionName == "" { diff --git a/act/runner/action_composite.go b/act/runner/action_composite.go index f2cd5609..645ef6d0 100644 --- a/act/runner/action_composite.go +++ b/act/runner/action_composite.go @@ -69,7 +69,9 @@ func newCompositeRunContext(ctx context.Context, parent *RunContext, step action Masks: parent.Masks, ExtraPath: parent.ExtraPath, Parent: parent, + EventJSON: parent.EventJSON, } + compositerc.ExprEval = compositerc.NewExpressionEvaluator(ctx) return compositerc } diff --git a/act/runner/container_mock_test.go b/act/runner/container_mock_test.go index a336d404..0de07815 100644 --- a/act/runner/container_mock_test.go +++ b/act/runner/container_mock_test.go @@ -11,6 +11,7 @@ import ( type containerMock struct { mock.Mock container.Container + container.LinuxContainerEnvironmentExtensions } func (cm *containerMock) Create(capAdd []string, capDrop []string) common.Executor { diff --git a/act/runner/expression.go b/act/runner/expression.go index adba5699..c2257b12 100644 --- a/act/runner/expression.go +++ b/act/runner/expression.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/nektos/act/pkg/common" - "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/exprparser" "github.com/nektos/act/pkg/model" "gopkg.in/yaml.v3" @@ -23,20 +22,22 @@ type ExpressionEvaluator interface { // NewExpressionEvaluator creates a new evaluator func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEvaluator { // todo: cleanup EvaluationEnvironment creation - job := rc.Run.Job() - strategy := make(map[string]interface{}) - if job.Strategy != nil { - strategy["fail-fast"] = job.Strategy.FailFast - strategy["max-parallel"] = job.Strategy.MaxParallel - } - - jobs := rc.Run.Workflow.Jobs - jobNeeds := rc.Run.Job().Needs() - using := make(map[string]map[string]map[string]string) - for _, needs := range jobNeeds { - using[needs] = map[string]map[string]string{ - "outputs": jobs[needs].Outputs, + strategy := make(map[string]interface{}) + if rc.Run != nil { + job := rc.Run.Job() + if job != nil && job.Strategy != nil { + strategy["fail-fast"] = job.Strategy.FailFast + strategy["max-parallel"] = job.Strategy.MaxParallel + } + + jobs := rc.Run.Workflow.Jobs + jobNeeds := rc.Run.Job().Needs() + + for _, needs := range jobNeeds { + using[needs] = map[string]map[string]string{ + "outputs": jobs[needs].Outputs, + } } } @@ -49,19 +50,16 @@ func (rc *RunContext) NewExpressionEvaluator(ctx context.Context) ExpressionEval Job: rc.getJobContext(), // todo: should be unavailable // but required to interpolate/evaluate the step outputs on the job - Steps: rc.getStepsContext(), - Runner: map[string]interface{}{ - "os": "Linux", - "arch": container.RunnerArch(ctx), - "temp": "/tmp", - "tool_cache": "/opt/hostedtoolcache", - }, + Steps: rc.getStepsContext(), Secrets: rc.Config.Secrets, Strategy: strategy, Matrix: rc.Matrix, Needs: using, Inputs: inputs, } + if rc.JobContainer != nil { + ee.Runner = rc.JobContainer.GetRunnerContext(ctx) + } return expressionEvaluator{ interpreter: exprparser.NewInterpeter(ee, exprparser.Config{ Run: rc.Run, @@ -95,16 +93,10 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) inputs := getEvaluatorInputs(ctx, rc, step, ghc) ee := &exprparser.EvaluationEnvironment{ - Github: step.getGithubContext(ctx), - Env: *step.getEnv(), - Job: rc.getJobContext(), - Steps: rc.getStepsContext(), - Runner: map[string]interface{}{ - "os": "Linux", - "arch": container.RunnerArch(ctx), - "temp": "/tmp", - "tool_cache": "/opt/hostedtoolcache", - }, + Github: step.getGithubContext(ctx), + Env: *step.getEnv(), + Job: rc.getJobContext(), + Steps: rc.getStepsContext(), Secrets: rc.Config.Secrets, Strategy: strategy, Matrix: rc.Matrix, @@ -113,6 +105,9 @@ func (rc *RunContext) NewStepExpressionEvaluator(ctx context.Context, step step) // but required to interpolate/evaluate the inputs in actions/composite Inputs: inputs, } + if rc.JobContainer != nil { + ee.Runner = rc.JobContainer.GetRunnerContext(ctx) + } return expressionEvaluator{ interpreter: exprparser.NewInterpeter(ee, exprparser.Config{ Run: rc.Run, @@ -331,15 +326,17 @@ func getEvaluatorInputs(ctx context.Context, rc *RunContext, step step, ghc *mod if ghc.EventName == "workflow_dispatch" { config := rc.Run.Workflow.WorkflowDispatchConfig() - for k, v := range config.Inputs { - value := nestedMapLookup(ghc.Event, "inputs", k) - if value == nil { - value = v.Default - } - if v.Type == "boolean" { - inputs[k] = value == "true" - } else { - inputs[k] = value + if config != nil && config.Inputs != nil { + for k, v := range config.Inputs { + value := nestedMapLookup(ghc.Event, "inputs", k) + if value == nil { + value = v.Default + } + if v.Type == "boolean" { + inputs[k] = value == "true" + } else { + inputs[k] = value + } } } } diff --git a/act/runner/expression_test.go b/act/runner/expression_test.go index b784a9a7..283b6cf9 100644 --- a/act/runner/expression_test.go +++ b/act/runner/expression_test.go @@ -117,7 +117,6 @@ func TestEvaluateRunContext(t *testing.T) { {"github.run_id", "1", ""}, {"github.run_number", "1", ""}, {"job.status", "success", ""}, - {"runner.os", "Linux", ""}, {"matrix.os", "Linux", ""}, {"matrix.foo", "bar", ""}, {"env.key", "value", ""}, diff --git a/act/runner/job_executor.go b/act/runner/job_executor.go index 3e175c04..5967dd12 100644 --- a/act/runner/job_executor.go +++ b/act/runner/job_executor.go @@ -38,6 +38,20 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo return common.NewDebugExecutor("No steps found") } + preSteps = append(preSteps, func(ctx context.Context) error { + // Have to be skipped for some Tests + if rc.Run == nil { + return nil + } + rc.ExprEval = rc.NewExpressionEvaluator(ctx) + // evaluate environment variables since they can contain + // GitHub's special environment variables. + for k, v := range rc.GetEnv() { + rc.Env[k] = rc.ExprEval.Interpolate(ctx, v) + } + return nil + }) + for i, stepModel := range infoSteps { stepModel := stepModel if stepModel == nil { @@ -120,7 +134,7 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo func useStepLogger(rc *RunContext, stepModel *model.Step, stage stepStage, executor common.Executor) common.Executor { return func(ctx context.Context) error { - ctx = withStepLogger(ctx, stepModel.Number, stepModel.ID, stepModel.String(), stage.String()) + ctx = withStepLogger(ctx, stepModel.Number, stepModel.ID, rc.ExprEval.Interpolate(ctx, stepModel.String()), stage.String()) rawLogger := common.Logger(ctx).WithField("raw_output", true) logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { diff --git a/act/runner/job_executor_test.go b/act/runner/job_executor_test.go index 8299f63f..e00a4fd6 100644 --- a/act/runner/job_executor_test.go +++ b/act/runner/job_executor_test.go @@ -79,6 +79,7 @@ func (jim *jobInfoMock) result(result string) { type jobContainerMock struct { container.Container + container.LinuxContainerEnvironmentExtensions } func (jcm *jobContainerMock) ReplaceLogWriter(stdout, stderr io.Writer) (io.Writer, io.Writer) { @@ -248,7 +249,17 @@ func TestNewJobExecutor(t *testing.T) { sfm := &stepFactoryMock{} rc := &RunContext{ JobContainer: &jobContainerMock{}, + Run: &model.Run{ + JobID: "test", + Workflow: &model.Workflow{ + Jobs: map[string]*model.Job{ + "test": {}, + }, + }, + }, + Config: &Config{}, } + rc.ExprEval = rc.NewExpressionEvaluator(ctx) executorOrder := make([]string, 0) jim.On("steps").Return(tt.steps) diff --git a/act/runner/run_context.go b/act/runner/run_context.go index ee7aaf7a..736a0b84 100644 --- a/act/runner/run_context.go +++ b/act/runner/run_context.go @@ -2,7 +2,10 @@ package runner import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -22,26 +25,25 @@ import ( "github.com/nektos/act/pkg/model" ) -const ActPath string = "/var/run/act" - // RunContext contains info about current job type RunContext struct { - Name string - Config *Config - Matrix map[string]interface{} - Run *model.Run - EventJSON string - Env map[string]string - ExtraPath []string - CurrentStep string - StepResults map[string]*model.StepResult - ExprEval ExpressionEvaluator - JobContainer container.Container - OutputMappings map[MappableOutput]MappableOutput - JobName string - ActionPath string - Parent *RunContext - Masks []string + Name string + Config *Config + Matrix map[string]interface{} + Run *model.Run + EventJSON string + Env map[string]string + ExtraPath []string + CurrentStep string + StepResults map[string]*model.StepResult + ExprEval ExpressionEvaluator + JobContainer container.ExecutionsEnvironment + OutputMappings map[MappableOutput]MappableOutput + JobName string + ActionPath string + Parent *RunContext + Masks []string + cleanUpJobContainer common.Executor } func (rc *RunContext) AddMask(mask string) { @@ -60,7 +62,13 @@ func (rc *RunContext) String() string { // GetEnv returns the env for the context func (rc *RunContext) GetEnv() map[string]string { if rc.Env == nil { - rc.Env = mergeMaps(rc.Run.Workflow.Env, rc.Run.Job().Environment(), rc.Config.Env) + rc.Env = map[string]string{} + if rc.Run != nil && rc.Run.Workflow != nil && rc.Config != nil { + job := rc.Run.Job() + if job != nil { + rc.Env = mergeMaps(rc.Run.Workflow.Env, job.Environment(), rc.Config.Env) + } + } } rc.Env["ACT"] = "true" return rc.Env @@ -82,9 +90,11 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { fmt.Sprintf("%s:%s", rc.Config.ContainerDaemonSocket, "/var/run/docker.sock"), } + ext := container.LinuxContainerEnvironmentExtensions{} + mounts := map[string]string{ "act-toolcache": "/toolcache", - name + "-env": ActPath, + name + "-env": ext.GetActPath(), } if job := rc.Run.Job(); job != nil { @@ -110,14 +120,84 @@ func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) { if selinux.GetEnabled() { bindModifiers = ":z" } - binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, rc.Config.ContainerWorkdir(), bindModifiers)) + binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, ext.ToContainerPath(rc.Config.Workdir), bindModifiers)) } else { - mounts[name] = rc.Config.ContainerWorkdir() + mounts[name] = ext.ToContainerPath(rc.Config.Workdir) } return binds, mounts } +func (rc *RunContext) startHostEnvironment() common.Executor { + return func(ctx context.Context) error { + logger := common.Logger(ctx) + rawLogger := logger.WithField("raw_output", true) + logWriter := common.NewLineWriter(rc.commandHandler(ctx), func(s string) bool { + if rc.Config.LogOutput { + rawLogger.Infof("%s", s) + } else { + rawLogger.Debugf("%s", s) + } + return true + }) + cacheDir := rc.ActionCacheDir() + randBytes := make([]byte, 8) + _, _ = rand.Read(randBytes) + miscpath := filepath.Join(cacheDir, hex.EncodeToString(randBytes)) + actPath := filepath.Join(miscpath, "act") + if err := os.MkdirAll(actPath, 0777); err != nil { + return err + } + path := filepath.Join(miscpath, "hostexecutor") + if err := os.MkdirAll(path, 0777); err != nil { + return err + } + runnerTmp := filepath.Join(miscpath, "tmp") + if err := os.MkdirAll(runnerTmp, 0777); err != nil { + return err + } + toolCache := filepath.Join(cacheDir, "tool_cache") + rc.JobContainer = &container.HostEnvironment{ + Path: path, + TmpDir: runnerTmp, + ToolCache: toolCache, + Workdir: rc.Config.Workdir, + ActPath: actPath, + CleanUp: func() { + os.RemoveAll(miscpath) + }, + StdOut: logWriter, + } + rc.cleanUpJobContainer = rc.JobContainer.Remove() + rc.Env["RUNNER_TOOL_CACHE"] = toolCache + rc.Env["RUNNER_OS"] = runtime.GOOS + rc.Env["RUNNER_ARCH"] = runtime.GOARCH + rc.Env["RUNNER_TEMP"] = runnerTmp + for _, env := range os.Environ() { + i := strings.Index(env, "=") + if i > 0 { + rc.Env[env[0:i]] = env[i+1:] + } + } + + return common.NewPipelineExecutor( + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ + Name: "workflow/event.json", + Mode: 0644, + Body: rc.EventJSON, + }, &container.FileEntry{ + Name: "workflow/envs.txt", + Mode: 0666, + Body: "", + }, &container.FileEntry{ + Name: "workflow/paths.txt", + Mode: 0666, + Body: "", + }), + )(ctx) + } +} + func (rc *RunContext) startJobContainer() common.Executor { return func(ctx context.Context) error { logger := common.Logger(ctx) @@ -147,12 +227,22 @@ func (rc *RunContext) startJobContainer() common.Executor { envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx))) envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp")) + ext := container.LinuxContainerEnvironmentExtensions{} binds, mounts := rc.GetBindsAndMounts() + rc.cleanUpJobContainer = func(ctx context.Context) error { + if rc.JobContainer != nil && !rc.Config.ReuseContainers { + return rc.JobContainer.Remove(). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). + Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + } + return nil + } + rc.JobContainer = container.NewContainer(&container.NewContainerInput{ Cmd: nil, Entrypoint: []string{"/bin/sleep", fmt.Sprint(rc.Config.ContainerMaxLifetime.Round(time.Second).Seconds())}, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: ext.ToContainerPath(rc.Config.Workdir), Image: image, Username: username, Password: password, @@ -169,6 +259,9 @@ func (rc *RunContext) startJobContainer() common.Executor { Options: rc.options(ctx), AutoRemove: rc.Config.AutoRemove, }) + if rc.JobContainer == nil { + return errors.New("Failed to create job container") + } return common.NewPipelineExecutor( rc.JobContainer.Pull(rc.Config.ForcePull), @@ -177,17 +270,17 @@ func (rc *RunContext) startJobContainer() common.Executor { rc.JobContainer.Start(false), rc.JobContainer.UpdateFromImageEnv(&rc.Env), rc.JobContainer.UpdateFromEnv("/etc/environment", &rc.Env), - rc.JobContainer.Copy(ActPath+"/", &container.FileEntry{ + rc.JobContainer.Copy(rc.JobContainer.GetActPath()+"/", &container.FileEntry{ Name: "workflow/event.json", - Mode: 0o644, + Mode: 0644, Body: rc.EventJSON, }, &container.FileEntry{ Name: "workflow/envs.txt", - Mode: 0o666, + Mode: 0666, Body: "", }, &container.FileEntry{ Name: "workflow/paths.txt", - Mode: 0o666, + Mode: 0666, Body: "", }), )(ctx) @@ -203,10 +296,8 @@ func (rc *RunContext) execJobContainer(cmd []string, env map[string]string, user // stopJobContainer removes the job container (if it exists) and its volume (if it exists) if !rc.Config.ReuseContainers func (rc *RunContext) stopJobContainer() common.Executor { return func(ctx context.Context) error { - if rc.JobContainer != nil && !rc.Config.ReuseContainers { - return rc.JobContainer.Remove(). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName(), false)). - Then(container.NewDockerVolumeRemoveExecutor(rc.jobContainerName()+"-env", false))(ctx) + if rc.cleanUpJobContainer != nil && !rc.Config.ReuseContainers { + return rc.cleanUpJobContainer(ctx) } return nil } @@ -243,7 +334,13 @@ func (rc *RunContext) interpolateOutputs() common.Executor { } func (rc *RunContext) startContainer() common.Executor { - return rc.startJobContainer() + return func(ctx context.Context) error { + image := rc.platformImage(ctx) + if strings.EqualFold(image, "-self-hosted") { + return rc.startHostEnvironment()(ctx) + } + return rc.startJobContainer()(ctx) + } } func (rc *RunContext) stopContainer() common.Executor { @@ -438,13 +535,11 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext logger := common.Logger(ctx) ghc := &model.GithubContext{ Event: make(map[string]interface{}), - EventPath: ActPath + "/workflow/event.json", Workflow: rc.Run.Workflow.Name, RunID: rc.Config.Env["GITHUB_RUN_ID"], RunNumber: rc.Config.Env["GITHUB_RUN_NUMBER"], Actor: rc.Config.Actor, EventName: rc.Config.EventName, - Workspace: rc.Config.ContainerWorkdir(), Action: rc.CurrentStep, Token: rc.Config.Token, ActionPath: rc.ActionPath, @@ -453,6 +548,10 @@ func (rc *RunContext) getGithubContext(ctx context.Context) *model.GithubContext RunnerPerflog: rc.Config.Env["RUNNER_PERFLOG"], RunnerTrackingID: rc.Config.Env["RUNNER_TRACKING_ID"], } + if rc.JobContainer != nil { + ghc.EventPath = rc.JobContainer.GetActPath() + "/workflow/event.json" + ghc.Workspace = rc.JobContainer.ToContainerPath(rc.Config.Workdir) + } if ghc.RunID == "" { ghc.RunID = "1" @@ -586,8 +685,8 @@ func nestedMapLookup(m map[string]interface{}, ks ...string) (rval interface{}) func (rc *RunContext) withGithubEnv(ctx context.Context, github *model.GithubContext, env map[string]string) map[string]string { env["CI"] = "true" - env["GITHUB_ENV"] = ActPath + "/workflow/envs.txt" - env["GITHUB_PATH"] = ActPath + "/workflow/paths.txt" + env["GITHUB_ENV"] = rc.JobContainer.GetActPath() + "/workflow/envs.txt" + env["GITHUB_PATH"] = rc.JobContainer.GetActPath() + "/workflow/paths.txt" env["GITHUB_WORKFLOW"] = github.Workflow env["GITHUB_RUN_ID"] = github.RunID env["GITHUB_RUN_NUMBER"] = github.RunNumber diff --git a/act/runner/run_context_test.go b/act/runner/run_context_test.go index 4d68dc8f..287cb46a 100644 --- a/act/runner/run_context_test.go +++ b/act/runner/run_context_test.go @@ -385,14 +385,12 @@ func TestGetGitHubContext(t *testing.T) { } assert.Equal(t, ghc.RunID, "1") - assert.Equal(t, ghc.Workspace, rc.Config.containerPath(cwd)) assert.Equal(t, ghc.RunNumber, "1") assert.Equal(t, ghc.RetentionDays, "0") assert.Equal(t, ghc.Actor, actor) assert.Equal(t, ghc.Repository, repo) assert.Equal(t, ghc.RepositoryOwner, owner) assert.Equal(t, ghc.RunnerPerflog, "/dev/null") - assert.Equal(t, ghc.EventPath, ActPath+"/workflow/event.json") assert.Equal(t, ghc.Token, rc.Config.Secrets["GITHUB_TOKEN"]) } diff --git a/act/runner/runner.go b/act/runner/runner.go index 0c2a988f..65dde5ef 100644 --- a/act/runner/runner.go +++ b/act/runner/runner.go @@ -4,10 +4,6 @@ import ( "context" "fmt" "os" - "path/filepath" - "regexp" - "runtime" - "strings" "time" log "github.com/sirupsen/logrus" @@ -65,46 +61,6 @@ type Config struct { PlatformPicker func(labels []string) string // platform picker, it will take precedence over Platforms if isn't nil } -// Resolves the equivalent host path inside the container -// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject -// For use in docker volumes and binds -func (config *Config) containerPath(path string) string { - if runtime.GOOS == "windows" && strings.Contains(path, "/") { - log.Error("You cannot specify linux style local paths (/mnt/etc) on Windows as it does not understand them.") - return "" - } - - abspath, err := filepath.Abs(path) - if err != nil { - log.Error(err) - return "" - } - - // Test if the path is a windows path - windowsPathRegex := regexp.MustCompile(`^([a-zA-Z]):\\(.+)$`) - windowsPathComponents := windowsPathRegex.FindStringSubmatch(abspath) - - // Return as-is if no match - if windowsPathComponents == nil { - return abspath - } - - // Convert to WSL2-compatible path if it is a windows path - // NOTE: Cannot use filepath because it will use the wrong path separators assuming we want the path to be windows - // based if running on Windows, and because we are feeding this to Docker, GoLang auto-path-translate doesn't work. - driveLetter := strings.ToLower(windowsPathComponents[1]) - translatedPath := strings.ReplaceAll(windowsPathComponents[2], `\`, `/`) - // Should make something like /mnt/c/Users/person/My Folder/MyActProject - result := strings.Join([]string{"/mnt", driveLetter, translatedPath}, `/`) - return result -} - -// Resolves the equivalent host path inside the container -// This is required for windows and WSL 2 to translate things like C:\Users\Myproject to /mnt/users/Myproject -func (config *Config) ContainerWorkdir() string { - return config.containerPath(config.Workdir) -} - type runnerImpl struct { config *Config eventJSON string @@ -173,11 +129,6 @@ func (runner *runnerImpl) NewPlanExecutor(plan *model.Plan) common.Executor { if len(matrixes) > 1 { rc.Name = fmt.Sprintf("%s-%d", rc.Name, i+1) } - // evaluate environment variables since they can contain - // GitHub's special environment variables. - for k, v := range rc.GetEnv() { - rc.Env[k] = rc.ExprEval.Interpolate(ctx, v) - } if len(rc.String()) > maxJobNameLen { maxJobNameLen = len(rc.String()) } diff --git a/act/runner/runner_test.go b/act/runner/runner_test.go index dbf8a007..9cb4ff40 100644 --- a/act/runner/runner_test.go +++ b/act/runner/runner_test.go @@ -183,6 +183,9 @@ func TestRunEvent(t *testing.T) { {workdir, "evalenv", "push", "", platforms}, {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms}, {workdir, "workflow_dispatch", "workflow_dispatch", "", platforms}, + {workdir, "workflow_dispatch_no_inputs_mapping", "workflow_dispatch", "", platforms}, + {workdir, "workflow_dispatch-scalar", "workflow_dispatch", "", platforms}, + {workdir, "workflow_dispatch-scalar-composite-action", "workflow_dispatch", "", platforms}, {"../model/testdata", "strategy", "push", "", platforms}, // TODO: move all testdata into pkg so we can validate it with planner and runner // {"testdata", "issue-228", "push", "", platforms, }, // TODO [igni]: Remove this once everything passes {"../model/testdata", "container-volumes", "push", "", platforms}, @@ -202,6 +205,95 @@ func TestRunEvent(t *testing.T) { } } +func TestRunEventHostEnvironment(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + ctx := context.Background() + + tables := []TestJobFileInfo{} + + if runtime.GOOS == "linux" { + platforms := map[string]string{ + "ubuntu-latest": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + // Shells + {workdir, "shells/defaults", "push", "", platforms}, + {workdir, "shells/pwsh", "push", "", platforms}, + {workdir, "shells/bash", "push", "", platforms}, + {workdir, "shells/python", "push", "", platforms}, + {workdir, "shells/sh", "push", "", platforms}, + + // Local action + {workdir, "local-action-js", "push", "", platforms}, + + // Uses + {workdir, "uses-composite", "push", "", platforms}, + {workdir, "uses-composite-with-error", "push", "Job 'failing-composite-action' failed", platforms}, + {workdir, "uses-nested-composite", "push", "", platforms}, + {workdir, "act-composite-env-test", "push", "", platforms}, + + // Eval + {workdir, "evalmatrix", "push", "", platforms}, + {workdir, "evalmatrixneeds", "push", "", platforms}, + {workdir, "evalmatrixneeds2", "push", "", platforms}, + {workdir, "evalmatrix-merge-map", "push", "", platforms}, + {workdir, "evalmatrix-merge-array", "push", "", platforms}, + {workdir, "issue-1195", "push", "", platforms}, + + {workdir, "fail", "push", "exit with `FAILURE`: 1", platforms}, + {workdir, "runs-on", "push", "", platforms}, + {workdir, "checkout", "push", "", platforms}, + {workdir, "remote-action-js", "push", "", platforms}, + {workdir, "matrix", "push", "", platforms}, + {workdir, "matrix-include-exclude", "push", "", platforms}, + {workdir, "commands", "push", "", platforms}, + {workdir, "defaults-run", "push", "", platforms}, + {workdir, "composite-fail-with-output", "push", "", platforms}, + {workdir, "issue-597", "push", "", platforms}, + {workdir, "issue-598", "push", "", platforms}, + {workdir, "if-env-act", "push", "", platforms}, + {workdir, "env-and-path", "push", "", platforms}, + {workdir, "non-existent-action", "push", "Job 'nopanic' failed", platforms}, + {workdir, "outputs", "push", "", platforms}, + {workdir, "steps-context/conclusion", "push", "", platforms}, + {workdir, "steps-context/outcome", "push", "", platforms}, + {workdir, "job-status-check", "push", "job 'fail' failed", platforms}, + {workdir, "if-expressions", "push", "Job 'mytest' failed", platforms}, + {workdir, "uses-action-with-pre-and-post-step", "push", "", platforms}, + {workdir, "evalenv", "push", "", platforms}, + {workdir, "ensure-post-steps", "push", "Job 'second-post-step-should-fail' failed", platforms}, + }...) + } + if runtime.GOOS == "windows" { + platforms := map[string]string{ + "windows-latest": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + {workdir, "windows-prepend-path", "push", "", platforms}, + {workdir, "windows-add-env", "push", "", platforms}, + }...) + } else { + platforms := map[string]string{ + "self-hosted": "-self-hosted", + } + + tables = append(tables, []TestJobFileInfo{ + {workdir, "nix-prepend-path", "push", "", platforms}, + }...) + } + + for _, table := range tables { + t.Run(table.workflowPath, func(t *testing.T) { + table.runTest(ctx, t, &Config{}) + }) + } +} + func TestDryrunEvent(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") @@ -317,60 +409,3 @@ func TestRunEventPullRequest(t *testing.T) { tjfi.runTest(context.Background(), t, &Config{EventPath: filepath.Join(workdir, workflowPath, "event.json")}) } - -func TestContainerPath(t *testing.T) { - type containerPathJob struct { - destinationPath string - sourcePath string - workDir string - } - - if runtime.GOOS == "windows" { - cwd, err := os.Getwd() - if err != nil { - log.Error(err) - } - - rootDrive := os.Getenv("SystemDrive") - rootDriveLetter := strings.ReplaceAll(strings.ToLower(rootDrive), `:`, "") - for _, v := range []containerPathJob{ - {"/mnt/c/Users/act/go/src/github.com/nektos/act", "C:\\Users\\act\\go\\src\\github.com\\nektos\\act\\", ""}, - {"/mnt/f/work/dir", `F:\work\dir`, ""}, - {"/mnt/c/windows/to/unix", "windows\\to\\unix", fmt.Sprintf("%s\\", rootDrive)}, - {fmt.Sprintf("/mnt/%v/act", rootDriveLetter), "act", fmt.Sprintf("%s\\", rootDrive)}, - } { - if v.workDir != "" { - if err := os.Chdir(v.workDir); err != nil { - log.Error(err) - t.Fail() - } - } - - runnerConfig := &Config{ - Workdir: v.sourcePath, - } - - assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir)) - } - - if err := os.Chdir(cwd); err != nil { - log.Error(err) - } - } else { - cwd, err := os.Getwd() - if err != nil { - log.Error(err) - } - for _, v := range []containerPathJob{ - {"/home/act/go/src/github.com/nektos/act", "/home/act/go/src/github.com/nektos/act", ""}, - {"/home/act", `/home/act/`, ""}, - {cwd, ".", ""}, - } { - runnerConfig := &Config{ - Workdir: v.sourcePath, - } - - assert.Equal(t, v.destinationPath, runnerConfig.containerPath(runnerConfig.Workdir)) - } - } -} diff --git a/act/runner/step.go b/act/runner/step.go index 8682d582..f730ac1a 100644 --- a/act/runner/step.go +++ b/act/runner/step.go @@ -3,9 +3,11 @@ package runner import ( "context" "fmt" + "path" "strings" "github.com/nektos/act/pkg/common" + "github.com/nektos/act/pkg/container" "github.com/nektos/act/pkg/exprparser" "github.com/nektos/act/pkg/model" ) @@ -88,12 +90,26 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo return nil } - stepString := stepModel.String() + stepString := rc.ExprEval.Interpolate(ctx, stepModel.String()) if strings.Contains(stepString, "::add-mask::") { stepString = "add-mask command" } logger.Infof("\u2B50 Run %s %s", stage, stepString) + // Prepare and clean Runner File Commands + actPath := rc.JobContainer.GetActPath() + outputFileCommand := path.Join("workflow", "outputcmd.txt") + stateFileCommand := path.Join("workflow", "statecmd.txt") + (*step.getEnv())["GITHUB_OUTPUT"] = path.Join(actPath, outputFileCommand) + (*step.getEnv())["GITHUB_STATE"] = path.Join(actPath, stateFileCommand) + _ = rc.JobContainer.Copy(actPath, &container.FileEntry{ + Name: outputFileCommand, + Mode: 0666, + }, &container.FileEntry{ + Name: stateFileCommand, + Mode: 0666, + })(ctx) + err = executor(ctx) if err == nil { @@ -117,6 +133,27 @@ func runStepExecutor(step step, stage stepStage, executor common.Executor) commo logger.WithField("stepResult", rc.StepResults[rc.CurrentStep].Outcome).Errorf(" \u274C Failure - %s %s", stage, stepString) } + // Process Runner File Commands + orgerr := err + state := map[string]string{} + err = rc.JobContainer.UpdateFromEnv(path.Join(actPath, stateFileCommand), &state)(ctx) + if err != nil { + return err + } + for k, v := range state { + rc.saveState(ctx, map[string]string{"name": k}, v) + } + output := map[string]string{} + err = rc.JobContainer.UpdateFromEnv(path.Join(actPath, outputFileCommand), &output)(ctx) + if err != nil { + return err + } + for k, v := range output { + rc.setOutput(ctx, map[string]string{"name": k}, v) + } + if orgerr != nil { + return orgerr + } return err } } @@ -162,13 +199,12 @@ func mergeEnv(ctx context.Context, step step) { mergeIntoMap(env, rc.GetEnv()) } - if (*env)["PATH"] == "" { - (*env)["PATH"] = `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` + path := rc.JobContainer.GetPathVariableName() + if (*env)[path] == "" { + (*env)[path] = rc.JobContainer.DefaultPathVariable() } if rc.ExtraPath != nil && len(rc.ExtraPath) > 0 { - p := (*env)["PATH"] - (*env)["PATH"] = strings.Join(rc.ExtraPath, `:`) - (*env)["PATH"] += `:` + p + (*env)[path] = rc.JobContainer.JoinPathVariable(append(rc.ExtraPath, (*env)[path])...) } rc.withGithubEnv(ctx, step.getGithubContext(ctx), *env) diff --git a/act/runner/step_action_local_test.go b/act/runner/step_action_local_test.go index 8180d6de..63902898 100644 --- a/act/runner/step_action_local_test.go +++ b/act/runner/step_action_local_test.go @@ -2,6 +2,7 @@ package runner import ( "context" + "path/filepath" "strings" "testing" @@ -63,7 +64,7 @@ func TestStepActionLocalTest(t *testing.T) { }, } - salm.On("readAction", sal.Step, "/tmp/path/to/action", "", mock.Anything, mock.Anything). + salm.On("readAction", sal.Step, filepath.Clean("/tmp/path/to/action"), "", mock.Anything, mock.Anything). Return(&model.Action{}, nil) cm.On("UpdateFromImageEnv", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { @@ -78,7 +79,19 @@ func TestStepActionLocalTest(t *testing.T) { return nil }) - salm.On("runAction", sal, "/tmp/path/to/action", (*remoteAction)(nil)).Return(func(ctx context.Context) error { + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + salm.On("runAction", sal, filepath.Clean("/tmp/path/to/action"), (*remoteAction)(nil)).Return(func(ctx context.Context) error { return nil }) @@ -262,6 +275,7 @@ func TestStepActionLocalPost(t *testing.T) { Step: tt.stepModel, action: tt.actionModel, } + sal.RunContext.ExprEval = sal.RunContext.NewExpressionEvaluator(ctx) if tt.mocks.env { cm.On("UpdateFromImageEnv", &sal.env).Return(func(ctx context.Context) error { return nil }) @@ -275,6 +289,18 @@ func TestStepActionLocalPost(t *testing.T) { }) } cm.On("Exec", suffixMatcher("pkg/runner/local/action/post.js"), sal.env, "", "").Return(func(ctx context.Context) error { return tt.err }) + + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) } err := sal.post()(ctx) diff --git a/act/runner/step_action_remote.go b/act/runner/step_action_remote.go index 78b4d732..26b178e7 100644 --- a/act/runner/step_action_remote.go +++ b/act/runner/step_action_remote.go @@ -115,7 +115,7 @@ func (sar *stepActionRemote) main() common.Executor { return nil } eval := sar.RunContext.NewExpressionEvaluator(ctx) - copyToPath := path.Join(sar.RunContext.Config.ContainerWorkdir(), eval.Interpolate(ctx, sar.Step.With["path"])) + copyToPath := path.Join(sar.RunContext.JobContainer.ToContainerPath(sar.RunContext.Config.Workdir), eval.Interpolate(ctx, sar.Step.With["path"])) return sar.RunContext.JobContainer.CopyDir(copyToPath, sar.RunContext.Config.Workdir+string(filepath.Separator)+".", sar.RunContext.Config.UseGitIgnore)(ctx) } diff --git a/act/runner/step_action_remote_test.go b/act/runner/step_action_remote_test.go index 1e117ea5..72b0eeeb 100644 --- a/act/runner/step_action_remote_test.go +++ b/act/runner/step_action_remote_test.go @@ -155,6 +155,7 @@ func TestStepActionRemote(t *testing.T) { readAction: sarm.readAction, runAction: sarm.runAction, } + sar.RunContext.ExprEval = sar.RunContext.NewExpressionEvaluator(ctx) suffixMatcher := func(suffix string) interface{} { return mock.MatchedBy(func(actionDir string) bool { @@ -172,6 +173,18 @@ func TestStepActionRemote(t *testing.T) { } if tt.mocks.run { sarm.On("runAction", sar, suffixMatcher("act/remote-action@v1"), newRemoteAction(sar.Step.Uses)).Return(func(ctx context.Context) error { return tt.runError }) + + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) } err := sar.pre()(ctx) @@ -574,6 +587,7 @@ func TestStepActionRemotePost(t *testing.T) { Step: tt.stepModel, action: tt.actionModel, } + sar.RunContext.ExprEval = sar.RunContext.NewExpressionEvaluator(ctx) if tt.mocks.env { cm.On("UpdateFromImageEnv", &sar.env).Return(func(ctx context.Context) error { return nil }) @@ -582,6 +596,18 @@ func TestStepActionRemotePost(t *testing.T) { } if tt.mocks.exec { cm.On("Exec", []string{"node", "/var/run/act/actions/remote-action@v1/post.js"}, sar.env, "", "").Return(func(ctx context.Context) error { return tt.err }) + + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) } err := sar.post()(ctx) diff --git a/act/runner/step_docker.go b/act/runner/step_docker.go index 4010b559..b83435f9 100644 --- a/act/runner/step_docker.go +++ b/act/runner/step_docker.go @@ -116,7 +116,7 @@ func (sd *stepDocker) newStepContainer(ctx context.Context, image string, cmd [] stepContainer := ContainerNewContainer(&container.NewContainerInput{ Cmd: cmd, Entrypoint: entrypoint, - WorkingDir: rc.Config.ContainerWorkdir(), + WorkingDir: rc.JobContainer.ToContainerPath(rc.Config.Workdir), Image: image, Username: rc.Config.Secrets["DOCKER_USERNAME"], Password: rc.Config.Secrets["DOCKER_PASSWORD"], diff --git a/act/runner/step_docker_test.go b/act/runner/step_docker_test.go index c7482718..2008357f 100644 --- a/act/runner/step_docker_test.go +++ b/act/runner/step_docker_test.go @@ -17,7 +17,7 @@ func TestStepDockerMain(t *testing.T) { // mock the new container call origContainerNewContainer := ContainerNewContainer - ContainerNewContainer = func(containerInput *container.NewContainerInput) container.Container { + ContainerNewContainer = func(containerInput *container.NewContainerInput) container.ExecutionsEnvironment { input = containerInput return cm } @@ -25,6 +25,8 @@ func TestStepDockerMain(t *testing.T) { ContainerNewContainer = origContainerNewContainer })() + ctx := context.Background() + sd := &stepDocker{ RunContext: &RunContext{ StepResults: map[string]*model.StepResult{}, @@ -51,8 +53,7 @@ func TestStepDockerMain(t *testing.T) { WorkingDirectory: "workdir", }, } - - ctx := context.Background() + sd.RunContext.ExprEval = sd.RunContext.NewExpressionEvaluator(ctx) cm.On("UpdateFromImageEnv", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { return nil @@ -86,6 +87,18 @@ func TestStepDockerMain(t *testing.T) { return nil }) + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + err := sd.main()(ctx) assert.Nil(t, err) diff --git a/act/runner/step_run.go b/act/runner/step_run.go index 85dfed64..a74f781b 100644 --- a/act/runner/step_run.go +++ b/act/runner/step_run.go @@ -68,7 +68,8 @@ func (sr *stepRun) setupShellCommandExecutor() common.Executor { return err } - return sr.RunContext.JobContainer.Copy(ActPath, &container.FileEntry{ + rc := sr.getRunContext() + return rc.JobContainer.Copy(rc.JobContainer.GetActPath(), &container.FileEntry{ Name: scriptName, Mode: 0755, Body: script, @@ -128,7 +129,8 @@ func (sr *stepRun) setupShellCommand(ctx context.Context) (name, script string, logger.Debugf("Wrote add-mask command to '%s'", name) } - scriptPath := fmt.Sprintf("%s/%s", ActPath, name) + rc := sr.getRunContext() + scriptPath := fmt.Sprintf("%s/%s", rc.JobContainer.GetActPath(), name) sr.cmd, err = shellquote.Split(strings.Replace(scCmd, `{0}`, scriptPath, 1)) return name, script, err diff --git a/act/runner/step_run_test.go b/act/runner/step_run_test.go index 081d8647..e5cde123 100644 --- a/act/runner/step_run_test.go +++ b/act/runner/step_run_test.go @@ -65,6 +65,18 @@ func TestStepRun(t *testing.T) { return nil }) + cm.On("Copy", "/var/run/act", mock.AnythingOfType("[]*container.FileEntry")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/statecmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + + cm.On("UpdateFromEnv", "/var/run/act/workflow/outputcmd.txt", mock.AnythingOfType("*map[string]string")).Return(func(ctx context.Context) error { + return nil + }) + ctx := context.Background() err := sr.main()(ctx) diff --git a/act/runner/testdata/nix-prepend-path/push.yml b/act/runner/testdata/nix-prepend-path/push.yml new file mode 100644 index 00000000..71fd5bc7 --- /dev/null +++ b/act/runner/testdata/nix-prepend-path/push.yml @@ -0,0 +1,26 @@ +on: + push: +defaults: + run: + shell: sh +jobs: + test: + runs-on: self-hosted + steps: + - run: | + mkdir build + echo '#!/usr/bin/env sh' > build/testtool + echo 'echo Hi' >> build/testtool + chmod +x build/testtool + - run: | + echo '${{ tojson(runner) }}' + ls + echo '${{ github.workspace }}' + working-directory: ${{ github.workspace }}/build + - run: | + echo "$GITHUB_PATH" + echo '${{ github.workspace }}/build' > "$GITHUB_PATH" + cat "$GITHUB_PATH" + - run: | + echo "$PATH" + testtool diff --git a/act/runner/testdata/pull-request/main.yaml b/act/runner/testdata/pull-request/main.yaml index b9946b94..eb81939b 100644 --- a/act/runner/testdata/pull-request/main.yaml +++ b/act/runner/testdata/pull-request/main.yaml @@ -5,6 +5,22 @@ jobs: build: runs-on: ubuntu-latest steps: - - run: echo '${{github.ref}}' + # test refs from event.json + - run: echo '${{github.ref}}' - run: echo '${{github.head_ref}}' | grep sample-head-ref - run: echo '${{github.base_ref}}' | grep sample-base-ref + # test main/composite context equality with data from event.json + - run: | + runs: + using: composite + steps: + - run: | + echo WORKFLOW_GITHUB_CONTEXT="$WORKFLOW_GITHUB_CONTEXT" + echo COMPOSITE_GITHUB_CONTEXT="$COMPOSITE_GITHUB_CONTEXT" + [[ "$WORKFLOW_GITHUB_CONTEXT" = "$COMPOSITE_GITHUB_CONTEXT" ]] + env: + WORKFLOW_GITHUB_CONTEXT: ${{ tojson(tojson(github.event)) }} + COMPOSITE_GITHUB_CONTEXT: ${{ '${{tojson(github.event)}}' }} + shell: bash + shell: cp {0} action.yml + - uses: ./ diff --git a/act/runner/testdata/windows-add-env/push.yml b/act/runner/testdata/windows-add-env/push.yml new file mode 100644 index 00000000..275c5f1f --- /dev/null +++ b/act/runner/testdata/windows-add-env/push.yml @@ -0,0 +1,27 @@ +on: + push: +defaults: + run: + shell: pwsh +jobs: + test: + runs-on: windows-latest + steps: + - run: | + echo $env:GITHUB_ENV + echo "key=val" > $env:GITHUB_ENV + echo "key2<> $env:GITHUB_ENV + echo "line1" >> $env:GITHUB_ENV + echo "line2" >> $env:GITHUB_ENV + echo "EOF" >> $env:GITHUB_ENV + cat $env:GITHUB_ENV + - run: | + ls env: + if($env:key -ne 'val') { + echo "Unexpected value for `$env:key: $env:key" + exit 1 + } + if($env:key2 -ne "line1`nline2") { + echo "Unexpected value for `$env:key2: $env:key2" + exit 1 + } diff --git a/act/runner/testdata/windows-prepend-path/push.yml b/act/runner/testdata/windows-prepend-path/push.yml new file mode 100644 index 00000000..176de691 --- /dev/null +++ b/act/runner/testdata/windows-prepend-path/push.yml @@ -0,0 +1,25 @@ +on: + push: +defaults: + run: + shell: pwsh +jobs: + test: + runs-on: windows-latest + steps: + - run: | + mkdir build + echo '@echo off' > build/test.cmd + echo 'echo Hi' >> build/test.cmd + - run: | + echo '${{ tojson(runner) }}' + ls + echo '${{ github.workspace }}' + working-directory: ${{ github.workspace }}\build + - run: | + echo $env:GITHUB_PATH + echo '${{ github.workspace }}\build' > $env:GITHUB_PATH + cat $env:GITHUB_PATH + - run: | + echo $env:PATH + test diff --git a/act/runner/testdata/workflow_dispatch-scalar-composite-action/workflow_dispatch.yml b/act/runner/testdata/workflow_dispatch-scalar-composite-action/workflow_dispatch.yml new file mode 100644 index 00000000..f8447b45 --- /dev/null +++ b/act/runner/testdata/workflow_dispatch-scalar-composite-action/workflow_dispatch.yml @@ -0,0 +1,17 @@ +name: workflow_dispatch + +on: workflow_dispatch + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: | + runs: + using: composite + steps: + - run: | + exit 0 + shell: bash + shell: cp {0} action.yml + - uses: ./ diff --git a/act/runner/testdata/workflow_dispatch-scalar/workflow_dispatch.yml b/act/runner/testdata/workflow_dispatch-scalar/workflow_dispatch.yml new file mode 100644 index 00000000..9c900e87 --- /dev/null +++ b/act/runner/testdata/workflow_dispatch-scalar/workflow_dispatch.yml @@ -0,0 +1,9 @@ +name: workflow_dispatch + +on: workflow_dispatch + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: exit 0 diff --git a/act/runner/testdata/workflow_dispatch_no_inputs_mapping/workflow_dispatch.yml b/act/runner/testdata/workflow_dispatch_no_inputs_mapping/workflow_dispatch.yml new file mode 100644 index 00000000..9fc6b097 --- /dev/null +++ b/act/runner/testdata/workflow_dispatch_no_inputs_mapping/workflow_dispatch.yml @@ -0,0 +1,10 @@ +name: workflow_dispatch + +on: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - run: exit 0