add test case for premature termination before health check completes

act/container/container_types.go

@@ -7,6 +7,7 @@ import (
 
 	"code.forgejo.org/forgejo/runner/v9/act/common"
 	"github.com/docker/go-connections/nat"
+	"github.com/pkg/errors"
 )
 
 // NewContainerInput the input for the New function
@@ -93,3 +94,5 @@ const (
 	HealthHealthy
 	HealthUnHealthy
 )
+
+var ErrContainerNotFound = errors.New("container not found")

act/container/docker_run.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	"github.com/Masterminds/semver"
+	cerrdefs "github.com/containerd/errdefs"
 	"github.com/docker/cli/cli/compose/loader"
 	"github.com/docker/cli/cli/connhelper"
 	"github.com/docker/docker/api/types"
@@ -194,6 +195,9 @@ func (cr *containerReference) Remove() common.Executor {
 
 func (cr *containerReference) GetHealth(ctx context.Context) (Health, error) {
 	resp, err := cr.cli.ContainerInspect(ctx, cr.id)
+	if cerrdefs.IsNotFound(err) {
+		return HealthUnHealthy, ErrContainerNotFound
+	}
 	logger := common.Logger(ctx)
 	if err != nil {
 		return HealthUnHealthy, err

act/runner/run_context.go

@@ -768,10 +768,16 @@ func (rc *RunContext) waitForServiceContainer(c container.ExecutionsEnvironment)
 				return fmt.Errorf("service container %s: timed out while waiting for healthy or unhealthy status to be reported", c.GetName())
 			} else if errors.Is(err, context.Canceled) {
 				return err
+			} else if errors.Is(err, container.ErrContainerNotFound) || (err == nil && health == container.HealthUnHealthy) {
+				// Container absent (terminated during health check) and unhealthy are difficult to consistently report
+				// differently from each other as, in docker, a terminated container will briefly appear unhealthy and
+				// then start reporting container not found; so, report both the same. Without any detection of the
+				// ErrContainerNotFound case we would just treat it as a transient failure and timeout, which would be a
+				// slower error mode that this is working to avoid.
+				return fmt.Errorf("service container %s: failed health check or terminated before becoming healthy", c.GetName())
 			} else if err != nil {
+				// assume transient error in the execution environment
 				logger.Warnf("service container %s: error while checking for health state, will retry: %v", c.GetName(), err)
-			} else if health == container.HealthUnHealthy {
-				return fmt.Errorf("service container %s failed health check", c.GetName())
 			} else if health == container.HealthHealthy {
 				return nil
 			}

act/runner/runner_test.go

@@ -323,7 +323,8 @@ func TestRunner_RunEvent(t *testing.T) {
 		{workdir, "services", "push", "", platforms, secrets},
 		{workdir, "services-with-container", "push", "", platforms, secrets},
 		{workdir, "mysql-service-container-with-health-check", "push", "", platforms, secrets},
-		{workdir, "mysql-service-container-failed-health-check", "push", "service container NAME failed health check", platforms, secrets},
+		{workdir, "mysql-service-container-failed-health-check", "push", "service container NAME: failed health check or terminated before becoming healthy", platforms, secrets},
+		{workdir, "mysql-service-container-premature-terminate", "push", "service container NAME: failed health check or terminated before becoming healthy", platforms, secrets},
 	}
 
 	for _, table := range tables {

act/runner/testdata/mysql-service-container-premature-terminate/push.yml

@@ -0,0 +1,21 @@
+name: service-container
+on: push
+jobs:
+  service-container-test:
+    runs-on: ubuntu-latest
+    container: code.forgejo.org/oci/mysql:8.4
+    services:
+      maindb:
+        image: code.forgejo.org/oci/mysql:8.4
+        # This container should immediately exit due to missing env variable for poassword config. ... [ERROR]
+        # [Entrypoint]: Database is uninitialized and password option is not specified You need to specify one of the
+        # following as an environment variable:
+        # - MYSQL_ROOT_PASSWORD
+        # - MYSQL_ALLOW_EMPTY_PASSWORD
+        # - MYSQL_RANDOM_ROOT_PASSWORD
+        #
+        # This container should retain the same health check config as the mysql-service-container-with-health-check
+        # case.
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+    steps:
+    - run: exit 100 # should never be hit since service will never be healthy