[rayci] add buildkite notification (#237)

Add buildkite notification upon build failures. Notification is sent to
the owner of the build (buildkite syntax:
https://buildkite.com/docs/pipelines/notifications).

Enable notification for microcheck for now. This will allow folks to
know build issues faster.

Test:
- CI 
- I added a failure into rayci CI in a previous commit and check that I
get notified

Signed-off-by: can <[email protected]>

Author: can-anyscale

raycicmd/bk_pipeline.go

@@ -5,6 +5,11 @@ import (
 	"time"
 )
 
+type bkNotify struct {
+	Email string `yaml:"email,omitempty"`
+	If    string `yaml:"if,omitempty"`
+}
+
 type bkPipelineGroup struct {
 	Group     string   `yaml:"group,omitempty"`
 	Key       string   `yaml:"key,omitempty"`
@@ -13,7 +18,8 @@ type bkPipelineGroup struct {
 }
 
 type bkPipeline struct {
-	Steps []*bkPipelineGroup `yaml:"steps,omitempty"`
+	Steps  []*bkPipelineGroup `yaml:"steps,omitempty"`
+	Notify []*bkNotify        `yaml:"notify,omitempty"`
 }
 
 func (p *bkPipeline) totalSteps() int {
@@ -28,6 +34,13 @@ func newBkAgents(queue string) map[string]any {
 	return map[string]any{"queue": queue}
 }
 
+func makeBuildFailureBkNotify(email string) *bkNotify {
+	return &bkNotify{
+		Email: email,
+		If:    `build.state == "failing"`,
+	}
+}
+
 func makeNoopBkPipeline(q string) *bkPipeline {
 	step := map[string]any{"command": "echo no pipeline steps"}
 	if q != "" {

raycicmd/build_info.go

@@ -10,10 +10,11 @@ import (
 )
 
 type buildInfo struct {
-	buildID        string
-	launcherBranch string
-	gitCommit      string
-	selects        []string
+	buildID          string
+	buildAuthorEmail string
+	launcherBranch   string
+	gitCommit        string
+	selects          []string
 }
 
 func makeBuildID(envs Envs) (string, error) {

raycicmd/config.go

@@ -126,6 +126,12 @@ type config struct {
 	// Optional.
 	MaxParallelism int `yaml:"max_parallelism"`
 
+	// NotifyOwnerOnFailure sets if the owner of the build should be notified
+	// when a build fails.
+	//
+	// Optional.
+	NotifyOwnerOnFailure bool `yaml:"notify_owner_on_failure"`
+
 	// DockerPlugin contains additional docker plugin configs, to fine tune
 	// the docker plugin's behavior.
 	//
@@ -283,11 +289,14 @@ func ciDefaultConfig(envs Envs) *config {
 	case rayPRPipeline, rayV2PremergePipeline, rayDevPipeline:
 		return prPipelineConfig("ray-pr", nil, -1)
 	case rayV2MicrocheckPipeline:
-		return prPipelineConfig(
+		mcPipelineConfig := prPipelineConfig(
 			"ray-pr-microcheck",
 			map[string]string{"RAYCI_MICROCHECK_RUN": "1"},
 			1,
 		)
+		mcPipelineConfig.NotifyOwnerOnFailure = true
+
+		return mcPipelineConfig
 	}
 
 	// By default, assume it is less privileged.

raycicmd/main.go

@@ -104,14 +104,19 @@ func makeBuildInfo(flags *Flags, envs Envs) (*buildInfo, error) {
 	}
 
 	rayciBranch, _ := envs.Lookup("RAYCI_BRANCH")
+
+	// buildAuthorEmail is the email of the user who triggered the buildkite webhook
+	// event; for most parts, it is the same as the github author email.
+	buildAuthorEmail, _ := envs.Lookup("BUILDKITE_BUILD_CREATOR_EMAIL")
 	commit := gitCommit(envs)
 	selects := stepSelects(flags.Select, envs)
 
 	return &buildInfo{
-		buildID:        buildID,
-		launcherBranch: rayciBranch,
-		gitCommit:      commit,
-		selects:        selects,
+		buildID:          buildID,
+		buildAuthorEmail: buildAuthorEmail,
+		launcherBranch:   rayciBranch,
+		gitCommit:        commit,
+		selects:          selects,
 	}, nil
 }
 

raycicmd/main_test.go

@@ -58,10 +58,11 @@ func TestExecWithInput(t *testing.T) {
 func TestMakeBuildInfo(t *testing.T) {
 	flags := &Flags{}
 	envs := newEnvsMap(map[string]string{
-		"RAYCI_BUILD_ID":   "fake-build-id",
-		"BUILDKITE_COMMIT": "abc123",
-		"RAYCI_BRANCH":     "foobar",
-		"RAYCI_SELECT":     "foo,bar,taz",
+		"RAYCI_BUILD_ID":                "fake-build-id",
+		"BUILDKITE_COMMIT":              "abc123",
+		"BUILDKITE_BUILD_CREATOR_EMAIL": "[email protected]",
+		"RAYCI_BRANCH":                  "foobar",
+		"RAYCI_SELECT":                  "foo,bar,taz",
 	})
 
 	info, err := makeBuildInfo(flags, envs)
@@ -70,10 +71,11 @@ func TestMakeBuildInfo(t *testing.T) {
 	}
 
 	want := &buildInfo{
-		buildID:        "fake-build-id",
-		launcherBranch: "foobar",
-		gitCommit:      "abc123",
-		selects:        []string{"foo", "bar", "taz"},
+		buildID:          "fake-build-id",
+		buildAuthorEmail: "[email protected]",
+		launcherBranch:   "foobar",
+		gitCommit:        "abc123",
+		selects:          []string{"foo", "bar", "taz"},
 	}
 	if !reflect.DeepEqual(info, want) {
 		t.Errorf("got %+v, want %+v", info, want)
@@ -85,10 +87,11 @@ func TestMakeBuildInfo(t *testing.T) {
 		t.Fatal("make build info with selects overwrite: ", err)
 	}
 	want = &buildInfo{
-		buildID:        "fake-build-id",
-		launcherBranch: "foobar",
-		gitCommit:      "abc123",
-		selects:        []string{"gee", "goo"},
+		buildID:          "fake-build-id",
+		buildAuthorEmail: "[email protected]",
+		launcherBranch:   "foobar",
+		gitCommit:        "abc123",
+		selects:          []string{"gee", "goo"},
 	}
 	if !reflect.DeepEqual(info, want) {
 		t.Errorf("got %+v, want %+v", info, want)

raycicmd/make.go

@@ -141,5 +141,9 @@ func makePipeline(repoDir string, config *config, info *buildInfo) (
 		return makeNoopBkPipeline(q), nil
 	}
 
+	if email := info.buildAuthorEmail; email != "" && config.NotifyOwnerOnFailure {
+		pl.Notify = append(pl.Notify, makeBuildFailureBkNotify(email))
+	}
+
 	return pl, nil
 }

raycicmd/make_test.go

@@ -293,6 +293,38 @@ func TestMakePipeline(t *testing.T) {
 			t.Errorf("got step keys %v, want %v", keys, want)
 		}
 	})
+
+	t.Run("notify", func(t *testing.T) {
+		buildID := "fakebuild"
+		info := &buildInfo{
+			buildID: buildID,
+		}
+
+		config := *commonConfig
+		config.NotifyOwnerOnFailure = false
+
+		got, err := makePipeline(tmp, &config, info)
+		if err != nil {
+			t.Fatalf("makePipeline: %v", err)
+		}
+		if len(got.Notify) != 0 {
+			t.Errorf("got %d notify, want 0", len(got.Notify))
+		}
+
+		const email = "[email protected]"
+		infoWithEmail := &buildInfo{
+			buildID:          buildID,
+			buildAuthorEmail: email,
+		}
+		config.NotifyOwnerOnFailure = true
+		got, err = makePipeline(tmp, &config, infoWithEmail)
+		if err != nil {
+			t.Fatalf("makePipeline: %v", err)
+		}
+		if len(got.Notify) == 0 || got.Notify[0].Email != email || got.Notify[0].If != `build.state == "failing"` {
+			t.Errorf(`got %v, want email %v, want if build.state == "failing"`, got.Notify, email)
+		}
+	})
 }
 
 func TestSortPipelineGroups(t *testing.T) {