[rayci] allow concurrency key in buildkite step (#239)

Allow `concurrency` and `concurrency_group`; this is require to rate
limit the concurrent run of certain release test types (see
anyscale/llm-forge#387)

Test:
- CI

---------

Signed-off-by: can <[email protected]>

Author: can-anyscale

raycicmd/converter_test.go

@@ -224,6 +224,39 @@ func TestConvertPipelineStep(t *testing.T) {
 			},
 		},
 	}, {
+		in: map[string]any{
+			"label":             "say hello",
+			"key":               "key",
+			"command":           "echo hello",
+			"depends_on":        "dep",
+			"concurrency":       2,
+			"concurrency_group": "group",
+		},
+		out: map[string]any{
+			"label":             "say hello",
+			"key":               "key",
+			"command":           "echo hello",
+			"depends_on":        "dep",
+			"concurrency":       2,
+			"concurrency_group": "group",
+
+			"agents": newBkAgents("fakerunner"),
+
+			"timeout_in_minutes": defaultTimeoutInMinutes,
+			"artifact_paths":     defaultArtifactPaths,
+			"retry":              defaultRayRetry,
+			"env": map[string]string{
+				"RAYCI_BUILD_ID":            buildID,
+				"RAYCI_TEMP":                "s3://ci-temp/abc123/",
+				"BUILDKITE_BAZEL_CACHE_URL": "https://bazel-build-cache",
+				"RAYCI_WORK_REPO":           "fakeecr",
+				"RAYCI_BRANCH":              "beta",
+				"RAYCI_STEP_ID":             fakeStepID,
+
+				"BUILDKITE_ARTIFACT_UPLOAD_DESTINATION": artifactDest,
+			},
+		},
+	}, {
 		in: map[string]any{
 			"name":       "forge",
 			"label":      "my forge",

raycicmd/rayci_pipeline.go

@@ -43,7 +43,7 @@ var (
 	commandStepAllowedKeys = []string{
 		"command", "commands", "priority", "parallelism", "if",
 		"label", "name", "key", "depends_on", "soft_fail", "matrix",
-		"allow_dependency_failure",
+		"allow_dependency_failure", "concurrency", "concurrency_group",
 
 		// The following keys will be processed by rayci and dropped.
 		"instance_type", "queue", "job_env", "tags",