From 31a60a1ba69eca526fd1605e247f6db583c7ecb1 Mon Sep 17 00:00:00 2001
From: Romain Beuque <556072+rbeuque74@users.noreply.github.com>
Date: Tue, 21 Dec 2021 17:15:02 +0000
Subject: [PATCH 1/2] feat: api: add a route to resolve a templating expression
When debugging a failed task, we often need to resolve templated value
that we struggle to inspect. POST /resolution/:id/templating will
execute a templating expression given as input, and returns the resolved
expression as output. This route can only be used by admins.
Signed-off-by: Romain Beuque <556072+rbeuque74@users.noreply.github.com>
---
api/api_test.go | 99 ++++++++++++++++++++++-----------------
api/handler/resolution.go | 78 ++++++++++++++++++++++++++++++
api/server.go | 8 ++++
engine/engine.go | 56 ++--------------------
utask.go | 74 +++++++++++++++++++++++++++++
5 files changed, 221 insertions(+), 94 deletions(-)
diff --git a/api/api_test.go b/api/api_test.go
index 9ae50c35..8db9707e 100644
--- a/api/api_test.go
+++ b/api/api_test.go
@@ -558,40 +558,6 @@ func waitChecker(dur time.Duration) iffy.Checker {
}
}
-func templatesWithInvalidInputs() []tasktemplate.TaskTemplate {
- var tt []tasktemplate.TaskTemplate
- for _, inp := range []input.Input{
- {
- Name: "input-with-redundant-regex",
- LegalValues: []interface{}{"a", "b", "c"},
- Regex: strPtr("^d.+$"),
- },
- {
- Name: "input-with-bad-regex",
- Regex: strPtr("^^[d.+$"),
- },
- {
- Name: "input-with-bad-type",
- Type: "bad-type",
- },
- {
- Name: "input-with-bad-legal-values",
- Type: "number",
- LegalValues: []interface{}{"a", "b", "c"},
- },
- } {
- tt = append(tt, tasktemplate.TaskTemplate{
- Name: "invalid-template",
- Description: "Invalid template",
- TitleFormat: "Invalid template",
- Inputs: []input.Input{
- inp,
- },
- })
- }
- return tt
-}
-
func templateWithPasswordInput() tasktemplate.TaskTemplate {
return tasktemplate.TaskTemplate{
Name: "input-password",
@@ -706,6 +672,63 @@ func dummyTemplate() tasktemplate.TaskTemplate {
}
}
+func clientErrorTemplate() tasktemplate.TaskTemplate {
+ return tasktemplate.TaskTemplate{
+ Name: "client-error-template",
+ Description: "does nothing",
+ TitleFormat: "this task does nothing at all",
+ Inputs: []input.Input{
+ {
+ Name: "id",
+ },
+ },
+ Variables: []values.Variable{
+ {
+ Name: "var1",
+ Value: "hello id {{.input.id }} for {{ .step.step1.output.foo }} and {{ .step.this.state | default \"BROKEN_TEMPLATING\" }}",
+ },
+ {
+ Name: "var2",
+ Expression: "var a = 3+2; a;",
+ },
+ },
+ Steps: map[string]*step.Step{
+ "step1": {
+ Action: executor.Executor{
+ Type: "echo",
+ Configuration: json.RawMessage(`{
+ "output": {"foo":"bar"}
+ }`),
+ },
+ },
+ "step2": {
+ Action: executor.Executor{
+ Type: "echo",
+ Configuration: json.RawMessage(`{
+ "output": {"foo":"bar"}
+ }`),
+ },
+ Dependencies: []string{"step1"},
+ Conditions: []*condition.Condition{
+ {
+ If: []*condition.Assert{
+ {
+ Expected: "1",
+ Value: "1",
+ Operator: "EQ",
+ },
+ },
+ Then: map[string]string{
+ "this": "CLIENT_ERROR",
+ },
+ Type: "skip",
+ },
+ },
+ },
+ },
+ }
+}
+
func blockedHidden(name string, blocked, hidden bool) tasktemplate.TaskTemplate {
return tasktemplate.TaskTemplate{
Name: name,
@@ -759,12 +782,4 @@ func expectStringPresent(value string) iffy.Checker {
}
}
-func marshalJSON(t *testing.T, i interface{}) string {
- jsonBytes, err := json.Marshal(i)
- if err != nil {
- t.Fatal(err)
- }
- return string(jsonBytes)
-}
-
func strPtr(s string) *string { return &s }
diff --git a/api/handler/resolution.go b/api/handler/resolution.go
index f94c09d9..d651d51a 100644
--- a/api/handler/resolution.go
+++ b/api/handler/resolution.go
@@ -9,6 +9,7 @@ import (
"github.com/loopfz/gadgeto/zesty"
"github.com/sirupsen/logrus"
+ "github.com/ovh/configstore"
"github.com/ovh/utask"
"github.com/ovh/utask/engine"
"github.com/ovh/utask/engine/step"
@@ -932,3 +933,80 @@ func UpdateResolutionStepState(c *gin.Context, in *updateResolutionStepStateIn)
return nil
}
+
+type resolveTemplatingResolutionIn struct {
+ PublicID string `path:"id" validate:"required"`
+ TemplatingExpression string `json:"templating_expression" validate:"required"`
+ StepName string `json:"step_name"`
+}
+
+// ResolveTemplatingResolutionOut is the output of the HTTP route
+// for ResolveTemplatingResolution
+type ResolveTemplatingResolutionOut struct {
+ Result string `json:"result"`
+ Error *string `json:"error,omitempty"`
+}
+
+// ResolveTemplatingResolution will use µtask templating engine for a given resolution
+// to validate a given template. Action is restricted to admin only, as it could be used
+// to exfiltrate configuration.
+func ResolveTemplatingResolution(c *gin.Context, in *resolveTemplatingResolutionIn) (*ResolveTemplatingResolutionOut, error) {
+ metadata.AddActionMetadata(c, metadata.ResolutionID, in.PublicID)
+
+ dbp, err := zesty.NewDBProvider(utask.DBName)
+ if err != nil {
+ return nil, err
+ }
+
+ r, err := resolution.LoadFromPublicID(dbp, in.PublicID)
+ if err != nil {
+ return nil, err
+ }
+
+ t, err := task.LoadFromID(dbp, r.TaskID)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata.AddActionMetadata(c, metadata.TaskID, t.PublicID)
+
+ tt, err := tasktemplate.LoadFromID(dbp, t.TemplateID)
+ if err != nil {
+ return nil, err
+ }
+
+ metadata.AddActionMetadata(c, metadata.TemplateName, tt.Name)
+
+ admin := auth.IsAdmin(c) == nil
+
+ if !admin {
+ return nil, errors.Forbiddenf("You are not allowed to resolve resolution variables")
+ }
+
+ metadata.SetSUDO(c)
+
+ // provide the resolution with values
+ t.ExportTaskInfos(r.Values)
+ r.Values.SetInput(t.Input)
+ r.Values.SetResolverInput(r.ResolverInput)
+ r.Values.SetVariables(tt.Variables)
+
+ config, err := utask.GetTemplatingConfig(configstore.DefaultStore)
+ if err != nil {
+ return nil, err
+ }
+
+ r.Values.SetConfig(config)
+
+ output, err := r.Values.Apply(in.TemplatingExpression, nil, in.StepName)
+ if err != nil {
+ errStr := err.Error()
+ return &ResolveTemplatingResolutionOut{
+ Error: &errStr,
+ }, nil
+ }
+
+ return &ResolveTemplatingResolutionOut{
+ Result: string(output),
+ }, nil
+}
diff --git a/api/server.go b/api/server.go
index 414eda3b..e7f8c6af 100644
--- a/api/server.go
+++ b/api/server.go
@@ -401,6 +401,14 @@ func (s *Server) build(ctx context.Context) {
},
maintenanceMode,
tonic.Handler(handler.UpdateResolutionStepState, 204))
+ resolutionRoutes.POST("/resolution/:id/templating",
+ []fizz.OperationOption{
+ fizz.ID("ResolveTemplatingResolution"),
+ fizz.Summary("Resolve templating of a resolution"),
+ fizz.Description("Resolve the templating of a string, within a task resolution. Admin users only."),
+ },
+ maintenanceMode,
+ tonic.Handler(handler.ResolveTemplatingResolution, 200))
// resolutionRoutes.POST("/resolution/:id/rollback",
// []fizz.OperationOption{
diff --git a/engine/engine.go b/engine/engine.go
index c0991373..a33e9ffb 100644
--- a/engine/engine.go
+++ b/engine/engine.go
@@ -3,14 +3,12 @@ package engine
import (
"bytes"
"context"
- "encoding/json"
"fmt"
"strings"
"sync"
"time"
"github.com/cenkalti/backoff"
- "github.com/ghodss/yaml"
expbk "github.com/jpillora/backoff"
"github.com/juju/errors"
"github.com/loopfz/gadgeto/zesty"
@@ -64,38 +62,13 @@ func Init(ctx context.Context, wg *sync.WaitGroup, store *configstore.Store) err
if err != nil {
return err
}
- // get all configuration items
- itemList, err := store.GetItemList()
- if err != nil {
- return err
- }
- // Squash to ensure that secrets with lower priority
- // are dismissed.
- itemList = configstore.Filter().Squash().Apply(itemList)
- // drop those that shouldnt be available for task execution
- // (don't let DB credentials leak, for instance...)
- config, err := filteredConfig(itemList, cfg.ConcealedSecrets...)
- if err != nil {
+ var engineCfg map[string]interface{}
+ if engineCfg, err = utask.GetTemplatingConfig(store); err != nil {
return err
}
- // attempt to deserialize json formatted config items
- // -> make it easier to access internal nodes/values when templating
- eng.config = make(map[string]interface{})
- for k, v := range config {
- var i interface{}
- if v != nil {
- err := yaml.Unmarshal([]byte(*v), &i, func(dec *json.Decoder) *json.Decoder {
- dec.UseNumber()
- return dec
- })
- if err != nil {
- eng.config[k] = v
- } else {
- eng.config[k] = i
- }
- }
- }
+
+ eng.config = engineCfg
// channels for handling graceful shutdown
shutdownCtx = ctx
@@ -150,27 +123,6 @@ func Init(ctx context.Context, wg *sync.WaitGroup, store *configstore.Store) err
return nil
}
-// filteredConfig takes a configstore item list, drops some items by key
-// then reduces the result into a map of key->values
-func filteredConfig(list *configstore.ItemList, dropAlias ...string) (map[string]*string, error) {
- cfg := make(map[string]*string)
- for _, i := range list.Items {
- if !utils.ListContainsString(dropAlias, i.Key()) {
- // assume only one value per alias
- if _, ok := cfg[i.Key()]; !ok {
- v, err := i.Value()
- if err != nil {
- return nil, err
- }
- if len(v) > 0 {
- cfg[i.Key()] = &v
- }
- }
- }
- }
- return cfg, nil
-}
-
// GetEngine returns the singleton instance of Engine
func GetEngine() Engine {
return eng
diff --git a/utask.go b/utask.go
index c786d747..4ce2921d 100644
--- a/utask.go
+++ b/utask.go
@@ -7,6 +7,7 @@ import (
"fmt"
"time"
+ "github.com/ghodss/yaml"
"golang.org/x/sync/semaphore"
"github.com/ovh/configstore"
@@ -386,3 +387,76 @@ func Config(store *configstore.Store) (*Cfg, error) {
return global, nil
}
+
+// GetTemplatingConfig returns the µTask configuration without sensible piece of configuration
+// such as encryption key, ... to be used by templating functions.
+func GetTemplatingConfig(store *configstore.Store) (map[string]interface{}, error) {
+ cfg, err := Config(store)
+ if err != nil {
+ return nil, err
+ }
+ // get all configuration items
+ itemList, err := store.GetItemList()
+ if err != nil {
+ return nil, err
+ }
+ // Squash to ensure that secrets with lower priority
+ // are dismissed.
+ itemList = configstore.Filter().Squash().Apply(itemList)
+
+ // drop those that shouldnt be available for task execution
+ // (don't let DB credentials leak, for instance...)
+ config, err := filteredConfig(itemList, cfg.ConcealedSecrets...)
+ if err != nil {
+ return nil, err
+ }
+ // attempt to deserialize json formatted config items
+ // -> make it easier to access internal nodes/values when templating
+ c := make(map[string]interface{})
+ for k, v := range config {
+ var i interface{}
+ if v != nil {
+ err := yaml.Unmarshal([]byte(*v), &i, func(dec *json.Decoder) *json.Decoder {
+ dec.UseNumber()
+ return dec
+ })
+ if err != nil {
+ c[k] = v
+ } else {
+ c[k] = i
+ }
+ }
+ }
+ return c, nil
+}
+
+// filteredConfig takes a configstore item list, drops some items by key
+// then reduces the result into a map of key->values
+func filteredConfig(list *configstore.ItemList, dropAlias ...string) (map[string]*string, error) {
+ cfg := make(map[string]*string)
+ for _, i := range list.Items {
+ if !listContainsString(dropAlias, i.Key()) {
+ // assume only one value per alias
+ if _, ok := cfg[i.Key()]; !ok {
+ v, err := i.Value()
+ if err != nil {
+ return nil, err
+ }
+ if len(v) > 0 {
+ cfg[i.Key()] = &v
+ }
+ }
+ }
+ }
+ return cfg, nil
+}
+
+// listContainsString asserts that a string slice contains a given string
+func listContainsString(list []string, item string) bool {
+ for _, i := range list {
+ if i == item {
+ return true
+ }
+ }
+ return false
+}
From 7c898db9a4716ae62ff4243e3fcea7629a10c42b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thomas=20B=C3=A9trancourt?= <thomas@betrancourt.net>
Date: Fri, 6 May 2022 10:22:42 +0000
Subject: [PATCH 2/2] feat: ui: add template expression widget
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Thomas Bétrancourt <thomas@betrancourt.net>
---
.../resolution-expression.component.ts | 71 +++++++++++++++++++
.../resolution-expression.html | 25 +++++++
.../resolution-expression.sass | 13 ++++
.../src/lib/@models/resolution.model.ts | 7 +-
.../utask-lib/src/lib/@routes/task/task.html | 11 +++
.../src/lib/@services/api.service.ts | 11 +++
.../utask-lib/src/lib/utask-lib.module.ts | 2 +
7 files changed, 139 insertions(+), 1 deletion(-)
create mode 100644 ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.component.ts
create mode 100644 ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.html
create mode 100644 ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.sass
diff --git a/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.component.ts b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.component.ts
new file mode 100644
index 00000000..62c024c9
--- /dev/null
+++ b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.component.ts
@@ -0,0 +1,71 @@
+import { ChangeDetectionStrategy, Component, Input } from "@angular/core";
+import { FormBuilder, FormGroup, Validators } from "@angular/forms";
+import { BehaviorSubject } from "rxjs";
+import Resolution from "../../@models/resolution.model";
+import { ApiService } from "../../@services/api.service";
+
+@Component({
+ selector: "lib-utask-resoution-expression",
+ templateUrl: "./resolution-expression.html",
+ styleUrls: ["./resolution-expression.sass"],
+ changeDetection: ChangeDetectionStrategy.OnPush,
+})
+export class ResolutionExpressionComponent {
+ private _resolution: Resolution;
+ private _steps$ = new BehaviorSubject<String[]>([]);
+ private _result$ = new BehaviorSubject<String | null>(null);
+ private _error$ = new BehaviorSubject<String | null>(null);
+
+ readonly formGroup: FormGroup;
+
+ readonly steps$ = this._steps$.asObservable();
+ readonly result$ = this._result$.asObservable();
+ readonly error$ = this._error$.asObservable();
+
+ @Input("resolution") set resolution(r: Resolution) {
+ if ((this._resolution = r)) {
+ this._steps$.next(Object.keys(r.steps));
+ } else {
+ this._steps$.next([]);
+ }
+ }
+
+ get resolution(): Resolution {
+ return this._resolution;
+ }
+
+ constructor(private _api: ApiService, _builder: FormBuilder) {
+ this.formGroup = _builder.group({
+ step: ["", [Validators.required]],
+ expression: ["", [Validators.required]],
+ });
+ }
+
+ reset(): void {
+ this.formGroup.reset();
+ this._result$.next(null);
+ this._error$.next(null);
+ }
+
+ submit(): void {
+ const { step, expression } = this.formGroup.value;
+
+ this._api.resolution
+ .templating(this._resolution, step, expression)
+ .subscribe(
+ (result) => {
+ if (result.error) {
+ this._result$.next(null);
+ this._error$.next(result.error);
+ } else {
+ this._result$.next(result.result);
+ this._error$.next(null);
+ }
+ },
+ (e) => {
+ this._result$.next(null);
+ this._error$.next(e.error.error);
+ }
+ );
+ }
+}
diff --git a/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.html b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.html
new file mode 100644
index 00000000..d43e3878
--- /dev/null
+++ b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.html
@@ -0,0 +1,25 @@
+<form *ngIf="formGroup" nz-form [formGroup]="formGroup" [nzLayout]="'vertical'" (ngSubmit)="submit()">
+ <nz-form-item>
+ <nz-form-label nzFor="step" [nzRequired]="true">Step</nz-form-label>
+ <nz-form-control nzErrorTip="Please select step!">
+ <nz-select formControlName="step" nzPlaceHolder="Please select">
+ <nz-option *ngFor="let step of steps$ | async" [nzLabel]="step" [nzValue]="step"></nz-option>
+ </nz-select>
+ </nz-form-control>
+ </nz-form-item>
+ <nz-form-item>
+ <nz-form-label nzFor="expression" [nzRequired]="true">Template expression</nz-form-label>
+ <nz-form-control nzErrorTip="Please set a valid value for expression!">
+ <input type="string" formControlName="expression" nz-input [placeholder]="'{{.input}}'"/>
+ </nz-form-control>
+ </nz-form-item>
+</form>
+<lib-utask-error-message *ngIf="error$ | async as error" [data]="error">
+</lib-utask-error-message>
+<div class="buttons">
+ <button nz-button nzType="secondary" type="button" (click)="reset()">Reset</button>
+ <button nz-button nzType="primary" [disabled]="!formGroup.valid" type="button" (click)="submit()">Submit</button>
+</div>
+<div *ngIf="result$ | async as result" class="result">
+ <pre>{{ result }}</pre>
+</div>
diff --git a/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.sass b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.sass
new file mode 100644
index 00000000..e200cc3e
--- /dev/null
+++ b/ui/dashboard/projects/utask-lib/src/lib/@components/resolution-expression/resolution-expression.sass
@@ -0,0 +1,13 @@
+.buttons
+ margin-top: 1em
+
+ button
+ margin-right: 0.5em
+
+.result
+ margin-top: 1em
+ border: 1px solid #d9d9d9
+ padding: 1em
+
+ pre
+ margin: 0
diff --git a/ui/dashboard/projects/utask-lib/src/lib/@models/resolution.model.ts b/ui/dashboard/projects/utask-lib/src/lib/@models/resolution.model.ts
index b6141999..aed64136 100644
--- a/ui/dashboard/projects/utask-lib/src/lib/@models/resolution.model.ts
+++ b/ui/dashboard/projects/utask-lib/src/lib/@models/resolution.model.ts
@@ -14,4 +14,9 @@ export default class Resolution {
task_id: string;
task_title: string;
steps: { [key: string]: Step };
-}
\ No newline at end of file
+}
+
+export class TemplateExpression {
+ result: string;
+ error?: string;
+}
diff --git a/ui/dashboard/projects/utask-lib/src/lib/@routes/task/task.html b/ui/dashboard/projects/utask-lib/src/lib/@routes/task/task.html
index 64a3dd80..7f3f8636 100644
--- a/ui/dashboard/projects/utask-lib/src/lib/@routes/task/task.html
+++ b/ui/dashboard/projects/utask-lib/src/lib/@routes/task/task.html
@@ -272,6 +272,17 @@
</div>
</lib-utask-box>
+ <!-- Templating expression box -->
+ <lib-utask-box [header]="{openable: true, init: false, class: 'primary'}"
+ *ngIf="meta.user_is_admin && resolution && resolution.steps">
+ <div app-box-header>
+ Template expression
+ <span *ngIf="loaders.execution || loaders.refreshTask"> <i nz-icon nzType="loading"></i></span>
+ </div>
+ <div app-box-content>
+ <lib-utask-resoution-expression [resolution]="resolution"></lib-utask-resoution-expression>
+ </div>
+ </lib-utask-box>
</ng-container>
diff --git a/ui/dashboard/projects/utask-lib/src/lib/@services/api.service.ts b/ui/dashboard/projects/utask-lib/src/lib/@services/api.service.ts
index c1dc5a10..4ecbabef 100644
--- a/ui/dashboard/projects/utask-lib/src/lib/@services/api.service.ts
+++ b/ui/dashboard/projects/utask-lib/src/lib/@services/api.service.ts
@@ -5,6 +5,7 @@ import { Observable } from 'rxjs';
import { HttpClient, HttpResponse } from '@angular/common/http';
import Meta from '../@models/meta.model';
import Template from '../@models/template.model';
+import Resolution, { TemplateExpression } from '../@models/resolution.model';
export class ParamsListTasks {
page_size?: number;
@@ -314,6 +315,16 @@ export class ApiServiceResolution {
resolution
);
}
+
+ templating(resolution: Resolution, step: string, expression: string) {
+ return this.http.post<TemplateExpression>(
+ `${this.base}resolution/${resolution.id}/templating`,
+ {
+ step_name: step,
+ templating_expression: expression,
+ }
+ );
+ }
}
@Injectable({
diff --git a/ui/dashboard/projects/utask-lib/src/lib/utask-lib.module.ts b/ui/dashboard/projects/utask-lib/src/lib/utask-lib.module.ts
index f7def510..0dfa4c73 100644
--- a/ui/dashboard/projects/utask-lib/src/lib/utask-lib.module.ts
+++ b/ui/dashboard/projects/utask-lib/src/lib/utask-lib.module.ts
@@ -15,6 +15,7 @@ import { InputsFormComponent } from './@components/inputs-form/inputs-form.compo
import { InputTagsComponent } from './@components/input-tags/input-tags.component';
import { InputEditorComponent } from './@components/input-editor/input-editor.component';
import { LoaderComponent } from './@components/loader/loader.component';
+import { ResolutionExpressionComponent } from './@components/resolution-expression/resolution-expression.component';
import { MetaResolve } from './@resolves/meta.resolve';
import { ModalApiYamlComponent } from './@modals/modal-api-yaml/modal-api-yaml.component';
import { ModalEditResolutionStepStateComponent } from './@modals/modal-edit-resolution-step-state/modal-edit-resolution-step-state.component';
@@ -77,6 +78,7 @@ const components: any[] = [
ModalEditResolutionStepStateComponent,
ModalApiYamlEditComponent,
NzModalContentWithErrorComponent,
+ ResolutionExpressionComponent,
StepNodeComponent,
StepsListComponent,
StepsViewerComponent,