Skip to content

Commit bc42335

Browse files
Separate success and failure flows in asoctl resource importing (#4452)
* Change interface report to minimize coupling * Create import_error.go * Separate success and failure information flows * Tweak error production * Simplify interfaces * Fix merge
1 parent c39cf82 commit bc42335

6 files changed

+155
-91
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/*
2+
* Copyright (c) Microsoft Corporation.
3+
* Licensed under the MIT license.
4+
*/
5+
6+
package importresources
7+
8+
import (
9+
"fmt"
10+
11+
"k8s.io/apimachinery/pkg/runtime/schema"
12+
)
13+
14+
type ImportError struct {
15+
err error // The error that caused the import to fail
16+
gk schema.GroupKind // The GroupKind of the resource that failed to import
17+
name string // The name of the resource that failed to import
18+
}
19+
20+
var _ error = &ImportError{}
21+
22+
func MakeImportError(err error, gk schema.GroupKind, name string) ImportError {
23+
return ImportError{
24+
err: err,
25+
gk: gk,
26+
name: name,
27+
}
28+
}
29+
30+
func (ie *ImportError) Error() string {
31+
return fmt.Sprintf("failed to import %s %s: %s", ie.gk, ie.name, ie.err.Error())
32+
}
33+
34+
func (ie *ImportError) Unwrap() error {
35+
return ie.err
36+
}

v2/cmd/asoctl/pkg/importresources/importable_arm_resource.go

+19-13
Original file line numberDiff line numberDiff line change
@@ -100,44 +100,50 @@ func (i *importableARMResource) Resource() genruntime.MetaObject {
100100
// ctx is the context to use for the import.
101101
func (i *importableARMResource) Import(
102102
ctx context.Context,
103+
progress importreporter.Interface,
103104
_ logr.Logger,
104-
) (ImportedResource, error) {
105+
) (ImportResourceResult, error) {
105106
// Create an importable blank object into which we capture the current state of the resource
106107
importable, err := i.createImportableObjectFromID(i.owner, i.armID)
107108
if err != nil {
108109
// Error doesn't need additional context
109-
return i, err
110+
return ImportResourceResult{}, err
110111
}
111112

112113
// Our resource might have an extension that can customize the import process,
113114
// so we have a factory to create the loader function we call.
114115
loader := i.createImportFunction(importable)
115-
result, err := loader(ctx, importable, i.owner)
116+
loaderResult, err := loader(ctx, importable, i.owner)
116117
if err != nil {
117118
i.err = err
118-
return i, err
119+
return ImportResourceResult{}, err
119120
}
120121

121-
if because, skipped := result.Skipped(); skipped {
122+
if because, skipped := loaderResult.Skipped(); skipped {
122123
gk := importable.GetObjectKind().GroupVersionKind().GroupKind()
123-
return i, NewSkippedError(gk, i.armID.Name, because, i)
124+
return ImportResourceResult{}, NewSkippedError(gk, i.armID.Name, because, i)
124125
}
125126

126127
i.resource = importable
127128

128-
return i, nil
129-
}
129+
result := ImportResourceResult{
130+
resource: i,
131+
}
132+
133+
if children, err := i.findChildren(ctx, progress); err != nil {
134+
return result, err
135+
} else {
136+
result.pending = children
137+
}
130138

131-
// Error returns any error that occurred during the import.
132-
func (i *importableARMResource) Error() error {
133-
return i.err
139+
return result, nil
134140
}
135141

136-
// FindChildren returns any child resources that need to be imported.
142+
// findChildren returns any child resources that need to be imported.
137143
// ctx allows for cancellation of the import.
138144
// Returns any additional resources that also need to be imported, as well as any errors that occur.
139145
// Partial success is allowed, but the caller should be notified of any errors.
140-
func (i *importableARMResource) FindChildren(
146+
func (i *importableARMResource) findChildren(
141147
ctx context.Context,
142148
progress importreporter.Interface,
143149
) ([]ImportableResource, error) {

v2/cmd/asoctl/pkg/importresources/importable_resource.go

+2-10
Original file line numberDiff line numberDiff line change
@@ -36,18 +36,10 @@ type ImportableResource interface {
3636
// ctx allows for cancellation of the import.
3737
// log allows information about progress to be reported
3838
Import(
39-
ctx context.Context,
40-
log logr.Logger,
41-
) (ImportedResource, error)
42-
43-
// FindChildren returns any child resources that need to be imported.
44-
// ctx allows for cancellation of the import.
45-
// Returns any additional resources that also need to be imported, as well as any errors that occur.
46-
// Partial success is allowed, but the caller should be notified of any errors.
47-
FindChildren(
4839
ctx context.Context,
4940
reporter importreporter.Interface,
50-
) ([]ImportableResource, error)
41+
log logr.Logger,
42+
) (ImportResourceResult, error)
5143
}
5244

5345
// importableResource is a core of common data and support methods for implementing ImportableResource

v2/cmd/asoctl/pkg/importresources/imported_resource.go

-3
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,4 @@ type ImportedResource interface {
2626

2727
// Resource returns the actual resource that has been imported.
2828
Resource() genruntime.MetaObject
29-
30-
// Error returns any error that occurred during the import.
31-
Error() error
3229
}

v2/cmd/asoctl/pkg/importresources/resource_importer.go

+80-56
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,8 @@ type ResourceImporterOptions struct {
3737
}
3838

3939
type ImportResourceResult struct {
40-
resource ImportedResource
41-
pending []ImportableResource
42-
err error
40+
resource ImportedResource // The resource that was imported
41+
pending []ImportableResource // Any child resources that need to be imported next
4342
}
4443

4544
// New creates a new factory with the scheme baked in
@@ -83,21 +82,25 @@ func (ri *ResourceImporter) Import(
8382
done chan struct{},
8483
) (*Result, error) {
8584
workersRequired := ri.desiredWorkers()
85+
8686
candidates := make(chan ImportableResource) // candidates that need to be deduped
8787
pending := make(chan ImportableResource) // importers that are pending import
88-
completed := make(chan ImportResourceResult) // importers that have been executed successfully
89-
report := make(chan *resourceImportReport) // summary report of the import
88+
successes := make(chan ImportResourceResult) // importers that have been executed successfully
89+
failures := make(chan ImportError) // errors from importers that failed
90+
completions := make(chan struct{}) // channel to signal completion
9091

9192
// Dedupe candidates so we import each distinct resource only once
9293
go ri.queueUniqueImporters(candidates, pending, ri.reporter)
9394

9495
// Create workers to run the import
9596
for i := 0; i < workersRequired; i++ {
96-
go ri.importWorker(ctx, pending, completed, ri.reporter)
97+
go ri.importWorker(ctx, pending, successes, failures, ri.reporter, completions)
9798
}
9899

99100
// Collate the results
100-
go ri.collateResults(completed, candidates, ri.reporter, report)
101+
report := newResourceImportReport()
102+
go ri.collateResults(successes, candidates, ri.reporter, report, completions)
103+
go ri.collateErrors(failures, report, completions)
101104

102105
// Set up by adding our initial resources; these will be completed when we collate their results
103106
for _, rsrc := range ri.resources {
@@ -117,11 +120,15 @@ func (ri *ResourceImporter) Import(
117120
// Close channels so final reporting and other cleanup occurs
118121
close(candidates)
119122
close(pending)
120-
close(completed)
123+
close(successes)
124+
125+
// Wait for everything to finish
126+
for i := 0; i < workersRequired+2; i++ {
127+
<-completions
128+
}
121129

122130
// Get the summary report and write it
123-
rpt := <-report
124-
rpt.WriteToLog(ri.log)
131+
report.WriteToLog(ri.log)
125132

126133
// Now we've imported everything, return the resources
127134
// We do this even if there's an error so that we can return partial results
@@ -186,11 +193,21 @@ func (ri *ResourceImporter) queueUniqueImporters(
186193
}
187194
}
188195

196+
// importerWorker is a goroutine for importing resources.
197+
// It reads from the pending channel, imports the resource, and sends the result to the completed
198+
// channel if it worked, or the failed channel if it didn't.
199+
// ctx is used to check for cancellation.
200+
// pending is a source of resources to import.
201+
// completed is where we send the result of a successful import.
202+
// failed is where we send the error from a failed import.
203+
// done is a channel we signal when we're finished.
189204
func (ri *ResourceImporter) importWorker(
190205
ctx context.Context,
191206
pending <-chan ImportableResource,
192207
completed chan<- ImportResourceResult,
208+
failed chan<- ImportError,
193209
progress importreporter.Interface,
210+
done chan<- struct{},
194211
) {
195212
for rsrc := range pending {
196213
if ctx.Err() != nil {
@@ -201,18 +218,24 @@ func (ri *ResourceImporter) importWorker(
201218

202219
// We have a resource to import
203220
ri.log.V(1).Info("Importing", "resource", rsrc.ID())
204-
result := ri.importResource(ctx, rsrc, progress)
205-
completed <- result
221+
222+
if imported, err := ri.importResource(ctx, rsrc, progress); err != nil {
223+
failed <- MakeImportError(err, rsrc.GroupKind(), rsrc.Name())
224+
} else {
225+
completed <- imported
226+
}
206227
}
228+
229+
done <- struct{}{}
207230
}
208231

209232
func (ri *ResourceImporter) collateResults(
210233
completed <-chan ImportResourceResult, // completed imports for us to collate
211234
candidates chan<- ImportableResource, // additional candidates for importing
212235
progress importreporter.Interface, // importreporter tracking
213-
publish chan<- *resourceImportReport, // publishing our final summary
236+
report *resourceImportReport, // report to write to
237+
done chan<- struct{}, // channel to signal completion
214238
) {
215-
report := newResourceImportReport()
216239
for importResult := range completed {
217240
rsrc := importResult.resource
218241
gk := rsrc.GroupKind()
@@ -223,46 +246,54 @@ func (ri *ResourceImporter) collateResults(
223246
candidates <- p
224247
}
225248

226-
if importResult.err != nil {
227-
var skipped *SkippedError
228-
if eris.As(importResult.err, &skipped) {
229-
ri.log.V(1).Info(
230-
"Skipped",
231-
"kind", gk,
232-
"name", rsrc.Name(),
233-
"because", skipped.Because)
234-
report.AddSkippedImport(rsrc, skipped.Because)
235-
} else {
236-
ri.log.Error(importResult.err,
237-
"Failed",
238-
"kind", gk,
239-
"name", rsrc.Name())
249+
ri.log.Info(
250+
"Imported",
251+
"kind", gk,
252+
"name", rsrc.Name())
240253

241-
report.AddFailedImport(rsrc, importResult.err.Error())
242-
}
243-
} else {
244-
ri.log.Info(
245-
"Imported",
246-
"kind", gk,
247-
"name", rsrc.Name())
248-
249-
report.AddSuccessfulImport(rsrc)
250-
ri.imported[rsrc.ID()] = rsrc
251-
}
254+
report.AddSuccessfulImport(gk)
255+
ri.imported[rsrc.ID()] = rsrc
252256

253257
// Flag the main resource as complete
254258
// We do this after everything else because it might indicate we're finished
255259
progress.Completed(1)
256260
}
257261

258-
publish <- report
262+
done <- struct{}{}
263+
}
264+
265+
func (ri *ResourceImporter) collateErrors(
266+
failures <-chan ImportError,
267+
report *resourceImportReport,
268+
done chan<- struct{},
269+
) {
270+
for ie := range failures {
271+
var skipped *SkippedError
272+
if eris.As(ie.err, &skipped) {
273+
ri.log.V(1).Info(
274+
"Skipped",
275+
"kind", ie.gk,
276+
"name", ie.name,
277+
"because", skipped.Because)
278+
report.AddSkippedImport(ie.gk, skipped.Because)
279+
} else {
280+
ri.log.Error(ie.err,
281+
"Failed",
282+
"kind", ie.gk,
283+
"name", ie.name)
284+
285+
report.AddFailedImport(ie.gk, ie.err.Error())
286+
}
287+
}
288+
289+
done <- struct{}{}
259290
}
260291

261292
func (ri *ResourceImporter) importResource(
262293
ctx context.Context,
263294
rsrc ImportableResource,
264295
parent importreporter.Interface,
265-
) ImportResourceResult {
296+
) (ImportResourceResult, error) {
266297
// Import it
267298
gk := rsrc.GroupKind()
268299
name := fmt.Sprintf("%s %s", gk, rsrc.Name())
@@ -273,23 +304,16 @@ func (ri *ResourceImporter) importResource(
273304
// Our main resource is pending
274305
progress.AddPending(1)
275306

276-
// Import the resource itself
277-
imported, err := rsrc.Import(ctx, ri.log)
278-
result := ImportResourceResult{
279-
resource: imported,
280-
err: err,
281-
}
282-
283-
// If the main resource was imported ok, look for any children
284-
if result.err == nil {
285-
result.pending, result.err = rsrc.FindChildren(ctx, progress)
286-
}
287-
288307
// Indicate the main resource is complete
289-
// (we must only do this after checking for children, to ensure we don't appear complete too early)
290-
progress.Completed(1)
308+
// (we must only do this when we return, to ensure we don't appear complete too early)
309+
defer progress.Completed(1)
291310

292-
return result
311+
// Import the resource itself
312+
if imported, err := rsrc.Import(ctx, progress, ri.log); err != nil {
313+
return ImportResourceResult{}, eris.Wrapf(err, "importing %s", name)
314+
} else {
315+
return imported, nil
316+
}
293317
}
294318

295319
// desiredWorkers returns the number of workers to use for importing resources.

0 commit comments

Comments
 (0)