@@ -37,9 +37,8 @@ type ResourceImporterOptions struct {
37
37
}
38
38
39
39
type ImportResourceResult struct {
40
- resource ImportedResource
41
- pending []ImportableResource
42
- err error
40
+ resource ImportedResource // The resource that was imported
41
+ pending []ImportableResource // Any child resources that need to be imported next
43
42
}
44
43
45
44
// New creates a new factory with the scheme baked in
@@ -83,21 +82,25 @@ func (ri *ResourceImporter) Import(
83
82
done chan struct {},
84
83
) (* Result , error ) {
85
84
workersRequired := ri .desiredWorkers ()
85
+
86
86
candidates := make (chan ImportableResource ) // candidates that need to be deduped
87
87
pending := make (chan ImportableResource ) // importers that are pending import
88
- completed := make (chan ImportResourceResult ) // importers that have been executed successfully
89
- report := make (chan * resourceImportReport ) // summary report of the import
88
+ successes := make (chan ImportResourceResult ) // importers that have been executed successfully
89
+ failures := make (chan ImportError ) // errors from importers that failed
90
+ completions := make (chan struct {}) // channel to signal completion
90
91
91
92
// Dedupe candidates so we import each distinct resource only once
92
93
go ri .queueUniqueImporters (candidates , pending , ri .reporter )
93
94
94
95
// Create workers to run the import
95
96
for i := 0 ; i < workersRequired ; i ++ {
96
- go ri .importWorker (ctx , pending , completed , ri .reporter )
97
+ go ri .importWorker (ctx , pending , successes , failures , ri .reporter , completions )
97
98
}
98
99
99
100
// Collate the results
100
- go ri .collateResults (completed , candidates , ri .reporter , report )
101
+ report := newResourceImportReport ()
102
+ go ri .collateResults (successes , candidates , ri .reporter , report , completions )
103
+ go ri .collateErrors (failures , report , completions )
101
104
102
105
// Set up by adding our initial resources; these will be completed when we collate their results
103
106
for _ , rsrc := range ri .resources {
@@ -117,11 +120,15 @@ func (ri *ResourceImporter) Import(
117
120
// Close channels so final reporting and other cleanup occurs
118
121
close (candidates )
119
122
close (pending )
120
- close (completed )
123
+ close (successes )
124
+
125
+ // Wait for everything to finish
126
+ for i := 0 ; i < workersRequired + 2 ; i ++ {
127
+ <- completions
128
+ }
121
129
122
130
// Get the summary report and write it
123
- rpt := <- report
124
- rpt .WriteToLog (ri .log )
131
+ report .WriteToLog (ri .log )
125
132
126
133
// Now we've imported everything, return the resources
127
134
// We do this even if there's an error so that we can return partial results
@@ -186,11 +193,21 @@ func (ri *ResourceImporter) queueUniqueImporters(
186
193
}
187
194
}
188
195
196
+ // importerWorker is a goroutine for importing resources.
197
+ // It reads from the pending channel, imports the resource, and sends the result to the completed
198
+ // channel if it worked, or the failed channel if it didn't.
199
+ // ctx is used to check for cancellation.
200
+ // pending is a source of resources to import.
201
+ // completed is where we send the result of a successful import.
202
+ // failed is where we send the error from a failed import.
203
+ // done is a channel we signal when we're finished.
189
204
func (ri * ResourceImporter ) importWorker (
190
205
ctx context.Context ,
191
206
pending <- chan ImportableResource ,
192
207
completed chan <- ImportResourceResult ,
208
+ failed chan <- ImportError ,
193
209
progress importreporter.Interface ,
210
+ done chan <- struct {},
194
211
) {
195
212
for rsrc := range pending {
196
213
if ctx .Err () != nil {
@@ -201,18 +218,24 @@ func (ri *ResourceImporter) importWorker(
201
218
202
219
// We have a resource to import
203
220
ri .log .V (1 ).Info ("Importing" , "resource" , rsrc .ID ())
204
- result := ri .importResource (ctx , rsrc , progress )
205
- completed <- result
221
+
222
+ if imported , err := ri .importResource (ctx , rsrc , progress ); err != nil {
223
+ failed <- MakeImportError (err , rsrc .GroupKind (), rsrc .Name ())
224
+ } else {
225
+ completed <- imported
226
+ }
206
227
}
228
+
229
+ done <- struct {}{}
207
230
}
208
231
209
232
func (ri * ResourceImporter ) collateResults (
210
233
completed <- chan ImportResourceResult , // completed imports for us to collate
211
234
candidates chan <- ImportableResource , // additional candidates for importing
212
235
progress importreporter.Interface , // importreporter tracking
213
- publish chan <- * resourceImportReport , // publishing our final summary
236
+ report * resourceImportReport , // report to write to
237
+ done chan <- struct {}, // channel to signal completion
214
238
) {
215
- report := newResourceImportReport ()
216
239
for importResult := range completed {
217
240
rsrc := importResult .resource
218
241
gk := rsrc .GroupKind ()
@@ -223,46 +246,54 @@ func (ri *ResourceImporter) collateResults(
223
246
candidates <- p
224
247
}
225
248
226
- if importResult .err != nil {
227
- var skipped * SkippedError
228
- if eris .As (importResult .err , & skipped ) {
229
- ri .log .V (1 ).Info (
230
- "Skipped" ,
231
- "kind" , gk ,
232
- "name" , rsrc .Name (),
233
- "because" , skipped .Because )
234
- report .AddSkippedImport (rsrc , skipped .Because )
235
- } else {
236
- ri .log .Error (importResult .err ,
237
- "Failed" ,
238
- "kind" , gk ,
239
- "name" , rsrc .Name ())
249
+ ri .log .Info (
250
+ "Imported" ,
251
+ "kind" , gk ,
252
+ "name" , rsrc .Name ())
240
253
241
- report .AddFailedImport (rsrc , importResult .err .Error ())
242
- }
243
- } else {
244
- ri .log .Info (
245
- "Imported" ,
246
- "kind" , gk ,
247
- "name" , rsrc .Name ())
248
-
249
- report .AddSuccessfulImport (rsrc )
250
- ri .imported [rsrc .ID ()] = rsrc
251
- }
254
+ report .AddSuccessfulImport (gk )
255
+ ri .imported [rsrc .ID ()] = rsrc
252
256
253
257
// Flag the main resource as complete
254
258
// We do this after everything else because it might indicate we're finished
255
259
progress .Completed (1 )
256
260
}
257
261
258
- publish <- report
262
+ done <- struct {}{}
263
+ }
264
+
265
+ func (ri * ResourceImporter ) collateErrors (
266
+ failures <- chan ImportError ,
267
+ report * resourceImportReport ,
268
+ done chan <- struct {},
269
+ ) {
270
+ for ie := range failures {
271
+ var skipped * SkippedError
272
+ if eris .As (ie .err , & skipped ) {
273
+ ri .log .V (1 ).Info (
274
+ "Skipped" ,
275
+ "kind" , ie .gk ,
276
+ "name" , ie .name ,
277
+ "because" , skipped .Because )
278
+ report .AddSkippedImport (ie .gk , skipped .Because )
279
+ } else {
280
+ ri .log .Error (ie .err ,
281
+ "Failed" ,
282
+ "kind" , ie .gk ,
283
+ "name" , ie .name )
284
+
285
+ report .AddFailedImport (ie .gk , ie .err .Error ())
286
+ }
287
+ }
288
+
289
+ done <- struct {}{}
259
290
}
260
291
261
292
func (ri * ResourceImporter ) importResource (
262
293
ctx context.Context ,
263
294
rsrc ImportableResource ,
264
295
parent importreporter.Interface ,
265
- ) ImportResourceResult {
296
+ ) ( ImportResourceResult , error ) {
266
297
// Import it
267
298
gk := rsrc .GroupKind ()
268
299
name := fmt .Sprintf ("%s %s" , gk , rsrc .Name ())
@@ -273,23 +304,16 @@ func (ri *ResourceImporter) importResource(
273
304
// Our main resource is pending
274
305
progress .AddPending (1 )
275
306
276
- // Import the resource itself
277
- imported , err := rsrc .Import (ctx , ri .log )
278
- result := ImportResourceResult {
279
- resource : imported ,
280
- err : err ,
281
- }
282
-
283
- // If the main resource was imported ok, look for any children
284
- if result .err == nil {
285
- result .pending , result .err = rsrc .FindChildren (ctx , progress )
286
- }
287
-
288
307
// Indicate the main resource is complete
289
- // (we must only do this after checking for children , to ensure we don't appear complete too early)
290
- progress .Completed (1 )
308
+ // (we must only do this when we return , to ensure we don't appear complete too early)
309
+ defer progress .Completed (1 )
291
310
292
- return result
311
+ // Import the resource itself
312
+ if imported , err := rsrc .Import (ctx , progress , ri .log ); err != nil {
313
+ return ImportResourceResult {}, eris .Wrapf (err , "importing %s" , name )
314
+ } else {
315
+ return imported , nil
316
+ }
293
317
}
294
318
295
319
// desiredWorkers returns the number of workers to use for importing resources.
0 commit comments