Skip to content

Commit fdc58f0

Browse files
tbroden84tbroden84
committed
ClientLibrary: Add retries for status 423 and 503 for older retry functions
1 parent c9a044e commit fdc58f0

File tree

8 files changed

+171
-3
lines changed

8 files changed

+171
-3
lines changed

‎api/disk.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,16 @@ func (api *API) resizeDiskWithRetry(ctx context.Context, id string, params map[s
5959
default:
6060
return nil, fmt.Errorf("failed to resize disk: %s", failed["error"].(string))
6161
}
62+
case 423:
63+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
64+
attempt++
65+
time.Sleep(time.Duration(sleep) * time.Second)
66+
return api.resizeDiskWithRetry(ctx, id, params, attempt, sleep, timeout)
67+
case 503:
68+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
69+
attempt++
70+
time.Sleep(time.Duration(sleep) * time.Second)
71+
return api.resizeDiskWithRetry(ctx, id, params, attempt, sleep, timeout)
6272
}
6373
return nil, fmt.Errorf("failed to resize disk, status=%d message=%s ",
6474
response.StatusCode, failed["error"].(string))

‎api/plugins.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,16 @@ func (api *API) listPluginsWithRetry(ctx context.Context, path string, attempt,
9595
// Instance not found - likely manually deleted
9696
tflog.Debug(ctx, fmt.Sprintf("instance not found (404), likely manually deleted: %s", path))
9797
return nil, fmt.Errorf("instance not found, status=404 message=%s", failed)
98+
case 423:
99+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
100+
attempt++
101+
time.Sleep(time.Duration(sleep) * time.Second)
102+
return api.listPluginsWithRetry(ctx, path, attempt, sleep, timeout)
103+
case 503:
104+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
105+
attempt++
106+
time.Sleep(time.Duration(sleep) * time.Second)
107+
return api.listPluginsWithRetry(ctx, path, attempt, sleep, timeout)
98108
}
99109
return nil, fmt.Errorf("failed to list plugins, status=%d message=%s ",
100110
response.StatusCode, failed)

‎api/plugins_community.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,16 @@ func (api *API) listPluginsCommunityWithRetry(ctx context.Context, path string,
9696
// Instance not found - likely manually deleted
9797
tflog.Debug(ctx, fmt.Sprintf("instance not found (404), likely manually deleted: %s", path))
9898
return nil, fmt.Errorf("instance not found, status=404 message=%s", failed)
99+
case 423:
100+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
101+
attempt++
102+
time.Sleep(time.Duration(sleep) * time.Second)
103+
return api.listPluginsCommunityWithRetry(ctx, path, attempt, sleep, timeout)
104+
case 503:
105+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
106+
attempt++
107+
time.Sleep(time.Duration(sleep) * time.Second)
108+
return api.listPluginsCommunityWithRetry(ctx, path, attempt, sleep, timeout)
99109
}
100110
return nil, fmt.Errorf("failed to list communit plugins, status=%d message=%s ",
101111
response.StatusCode, failed)

‎api/privatelink.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,16 @@ func (api *API) readPrivateLinkWithRetry(ctx context.Context, path string, attem
7878
case 404:
7979
tflog.Warn(ctx, "Privatelink not found")
8080
return nil, nil
81+
case 423:
82+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
83+
attempt++
84+
time.Sleep(time.Duration(sleep) * time.Second)
85+
return api.readPrivateLinkWithRetry(ctx, path, attempt, sleep, timeout)
86+
case 503:
87+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
88+
attempt++
89+
time.Sleep(time.Duration(sleep) * time.Second)
90+
return api.readPrivateLinkWithRetry(ctx, path, attempt, sleep, timeout)
8191
}
8292

8393
return nil, fmt.Errorf("failed to read PrivateLink, status=%d message=%s ",
@@ -160,6 +170,16 @@ func (api *API) waitForEnablePrivatelinkWithRetry(ctx context.Context, instanceI
160170
time.Sleep(time.Duration(sleep) * time.Second)
161171
return api.waitForEnablePrivatelinkWithRetry(ctx, instanceID, attempt, sleep, timeout)
162172
}
173+
case 423:
174+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
175+
attempt++
176+
time.Sleep(time.Duration(sleep) * time.Second)
177+
return api.waitForEnablePrivatelinkWithRetry(ctx, instanceID, attempt, sleep, timeout)
178+
case 503:
179+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
180+
attempt++
181+
time.Sleep(time.Duration(sleep) * time.Second)
182+
return api.waitForEnablePrivatelinkWithRetry(ctx, instanceID, attempt, sleep, timeout)
163183
}
164184

165185
return fmt.Errorf("failed to enable PrivateLink, status=%d message=%s ",

‎api/security_firewall.go‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,10 @@ func (api *API) createFirewallSettingsWithRetry(ctx context.Context, path string
7575
"settings", timeout)
7676
}
7777

78-
switch {
79-
case response.StatusCode == 201:
78+
switch response.StatusCode {
79+
case 201:
8080
return attempt, nil
81-
case response.StatusCode == 400:
81+
case 400:
8282
switch {
8383
case failed["error_code"] == nil:
8484
break
@@ -92,6 +92,16 @@ func (api *API) createFirewallSettingsWithRetry(ctx context.Context, path string
9292
return attempt, fmt.Errorf("firewall rules validation failed due to: %s",
9393
failed["error"].(string))
9494
}
95+
case 423:
96+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
97+
attempt++
98+
time.Sleep(time.Duration(sleep) * time.Second)
99+
return api.createFirewallSettingsWithRetry(ctx, path, params, attempt, sleep, timeout)
100+
case 503:
101+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
102+
attempt++
103+
time.Sleep(time.Duration(sleep) * time.Second)
104+
return api.createFirewallSettingsWithRetry(ctx, path, params, attempt, sleep, timeout)
95105
}
96106
return attempt, fmt.Errorf("failed to create new firewall, status=%d message=%s ",
97107
response.StatusCode, failed)
@@ -170,6 +180,16 @@ func (api *API) updateFirewallSettingsWithRetry(ctx context.Context, path string
170180
return attempt, fmt.Errorf("firewall rules validation failed due to: %s",
171181
failed["error"].(string))
172182
}
183+
case 423:
184+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
185+
attempt++
186+
time.Sleep(time.Duration(sleep) * time.Second)
187+
return api.updateFirewallSettingsWithRetry(ctx, path, params, attempt, sleep, timeout)
188+
case 503:
189+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
190+
attempt++
191+
time.Sleep(time.Duration(sleep) * time.Second)
192+
return api.updateFirewallSettingsWithRetry(ctx, path, params, attempt, sleep, timeout)
173193
}
174194
return attempt, fmt.Errorf("failed to update firewall settings, status=%d message=%s ",
175195
response.StatusCode, failed)
@@ -226,6 +246,16 @@ func (api *API) deleteFirewallSettingsWithRetry(ctx context.Context, path string
226246
return attempt, fmt.Errorf("firewall rules validation failed due to: %s",
227247
failed["error"].(string))
228248
}
249+
case 423:
250+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
251+
attempt++
252+
time.Sleep(time.Duration(sleep) * time.Second)
253+
return api.deleteFirewallSettingsWithRetry(ctx, path, attempt, sleep, timeout)
254+
case 503:
255+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
256+
attempt++
257+
time.Sleep(time.Duration(sleep) * time.Second)
258+
return api.deleteFirewallSettingsWithRetry(ctx, path, attempt, sleep, timeout)
229259
}
230260
return attempt, fmt.Errorf("failed to reset firewall, status=%d message=%s ",
231261
response.StatusCode, failed)

‎api/vpc_connect.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,16 @@ func (api *API) waitForEnableVpcConnectWithRetry(ctx context.Context, instanceID
140140
time.Sleep(time.Duration(sleep) * time.Second)
141141
return api.waitForEnableVpcConnectWithRetry(ctx, instanceID, attempt, sleep, timeout)
142142
}
143+
case 423:
144+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
145+
attempt++
146+
time.Sleep(time.Duration(sleep) * time.Second)
147+
return api.waitForEnableVpcConnectWithRetry(ctx, instanceID, attempt, sleep, timeout)
148+
case 503:
149+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
150+
attempt++
151+
time.Sleep(time.Duration(sleep) * time.Second)
152+
return api.waitForEnableVpcConnectWithRetry(ctx, instanceID, attempt, sleep, timeout)
143153
}
144154

145155
return fmt.Errorf("failed to enable VPC connect, status=%d message=%s ",

‎api/vpc_gcp_peering.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,16 @@ func (api *API) requestVpcGcpPeeringWithRetry(ctx context.Context, path string,
116116
return api.requestVpcGcpPeeringWithRetry(ctx, path, params, waitOnStatus, attempt, sleep,
117117
timeout)
118118
}
119+
case 423:
120+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
121+
attempt++
122+
time.Sleep(time.Duration(sleep) * time.Second)
123+
return api.requestVpcGcpPeeringWithRetry(ctx, path, params, waitOnStatus, attempt, sleep, timeout)
124+
case 503:
125+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
126+
attempt++
127+
time.Sleep(time.Duration(sleep) * time.Second)
128+
return api.requestVpcGcpPeeringWithRetry(ctx, path, params, waitOnStatus, attempt, sleep, timeout)
119129
}
120130
return attempt, nil, fmt.Errorf("failed to request VPC peering, status=%d message=%s ",
121131
response.StatusCode, failed)
@@ -159,6 +169,16 @@ func (api *API) readVpcGcpPeeringWithRetry(ctx context.Context, path string, att
159169
time.Sleep(time.Duration(sleep) * time.Second)
160170
return api.readVpcGcpPeeringWithRetry(ctx, path, attempt, sleep, timeout)
161171
}
172+
case 423:
173+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
174+
attempt++
175+
time.Sleep(time.Duration(sleep) * time.Second)
176+
return api.readVpcGcpPeeringWithRetry(ctx, path, attempt, sleep, timeout)
177+
case 503:
178+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
179+
attempt++
180+
time.Sleep(time.Duration(sleep) * time.Second)
181+
return api.readVpcGcpPeeringWithRetry(ctx, path, attempt, sleep, timeout)
162182
}
163183
return attempt, nil, fmt.Errorf("failed to read VPC peering, status=%d message=%s ",
164184
response.StatusCode, failed)
@@ -210,6 +230,16 @@ func (api *API) removeVpcGcpPeeringWithRetry(ctx context.Context, path string, a
210230
time.Sleep(time.Duration(sleep) * time.Second)
211231
return api.removeVpcGcpPeeringWithRetry(ctx, path, attempt, sleep, timeout)
212232
}
233+
case 423:
234+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
235+
attempt++
236+
time.Sleep(time.Duration(sleep) * time.Second)
237+
return api.removeVpcGcpPeeringWithRetry(ctx, path, attempt, sleep, timeout)
238+
case 503:
239+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
240+
attempt++
241+
time.Sleep(time.Duration(sleep) * time.Second)
242+
return api.removeVpcGcpPeeringWithRetry(ctx, path, attempt, sleep, timeout)
213243
}
214244
return fmt.Errorf("failed to remove VPC peering, status=%d message=%s", response.StatusCode, failed)
215245
}
@@ -250,6 +280,16 @@ func (api *API) readVpcGcpInfoWithRetry(ctx context.Context, path string, attemp
250280
time.Sleep(time.Duration(sleep) * time.Second)
251281
return api.readVpcGcpInfoWithRetry(ctx, path, attempt, sleep, timeout)
252282
}
283+
case 423:
284+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
285+
attempt++
286+
time.Sleep(time.Duration(sleep) * time.Second)
287+
return api.readVpcGcpInfoWithRetry(ctx, path, attempt, sleep, timeout)
288+
case 503:
289+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
290+
attempt++
291+
time.Sleep(time.Duration(sleep) * time.Second)
292+
return api.readVpcGcpInfoWithRetry(ctx, path, attempt, sleep, timeout)
253293
}
254294
return nil, fmt.Errorf("failed to read VPC info, status=%d message=%s ",
255295
response.StatusCode, failed)

‎api/vpc_peering.go‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,16 @@ func (api *API) retryAcceptVpcPeering(ctx context.Context, path string, attempt,
9191
time.Sleep(time.Duration(sleep) * time.Second)
9292
return api.retryAcceptVpcPeering(ctx, path, attempt, sleep, timeout)
9393
}
94+
case 423:
95+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
96+
attempt++
97+
time.Sleep(time.Duration(sleep) * time.Second)
98+
return api.retryAcceptVpcPeering(ctx, path, attempt, sleep, timeout)
99+
case 503:
100+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
101+
attempt++
102+
time.Sleep(time.Duration(sleep) * time.Second)
103+
return api.retryAcceptVpcPeering(ctx, path, attempt, sleep, timeout)
94104
}
95105

96106
return nil, fmt.Errorf("failed to accept VPC peering, status=%d message=%s ",
@@ -125,6 +135,14 @@ func (api *API) readVpcInfoWithRetry(ctx context.Context, path string, attempts,
125135
return nil, fmt.Errorf("failed to read VPC info, status=%d message=%s ",
126136
response.StatusCode, failed)
127137
}
138+
case 423:
139+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, after=%d ", sleep))
140+
time.Sleep(time.Duration(sleep) * time.Second)
141+
return api.readVpcInfoWithRetry(ctx, path, attempts, 2*sleep)
142+
case 503:
143+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, after=%d ", sleep))
144+
time.Sleep(time.Duration(sleep) * time.Second)
145+
return api.readVpcInfoWithRetry(ctx, path, attempts, 2*sleep)
128146
}
129147

130148
return nil, fmt.Errorf("failed to read VPC info, status=%d message=%s ",
@@ -157,6 +175,16 @@ func (api *API) retryRemoveVpcPeering(ctx context.Context, path string, attempt,
157175
time.Sleep(time.Duration(sleep) * time.Second)
158176
return api.retryRemoveVpcPeering(ctx, path, attempt, sleep, timeout)
159177
}
178+
case 423:
179+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
180+
attempt++
181+
time.Sleep(time.Duration(sleep) * time.Second)
182+
return api.retryRemoveVpcPeering(ctx, path, attempt, sleep, timeout)
183+
case 503:
184+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
185+
attempt++
186+
time.Sleep(time.Duration(sleep) * time.Second)
187+
return api.retryRemoveVpcPeering(ctx, path, attempt, sleep, timeout)
160188
}
161189

162190
return fmt.Errorf("failed to remove VPC peering, status=%d message=%s ",
@@ -207,6 +235,16 @@ func (api *API) waitForPeeringStatusWithRetry(ctx context.Context, path, peering
207235
time.Sleep(time.Duration(sleep) * time.Second)
208236
return api.waitForPeeringStatusWithRetry(ctx, path, peeringID, attempt, sleep, timeout)
209237
}
238+
case 423:
239+
tflog.Debug(ctx, fmt.Sprintf("resource is locked, will try again, attempt=%d ", attempt))
240+
attempt++
241+
time.Sleep(time.Duration(sleep) * time.Second)
242+
return api.waitForPeeringStatusWithRetry(ctx, path, peeringID, attempt, sleep, timeout)
243+
case 503:
244+
tflog.Debug(ctx, fmt.Sprintf("service unavailable, will try again, attempt=%d ", attempt))
245+
attempt++
246+
time.Sleep(time.Duration(sleep) * time.Second)
247+
return api.waitForPeeringStatusWithRetry(ctx, path, peeringID, attempt, sleep, timeout)
210248
}
211249

212250
return attempt, fmt.Errorf("failed to accept VPC peering, status=%d message=%s ",

0 commit comments

Comments
 (0)