Vanguard: Keep batchId and accept replayed batches

README.md

@@ -118,6 +118,8 @@ nexrender batch submit --file campaign.json
 nexrender batch wait BATCH_ID
 ```
 
+To make a batch safe to resubmit, give it a `batchId` (a ULID you choose) and, optionally, each job a unique `batchItemId`. Submitting the same file again creates only the jobs that are missing. Wait until the first attempt has finished on the server first: a local timeout is not enough, so check `nexrender batch show BATCH_ID` until it stops changing.
+
 Fonts and encrypted secrets are along for the ride too:
 
 ```bash

internal/command/batch.go

@@ -58,10 +58,21 @@ func newBatchSubmitCommand(state *State) *cobra.Command {
 			if err != nil {
 				return client.NetworkError(err)
 			}
-			if response.StatusCode() != http.StatusCreated || response.JSON201 == nil {
-				return client.HTTPError(response.StatusCode(), response.Body, responseHeaders(response.HTTPResponse))
+			switch {
+			case response.StatusCode() == http.StatusCreated && response.JSON201 != nil:
+				return state.Printer(state.Out).Success(*response.JSON201, "Submitted batch "+response.JSON201.BatchId, nil)
+			case response.StatusCode() == http.StatusOK && response.JSON200 != nil:
+				// A retry with a batchId the API already knows: it created only what was missing.
+				result := *response.JSON200
+				return state.Printer(state.Out).Success(result, replaySummary(result), nil)
+			}
+			failure := client.HTTPError(response.StatusCode(), response.Body, responseHeaders(response.HTTPResponse))
+			if response.StatusCode() == http.StatusServiceUnavailable && payload.BatchId != nil {
+				if structured, ok := failure.(*clierr.Error); ok {
+					structured.Hint = "Some jobs may already exist. Run the same command again with the same batchId; only the missing jobs are created."
+				}
 			}
-			return state.Printer(state.Out).Success(*response.JSON201, "Submitted batch "+response.JSON201.BatchId, nil)
+			return failure
 		},
 	}
 	cmd.Flags().StringVarP(&file, "file", "f", "-", "batch JSON file, or - for stdin")
@@ -174,3 +185,19 @@ func newBatchCancelCommand(state *State) *cobra.Command {
 	cmd.Flags().BoolVarP(&yes, "yes", "y", false, "confirm cancellation")
 	return cmd
 }
+
+// replaySummary describes what a retry with a known batchId did. A partial replay can still
+// report jobs that could not be created, as a 201 partial batch does.
+func replaySummary(result api.BatchCreationResponse) string {
+	if result.Status == api.BatchCreationResponseStatusReplayed {
+		return "Batch " + result.BatchId + " already existed; no new jobs were created"
+	}
+	failed := 0
+	if result.Errors != nil {
+		failed = len(*result.Errors)
+	}
+	if failed > 0 {
+		return fmt.Sprintf("Batch %s already existed; created some missing jobs, but %d could not be created (see errors). %d of %d jobs exist", result.BatchId, failed, result.SuccessCount, result.TotalJobs)
+	}
+	return fmt.Sprintf("Batch %s already existed; created the missing jobs (%d of %d now exist)", result.BatchId, result.SuccessCount, result.TotalJobs)
+}

internal/command/command_test.go

@@ -80,6 +80,53 @@ func TestJobListUsesV2AndBearerAuth(t *testing.T) {
 	}
 }
 
+func TestBatchSubmitKeepsBatchIDAndAcceptsReplay(t *testing.T) {
+	var sent map[string]any
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost || r.URL.Path != "/api/v2/batches" {
+			t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+		}
+		if err := json.NewDecoder(r.Body).Decode(&sent); err != nil {
+			t.Fatal(err)
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		io.WriteString(w, `{"batchId":"01ARZ3NDEKTSV4RRFFQ69G5FAV","status":"replayed","totalJobs":1,"successCount":1,"jobs":[{"index":0,"id":"01ARZ3NDEKTSV4RRFFQ69G5FA0","status":"queued","outputUrl":"https://out.example.test/a.mp4"}]}`)
+	}))
+	defer server.Close()
+	batch := `{"batchId":"01ARZ3NDEKTSV4RRFFQ69G5FAV","jobs":[{"template":{"id":"01ARZ3NDEKTSV4RRFFQ69G5FAW","composition":"Main"},"batchItemId":"hero"}]}`
+	state, out, _ := testState(t, server.URL+"/api/v2", strings.NewReader(batch))
+	if code := Execute(context.Background(), state, []string{"--json", "batch", "submit"}); code != 0 {
+		t.Fatalf("exit = %d, output = %s", code, out.String())
+	}
+	if sent["batchId"] != "01ARZ3NDEKTSV4RRFFQ69G5FAV" {
+		t.Fatalf("batchId was not sent: %v", sent)
+	}
+	jobs, _ := sent["jobs"].([]any)
+	if len(jobs) != 1 || jobs[0].(map[string]any)["batchItemId"] != "hero" {
+		t.Fatalf("batchItemId was not sent: %v", sent["jobs"])
+	}
+	if !strings.Contains(out.String(), `"replayed"`) || !strings.Contains(out.String(), "no new jobs were created") {
+		t.Fatalf("unexpected output: %s", out.String())
+	}
+}
+
+func TestBatchReplaySummaryReportsJobsThatStillFailed(t *testing.T) {
+	var result api.BatchCreationResponse
+	body := `{"batchId":"01ARZ3NDEKTSV4RRFFQ69G5FAV","status":"partial","totalJobs":3,"successCount":2,"jobs":[],"errors":[{"index":2,"error":"Template not found","errorCode":"TEMPLATE_NOT_FOUND"}]}`
+	if err := json.Unmarshal([]byte(body), &result); err != nil {
+		t.Fatal(err)
+	}
+	summary := replaySummary(result)
+	if !strings.Contains(summary, "1 could not be created") || !strings.Contains(summary, "2 of 3 jobs exist") {
+		t.Fatalf("summary does not report the failure: %q", summary)
+	}
+	result.Errors = nil
+	if summary := replaySummary(result); !strings.Contains(summary, "created the missing jobs (2 of 3 now exist)") {
+		t.Fatalf("unexpected summary without errors: %q", summary)
+	}
+}
+
 func TestTemplateUploadDoesNotForwardBearer(t *testing.T) {
 	var uploaded []byte
 	uploadServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

openapi/openapi.json

@@ -921,9 +921,14 @@
         "type": "object",
         "description": "Configuration for creating a batch of render jobs in a single request (1 to 1000 jobs)",
         "properties": {
+          "batchId": {
+            "type": "string",
+            "pattern": "^[0-9A-HJKMNP-TV-Za-hjkmnp-tv-z]{26}$",
+            "description": "Optional client-chosen ULID that makes the request safe to retry. Sending a batch again with the same batchId creates only the jobs that are not in it yet: the response is 200 with status 'replayed' when nothing was missing, or 'partial' when the missing jobs were created. Retries are recognised for 30 days, while the batch's jobs are kept. Requests with the same batchId must not run at the same time: two that overlap on the server can both create jobs. A timeout on your side does not mean the server stopped processing, so after a timeout wait until GET /batches/{id} stops changing before retrying. Without it the API generates one, and a retry creates a new batch."
+          },
           "jobs": {
             "type": "array",
-            "description": "Array of job configurations to create as a batch. Each item follows the same schema as a single job creation request.",
+            "description": "Array of job configurations to create as a batch. Each item follows the same schema as a single job creation request, plus an optional `batchItemId` string (at most 255 characters) that identifies the job within the batch when it is retried. Items without one are matched by their position.",
             "items": {
               "$ref": "#/components/schemas/JobCreation"
             },
@@ -943,8 +948,8 @@
           },
           "status": {
             "type": "string",
-            "enum": ["created", "partial", "error"],
-            "description": "'created' if all jobs succeeded, 'partial' if some failed, 'error' if all failed"
+            "enum": ["created", "partial", "error", "replayed"],
+            "description": "'created' if all jobs succeeded, 'partial' if some failed or a retry created only the missing jobs, 'error' if all failed, 'replayed' if a retry with the same batchId found every job already created"
           },
           "totalJobs": {
             "type": "integer",
@@ -954,6 +959,14 @@
             "type": "integer",
             "description": "Number of successfully created jobs"
           },
+          "error": {
+            "type": "string",
+            "description": "Present on 409 and 503 responses: why the batch could not be completed"
+          },
+          "errorCode": {
+            "type": "string",
+            "description": "Present on 409 and 503 responses, for example BATCH_ALREADY_PARTIAL or BATCH_RETRYABLE"
+          },
           "jobs": {
             "type": "array",
             "description": "Array of successfully created job details",
@@ -3161,8 +3174,44 @@
               }
             }
           },
+          "200": {
+            "description": "A retry with an existing batchId. Status 'replayed' when every job already existed, 'partial' when the missing jobs were created now.",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/BatchCreationResponse"
+                }
+              }
+            }
+          },
           "400": {
-            "description": "Invalid request or all jobs in the batch failed",
+            "description": "Invalid request (including INVALID_BATCH_ID) or all jobs in the batch failed",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/BatchCreationResponse"
+                }
+              }
+            }
+          },
+          "409": {
+            "description": "BATCH_ALREADY_PARTIAL: a retry with an existing batchId could not reconcile the jobs already in the batch with the request.",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/BatchCreationResponse"
+                }
+              }
+            }
+          },
+          "503": {
+            "description": "BATCH_RETRYABLE: the database was briefly unavailable part-way through. The jobs created so far are listed. Retry with the same batchId after Retry-After seconds to create only the rest.",
+            "headers": {
+              "Retry-After": {
+                "description": "Seconds to wait before retrying",
+                "schema": { "type": "integer" }
+              }
+            },
             "content": {
               "application/json": {
                 "schema": {