Scale imports and evaluations ,presign uploads(csv), FlexPrice batching

.github/workflows/docker-build-check.yml

@@ -110,3 +110,54 @@ jobs:
           platforms: linux/amd64
           build-args: |
             INSTALL_EXTRAS=qualitative-voice
+
+  build-worker-harbor:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU (for multi-platform builds)
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        if: github.event_name == 'workflow_dispatch' && inputs.publish == true
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Harbor worker image
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          # Same GHCR package as worker; tags end with -harbor (matches compose + release-and-publish).
+          images: ${{ env.REGISTRY }}/${{ env.WORKER_IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch,suffix=-harbor
+            type=ref,event=pr,suffix=-harbor
+            type=raw,value=latest-harbor,enable={{is_default_branch}}
+
+      - name: Build Harbor worker image (amd64 only for speed)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/Dockerfile.worker
+          push: ${{ github.event_name == 'workflow_dispatch' && inputs.publish == true }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=worker-harbor-check
+          cache-to: type=gha,scope=worker-harbor-check,mode=max
+          platforms: linux/amd64
+          build-args: |
+            PYTHON_VERSION=3.12
+            INSTALL_EXTRAS=harbor
+            TORCH_CPU_ONLY=false

.github/workflows/pr-dev-images.yml

@@ -99,14 +99,26 @@ jobs:
           - service: api
             dockerfile: docker/Dockerfile.api
             image_name_var: API_IMAGE_NAME
+            tag_suffix: ""
             cache_scope: api-pr-dev
             build_args: ""
           - service: worker
             dockerfile: docker/Dockerfile.worker
             image_name_var: WORKER_IMAGE_NAME
+            tag_suffix: ""
             cache_scope: worker-pr-dev
             build_args: |
               INSTALL_EXTRAS=qualitative-voice
+          # Same GHCR package as worker; tags end with -harbor (avoids creating a new package).
+          - service: worker-harbor
+            dockerfile: docker/Dockerfile.worker
+            image_name_var: WORKER_IMAGE_NAME
+            tag_suffix: "-harbor"
+            cache_scope: worker-harbor-pr-dev
+            build_args: |
+              PYTHON_VERSION=3.12
+              INSTALL_EXTRAS=harbor
+              TORCH_CPU_ONLY=false
 
     steps:
       - name: Checkout repository
@@ -141,10 +153,10 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }}
           tags: |
-            type=raw,value=pr-${{ needs.compute-metadata.outputs.pr_number }}
-            type=raw,value=pr-${{ needs.compute-metadata.outputs.pr_number }}-${{ needs.compute-metadata.outputs.short_sha }}
-            type=raw,value=${{ needs.compute-metadata.outputs.dev_version }}
-            type=raw,value=dev-latest-pr-${{ needs.compute-metadata.outputs.pr_number }}
+            type=raw,value=pr-${{ needs.compute-metadata.outputs.pr_number }}${{ matrix.tag_suffix }}
+            type=raw,value=pr-${{ needs.compute-metadata.outputs.pr_number }}-${{ needs.compute-metadata.outputs.short_sha }}${{ matrix.tag_suffix }}
+            type=raw,value=${{ needs.compute-metadata.outputs.dev_version }}${{ matrix.tag_suffix }}
+            type=raw,value=dev-latest-pr-${{ needs.compute-metadata.outputs.pr_number }}${{ matrix.tag_suffix }}
 
       - name: Build and optionally push image
         uses: docker/build-push-action@v5

.github/workflows/release-and-publish.yml

@@ -9,6 +9,7 @@ env:
   REGISTRY: ghcr.io
   API_IMAGE_NAME: ${{ github.repository }}-api
   WORKER_IMAGE_NAME: ${{ github.repository }}-worker
+  WORKER_HARBOR_IMAGE_NAME: ${{ github.repository }}-worker-harbor
 
 permissions:
   contents: write
@@ -180,6 +181,7 @@ jobs:
         run: |
           echo "API_IMAGE_NAME=${API_IMAGE_NAME,,}" >> $GITHUB_ENV
           echo "WORKER_IMAGE_NAME=${WORKER_IMAGE_NAME,,}" >> $GITHUB_ENV
+          echo "WORKER_HARBOR_IMAGE_NAME=${WORKER_HARBOR_IMAGE_NAME,,}" >> $GITHUB_ENV
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -215,13 +217,30 @@ jobs:
           build-args: |
             INSTALL_EXTRAS=qualitative-voice
 
+      - name: Build and push Harbor worker image (by digest)
+        id: build-worker-harbor
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/Dockerfile.worker
+          platforms: ${{ matrix.platform }}
+          outputs: type=image,name=${{ env.REGISTRY }}/${{ env.WORKER_IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=gha,scope=worker-harbor-${{ matrix.slug }}
+          cache-to: type=gha,scope=worker-harbor-${{ matrix.slug }},mode=max
+          build-args: |
+            PYTHON_VERSION=3.12
+            INSTALL_EXTRAS=harbor
+            TORCH_CPU_ONLY=false
+
       - name: Export digests
         run: |
-          mkdir -p /tmp/digests/api /tmp/digests/worker
+          mkdir -p /tmp/digests/api /tmp/digests/worker /tmp/digests/worker-harbor
           api_digest="${{ steps.build-api.outputs.digest }}"
           worker_digest="${{ steps.build-worker.outputs.digest }}"
+          harbor_digest="${{ steps.build-worker-harbor.outputs.digest }}"
           touch "/tmp/digests/api/${api_digest#sha256:}"
           touch "/tmp/digests/worker/${worker_digest#sha256:}"
+          touch "/tmp/digests/worker-harbor/${harbor_digest#sha256:}"
 
       - name: Upload API digest
         uses: actions/upload-artifact@v4
@@ -239,6 +258,14 @@ jobs:
           if-no-files-found: error
           retention-days: 1
 
+      - name: Upload Harbor worker digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: worker-harbor-digest-${{ matrix.slug }}
+          path: /tmp/digests/worker-harbor/*
+          if-no-files-found: error
+          retention-days: 1
+
   # ── Step 3: Merge platform digests into multi-arch manifests ─
   merge:
     needs: [release, build]
@@ -252,6 +279,7 @@ jobs:
         run: |
           echo "API_IMAGE_NAME=${API_IMAGE_NAME,,}" >> $GITHUB_ENV
           echo "WORKER_IMAGE_NAME=${WORKER_IMAGE_NAME,,}" >> $GITHUB_ENV
+          echo "WORKER_HARBOR_IMAGE_NAME=${WORKER_HARBOR_IMAGE_NAME,,}" >> $GITHUB_ENV
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -277,6 +305,13 @@ jobs:
           pattern: worker-digest-*
           merge-multiple: true
 
+      - name: Download Harbor worker digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests/worker-harbor
+          pattern: worker-harbor-digest-*
+          merge-multiple: true
+
       - name: Create API multi-arch manifest
         env:
           VERSION: ${{ needs.release.outputs.version }}
@@ -303,6 +338,19 @@ jobs:
             -t "$IMAGE:latest" \
             $(printf "$IMAGE@sha256:%s " *)
 
+      - name: Create Harbor worker multi-arch manifest
+        env:
+          VERSION: ${{ needs.release.outputs.version }}
+          MAJOR_MINOR: ${{ needs.release.outputs.major_minor }}
+          IMAGE: ${{ env.REGISTRY }}/${{ env.WORKER_IMAGE_NAME }}
+        run: |
+          cd /tmp/digests/worker-harbor
+          docker buildx imagetools create \
+            -t "$IMAGE:${VERSION}-harbor" \
+            -t "$IMAGE:${MAJOR_MINOR}-harbor" \
+            -t "$IMAGE:latest-harbor" \
+            $(printf "$IMAGE@sha256:%s " *)
+
       - name: Create GitHub release
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -137,5 +137,8 @@ enterprise/keys/*.pem
 celerybeat-schedule
 celerybeat-schedule-*
 celerybeat-schedule.*
+.claude/
+.cursor/
+
 
 #

README.md

@@ -520,8 +520,8 @@ gcs:
 # CORS Settings
 cors:
   origins:
-    - "http://localhost:3000"
     - "http://localhost:8000"
+    - "https://sandbox.efficientai.cloud"
 
 # API Settings
 api: