Make the live management tests reliable: avoid us-east-1, don't replay upload PUTs

singlestoredb/management/manager.py

@@ -52,7 +52,12 @@ def set_organization(kwargs: Dict[str, Any]) -> None:
 #:
 #: :func:`retry_on_lock` is the one POST replay, keyed on an error message this
 #: policy never sees.
-RETRY_METHODS = frozenset(['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE'])
+#:
+#: PUT is absent too. Every PUT the API takes is a Files or Stage create, which
+#: answers a replay of one that landed with 409 "File with the same name
+#: already exists" -- hiding the timeout that caused the replay. A lost
+#: response on an upload is raised as it is, and the caller decides.
+RETRY_METHODS = frozenset(['GET', 'HEAD', 'OPTIONS', 'DELETE'])
 
 #: Status codes worth retrying. These are the transient ones; a 4xx other
 #: than 429 is a client error that will fail again identically.

singlestoredb/tests/test_fusion.py

@@ -1501,12 +1501,7 @@ def setUpClass(cls):
         mgr = s2.manage_clusters(version='v2')
         cls.manager = mgr
 
-        cls.us_regions = [
-            x for x in mgr.regions
-            if 'US' in x.name or 'us-' in (x.region_name or '')
-        ]
-        if not cls.us_regions:
-            raise unittest.SkipTest('No US regions reported by the v2 API')
+        cls.us_regions = utils.us_regions(mgr)
 
         cls.project_id = cls._project_id(mgr)
 
@@ -1765,7 +1760,7 @@ def test_create_drop_cluster(self):
 
         mgr = type(self).manager
         name = f'd-fusion-cluster-{self.id}'
-        region = type(self).us_regions[0]
+        region = random.choice(type(self).us_regions)
 
         try:
             self.cur.execute(

singlestoredb/tests/test_management_utils.py

@@ -26,6 +26,7 @@
 from singlestoredb.tests.utils import admin_password
 from singlestoredb.tests.utils import counting_file_space
 from singlestoredb.tests.utils import counting_stage
+from singlestoredb.tests.utils import us_regions
 
 
 TEST_DIR = pathlib.Path(os.path.dirname(__file__))
@@ -716,6 +717,41 @@ def test_listdir_style_suffix(self):
         assert normalize_remote_path('/', strip_leading=True) + '/' == '/'
 
 
+class TestUSRegions(unittest.TestCase):
+    """Coverage for the region filter the live v2 suites deploy through."""
+
+    @staticmethod
+    def _region(name, provider, region_name):
+        return SimpleNamespace(
+            name=name, provider=provider, region_name=region_name,
+        )
+
+    def test_keeps_us_regions_and_drops_avoided_ones(self):
+        mgr = SimpleNamespace(
+            regions=[
+                self._region('US East 1 (N. Virginia)', 'AWS', 'us-east-1'),
+                self._region('US East 2 (Ohio)', 'AWS', 'us-east-2'),
+                self._region('US East 1 (S. Carolina)', 'GCP', 'us-east1'),
+                self._region('East US 2 (Virginia)', 'Azure', 'eastus2'),
+                self._region('Europe West 1 (Ireland)', 'AWS', 'eu-west-1'),
+            ],
+        )
+        got = [(x.provider, x.region_name) for x in us_regions(mgr)]
+        self.assertEqual(
+            got,
+            [('AWS', 'us-east-2'), ('GCP', 'us-east1'), ('Azure', 'eastus2')],
+        )
+
+    def test_skips_when_only_avoided_regions_are_left(self):
+        mgr = SimpleNamespace(
+            regions=[
+                self._region('US East 1 (N. Virginia)', 'AWS', 'us-east-1'),
+            ],
+        )
+        with self.assertRaises(unittest.SkipTest):
+            us_regions(mgr)
+
+
 class TestSecretFromDictTimestamps(unittest.TestCase):
     """
     Coverage for ``Secret.from_dict`` running its timestamp fields
@@ -2202,6 +2238,7 @@ class Region:
             def __init__(self, name):
                 self.name = name
                 self.region_name = name
+                self.provider = 'AWS'
 
         class Project:
             def __init__(self, edition):
@@ -2320,6 +2357,21 @@ def test_no_us_region_skips_rather_than_failing(self):
 
         self.assertEqual(self.created, [])
 
+    def test_pool_never_deploys_to_an_avoided_region(self):
+        with self._patched(regions=('us-east-1', 'us-east-2')):
+            self.utils.shared_clusters(3)
+
+        self.assertEqual(
+            [kwargs['region'].region_name for _, _, kwargs in self.created],
+            ['us-east-2'] * 3,
+        )
+
+    def test_only_avoided_regions_skips_rather_than_failing(self):
+        with self._patched(regions=('us-east-1',)):
+            with self.assertRaises(unittest.SkipTest):
+                self.utils.shared_clusters(1)
+        self.assertEqual(self.created, [])
+
     def test_no_standard_project_skips_rather_than_failing(self):
         with self._patched(projects=('SHARED',)):
             with self.assertRaises(unittest.SkipTest) as cm:

singlestoredb/tests/test_management_v2.py

@@ -74,14 +74,6 @@ def shared_database_name(s):
     return re.sub(r'[^\w]', '', s).replace('-', '_').lower()
 
 
-def _us_regions(manager):
-    """Return the US regions a v2 manager reports, or skip the test."""
-    out = [x for x in manager.regions if 'US' in x.name or 'us-' in x.name]
-    if not out:
-        raise unittest.SkipTest('No US regions reported by the v2 API')
-    return out
-
-
 def _project_id(manager):
     """
     Return the project ID the live v2 suites deploy into, or skip the test.
@@ -1320,7 +1312,7 @@ class TestCluster(unittest.TestCase):
     def setUpClass(cls):
         cls.manager = s2.manage_clusters(version='v2')
 
-        us_regions = _us_regions(cls.manager)
+        us_regions = utils.us_regions(cls.manager)
 
         name = clean_name(secrets.token_urlsafe(20)[:20])
         region = random.choice(us_regions)

singlestoredb/tests/utils.py

@@ -376,6 +376,32 @@ def admin_password(length: int = 24) -> str:
 #: and the starter deployments accept no such argument.
 DEPLOYMENT_EXPIRES_AT = '2h'
 
+#: (provider, region_name) pairs the live v2 suites never deploy into.
+#:
+#: AWS us-east-1: from 2026-11-04 every S-00 cluster the nightly coverage run
+#: created there sat in PENDING past the 1200s wait, while clusters in the
+#: other US regions came up in minutes. Nothing the suites assert depends on
+#: the region, so leaving it out costs no coverage.
+AVOIDED_REGIONS = {('AWS', 'us-east-1')}
+
+
+def us_regions(manager: Any) -> List[Any]:
+    """
+    Return the US regions a v2 manager reports, less ``AVOIDED_REGIONS``.
+
+    Raises ``unittest.SkipTest`` if none are left.
+
+    """
+    out = [
+        x for x in manager.regions
+        if ('US' in x.name or 'us-' in (x.region_name or ''))
+        and (x.provider, x.region_name) not in AVOIDED_REGIONS
+    ]
+    if not out:
+        raise unittest.SkipTest('No usable US regions reported by the v2 API')
+    return out
+
+
 #: (owner, label, object) for every deployment created so far and not yet
 #: swept. The owner is the test class that was running at creation time, so
 #: a class's leftovers can be dropped when the run leaves that class rather
@@ -1114,13 +1140,11 @@ def setUpClass(cls):
     # not follow the management.version option out of v2 either.
     mgr = s2.manage_clusters(version='v2')
 
-    us_regions = [
-        x for x in mgr.regions
-        if 'US' in x.name or 'us-' in (x.region_name or '')
-    ]
-    if not us_regions:
-        _pool_skip = 'No US regions reported by the v2 API'
-        raise unittest.SkipTest(_pool_skip)
+    try:
+        regions = us_regions(mgr)
+    except unittest.SkipTest as exc:
+        _pool_skip = str(exc)
+        raise
 
     project_id = os.environ.get('SINGLESTOREDB_TEST_PROJECT')
     if not project_id:
@@ -1146,7 +1170,7 @@ def setUpClass(cls):
             _pool.append(
                 mgr.create_cluster(
                     f'cl-test-shared-{len(_pool)}-{_pool_id}',
-                    region=random.choice(us_regions),
+                    region=random.choice(regions),
                     size='S-00',
                     # The v2 suites that deploy their own ask for this, and a
                     # pool cluster stands in for those, so it has to be at
@@ -1258,7 +1282,7 @@ def _get(self, path: Any, params: Any = None, **kwargs: Any) -> Any:
     def _put(self, path: Any, **kwargs: Any) -> Any:
         key = self._key(path)
         if 'isFile=false' in str(path):
-            key = re.sub(r'/*$', r'/', key)
+            key = key.rstrip('/') + '/'
         self.calls.append(('PUT', key))
         self.existing.add(key)
         return SimpleNamespace(