diff --git a/.gitignore b/.gitignore index 696f4e89b..edb9006f9 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,7 @@ __pycache__/ build/ develop-eggs/ dist/ +node_modules/ downloads/ eggs/ .eggs/ diff --git a/AGENTS.md b/AGENTS.md index c2e8b4ff5..aa980e2e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,9 +117,9 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`, one site: a server image, `backend_image` in its env.toml, serving the pages and API, checked at its `health` path; a separate frontend image is deprecated, warned of at put and optional in a bundle, and a stored `backend_docker_image_artifact` reads as the server image), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`), a run only the forms its sandbox's deploys boot (`booted_forms`: on each link, the first form it boots natively, by the deploy's own rule, `boot.first_native`; a compose of a disk adds its container), and a deploy composes first what it boots that no compose started (`ComputerEnv.compose_unstarted`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 (`utils/probing.py`) then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference, over its env.toml's `[settings]` and a computer env's `settings` beside it in `apps`, each by top-level field (`params.app_settings`; checked at put, a required one left to the deploy; the deploy record keeps the merged values as `app_settings`, secrets withheld); its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. Its site route, `/website//`, never forwards a site's data plane or card (`/agentenv/…`, `/.well-known/agent-env.json`), which agentenv reaches through `/svc//`; a site with no frontend image is served by its server image there. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | -| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | +| `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, among them `website_state_verifier`, which reads each checked session of a deployed website env's sites, or a multi env's, at the site's `/go?sid=` through `/svc//` and checks one dot path of its `current_state`, `initial_state` or `state_diff` each, optionally reduced by `len`, `keys` or `bool`, with `eq`, `ne`, `ge`, `gt`, `le`, `lt`, `in` or `contains`, its sites checked before a run against the `env_version` it pins, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index e85d7ac82..71ff5c527 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -322,6 +322,8 @@ def _image(self, entry: BundleEntry, config: dict, ref: EntityRef, references: l return True if value is None: dockerfile = default_dockerfile(ref.path) + if ref.optional and not (entry.path / dockerfile).is_file(): + return True elif isinstance(value, dict) and value.keys() == {"dockerfile"}: dockerfile = value["dockerfile"] elif isinstance(value, dict) and "dockerfile" in value: diff --git a/src/agent_env/cli/env/website.py b/src/agent_env/cli/env/website.py index 9045413b7..7332e3df9 100644 --- a/src/agent_env/cli/env/website.py +++ b/src/agent_env/cli/env/website.py @@ -17,7 +17,8 @@ ) from agent_env.utils.card_naming import card_name_from_github, card_name_from_source from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, build_image -from agent_env.env.envs.website import WebsiteEnv +from agent_env.env.envs.website import WebsiteEnv, health_path_problem +from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_HEALTH_PATH from agent_env.store.ids import derive_id, image_repository @@ -30,14 +31,24 @@ def website(): @website.command() @click.option("--id", "env_id", required=True, help="Env id") -@click.option("--backend-dockerfile", default=None, type=click.Path(exists=True), help="Path to backend Dockerfile") -@click.option("--backend-docker-context", default=None, type=click.Path(exists=True), help="Backend Docker build context (defaults to Dockerfile's directory)") -@click.option("--backend-dockerfile-github-url", default=None, help="GitHub URL to backend Dockerfile") -@click.option("--backend-docker-context-github-url", default=None, help="GitHub URL to backend build context directory") -@click.option("--frontend-dockerfile", default=None, type=click.Path(exists=True), help="Path to frontend Dockerfile") +@click.option("--server-dockerfile", "--backend-dockerfile", "backend_dockerfile", default=None, type=click.Path(exists=True), + help="Path to the Dockerfile of the server image, which serves the site's pages and API") +@click.option("--server-docker-context", "--backend-docker-context", "backend_docker_context", default=None, + type=click.Path(exists=True), help="The server image's build context (defaults to its Dockerfile's directory)") +@click.option("--server-dockerfile-github-url", "--backend-dockerfile-github-url", "backend_dockerfile_github_url", + default=None, help="GitHub URL to the server image's Dockerfile") +@click.option("--server-docker-context-github-url", "--backend-docker-context-github-url", + "backend_docker_context_github_url", default=None, help="GitHub URL to the server image's build context directory") +@click.option("--server-image-ref", "server_image_ref", default=None, + help="A server image already in a registry, e.g. ghcr.io/org/site:v1: registered as it is, with its tag " + "pinned to the digest the registry serves now (instead of a Dockerfile)") +@click.option("--frontend-dockerfile", default=None, type=click.Path(exists=True), + help="Path to a separate frontend Dockerfile (deprecated: build the pages into the server image)") @click.option("--frontend-docker-context", default=None, type=click.Path(exists=True), help="Frontend Docker build context (defaults to Dockerfile's directory)") -@click.option("--frontend-dockerfile-github-url", default=None, help="GitHub URL to frontend Dockerfile") +@click.option("--frontend-dockerfile-github-url", default=None, help="GitHub URL to a separate frontend Dockerfile (deprecated)") @click.option("--frontend-docker-context-github-url", default=None, help="GitHub URL to frontend build context directory") +@click.option("--health-path", default=AGENT_ENV_WEBSITE_HEALTH_PATH, show_default=True, + help="The path that answers 200 once the site is usable") @environment_name_options @env_provider_type_option("What deploys the env: 'gateway' (a gateway in front of the website), or the type of an installed " "agent_env.env_providers plugin", "website") @@ -50,36 +61,44 @@ def put( backend_docker_context: str | None, backend_dockerfile_github_url: str | None, backend_docker_context_github_url: str | None, + server_image_ref: str | None, frontend_dockerfile: str | None, frontend_docker_context: str | None, frontend_dockerfile_github_url: str | None, frontend_docker_context_github_url: str | None, + health_path: str, environment_name: str | None, env_provider_type: str, metadata_pairs: tuple[str, ...], skip_validation: bool, build_platform: str, ): - """Build and upload a website environment (frontend + backend).""" + """Build and upload a website environment: its server image, which serves the site's pages and API, from a + Dockerfile, GitHub or a registry, and any deprecated separate frontend image.""" environment_name = resolve_environment_name(environment_name, allow_missing=True) - # Validate mutual exclusion - if backend_dockerfile and backend_dockerfile_github_url: - click.echo("Error: --backend-dockerfile and --backend-dockerfile-github-url are mutually exclusive", err=True) - sys.exit(1) - if not backend_dockerfile and not backend_dockerfile_github_url: - click.echo("Error: either --backend-dockerfile or --backend-dockerfile-github-url is required", err=True) + if sum(map(bool, (backend_dockerfile, backend_dockerfile_github_url, server_image_ref))) != 1: + click.echo("Error: exactly one of --server-dockerfile, --server-dockerfile-github-url and --server-image-ref is " + "required", err=True) sys.exit(1) if frontend_dockerfile and frontend_dockerfile_github_url: click.echo("Error: --frontend-dockerfile and --frontend-dockerfile-github-url are mutually exclusive", err=True) sys.exit(1) - if not frontend_dockerfile and not frontend_dockerfile_github_url: - click.echo("Error: either --frontend-dockerfile or --frontend-dockerfile-github-url is required", err=True) - sys.exit(1) use_github = bool(backend_dockerfile_github_url) - if use_github != bool(frontend_dockerfile_github_url): - click.echo("Error: backend and frontend must both use local Dockerfiles or both use GitHub URLs", err=True) + if (frontend_dockerfile and not backend_dockerfile) or (frontend_dockerfile_github_url and not use_github): + click.echo("Error: a frontend image is built the way the server image is, both from local Dockerfiles or both " + "from GitHub URLs", err=True) + sys.exit(1) + if server_image_ref and not environment_name: + click.echo("Error: --server-image-ref needs --environment-name, which a registry image's card can't give here", + err=True) + sys.exit(1) + if problem := health_path_problem(health_path): + click.echo(f"Error: {problem}", err=True) sys.exit(1) + if frontend_dockerfile or frontend_dockerfile_github_url: + click.echo("Warning: a separate frontend image is deprecated; build the site's pages into its server image", + err=True) user_metadata = {} for pair in metadata_pairs: @@ -88,12 +107,13 @@ def put( sys.exit(1) key, value = pair.split("=", 1) user_metadata[key] = value + site = {"health_path": health_path} if use_github: if environment_name is None: environment_name = card_name_from_github(backend_dockerfile_github_url, backend_docker_context_github_url, github_token=os.environ.get("GITHUB_TOKEN")) if not environment_name: - click.echo("Error: could not read @environment_card(name=...) from the backend GitHub source; pass --environment-name.", err=True) + click.echo("Error: could not read @environment_card(name=...) from the server image's GitHub source; pass --environment-name.", err=True) sys.exit(1) click.echo(f"Derived environment_name={environment_name!r} from the environment card.") if build_platform != DEFAULT_BUILD_PLATFORM: @@ -105,7 +125,7 @@ def put( def _redraw(): if lines_printed[0] > 0: click.echo(f"\033[{lines_printed[0]}A", nl=False) - click.echo(f"\033[2K {click.style('Backend:', fg='cyan')} {backend_status[0]}") + click.echo(f"\033[2K {click.style('Server:', fg='cyan')} {backend_status[0]}") click.echo(f"\033[2K {click.style('Frontend:', fg='magenta')} {frontend_status[0]}") lines_printed[0] = 2 @@ -121,7 +141,7 @@ def _frontend_progress(step: str, message: str, percent: int) -> None: id=env_id, backend_dockerfile_github_url=backend_dockerfile_github_url, backend_docker_context_github_url=backend_docker_context_github_url, - frontend_dockerfile_github_url=frontend_dockerfile_github_url, + frontend_dockerfile_github_url=frontend_dockerfile_github_url or "", frontend_docker_context_github_url=frontend_docker_context_github_url, environment_name=environment_name, metadata=user_metadata if user_metadata else None, @@ -129,6 +149,7 @@ def _frontend_progress(step: str, message: str, percent: int) -> None: on_frontend_progress=_frontend_progress, github_token=os.environ.get("GITHUB_TOKEN"), env_provider_type=env_provider_type, + **site, )) click.echo(f"Created WebsiteEnv: id={env.id} version={env.version} environment_name={env.environment_name}") if not skip_validation: @@ -137,64 +158,68 @@ def _frontend_progress(step: str, message: str, percent: int) -> None: click.echo(f"Validation task: {instance_id}") return - # Local Docker build path - backend_dockerfile_path = Path(backend_dockerfile) - backend_ctx = Path(backend_docker_context) if backend_docker_context else backend_dockerfile_path.parent backend_id, frontend_id = derive_id(env_id, "backend_image"), derive_id(env_id, "frontend_image") - refuse_unwritable_ids(env_id, backend_id, frontend_id) - backend_tag = image_repository(backend_id) - if environment_name is None: - environment_name = card_name_from_source(str(backend_dockerfile_path), str(backend_ctx)) - if not environment_name: - click.echo("Error: no @environment_card(name=...) found in the backend source; pass --environment-name.", err=True) - sys.exit(1) - click.echo(f"Derived environment_name={environment_name!r} from the environment card.") - - click.echo("Building website backend Docker image...") - build_image(backend_dockerfile_path, backend_ctx, backend_tag, platform=build_platform) - - click.echo("Creating backend DockerImageArtifact...") - backend_artifact = DockerImageArtifact.put( - id=backend_id, - description="Website backend image created from agent-env CLI", - image_name=backend_tag, - ) - click.echo(f"Created backend artifact: id={backend_artifact.id} version={backend_artifact.version}") - - frontend_dockerfile_path = Path(frontend_dockerfile) - frontend_ctx = Path(frontend_docker_context) if frontend_docker_context else frontend_dockerfile_path.parent - frontend_tag = image_repository(frontend_id) - - click.echo("Building website frontend Docker image...") - build_image(frontend_dockerfile_path, frontend_ctx, frontend_tag, platform=build_platform) - - click.echo("Creating frontend DockerImageArtifact...") - frontend_artifact = DockerImageArtifact.put( - id=frontend_id, - description="Website frontend image created from agent-env CLI", - image_name=frontend_tag, - ) - click.echo(f"Created frontend artifact: id={frontend_artifact.id} version={frontend_artifact.version}") - - metadata = detect_env_metadata(backend_dockerfile_path, backend_ctx) + refuse_unwritable_ids(env_id, backend_id, *([frontend_id] if frontend_dockerfile else [])) + metadata: dict[str, str] = {} + if server_image_ref: + if build_platform != DEFAULT_BUILD_PLATFORM: + click.echo(f"Warning: --platform {build_platform!r} is ignored for --server-image-ref, which builds nothing", err=True) + backend_artifact = DockerImageArtifact.put_ref(backend_id, description=f"Website server image for {env_id}", + image_name=server_image_ref) + click.echo(f"Registered server image: id={backend_artifact.id} version={backend_artifact.version} image={backend_artifact.image_name}") + else: + backend_dockerfile_path = Path(backend_dockerfile) + backend_ctx = Path(backend_docker_context) if backend_docker_context else backend_dockerfile_path.parent + if environment_name is None: + environment_name = card_name_from_source(str(backend_dockerfile_path), str(backend_ctx)) + if not environment_name: + click.echo("Error: no @environment_card(name=...) found in the server image's source; pass --environment-name.", err=True) + sys.exit(1) + click.echo(f"Derived environment_name={environment_name!r} from the environment card.") + backend_artifact = _build(backend_id, backend_dockerfile_path, backend_ctx, "server", build_platform) + metadata = detect_env_metadata(backend_dockerfile_path, backend_ctx) + + frontend_artifact = None + if frontend_dockerfile: + frontend_dockerfile_path = Path(frontend_dockerfile) + frontend_ctx = Path(frontend_docker_context) if frontend_docker_context else frontend_dockerfile_path.parent + frontend_artifact = _build(frontend_id, frontend_dockerfile_path, frontend_ctx, "frontend", build_platform) metadata.update(user_metadata) click.echo("Creating WebsiteEnv...") - env = WebsiteEnv.put( - id=env_id, - backend_docker_image_artifact=backend_artifact, - frontend_docker_image_artifact=frontend_artifact, - environment_name=environment_name, - metadata=metadata if metadata else None, - env_provider_type=env_provider_type, - ) - click.echo(f"Created WebsiteEnv: id={env.id} version={env.version} environment_name={env.environment_name} env_provider_type={env.env_provider_type}") + try: + env = WebsiteEnv.put( + id=env_id, + server_docker_image_artifact=backend_artifact, + frontend_docker_image_artifact=frontend_artifact, + environment_name=environment_name, + metadata=metadata if metadata else None, + env_provider_type=env_provider_type, + **site, + ) + except ValueError as e: + raise click.ClickException(str(e)) from e + click.echo(f"Created WebsiteEnv: id={env.id} version={env.version} environment_name={env.environment_name} " + f"env_provider_type={env.env_provider_type}") if not skip_validation: click.echo("\nValidating environment (use --skip-validation to skip)...") instance_id = asyncio.run(env.validate(on_progress=click.echo)) click.echo(f"Validation task: {instance_id}") +def _build(image_id: str, dockerfile: Path, context: Path, role: str, platform: str) -> DockerImageArtifact: + """Build ``dockerfile`` into the image ``image_id`` and store it with its build context, as an a2a agent's image + is stored.""" + tag = image_repository(image_id) + click.echo(f"Building website {role} Docker image...") + build_image(dockerfile, context, tag, platform=platform) + artifact = DockerImageArtifact.put(id=image_id, description=f"Website {role} image created from agent-env CLI", + image_name=tag, build_context_path=str(context), dockerfile_path=str(dockerfile), + platform=platform) + click.echo(f"Created {role} artifact: id={artifact.id} version={artifact.version}") + return artifact + + @website.command(name="load-environment-artifact") @click.option("--id", "env_id", default=None, help="WebsiteEnv id (optional; cross-checked against the instance)") @click.option("--environment-artifact-id", "environment_artifact_id", diff --git a/src/agent_env/entity/refs.py b/src/agent_env/entity/refs.py index e611ca426..4b6c359a6 100644 --- a/src/agent_env/entity/refs.py +++ b/src/agent_env/entity/refs.py @@ -56,7 +56,8 @@ class EntityRef: ref, in the same dict, that pins its version. ``beside`` names the keys a toml table holding the ref may carry next to it, which its type reads (a computer env's app takes ``install``). ``walk``, for a shape a path cannot spell, maps the value at the path's parent to the dicts that hold the - last key, each with its path parts. + last key, each with its path parts. ``optional``, for an image a toml may leave out, means a + folder with no Dockerfile for it builds none. Declarations build one with ``EntityRef.env``, ``EntityRef.agent`` or ``EntityRef.artifact``, passing the keyword fields.""" @@ -69,6 +70,7 @@ class EntityRef: env_type: str | None = None beside: tuple[str, ...] = () walk: Walker | None = None + optional: bool = False @classmethod def env(cls, path: str, **options: Any) -> EntityRef: diff --git a/src/agent_env/env/envs/website.py b/src/agent_env/env/envs/website.py index dd2770c49..64d4b2d5c 100644 --- a/src/agent_env/env/envs/website.py +++ b/src/agent_env/env/envs/website.py @@ -1,4 +1,9 @@ -"""Website environment with frontend and backend containers.""" +"""Website environment: one site, served by one server image that holds its pages and API. A site +built before the server image held its pages may still name a separate frontend image, which is deprecated. + +A deploy reads whether the site is ready from its env card, the protocol layer: ``website/v1``'s ``ready``, as +``computer-app/v1``'s says of an app. ``health_path`` (env.toml's ``health``) is read only for a site whose card has no +``ready``, or that serves no card, and is deprecated.""" from __future__ import annotations @@ -7,6 +12,7 @@ import os import posixpath import shlex +import re import uuid from importlib.metadata import version as pkg_version from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional @@ -24,6 +30,7 @@ ) from agent_env.env.gateway import GatewayMode from agent_env.attribution import Attribution +from agent_env.providers.env_providers.constants import AGENT_ENV_WEBSITE_HEALTH_PATH from agent_env.utils.paths import validate_file_path if TYPE_CHECKING: from agent_env.bundle.authoring import AuthoringContext @@ -33,25 +40,29 @@ logger = logging.getLogger(__name__) - +DEFAULT_HEALTH_PATH = AGENT_ENV_WEBSITE_HEALTH_PATH +_HEALTH_PATH = re.compile(r"/[A-Za-z0-9._~/-]*") class WebsiteEnv(Env): type: ClassVar[str] = "website" - description = "Website environment with frontend and backend containers, deployed through the environment provider its env_provider_type names" + description = "Website environment: one server image serving a site's pages and API, deployed through the environment provider its env_provider_type names" toml_refs: ClassVar[tuple[EntityRef, ...]] = (EntityRef.artifact("backend_image", artifact_type="docker_image"), - EntityRef.artifact("frontend_image", artifact_type="docker_image")) + EntityRef.artifact("frontend_image", artifact_type="docker_image", + optional=True)) toml_keys: ClassVar[dict[str, type]] = {"backend_image": object, "frontend_image": object, "environment_name": str, - "env_provider_type": str} + "health": str, "env_provider_type": str} def __init__( self, id: str, version: Optional[int], - backend_docker_image_artifact: DockerImageArtifact, - frontend_docker_image_artifact: DockerImageArtifact, + server_docker_image_artifact: Optional[DockerImageArtifact] = None, + frontend_docker_image_artifact: Optional[DockerImageArtifact] = None, environment_name: Optional[str] = None, *, + backend_docker_image_artifact: Optional[DockerImageArtifact] = None, + health_path: str = DEFAULT_HEALTH_PATH, metadata: Optional[dict[str, str]] = None, env_provider_type: str = "gateway", ): @@ -60,8 +71,14 @@ def __init__( raise ValueError("environment_name cannot be empty") if not env_provider_type: raise ValueError("env_provider_type cannot be empty") - self.backend_docker_image_artifact = backend_docker_image_artifact + if server_docker_image_artifact is not None and backend_docker_image_artifact is not None: + raise ValueError("a website env names its server image once, as server_docker_image_artifact or as its " + "older name backend_docker_image_artifact") + self.server_docker_image_artifact = server_docker_image_artifact or backend_docker_image_artifact self.frontend_docker_image_artifact = frontend_docker_image_artifact + if problem := health_path_problem(health_path): + raise ValueError(f"website env {id!r}: {problem}") + self.health_path = health_path self.environment_name = environment_name self.env_provider_type = env_provider_type self._sandbox = None @@ -71,18 +88,24 @@ def __init__( self._deployed: Optional[DeployedEnv] = None self._replaced: list[tuple] = [] # the provider and sandbox of each deployment a later deploy replaced, for close() + @property + def backend_docker_image_artifact(self) -> DockerImageArtifact: + """The server image, under the name it had while a site's pages came from a separate frontend image.""" + return self.server_docker_image_artifact + def to_dict(self) -> dict: base = super().to_dict() - base["backend_docker_image_artifact"] = { - "id": self.backend_docker_image_artifact.id, - "version": self.backend_docker_image_artifact.version, - "type": self.backend_docker_image_artifact.type, - } - base["frontend_docker_image_artifact"] = { - "id": self.frontend_docker_image_artifact.id, - "version": self.frontend_docker_image_artifact.version, - "type": self.frontend_docker_image_artifact.type, - } + server = {"id": self.server_docker_image_artifact.id, "version": self.server_docker_image_artifact.version, + "type": self.server_docker_image_artifact.type} + base["server_docker_image_artifact"] = server + base["backend_docker_image_artifact"] = server + if self.frontend_docker_image_artifact is not None: + base["frontend_docker_image_artifact"] = { + "id": self.frontend_docker_image_artifact.id, + "version": self.frontend_docker_image_artifact.version, + "type": self.frontend_docker_image_artifact.type, + } + base["health_path"] = self.health_path base["service_name"] = self.environment_name base["environment_name"] = self.environment_name # dual-write with service_name during the rename base["env_provider_type"] = self.env_provider_type @@ -90,41 +113,60 @@ def to_dict(self) -> dict: @classmethod def from_dict(cls, data: dict) -> WebsiteEnv: - backend_ref = data["backend_docker_image_artifact"] - backend_artifact = Artifact.get(backend_ref["id"], version=backend_ref["version"]) - frontend_ref = data["frontend_docker_image_artifact"] - frontend_artifact = Artifact.get(frontend_ref["id"], version=frontend_ref["version"]) + server_ref = data.get("server_docker_image_artifact") or data["backend_docker_image_artifact"] + frontend_ref = data.get("frontend_docker_image_artifact") return cls( id=data["id"], version=data.get("version"), - backend_docker_image_artifact=backend_artifact, - frontend_docker_image_artifact=frontend_artifact, + server_docker_image_artifact=Artifact.get(server_ref["id"], version=server_ref["version"]), + frontend_docker_image_artifact=(Artifact.get(frontend_ref["id"], version=frontend_ref["version"]) + if frontend_ref else None), environment_name=data["environment_name"] if "environment_name" in data else data["service_name"], + health_path=data.get("health_path", DEFAULT_HEALTH_PATH), metadata=data.get("metadata", {}), env_provider_type=data.get("env_provider_type", "gateway"), ) + @classmethod + def put(cls, **kwargs: Any) -> WebsiteEnv: + """Write the website env, warning when it names a separate frontend image: a site's server image serves its + pages, and a frontend image is deprecated, still deployed for the sites built on one.""" + if kwargs.get("frontend_docker_image_artifact") is not None: + logger.warning(f"website env {kwargs.get('id')!r} names a separate frontend image, which is deprecated: " + "build the site's pages into its server image instead") + return super().put(**kwargs) + @classmethod def from_toml(cls, data: dict, ctx: AuthoringContext) -> WebsiteEnv: - """Write the env authored as ``data`` (its env.toml, with ``backend_image`` and ``frontend_image`` resolved to - image artifacts' ids) under ``ctx.id`` and return it.""" + """Write the env authored as ``data`` (its env.toml, with ``backend_image``, the server image, and any + ``frontend_image`` resolved to image artifacts' ids) under ``ctx.id`` and return it.""" fields = cls.accept_toml(data, ctx) + frontend = fields.get("frontend_image") return cls.put(id=ctx.id, - backend_docker_image_artifact=ctx.artifact(fields["backend_image"], DockerImageArtifact), - frontend_docker_image_artifact=ctx.artifact(fields["frontend_image"], DockerImageArtifact), - environment_name=fields["environment_name"], env_provider_type=fields["env_provider_type"]) + server_docker_image_artifact=ctx.artifact(fields["backend_image"], DockerImageArtifact), + frontend_docker_image_artifact=ctx.artifact(frontend, DockerImageArtifact) if frontend else None, + environment_name=fields["environment_name"], + health_path=fields.get("health", DEFAULT_HEALTH_PATH), + env_provider_type=fields["env_provider_type"]) def to_toml(self, out: ExportContext) -> dict: - """The env.toml of a folder both of this env's images are built from, which ``from_toml`` reads back as it.""" - return {"type": self.type, "environment_name": self.environment_name, - "backend_image": out.image(self.backend_docker_image_artifact, "backend_image"), - "frontend_image": out.image(self.frontend_docker_image_artifact, "frontend_image"), - **self.provider_toml(out, self.env_provider_type)} + """The env.toml of a folder this env's images are built from, which ``from_toml`` reads back as it.""" + toml = {"type": self.type, "environment_name": self.environment_name, + "backend_image": out.image(self.server_docker_image_artifact, "backend_image")} + if self.frontend_docker_image_artifact is not None: + toml["frontend_image"] = out.image(self.frontend_docker_image_artifact, "frontend_image") + if self.health_path != DEFAULT_HEALTH_PATH: + toml["health"] = self.health_path + return {**toml, **self.provider_toml(out, self.env_provider_type)} @classmethod def accept_toml(cls, data: dict, ctx: AuthoringContext) -> dict: - """The keys of ``data``, an env.toml, a website env takes, its environment_name read from the backend image's - @environment_card when it isn't set. Raises BundleError listing every problem.""" + """The keys of ``data``, an env.toml, a website env takes, ``backend_image`` naming its server image, its + environment_name read from that image's @environment_card when it isn't set. Raises BundleError listing every + problem.""" + health = data.get("health", DEFAULT_HEALTH_PATH) + if isinstance(health, str) and (problem := health_path_problem(health)): + ctx.refuse([ctx.config_problem(problem)]) return ctx.accept_env(data, cls, named_by="backend_image") async def deploy(self, ttl_seconds: int = 10800, disk_size_gb: float = 10, gateway_mode: GatewayMode = GatewayMode.PERFORMANCE, cpu: float | None = None, memory_mb: int | None = None, sandbox_type: str | None = None, env_state_type: str | None = None, env_state_instance_id: str | None = None, artifact_id: str | None = None, artifact_version: int | None = None, *, attribution: Optional[Attribution] = None) -> DeployedEnv: @@ -292,25 +334,27 @@ async def put_from_github( on_frontend_progress: ProgressCallback | None = None, github_token: str | None = None, env_provider_type: str = "gateway", + health_path: str = DEFAULT_HEALTH_PATH, ) -> WebsiteEnv: - """Build backend and frontend Docker images from GitHub in parallel and create a WebsiteEnv.""" + """Build the server image, from ``backend_dockerfile_github_url``, and any deprecated frontend image from + GitHub in parallel, and create a WebsiteEnv.""" refuse_local_github_build(id) - backend, frontend = await asyncio.gather( - DockerImageArtifact.put_from_github( - id=derive_id(id, "backend_image"), - dockerfile_github_url=backend_dockerfile_github_url, - docker_context_github_url=backend_docker_context_github_url, - on_progress=on_backend_progress, - github_token=github_token, - ), - DockerImageArtifact.put_from_github( + builds = [DockerImageArtifact.put_from_github( + id=derive_id(id, "backend_image"), + dockerfile_github_url=backend_dockerfile_github_url, + docker_context_github_url=backend_docker_context_github_url, + on_progress=on_backend_progress, + github_token=github_token, + )] + if frontend_dockerfile_github_url: + builds.append(DockerImageArtifact.put_from_github( id=derive_id(id, "frontend_image"), dockerfile_github_url=frontend_dockerfile_github_url, docker_context_github_url=frontend_docker_context_github_url, on_progress=on_frontend_progress, github_token=github_token, - ), - ) + )) + backend, *frontend = await asyncio.gather(*builds) combined_metadata: dict[str, str] = {} user = os.getenv("USER") @@ -320,7 +364,7 @@ async def put_from_github( combined_metadata["agent_env_version"] = pkg_version("agentenv-framework") except Exception: pass - for prefix, build in [("backend", backend), ("frontend", frontend)]: + for prefix, build in [("backend", backend), *(("frontend", build) for build in frontend)]: combined_metadata[f"{prefix}_dockerfile_github_url"] = build.dockerfile_github_url combined_metadata[f"{prefix}_github_owner"] = build.github_owner combined_metadata[f"{prefix}_github_repo"] = build.github_repo @@ -334,10 +378,19 @@ async def put_from_github( env = cls.put( id=id, - backend_docker_image_artifact=backend.artifact, - frontend_docker_image_artifact=frontend.artifact, + server_docker_image_artifact=backend.artifact, + frontend_docker_image_artifact=frontend[0].artifact if frontend else None, environment_name=environment_name, metadata=combined_metadata, env_provider_type=env_provider_type, + health_path=health_path, ) return env + + +def health_path_problem(health_path: str) -> str | None: + """Why ``health_path`` can't be where a site answers once it's usable, or None: it's a path of plain characters, + since the gateway writes it into a health check.""" + if _HEALTH_PATH.fullmatch(health_path): + return None + return f"health: {health_path!r} is not a path of letters, digits, '.', '_', '~', '-' and '/' starting with '/'" diff --git a/src/agent_env/env/gateway/constants.py b/src/agent_env/env/gateway/constants.py index c20c5de23..30bd00031 100644 --- a/src/agent_env/env/gateway/constants.py +++ b/src/agent_env/env/gateway/constants.py @@ -79,6 +79,8 @@ def data_plane_load_timeout_s(size_bytes: int | None) -> int: EXT_STATE_URI = "urn:agentenv:state/v1" EXT_CHANGES_URI = "urn:agentenv:changes/v1" EXT_NOTIFICATIONS_URI = "urn:agentenv:notifications/v1" +EXT_WEBSITE_URI = "urn:agentenv:website/v1" +WEBSITE_READY_OP = "ready" # Rows per GET /changes page: the default, and the most a client may ask for. CHANGES_PAGE_DEFAULT = 1000 diff --git a/src/agent_env/env/gateway/gateway.py b/src/agent_env/env/gateway/gateway.py index e620645c3..73f6358be 100644 --- a/src/agent_env/env/gateway/gateway.py +++ b/src/agent_env/env/gateway/gateway.py @@ -7,6 +7,7 @@ import json import logging import os +import posixpath import socket import subprocess from contextlib import asynccontextmanager @@ -71,7 +72,7 @@ from .child_transport import child_http_client from .clock import Clock, ClockError, _iso from .get_time import GET_TIME_TOOL_NAME, build_get_time_tool -from .rest_proxy import ROLE_HEADER_BYTES, UpstreamResponse, fetch_child_card, forward, rewrite_child_card +from .rest_proxy import ROLE_HEADER_BYTES, UpstreamResponse, fetch_child_card, forward, rewrite_child_card, upstream_url from .triggers import TriggerEngine, TriggerError from .notifications import CHECK_TOOL_NAMES, NotificationError, Notifications, build_check_tools @@ -625,7 +626,11 @@ async def log_write(status: int) -> None: on_done=log_write) async def _proxy_website_request(self, request: Request) -> Response: - """Proxy a website request; with a website or state trigger registered, feed it to the engine and hold barriers.""" + """Proxy a website request; with a website or state trigger registered, feed it to the engine and hold barriers. + A site's data plane and card are never reached through its route, so an agent browsing the site can't read + what grades it; agentenv reaches them through /svc//.""" + if reserved_site_request(request): + return self._json_response({"error": "not found"}, 404) response = await self._proxy_rest_request(request, url_map=self.website_urls) app = request.path_params["service_name"] engine = self._trigger_engine @@ -1500,3 +1505,23 @@ def _call_never_reached_child(exc: BaseException) -> bool: if isinstance(exc, (ClosedResourceError, anyio.BrokenResourceError)): return True return isinstance(exc, McpError) and exc.error.code == 32600 + + +def reserved_site_path(path: str) -> bool: + """Whether ``path``, as a site's server would route it, is its data plane or its env card.""" + normalized = posixpath.normpath("/" + path.lstrip("/")) + return normalized == RPC_PATH or normalized.startswith(RPC_PATH + "/") or normalized == WELL_KNOWN_PATH + + +def reserved_site_request(request: Request) -> bool: + """Whether a request on a site's route reaches its data plane or env card: by the path as routed here, or by the + one its server receives, which is the raw path forwarded, its dot segments removed by httpx and then decoded. The + two differ when an encoded slash sits beside a dot segment (``public%2Fx/../agentenv``). A request-target the proxy + can't build a URL for isn't reserved here: ``forward()`` refuses it.""" + if reserved_site_path("/" + request.path_params.get("path", "")): + return True + try: + sent = httpx.URL(upstream_url("http://site", request)).path + except (ValueError, httpx.InvalidURL): + return False + return reserved_site_path(sent) diff --git a/src/agent_env/providers/env_providers/constants.py b/src/agent_env/providers/env_providers/constants.py index 110ef4446..7a6a00e97 100644 --- a/src/agent_env/providers/env_providers/constants.py +++ b/src/agent_env/providers/env_providers/constants.py @@ -12,6 +12,7 @@ GATEWAY_APP_DIR = "/app" AGENT_ENV_WEBSITE_BACKEND_PORT = 8000 +AGENT_ENV_WEBSITE_HEALTH_PATH = "/api/health" AGENT_ENV_WEBSITE_BACKEND_SUFFIX = "website-backend" AGENT_ENV_WEBSITE_FRONTEND_SUFFIX = "website-frontend" DOCKER_COMPOSE_PATH = f"{GATEWAY_APP_DIR}/docker-compose.yml" diff --git a/src/agent_env/providers/env_providers/env_gateway_provider.py b/src/agent_env/providers/env_providers/env_gateway_provider.py index e9c92fb03..84ca8cb11 100644 --- a/src/agent_env/providers/env_providers/env_gateway_provider.py +++ b/src/agent_env/providers/env_providers/env_gateway_provider.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Optional import httpx +from agentenv_protocol import client as protocol_v1 from agent_env.artifact.artifacts.docker_image import pull_problem from agent_env.attribution import Attribution @@ -22,7 +23,9 @@ from agent_env.env.gateway import AGENT_ENV_GATEWAY_MCP_PORT, GatewayMode from agent_env.env.gateway.constants import ( DEFAULT_MCP_SERVER_NAME, + EXT_WEBSITE_URI, STATE_TOOL_NAME_CLASHES_KEY, + WEBSITE_READY_OP, WELL_KNOWN_PATH, random_mcp_server_name, tool_name_clashes_text, @@ -40,6 +43,7 @@ AGENT_ENV_WEBSITE_BACKEND_PORT, AGENT_ENV_WEBSITE_BACKEND_SUFFIX, AGENT_ENV_WEBSITE_FRONTEND_SUFFIX, + AGENT_ENV_WEBSITE_HEALTH_PATH, DOCKER_COMPOSE_PATH, GATEWAY_APP_DIR, GATEWAY_SERVICE_NAME, @@ -66,6 +70,7 @@ from agent_env.artifact import DockerImageArtifact from agent_env.env.env import DeployedEnv, Env + from agent_env.env.envs.website import WebsiteEnv from agent_env.providers.env_state import DatabaseStateProvider, EnvStateInstance, LocalPostgresStoreSpec logger = logging.getLogger(__name__) @@ -84,6 +89,9 @@ MCP_HC_START_PERIOD = "60s" +WEBSITE_READY_TIMEOUT_S = 300.0 +WEBSITE_READY_POLL_S = 2.0 + @dataclass class MCPServerConfig: @@ -115,11 +123,14 @@ class DeployedGateway: @dataclass class WebsiteConfig: - """Configuration for a website (frontend + backend) in a gateway deployment.""" + """Configuration for a website in a gateway deployment: its server image, which serves the site's pages and API and + says when it's ready by its card's ``website/v1`` ``ready`` (``health_path``, deprecated, for a site whose card + has none), and a deprecated separate frontend image for a site built on one.""" backend_image: str - frontend_image: str + frontend_image: str | None environment_name: str + health_path: str = AGENT_ENV_WEBSITE_HEALTH_PATH @dataclass @@ -467,6 +478,8 @@ def create_docker_compose( if website_configs: website_urls = ",".join( f"{wc.environment_name}=http://{wc.environment_name}-{AGENT_ENV_WEBSITE_FRONTEND_SUFFIX}:80" + if wc.frontend_image else + f"{wc.environment_name}=http://{wc.environment_name}-{AGENT_ENV_WEBSITE_BACKEND_SUFFIX}:{AGENT_ENV_WEBSITE_BACKEND_PORT}" for wc in website_configs ) gateway_env.append(f" - WEBSITE_URLS={website_urls}") @@ -524,7 +537,8 @@ def create_docker_compose( f" - PORT={AGENT_ENV_WEBSITE_BACKEND_PORT}", " - PYTHONUNBUFFERED=1", " healthcheck:", - f' test: ["CMD", "curl", "-f", "http://localhost:{AGENT_ENV_WEBSITE_BACKEND_PORT}/api/health"]', + f' test: ["CMD-SHELL", "curl -sf -o /dev/null http://localhost:{AGENT_ENV_WEBSITE_BACKEND_PORT}{WELL_KNOWN_PATH}' + f' || curl -sf -o /dev/null http://localhost:{AGENT_ENV_WEBSITE_BACKEND_PORT}{wc.health_path}"]', " interval: 10s", " timeout: 5s", " start_period: 30s", @@ -532,6 +546,10 @@ def create_docker_compose( " networks:", " - env-network", "", + ]) + if not wc.frontend_image: + continue + lines.extend([ f" {frontend_name}:", f" image: {wc.frontend_image}", " depends_on:", @@ -886,6 +904,7 @@ async def _stand_up(_spec): card = await self._wait_for_tunnel(gateway_url, timeout=120) if card is None: raise RuntimeError("Tunnel did not become ready in time") + await _wait_for_websites(gateway_url, website_configs or []) card_read_at_utc = datetime.now(timezone.utc).isoformat() await self._refuse_tool_name_clashes(gateway_url) mcp_url = _mcp_url(gateway_url, card) @@ -1482,6 +1501,14 @@ class _GatewayTopology: names_gateway_on_vm: bool # a MultiEnv's record names its gateway sandbox even on a VM +def website_config(env: WebsiteEnv) -> WebsiteConfig: + """A website env as the gateway runs it: its server image, any deprecated frontend image, and its health path.""" + frontend = env.frontend_docker_image_artifact + return WebsiteConfig(backend_image=env.server_docker_image_artifact.image_name, + frontend_image=frontend.image_name if frontend else None, + environment_name=env.environment_name, health_path=env.health_path) + + def _gateway_topology(env: Env) -> _GatewayTopology: """What a gateway fronts for env: a MultiEnv's MCP servers and websites (whatever their own env_provider_type) under its name, a WebsiteEnv's website, or else the env's one MCP server.""" @@ -1498,12 +1525,9 @@ def _gateway_topology(env: Env) -> _GatewayTopology: return _GatewayTopology( mcp_servers=[MCPServerConfig(image=s.docker_image_artifact.image_name, environment_name=s.environment_name) for s in servers], mcp_server_images=[s.docker_image_artifact for s in servers], - website_configs=[ - WebsiteConfig(backend_image=w.backend_docker_image_artifact.image_name, frontend_image=w.frontend_docker_image_artifact.image_name, - environment_name=w.environment_name) - for w in websites - ] or None, - website_images=[image for w in websites for image in (w.backend_docker_image_artifact, w.frontend_docker_image_artifact)] or None, + website_configs=[website_config(w) for w in websites] or None, + website_images=[image for w in websites + for image in (w.server_docker_image_artifact, w.frontend_docker_image_artifact) if image] or None, mcp_server_name=name, names_gateway_on_vm=is_multi, ) @@ -1529,7 +1553,7 @@ def _compose_name_problems(mcp_servers: list[MCPServerConfig], website_configs: MCP server's name), or one is a service every gateway deploy runs.""" names = [c.environment_name for c in [*mcp_servers, *(website_configs or []), *(sidecars or [])]] refs = [*images, *(s.image for s in mcp_servers), *(s.image for s in sidecars or []), - *(ref for w in website_configs or [] for ref in (w.backend_image, w.frontend_image))] + *(ref for w in website_configs or [] for ref in (w.backend_image, w.frontend_image) if ref is not None)] problems = [problem for problem in [*map(environment_name_problem, names), *map(image_reference_problem, refs)] if problem] services = Counter([*(c.environment_name for c in [*mcp_servers, *(sidecars or [])]), @@ -1589,6 +1613,74 @@ def _redact_compose_secrets(compose_content: str) -> str: _TOOLS_PROBE_TIMEOUT_S = 10.0 +class WebsiteContractError(RuntimeError): + """A site breaks ``website/v1`` in a way waiting can't mend, so the deploy refuses it at once.""" + + +async def _wait_for_websites(gateway_url: str, website_configs: list[WebsiteConfig], *, + timeout_s: float = WEBSITE_READY_TIMEOUT_S, poll_s: float = WEBSITE_READY_POLL_S) -> None: + """Wait until each site is ready, as its card says through ``/svc//``: by ``website/v1``'s ``ready``, or, + for a card without it or a site serving none, by its stored health path answering 200, which is deprecated. + Raises naming each site still not ready, and why, once ``timeout_s`` has passed, and at once, with + ``WebsiteContractError``, for a site that answers its card's path with something that isn't a card.""" + deadline = time.monotonic() + timeout_s + waiting = {wc.environment_name: wc for wc in website_configs} + why: dict[str, str] = {} + warned: set[str] = set() + while waiting: + for name, wc in list(waiting.items()): + ready, why[name] = await _website_ready(f"{gateway_url}/svc/{name}", wc.health_path) + if ready is None and name not in warned: + warned.add(name) + logger.warning(f"website {name!r} has no {EXT_WEBSITE_URI} {WEBSITE_READY_OP} on its card, so its " + f"readiness is read at its health path {wc.health_path}, which is deprecated") + if ready is not False: + del waiting[name] + if not waiting: + return + if time.monotonic() >= deadline: + raise RuntimeError("websites not ready after %ds: %s" % ( + timeout_s, "; ".join(f"{name} ({why[name]})" for name in waiting))) + await asyncio.sleep(poll_s) + + +async def _website_ready(base_url: str, health_path: str) -> tuple[Optional[bool], str]: + """Whether the site at ``base_url`` is ready and, when not, why: True or False by its card's ``website/v1`` + ``ready``; with no card or no ``ready`` on it, None once its health path answers 200 and False before.""" + async with httpx.AsyncClient() as client: + try: + card = _card_of(await client.get(base_url + WELL_KNOWN_PATH, timeout=10), base_url) + if card is not None and protocol_v1.find_extension_method(card, EXT_WEBSITE_URI, WEBSITE_READY_OP): + answer = await protocol_v1.invoke_extension(base_url, card, EXT_WEBSITE_URI, method=WEBSITE_READY_OP, + timeout=10) + if isinstance(answer, dict) and answer.get("ready") is True: + return True, "" + problems = answer.get("problems") if isinstance(answer, dict) else None + return False, ", ".join(map(str, problems)) if problems else f"{WEBSITE_READY_OP} says not ready" + health = await client.get(base_url + health_path, timeout=10) + except (httpx.HTTPError, ValueError) as e: + return False, f"{type(e).__name__}: {e}" + return (None, "") if health.status_code == 200 else (False, f"{health_path} answered {health.status_code}") + + +def _card_of(response: httpx.Response, base_url: str) -> Optional[dict]: + """The env card a site answers its card's path with, None when it serves none there (any answer but 200). + ``website/v1`` reserves that path for the card, so a 200 that isn't a JSON object, such as a single-page app's + ``index.html``, breaks the contract and raises ``WebsiteContractError``.""" + if response.status_code != 200: + return None + try: + card = response.json() + except ValueError: + card = None + if not isinstance(card, dict): + kind = response.headers.get("content-type", "").split(";")[0] or "no content type" + raise WebsiteContractError( + f"the website at {base_url} answers {WELL_KNOWN_PATH} with {kind}, not an env card; website/v1 reserves " + f"that path for the card, so serve the card there or answer 404") + return card + + async def _probe_tools(env_id: str | None, gateway: DeployedGateway) -> None: """Log whether the gateway's MCP tools cover every tool its card's children advertise; never raises.""" start = time.monotonic() diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index d7409bb28..10f3d78c8 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -153,7 +153,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s "modify_env_tool_access", "reset_env", "review", "run_container_unit_tests_verifier", "run_docker_container", "snapshot_env", "snapshot_computer_env", "sync_env_clock", "validation_gate_aggregator", "verify_env_card", "verify_env_core_protocol", "verify_mcp_tool_schema", - "verify_spec_conformance", "verify_universe_load_export_roundtrip", + "verify_spec_conformance", "verify_universe_load_export_roundtrip", "website_state_verifier", # They read only what an outside turn leaves: its reply. "agent_prompt_response_verifier", "verify_mcp_env_assessment", # It skips an agent that isn't deployed. diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index d343df2c2..a4457a320 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -50,6 +50,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep from agent_env.task_step.task_steps.verifiers.rubrics_verifier import RubricsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep + from agent_env.task_step.task_steps.verifiers.website_state_verifier import WebsiteStateVerifierTaskStep from agent_env.task_step.task_steps.run_docker_container import RunDockerContainerTaskStep from agent_env.task_step.task_steps.mcp_env_validator.verify_mcp_tool_schema import VerifyMCPToolSchemaTaskStep from agent_env.task_step.task_steps.mcp_env_validator.verify_spec_conformance import VerifySpecConformanceTaskStep @@ -107,6 +108,7 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(InstallAgentTaskStep): InstallAgentTaskStep, _get_type(RubricsVerifierTaskStep): RubricsVerifierTaskStep, _get_type(RunContainerUnitTestsVerifierTaskStep): RunContainerUnitTestsVerifierTaskStep, + _get_type(WebsiteStateVerifierTaskStep): WebsiteStateVerifierTaskStep, _get_type(RunDockerContainerTaskStep): RunDockerContainerTaskStep, _get_type(VerifyMCPToolSchemaTaskStep): VerifyMCPToolSchemaTaskStep, _get_type(VerifySpecConformanceTaskStep): VerifySpecConformanceTaskStep, diff --git a/src/agent_env/task_step/task_steps/verifiers/__init__.py b/src/agent_env/task_step/task_steps/verifiers/__init__.py index 781c29c39..e3e284145 100644 --- a/src/agent_env/task_step/task_steps/verifiers/__init__.py +++ b/src/agent_env/task_step/task_steps/verifiers/__init__.py @@ -10,6 +10,7 @@ from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep from agent_env.task_step.task_steps.verifiers.run_container_unit_tests_verifier import RunContainerUnitTestsVerifierTaskStep from agent_env.task_step.task_steps.verifiers.verify_sandbox import VerifySandboxTaskStep +from agent_env.task_step.task_steps.verifiers.website_state_verifier import WebsiteStateVerifierTaskStep from agent_env.task_step.task_steps.verifiers.aggregate_verifiers import AggregateVerifiersTaskStep __all__ = [ @@ -26,5 +27,6 @@ "ScoreAggregator", "Turn", "VerifySandboxTaskStep", + "WebsiteStateVerifierTaskStep", "aggregate_score", ] diff --git a/src/agent_env/task_step/task_steps/verifiers/website_state_verifier.py b/src/agent_env/task_step/task_steps/verifiers/website_state_verifier.py new file mode 100644 index 000000000..2546c9141 --- /dev/null +++ b/src/agent_env/task_step/task_steps/verifiers/website_state_verifier.py @@ -0,0 +1,217 @@ +"""Verify the state of deployed websites, a website env or a multi env's: each check reads one value from one session +of one site, as the site's /go?sid= reports it ({initial_state, current_state, state_diff}, the verifier +endpoint CUA-Gym sites serve), reached through the gateway's /svc//, and compares it with an expected +value.""" + +from __future__ import annotations + +import logging +import re +import uuid + +import httpx +from dataclasses import asdict, dataclass +from typing import Any, ClassVar, Optional + +from agent_env.entity.refs import EntityRef +from agent_env.env import legacy_protocol +from agent_env.env.env import Env, gateway_url_of +from agent_env.store.base import NotFoundError +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep, TaskStepDependency, attach_retry_config +from agent_env.task_step.task_steps.verifiers.scoring import ScoreAggregator, aggregate_score + +logger = logging.getLogger(__name__) + +SOURCES = ("current_state", "initial_state", "state_diff") +OPS = ("eq", "ne", "ge", "gt", "le", "lt", "in", "contains") +REDUCES = ("len", "keys", "bool") +DEFAULT_SID = "default" +GO_TIMEOUT_S = 30 + + +@dataclass(frozen=True) +class StateCheck: + """One check: ``op`` of the value at ``path`` (dot-separated, a key's ``.`` and ``\\`` escaped with ``\\``, a list + index by number; empty for the whole source) in session ``sid`` of ``site``, a website env's environment_name, + reduced by ``reduce`` first when given, and ``value``. A path that isn't there reads as None.""" + + site: str + path: str + op: str + value: Any = None + sid: str = DEFAULT_SID + source: str = "current_state" + reduce: Optional[str] = None + weight: float = 1.0 + + def problems(self) -> list[str]: + named = f"check of {self.site!r} {self.path!r}" + return ([f"{named}: op {self.op!r} is none of {', '.join(OPS)}"] if self.op not in OPS else []) + ( + [f"{named}: source {self.source!r} is none of {', '.join(SOURCES)}"] if self.source not in SOURCES else []) + ( + [f"{named}: reduce {self.reduce!r} is none of {', '.join(REDUCES)}"] + if self.reduce is not None and self.reduce not in REDUCES else []) + + def result(self, sessions: dict) -> dict: + """The check's result row against ``sessions``, each site's ``/go`` answer by its sid.""" + session = sessions.get(self.site, {}).get(self.sid) + actual = None if session is None else _read(session.get(self.source), self.path, self.source == "state_diff") + if self.reduce is not None: + actual = _reduce(actual, self.reduce) + try: + passed = _compare(actual, self.op, self.value) + except TypeError: + passed = False + return {"criterion": f"{self.site}[{self.sid}].{self.source}{'.' + self.path if self.path else ''}" + f"{' |' + self.reduce if self.reduce else ''} {self.op} {self.value!r}", + "result": passed, "actual": actual, "weight": self.weight, + **({} if session is not None else {"detail": f"{self.site!r} has no session {self.sid!r}"})} + + +def _read(source: Any, path: str, whole_keys: bool) -> Any: + if not path: + return source + if whole_keys: + return source.get(path) if isinstance(source, dict) else None + value = source + for part in _parts(path): + if isinstance(value, dict): + value = value.get(part) + elif isinstance(value, list) and re.fullmatch(r"-?\d+", part) and -len(value) <= int(part) < len(value): + value = value[int(part)] + else: + return None + return value + + +def _parts(path: str) -> list[str]: + """``path``'s keys, split at each ``.`` that no ``\\`` escapes, as the runtime escapes a key's ``.`` and ``\\``.""" + parts, part, chars = [], [], iter(path) + for char in chars: + if char == "\\": + part.append(next(chars, "")) + elif char == ".": + parts.append("".join(part)) + part = [] + else: + part.append(char) + return [*parts, "".join(part)] + + +def _reduce(value: Any, how: str) -> Any: + if how == "bool": + return bool(value) + if how == "keys": + return sorted(value) if isinstance(value, dict) else [] + return len(value) if isinstance(value, (dict, list, str)) else 0 + + +def _compare(actual: Any, op: str, expected: Any) -> bool: + if op == "eq": + return actual == expected + if op == "ne": + return actual != expected + if op == "in": + return actual in expected + if op == "contains": + return actual is not None and expected in actual + if actual is None or expected is None: + return False + return {"ge": actual >= expected, "gt": actual > expected, "le": actual <= expected, "lt": actual < expected}[op] + + +class WebsiteStateVerifierTaskStep(TaskStep): + """Runs ``checks`` against the state the websites of the deployed env ``env_id``, a website env or a multi env, + report at ``/go?sid=``, one result row each, and scores them with ``score_aggregator``. The sites checks name are + checked before a run against ``env_version``, the version the task deploys, when it's given.""" + + type: ClassVar[str] = "website_state_verifier" + entity_refs = (EntityRef.env("env_id", version_field="env_version"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, checks: list[StateCheck | dict], + score_aggregator: Optional[ScoreAggregator | str] = None, verifier_id: Optional[str] = None, + depends_on: Optional[list[TaskStepDependency]] = None, fail_task_on_error: bool = True, + env_version: Optional[int] = None): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id = env_id + self.env_version = env_version + self.checks = [check if isinstance(check, StateCheck) else StateCheck(**check) for check in checks] + self.score_aggregator = ScoreAggregator(score_aggregator) if isinstance(score_aggregator, str) else ( + score_aggregator or ScoreAggregator.ALL_PASS) + self.verifier_id = verifier_id or uuid.uuid4().hex + if problems := [problem for check in self.checks for problem in check.problems()]: + raise ValueError(f"website_state_verifier {id!r}: " + "; ".join(problems)) + + def to_dict(self) -> dict: + return {**super().to_dict(), "env_id": self.env_id, "env_version": self.env_version, + "checks": [asdict(check) for check in self.checks], "score_aggregator": self.score_aggregator.value, + "verifier_id": self.verifier_id} + + @classmethod + def from_dict(cls, data: dict) -> WebsiteStateVerifierTaskStep: + return cls(**cls._base_from_dict(data), env_id=data["env_id"], env_version=data.get("env_version"), + checks=data["checks"], score_aggregator=data.get("score_aggregator"), + verifier_id=data.get("verifier_id")) + + def preflight(self) -> list[str]: + """The env ``env_id`` when it holds no website, and the sites checks name that it doesn't serve, read from + ``env_version``; with no ``env_version`` the sites are left to the run, which reads the version it deploys.""" + try: + env = Env.get(self.env_id, self.env_version) + except NotFoundError: + return [] + if not (sites := _sites(env)): + return [f"website_state_verifier {self.id!r}: {self.env_id} is a {env.type} env with no website; name a " + "website env or a multi env of websites"] + if self.env_version is None: + return [] + if unknown := sorted({check.site for check in self.checks} - sites): + return [f"website_state_verifier {self.id!r}: {', '.join(map(repr, unknown))} " + f"{'is not a site' if len(unknown) == 1 else 'are not sites'} of {self.env_id} " + f"v{self.env_version}, whose sites are {sorted(sites)}"] + return [] + + @classmethod + def put(cls, **kwargs: Any) -> WebsiteStateVerifierTaskStep: + """Store the step once its preflight passes. Refused, naming every problem, before anything is stored.""" + from agent_env.task_step.store import get_task_step_store + + kwargs.setdefault("version", None) + retry_config = kwargs.pop("retry_config", None) + step = cls(**kwargs) + if problems := step.preflight(): + raise ValueError("; ".join(problems)) + if retry_config is not None: + attach_retry_config(step, {"retry_config": retry_config}) + return get_task_step_store().put_document(step) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + deployed = next((d for d in context.deployed_envs if d.env_id == self.env_id), None) + if deployed is None: + raise RuntimeError(f"Env '{self.env_id}' not found in context.deployed_envs") + sites = _sites(Env.get(deployed.env_id, deployed.env_version)) + gateway_url = gateway_url_of(deployed) + sessions: dict[str, dict] = {} + async with httpx.AsyncClient() as client: + for site, sid in sorted({(check.site, check.sid) for check in self.checks}): + if site not in sites: + raise RuntimeError(f"env '{self.env_id}' serves no website {site!r}; its sites are {sorted(sites)}") + base_url = legacy_protocol.environment_base_url(gateway_url, site, mcp=False) + response = await client.get(f"{base_url}/go", params={"sid": sid}, timeout=GO_TIMEOUT_S) + response.raise_for_status() + sessions.setdefault(site, {})[sid] = response.json() + results = [check.result(sessions) for check in self.checks] + score = aggregate_score(results, self.score_aggregator) + context.metadata.setdefault("verifications", {})[self.verifier_id] = {"results": results, "score": score} + logger.info(f"Verification '{self.verifier_id}' complete: {len(results)} checks, score={score}") + return context + + +def _sites(env: Env) -> set[str]: + """The environment_names of the website envs ``env`` is or holds.""" + from agent_env.env.envs.multi_env import MultiEnv + from agent_env.env.envs.website import WebsiteEnv + + if isinstance(env, WebsiteEnv): + return {env.environment_name} + return {child.environment_name for child in env.website_envs} if isinstance(env, MultiEnv) else set() diff --git a/tst/integration/cli/cli_test.py b/tst/integration/cli/cli_test.py index 26b842f2e..1a30c98a7 100644 --- a/tst/integration/cli/cli_test.py +++ b/tst/integration/cli/cli_test.py @@ -157,11 +157,11 @@ def test_multi_put_missing_both_options(self): assert result.exit_code != 0 assert 'At least one' in result.output - def test_website_put_missing_backend_dockerfile(self): + def test_website_put_without_a_server_image_source(self): runner = CliRunner() result = runner.invoke(cli, ['env', 'website', 'put', '--id', 'test']) assert result.exit_code != 0 - assert 'Missing option' in result.output or '--backend-dockerfile' in result.output + assert 'exactly one of --server-dockerfile, --server-dockerfile-github-url and --server-image-ref is required' in result.output @pytest.mark.integration diff --git a/tst/unit/bundle/env_toml_test.py b/tst/unit/bundle/env_toml_test.py index 25c27f82f..236be7b99 100644 --- a/tst/unit/bundle/env_toml_test.py +++ b/tst/unit/bundle/env_toml_test.py @@ -143,6 +143,18 @@ def test_a_website_builds_dockerfile_backend_and_dockerfile_frontend_and_takes_i assert [call["build"][0] for call in builds] == ["Dockerfile.backend", "Dockerfile.frontend"] +def test_a_website_with_only_a_server_dockerfile_builds_one_image(bundle_dir, builds): + _bundle(bundle_dir, {"envs/mail/Dockerfile.backend": DOCKERFILE, "envs/mail/server.py": CARD.format("mail"), + "envs/mail/env.toml": 'type = "website"\nhealth = "/healthz"\n'}, "mail") + + _run(bundle_dir) + env = Env.get(f"{ROOT}/mail") + + assert (env.server_docker_image_artifact.id, env.frontend_docker_image_artifact) == (f"{ROOT}/mail__backend_image", None) + assert env.health_path == "/healthz" + assert [call["build"][0] for call in builds] == ["Dockerfile.backend"] + + def test_a_multi_names_bundle_and_store_envs_and_is_rewritten_when_a_child_is(bundle_dir, builds): with namespace_routing(): MCPServerEnv.put(id="inbox", docker_image_artifact=_store_image("inbox-image"), environment_name="inbox") diff --git a/tst/unit/bundle/website_optional_frontend_test.py b/tst/unit/bundle/website_optional_frontend_test.py new file mode 100644 index 000000000..76672c325 --- /dev/null +++ b/tst/unit/bundle/website_optional_frontend_test.py @@ -0,0 +1,23 @@ +"""A website's frontend image is optional in a bundle: a folder with no Dockerfile for it builds none.""" + +from pathlib import Path +from types import SimpleNamespace + +from agent_env.bundle.resolve import _Resolver +from agent_env.entity.refs import EntityRef + + +def test_an_optional_image_key_with_no_dockerfile_builds_nothing_and_a_required_one_is_refused(tmp_path): + (tmp_path / "Dockerfile.backend").write_text("FROM scratch\n") + entry = SimpleNamespace(path=Path(tmp_path), id="site", kind=SimpleNamespace(store="env")) + resolver = _Resolver.__new__(_Resolver) + resolver.problems, resolver.images = [], [] + resolver._problem = lambda entry, problem: resolver.problems.append(problem) + resolver._implied = lambda image, owner, key, ref, references: resolver.images.append(image.dockerfile) + + for ref in (EntityRef.artifact("backend_image", artifact_type="docker_image"), + EntityRef.artifact("frontend_image", artifact_type="docker_image", optional=True)): + assert resolver._image(entry, {}, ref, []) + assert (resolver.images, resolver.problems) == (["Dockerfile.backend"], []) + assert resolver._image(entry, {}, EntityRef.artifact("frontend_image", artifact_type="docker_image"), []) + assert resolver.problems == ["frontend_image: there is no 'Dockerfile.frontend' in this folder to build"] diff --git a/tst/unit/cli/environment_name_alias_test.py b/tst/unit/cli/environment_name_alias_test.py index 9a598ac2b..6954b728b 100644 --- a/tst/unit/cli/environment_name_alias_test.py +++ b/tst/unit/cli/environment_name_alias_test.py @@ -28,7 +28,6 @@ [ "env", "website", "put", "--id", "x", "--backend-dockerfile-github-url", "https://github.com/o/r/tree/main/b/Dockerfile", - "--frontend-dockerfile-github-url", "https://github.com/o/r/tree/main/f/Dockerfile", "--skip-validation", ], "agent_env.cli.env.website.WebsiteEnv.put_from_github", diff --git a/tst/unit/cli/website_server_image_cli_test.py b/tst/unit/cli/website_server_image_cli_test.py new file mode 100644 index 000000000..1a04d38c9 --- /dev/null +++ b/tst/unit/cli/website_server_image_cli_test.py @@ -0,0 +1,85 @@ +"""agent-env env website put: a site's server image, from a Dockerfile stored with its build context as an a2a +agent's is, or from a registry, checked at its health path; a separate frontend image is optional and warned of.""" + +from unittest.mock import patch + +from click.testing import CliRunner + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store + +from agent_env.cli.env.website import website +from agent_env.env import Env + + +def _stored(id, **_): + return get_artifact_store().put_document(DockerImageArtifact( + id=id, description="", image_name=f"registry.example/{id}@sha256:" + "a" * 64)) + + +def _put(*args): + return CliRunner().invoke(website, ["put", "--id", "mail-site", *args, "--skip-validation"]) + + +def test_a_site_is_built_from_its_server_dockerfile_alone_with_its_build_context(local_stores, tmp_path): + (tmp_path / "Dockerfile").write_text("FROM scratch\n") + built = [] + with patch("agent_env.cli.env.website.build_image", side_effect=lambda *a, **k: built.append(a[2])), \ + patch("agent_env.artifact.artifacts.docker_image.DockerImageArtifact.put", + side_effect=lambda **k: _stored(k["id"])) as stored: + result = _put("--server-dockerfile", str(tmp_path / "Dockerfile"), "--environment-name", "mail", + "--health-path", "/healthz") + assert result.exit_code == 0, result.output + assert len(built) == 1 and "deprecated" not in result.output + assert stored.call_args.kwargs["build_context_path"] == str(tmp_path) + assert stored.call_args.kwargs["dockerfile_path"] == str(tmp_path / "Dockerfile") + env = Env.get("mail-site") + assert (env.frontend_docker_image_artifact, env.health_path) == (None, "/healthz") + + +def test_a_site_registers_a_server_image_already_in_a_registry(local_stores): + with patch("agent_env.cli.env.website.DockerImageArtifact.put_ref", + side_effect=lambda id, **k: _stored(id)) as put_ref: + result = _put("--server-image-ref", "ghcr.io/acme/mail:3", "--environment-name", "mail") + assert result.exit_code == 0, result.output + assert put_ref.call_args.kwargs["image_name"] == "ghcr.io/acme/mail:3" + assert Env.get("mail-site").server_docker_image_artifact.id == "mail-site__backend_image" + + +def test_put_refuses_no_server_image_two_of_them_an_unnamed_registry_image_and_a_bad_health_path(local_stores, tmp_path): + (tmp_path / "Dockerfile").write_text("FROM scratch\n") + assert "exactly one of --server-dockerfile" in _put("--environment-name", "mail").output + assert "exactly one of" in _put("--server-dockerfile", str(tmp_path / "Dockerfile"), + "--server-image-ref", "ghcr.io/acme/mail:3").output + assert "needs --environment-name" in _put("--server-image-ref", "ghcr.io/acme/mail:3").output + with patch("agent_env.cli.env.website.DockerImageArtifact.put_ref", side_effect=lambda id, **k: _stored(id)) as put_ref: + refused = _put("--server-image-ref", "ghcr.io/acme/mail:3", "--environment-name", "mail", "--health-path", "ok") + assert refused.exit_code == 1 and "health: 'ok' is not a path" in refused.output + put_ref.assert_not_called() + + +def test_a_bad_health_path_fails_before_any_build_or_github_clone(local_stores, tmp_path): + (tmp_path / "Dockerfile").write_text("FROM scratch\n") + with patch("agent_env.cli.env.website.build_image") as build, \ + patch("agent_env.cli.env.website.WebsiteEnv.put_from_github") as from_github: + local = _put("--server-dockerfile", str(tmp_path / "Dockerfile"), "--environment-name", "mail", + "--health-path", "healthz") + github = _put("--server-dockerfile-github-url", "https://github.com/acme/mail/blob/main/Dockerfile", + "--environment-name", "mail", "--health-path", "healthz") + for refused in (local, github): + assert refused.exit_code == 1 and "health: 'healthz' is not a path" in refused.output + build.assert_not_called() + from_github.assert_not_called() + + +def test_a_separate_frontend_image_still_builds_and_is_warned_of(local_stores, tmp_path): + (tmp_path / "server.Dockerfile").write_text("FROM scratch\n") + (tmp_path / "frontend.Dockerfile").write_text("FROM scratch\n") + with patch("agent_env.cli.env.website.build_image"), \ + patch("agent_env.artifact.artifacts.docker_image.DockerImageArtifact.put", + side_effect=lambda **k: _stored(k["id"])): + result = _put("--backend-dockerfile", str(tmp_path / "server.Dockerfile"), "--frontend-dockerfile", + str(tmp_path / "frontend.Dockerfile"), "--environment-name", "mail") + assert result.exit_code == 0, result.output + assert "separate frontend image is deprecated" in result.output + assert Env.get("mail-site").frontend_docker_image_artifact.id == "mail-site__frontend_image" diff --git a/tst/unit/env/envs/test_website_server_image.py b/tst/unit/env/envs/test_website_server_image.py new file mode 100644 index 000000000..8bfc78af5 --- /dev/null +++ b/tst/unit/env/envs/test_website_server_image.py @@ -0,0 +1,73 @@ +"""A website env is one server image serving the site's pages and API, checked at its health path; a site still built +on a separate frontend image keeps loading and deploying, and put warns that it's deprecated.""" + +import logging +from types import SimpleNamespace + +import pytest + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.env import Env +from agent_env.env.envs.website import WebsiteEnv + + +def _image(id): + return get_artifact_store().put_document(DockerImageArtifact( + id=id, description="", image_name=f"registry.example/{id}@sha256:" + "a" * 64)) + + +def test_a_site_is_one_server_image_checked_at_its_health_path(local_stores): + env = WebsiteEnv.put(id="shop-site", server_docker_image_artifact=_image("shop-server"), environment_name="shop") + stored = Env.get("shop-site") + assert (stored.server_docker_image_artifact.id, stored.frontend_docker_image_artifact) == ("shop-server", None) + assert stored.health_path == "/api/health" + assert stored.backend_docker_image_artifact is stored.server_docker_image_artifact + doc = env.to_dict() + assert doc["server_docker_image_artifact"] == doc["backend_docker_image_artifact"] + assert "frontend_docker_image_artifact" not in doc + + +def test_a_site_declares_its_health_path_and_a_bad_one_is_refused(local_stores): + WebsiteEnv.put(id="slack-site", server_docker_image_artifact=_image("slack-server"), environment_name="slack", + health_path="/healthz") + assert Env.get("slack-site").health_path == "/healthz" + with pytest.raises(ValueError) as refused: + WebsiteEnv(id="bad", version=None, server_docker_image_artifact=_image("bad-server"), environment_name="bad", + health_path="healthz") + assert str(refused.value) == ("website env 'bad': health: 'healthz' is not a path of letters, digits, '.', '_', " + "'~', '-' and '/' starting with '/'") + + +def test_a_site_stored_before_the_server_image_reads_its_backend_image_as_it(local_stores): + backend, frontend = _image("old-backend"), _image("old-frontend") + old = {"type": "website", "id": "old-site", "version": 1, "environment_name": "old", "metadata": {}, + "backend_docker_image_artifact": {"id": backend.id, "version": backend.version}, + "frontend_docker_image_artifact": {"id": frontend.id, "version": frontend.version}} + env = WebsiteEnv.from_dict(old) + assert (env.server_docker_image_artifact.id, env.frontend_docker_image_artifact.id) == ("old-backend", "old-frontend") + with pytest.raises(ValueError, match="names its server image once"): + WebsiteEnv(id="x", version=None, server_docker_image_artifact=backend, backend_docker_image_artifact=backend, + environment_name="x") + + +def test_a_separate_frontend_image_is_deprecated_but_still_deploys(local_stores, caplog): + with caplog.at_level(logging.WARNING, logger="agent_env.env.envs.website"): + env = WebsiteEnv.put(id="shop-site", backend_docker_image_artifact=_image("shop-backend"), + frontend_docker_image_artifact=_image("shop-frontend"), environment_name="shop") + assert "separate frontend image, which is deprecated" in caplog.text + assert env.deploy_refusal() is None + + +def test_an_env_toml_takes_a_health_path_and_refuses_a_bad_one(): + def refuse(problems): + raise ValueError(problems) + + ctx = SimpleNamespace(config_problem=lambda problem: f"env.toml: {problem}", refuse=refuse, + accept_env=lambda data, cls, named_by: data) + data = {"backend_image": "server", "environment_name": "slack", "health": "/healthz"} + assert WebsiteEnv.accept_toml(data, ctx) == data + with pytest.raises(ValueError, match="env.toml: health: 'healthz' is not a path"): + WebsiteEnv.accept_toml({**data, "health": "healthz"}, ctx) + with pytest.raises(ValueError, match="is not a path of letters"): + WebsiteEnv.accept_toml({**data, "health": '/ok" x'}, ctx) diff --git a/tst/unit/env/gateway/website_reserved_paths_test.py b/tst/unit/env/gateway/website_reserved_paths_test.py new file mode 100644 index 000000000..063007984 --- /dev/null +++ b/tst/unit/env/gateway/website_reserved_paths_test.py @@ -0,0 +1,45 @@ +"""A site's route never reaches its data plane or its env card, however the path is spelled, so an agent browsing the +site can't read what grades it; agentenv still reaches both through /svc//.""" + +import httpx +import pytest + +from agent_env.env.gateway import rest_proxy as rest_proxy_module +from agent_env.env.gateway.gateway import reserved_site_path +from tst.unit.env.gateway.changes_test import _Changelog, _gateway, _http, _RealAsyncClient + + +@pytest.mark.parametrize(("path", "reserved"), [ + ("/agentenv", True), ("/agentenv/", True), ("/agentenv/data", True), ("//agentenv", True), + ("/x/../agentenv", True), ("/./.well-known/agent-env.json", True), ("/.well-known/agent-env.json", True), + ("/agentenv-help", False), ("/api/agentenv", False), ("/.well-known/other.json", False), ("/", False), +]) +def test_the_data_plane_and_card_are_reserved_however_their_path_is_spelled(path, reserved): + assert reserved_site_path(path) is reserved + + +@pytest.mark.asyncio +async def test_a_sites_route_refuses_its_reserved_paths_and_svc_still_reaches_them(monkeypatch): + reached = [] + + async def body(): + yield b"ok" + + def site(request): + reached.append(request.url.path) + return httpx.Response(200, content=body()) + + transport = httpx.MockTransport(site) + monkeypatch.setattr(rest_proxy_module.httpx, "AsyncClient", lambda **kw: _RealAsyncClient(transport=transport, **kw)) + gw = _gateway(_Changelog(), website_urls={"mail": "http://mail-website-backend:8000"}, + rest_proxy_urls={"mail": "http://mail-website-backend:8000"}) + + assert (await _http(gw, "GET", "/website/mail/.well-known/agent-env.json"))[0] == 404 + assert (await _http(gw, "POST", "/website/mail/agentenv", body=b"{}"))[0] == 404 + assert (await _http(gw, "POST", "/website/mail/%61gentenv", body=b"{}"))[0] == 404 + for spelled in ("public%2Fx/../agentenv", "public%2Fx/../.well-known/agent-env.json", "a%2F..%2Fagentenv", + "a/%2E%2E/agentenv", "x/..%2Fagentenv"): + assert (await _http(gw, "POST", f"/website/mail/{spelled}", body=b"{}"))[0] == 404, spelled + assert (await _http(gw, "GET", "/website/mail/inbox"))[0] == 200 + assert (await _http(gw, "GET", "/svc/mail/.well-known/agent-env.json"))[0] == 200 + assert reached == ["/inbox", "/.well-known/agent-env.json"] diff --git a/tst/unit/providers/env_providers/env_gateway_provider_test.py b/tst/unit/providers/env_providers/env_gateway_provider_test.py index 9941bdb5b..a2f9958bc 100644 --- a/tst/unit/providers/env_providers/env_gateway_provider_test.py +++ b/tst/unit/providers/env_providers/env_gateway_provider_test.py @@ -2426,3 +2426,137 @@ async def stalls(*args): assert raised.value is _COMPOSE_ERROR assert "Can't list the stack's services: no answer in 0s" in caplog.text + + +def test_a_one_image_site_runs_alone_on_its_port_up_once_it_answers_and_its_route_reaches_it(): + """A site whose server image serves its pages gets one container: the gateway's /website// and /svc// + both reach it, up once it answers its card or, serving none, its declared health path; a site still built on a + frontend image keeps both.""" + from agent_env.env.envs.service_db import ServiceDBConfig + + compose = EnvironmentGatewayProvider().create_docker_compose( + mcp_servers=[], + gateway_image="agent-gateway", + website_configs=[WebsiteConfig(backend_image="mail-server", frontend_image=None, environment_name="mail", + health_path="/healthz"), + WebsiteConfig(backend_image="shop-be", frontend_image="shop-fe", environment_name="shop")], + state_provider=LocalPostgresStateProvider(service_db_config=ServiceDBConfig()), + state_instance=LocalPostgresStateProvider.default_instance(), + ) + + services = yaml.safe_load(compose)["services"] + assert {"mail-website-backend", "shop-website-backend", "shop-website-frontend"} <= set(services) + assert "mail-website-frontend" not in services + assert services["mail-website-backend"]["healthcheck"]["test"] == [ + "CMD-SHELL", "curl -sf -o /dev/null http://localhost:8000/.well-known/agent-env.json" + " || curl -sf -o /dev/null http://localhost:8000/healthz"] + assert services["shop-website-backend"]["healthcheck"]["test"][1].endswith("http://localhost:8000/api/health") + gateway = dict(item.split("=", 1) for item in services[GATEWAY_SERVICE_NAME]["environment"]) + assert gateway["WEBSITE_URLS"] == "mail=http://mail-website-backend:8000,shop=http://shop-website-frontend:80" + assert gateway["REST_PROXY_URLS"] == "mail=http://mail-website-backend:8000,shop=http://shop-website-backend:8000" + + +def test_a_one_image_website_env_is_fronted_with_only_its_server_image(): + server = MagicMock(image_name="mail-server") + env = WebsiteEnv(id="web-mail", version=1, server_docker_image_artifact=server, environment_name="mail", + health_path="/healthz") + topology = env_gateway_provider._gateway_topology(env) + assert topology.website_configs == [WebsiteConfig(backend_image="mail-server", frontend_image=None, + environment_name="mail", health_path="/healthz")] + assert topology.website_images == [server] + + +def _sites(handler): + """Route the deploy's site checks to ``handler(request) -> httpx.Response``.""" + real = httpx.AsyncClient + return patch.object(env_gateway_provider.httpx, "AsyncClient", + lambda *a, **k: real(transport=httpx.MockTransport(handler))) + + +_READY_CARD = {"name": "mail", "capabilities": {"extensions": [{"uri": "urn:agentenv:website/v1", "params": { + "health": "/healthz", "methods": {"ready": {"method": "GET", "endpoint": "/agentenv/ext/website/ready"}}}}]}} + + +@pytest.mark.asyncio +async def test_a_deploy_waits_for_a_site_until_its_cards_ready_says_so_and_never_reads_its_health_path(): + asked = [] + + def site(request: httpx.Request) -> httpx.Response: + asked.append(request.url.path) + if request.url.path == "/svc/mail/.well-known/agent-env.json": + return httpx.Response(200, json=_READY_CARD) + if request.url.path == "/svc/mail/agentenv/ext/website/ready": + ready = asked.count(request.url.path) >= 2 + return httpx.Response(200, json={"ready": ready, "problems": [] if ready else ["seeded"]}) + return httpx.Response(500) + + with _sites(site): + await env_gateway_provider._wait_for_websites( + "http://gw", [WebsiteConfig(backend_image="m", frontend_image=None, environment_name="mail", + health_path="/healthz")], poll_s=0) + assert asked.count("/svc/mail/agentenv/ext/website/ready") == 2 and "/svc/mail/healthz" not in asked + + +@pytest.mark.asyncio +async def test_a_site_with_no_ready_on_its_card_is_waited_for_at_its_health_path_with_a_deprecation_warning(caplog): + caplog.set_level(logging.WARNING, logger="agent_env.providers.env_providers") + health = iter([503, 200]) + + def site(request: httpx.Request) -> httpx.Response: + if request.url.path == "/svc/shop/.well-known/agent-env.json": + return httpx.Response(404) + return httpx.Response(next(health)) if request.url.path == "/svc/shop/api/health" else httpx.Response(500) + + with _sites(site): + await env_gateway_provider._wait_for_websites( + "http://gw", [WebsiteConfig(backend_image="s", frontend_image=None, environment_name="shop")], poll_s=0) + assert next(health, "done") == "done" + assert [r.getMessage() for r in caplog.records].count( + "website 'shop' has no urn:agentenv:website/v1 ready on its card, so its readiness is read at its health path " + "/api/health, which is deprecated") == 1 + + +@pytest.mark.asyncio +async def test_a_site_that_never_says_it_is_ready_fails_the_deploy_naming_why(): + def site(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/.well-known/agent-env.json"): + return httpx.Response(200, json=_READY_CARD) + return httpx.Response(200, json={"ready": False, "problems": ["seeded"]}) + + with _sites(site), pytest.raises(RuntimeError, match=r"websites not ready after 0s: mail \(seeded\)"): + await env_gateway_provider._wait_for_websites( + "http://gw", [WebsiteConfig(backend_image="m", frontend_image=None, environment_name="mail")], + timeout_s=0, poll_s=0) + + +@pytest.mark.asyncio +async def test_a_site_serving_a_page_at_its_cards_path_is_refused_at_once_for_breaking_the_contract(): + """website/v1 reserves the card's path, so a 200 page there (a single-page app's index.html) is refused at once, + however long the deploy would wait and though its health path answers 200.""" + asked = [] + + def site(request: httpx.Request) -> httpx.Response: + asked.append(request.url.path) + if request.url.path == "/svc/shop/.well-known/agent-env.json": + return httpx.Response(200, text="Shop", headers={"content-type": "text/html"}) + return httpx.Response(200) + + with _sites(site), pytest.raises(env_gateway_provider.WebsiteContractError, + match=r"answers /\.well-known/agent-env\.json with text/html, not an env card"): + await env_gateway_provider._wait_for_websites( + "http://gw", [WebsiteConfig(backend_image="s", frontend_image=None, environment_name="shop")], + timeout_s=300, poll_s=0) + assert asked == ["/svc/shop/.well-known/agent-env.json"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [404, 503]) +async def test_a_site_whose_cards_path_answers_no_card_falls_back_to_its_health_path(status): + def site(request: httpx.Request) -> httpx.Response: + if request.url.path == "/svc/shop/.well-known/agent-env.json": + return httpx.Response(status, text="missing", headers={"content-type": "text/html"}) + return httpx.Response(200) if request.url.path == "/svc/shop/api/health" else httpx.Response(500) + + with _sites(site): + await env_gateway_provider._wait_for_websites( + "http://gw", [WebsiteConfig(backend_image="s", frontend_image=None, environment_name="shop")], poll_s=0) diff --git a/tst/unit/serialization/golden/env/website.json b/tst/unit/serialization/golden/env/website.json index c58fccd16..755b4f36a 100644 --- a/tst/unit/serialization/golden/env/website.json +++ b/tst/unit/serialization/golden/env/website.json @@ -11,10 +11,16 @@ "type": "docker_image", "version": 3 }, + "health_path": "/api/health", "id": "shop-site-env", "metadata": { "owner": "example-team" }, + "server_docker_image_artifact": { + "id": "shop-backend-image", + "type": "docker_image", + "version": 3 + }, "service_name": "shop", "type": "website", "version": 2 diff --git a/tst/unit/store/local_id_acceptance_test.py b/tst/unit/store/local_id_acceptance_test.py index 6f9664ede..8f3498a05 100644 --- a/tst/unit/store/local_id_acceptance_test.py +++ b/tst/unit/store/local_id_acceptance_test.py @@ -311,7 +311,7 @@ def test_a_put_with_an_local_id_builds_and_writes_its_images_under_ids_derived_f saved = {d["id"]: d["image_name"] for d in _local().query("artifacts", Filter())} assert saved == {image: image_repository(image) for image in images} (owner,) = _local().query("a2a_agents" if module.endswith(".put") else "envs", Filter()) - assert sorted(ref["id"] for ref in owner.values() if isinstance(ref, dict) and "id" in ref) == images + assert sorted({ref["id"] for ref in owner.values() if isinstance(ref, dict) and "id" in ref}) == images assert not _documents().path.exists() or _documents().count("artifacts", Filter()) == 0 diff --git a/tst/unit/task_step/verifiers/test_website_state_verifier_step.py b/tst/unit/task_step/verifiers/test_website_state_verifier_step.py new file mode 100644 index 000000000..68cc4847c --- /dev/null +++ b/tst/unit/task_step/verifiers/test_website_state_verifier_step.py @@ -0,0 +1,132 @@ +"""The website state verifier: each check reads one value of one session of one site, as the site's /go?sid= reports it +through the gateway, and compares it; the step is refused at put for an op it doesn't know, and for a site the env +version it pins doesn't serve.""" + +import asyncio +from types import SimpleNamespace + +import httpx +import pytest + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.env.envs.multi_env import MultiEnv +from agent_env.env.envs.website import WebsiteEnv +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps.verifiers import website_state_verifier +from agent_env.task_step.task_steps.verifiers.website_state_verifier import StateCheck, WebsiteStateVerifierTaskStep + +APPS = {"mail": {"s1": { + "initial_state": {"inbox": [{"id": 1, "read": False}], "user": {"name": "Ada"}}, + "current_state": {"inbox": [{"id": 1, "read": True}, {"id": 2, "read": False}], "user": {"name": "Ada"}, + "drafts": {"d1": "hi"}, "contacts": {"ada@example.com": {"vip": True}}}, + "state_diff": {"inbox": {"initial": [{"id": 1, "read": False}], + "current": [{"id": 1, "read": True}, {"id": 2, "read": False}]}, + "drafts.d1": {"current": "hi"}}}}} + + +@pytest.mark.parametrize(("check", "passed"), [ + (StateCheck(site="mail", sid="s1", path="user.name", op="eq", value="Ada"), True), + (StateCheck(site="mail", sid="s1", path="inbox.0.read", op="eq", value=True), True), + (StateCheck(site="mail", sid="s1", path="inbox.-1.id", op="eq", value=2), True), + (StateCheck(site="mail", sid="s1", path="inbox", op="ge", value=2, reduce="len"), True), + (StateCheck(site="mail", sid="s1", path="inbox", op="eq", value=1, reduce="len", source="initial_state"), True), + (StateCheck(site="mail", sid="s1", path="drafts", op="eq", value=["d1"], reduce="keys"), True), + (StateCheck(site="mail", sid="s1", path="drafts.d1", op="contains", value="h", source="state_diff"), False), + (StateCheck(site="mail", sid="s1", path="drafts.d1", op="eq", value={"current": "hi"}, source="state_diff"), True), + (StateCheck(site="mail", sid="s1", path="", op="eq", value=2, reduce="len", source="state_diff"), True), + (StateCheck(site="mail", sid="s1", path="user.tz", op="eq", value=None), True), + (StateCheck(site="mail", sid="s1", path="user.tz", op="lt", value=3), False), + (StateCheck(site="mail", sid="s1", path="user.name", op="in", value=["Ada", "Grace"]), True), + (StateCheck(site="mail", sid="s1", path="user.name", op="gt", value=1), False), + (StateCheck(site="mail", sid="s1", path="inbox.5", op="ne", value=None), False), + (StateCheck(site="mail", sid="s1", path="inbox.--1", op="eq", value=None), True), + (StateCheck(site="mail", sid="s2", path="", op="eq", value=False, reduce="bool"), True), + (StateCheck(site="mail", sid="s1", path=r"contacts.ada@example\.com.vip", op="eq", value=True), True), + (StateCheck(site="mail", sid="s1", path="contacts.ada@example.com.vip", op="eq", value=None), True), +]) +def test_a_check_reads_its_value_and_compares_it(check, passed): + assert check.result(APPS)["result"] is passed + + +def test_a_check_of_a_session_with_no_state_says_so(): + row = StateCheck(site="mail", sid="s9", path="user.name", op="eq", value="Ada").result(APPS) + assert (row["result"], row["actual"], row["detail"]) == (False, None, "'mail' has no session 's9'") + assert row["criterion"] == "mail[s9].current_state.user.name eq 'Ada'" + + +def test_every_check_that_names_an_unknown_op_source_or_reduce_is_refused_at_once(): + with pytest.raises(ValueError) as refused: + WebsiteStateVerifierTaskStep(id="v", version=None, env_id="office", checks=[ + {"site": "mail", "path": "a", "op": "matches"}, + {"site": "mail", "path": "b", "op": "eq", "source": "final_state", "reduce": "sum"}]) + assert str(refused.value).count("check of 'mail'") == 3 + + +def test_the_step_round_trips_and_is_registered(): + step = WebsiteStateVerifierTaskStep(id="v", version=None, env_id="office", score_aggregator="weighted_average", + checks=[{"site": "mail", "sid": "s1", "path": "inbox", "op": "ge", "value": 2, + "reduce": "len", "weight": 2.0}]) + assert WebsiteStateVerifierTaskStep.from_dict(step.to_dict()).to_dict() == step.to_dict() + assert get_task_step_registry()["website_state_verifier"] is WebsiteStateVerifierTaskStep + + +def _site(name): + image = get_artifact_store().put_document(DockerImageArtifact( + id=f"{name}-server", description="", image_name=f"registry.example/{name}@sha256:" + "d" * 64)) + return WebsiteEnv.put(id=f"{name}-site", server_docker_image_artifact=image, environment_name=name) + + +def test_put_refuses_a_check_of_a_site_the_pinned_env_version_doesnt_serve(local_stores): + mail, calendar = _site("mail"), _site("calendar") + MultiEnv.put(id="office", mcp_server_envs=[], website_envs=[mail, calendar]) + MultiEnv.put(id="office", mcp_server_envs=[], website_envs=[calendar]) + with pytest.raises(ValueError, match=r"'notes' is not a site of office v1, whose sites are \['calendar', 'mail'\]"): + WebsiteStateVerifierTaskStep.put(id="v", env_id="office", env_version=1, checks=[ + {"site": "notes", "path": "", "op": "eq"}, {"site": "mail", "path": "", "op": "eq"}]) + stored = WebsiteStateVerifierTaskStep.put(id="v", env_id="office", env_version=1, retry_config={ + "retry_from_step_id": "deploy"}, checks=[{"site": "mail", "path": "", "op": "eq"}]) + assert stored.version == 1 and stored.retry_config.retry_from_step_id == "deploy" + assert WebsiteStateVerifierTaskStep.from_dict(stored.to_dict()).env_version == 1 + pinned = WebsiteStateVerifierTaskStep(id="w", version=None, env_id="office", env_version=2, + checks=[{"site": "mail", "path": "", "op": "eq"}]) + assert pinned.preflight() == ["website_state_verifier 'w': 'mail' is not a site of office v2, whose sites are " + "['calendar']"] + assert WebsiteStateVerifierTaskStep(id="u", version=None, env_id="office", + checks=[{"site": "mail", "path": "", "op": "eq"}]).preflight() == [] + assert WebsiteStateVerifierTaskStep(id="s", version=None, env_id="mail-site", env_version=1, + checks=[{"site": "mail", "path": "", "op": "eq"}]).preflight() == [] + plain = MultiEnv.put(id="plain", mcp_server_envs=[]) + assert "with no website" in WebsiteStateVerifierTaskStep(id="y", version=None, env_id=plain.id, checks=[]).preflight()[0] + assert WebsiteStateVerifierTaskStep(id="x", version=None, env_id="missing", checks=[]).preflight() == [] + + +def test_execute_reads_each_sites_go_through_the_gateway_and_scores_the_checks(monkeypatch): + by_site = {"mail": APPS["mail"], "calendar": {"s1": {"current_state": {"events": [1, 2]}}}} + asked = [] + + def gateway(request): + site = request.url.path.split("/")[2] + sid = request.url.params["sid"] + asked.append((request.url.path, sid)) + return httpx.Response(200, json=by_site[site][sid]) + + real_client = httpx.AsyncClient + monkeypatch.setattr(website_state_verifier.httpx, "AsyncClient", + lambda **kw: real_client(transport=httpx.MockTransport(gateway), **kw)) + monkeypatch.setattr(website_state_verifier, "gateway_url_of", lambda deployed: "http://gateway") + monkeypatch.setattr(website_state_verifier, "_sites", lambda env: {"mail", "calendar"}) + monkeypatch.setattr(website_state_verifier.Env, "get", lambda id, version=None: None) + step = WebsiteStateVerifierTaskStep(id="v", version=None, env_id="office", verifier_id="office-check", checks=[ + {"site": "mail", "sid": "s1", "path": "inbox.0.read", "op": "eq", "value": True}, + {"site": "mail", "sid": "s1", "path": "drafts", "op": "eq", "value": 0, "reduce": "len"}, + {"site": "calendar", "sid": "s1", "path": "events", "op": "eq", "value": 2, "reduce": "len"}]) + context = TaskStepContext() + context.deployed_envs.append(SimpleNamespace(env_id="office", env_version=1)) + + asyncio.run(step.execute(context)) + + assert asked == [("/svc/calendar/go", "s1"), ("/svc/mail/go", "s1")] + verification = context.metadata["verifications"]["office-check"] + assert [row["result"] for row in verification["results"]] == [True, False, True]