diff --git a/README.md b/README.md index b3fa882c0..819b47422 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![CI | Unit Tests](https://github.com/runpod/runpod-python/actions/workflows/CI-pytests.yml/badge.svg)](https://github.com/runpod/runpod-python/actions/workflows/CI-pytests.yml) [![License](https://img.shields.io/badge/License-MIT-blue)](LICENSE) -[Documentation](https://docs.runpod.io) • [Examples](examples/apps) • [Discord](https://discord.gg/pJ3P2DbUUq) +[Documentation](https://docs.runpod.io) • [Examples](examples) • [Discord](https://discord.gg/pJ3P2DbUUq) diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 000000000..b71aecef5 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,63 @@ +# Examples + +## Before you start + +```bash +pip install runpod +rp login +``` + +`rp login` stores your API key, and every example picks it up from there. If +`RUNPOD_API_KEY` is set in your environment, the examples use that key +instead. + +## Which folder to start with + +| Folder | What it shows | Needs | +|---|---|---| +| [`apps/`](apps) | Python functions that run on cloud GPUs and CPUs, started with `rp flash dev` | An account. Start here if you are new. | +| [`serverless/`](serverless) | Handlers for your own serverless worker image | Nothing. They run locally. | +| [`endpoints/`](endpoints) | Sending jobs to an existing serverless endpoint | An endpoint id | +| [`api/`](api) | Creating templates, endpoints, and registry credentials | An account | +| [`rest_wrapper.py`](rest_wrapper.py) | Pod lifecycle: create, stop, resume, terminate | An account. Creates a billable GPU pod. | + +## Running the serverless handlers locally + +```bash +cd examples/serverless +python simple_handler.py --test_input '{"input": {"name": "world"}}' +``` + +Or start a local API server and call it: + +```bash +python simple_handler.py --rp_serve_api +curl -X POST http://localhost:8000/runsync \ + -H "Content-Type: application/json" \ + -d '{"input": {"name": "world"}}' +``` + +Run with no arguments, a handler looks for a `test_input.json` file in the +current directory. + +## Getting an endpoint id for `endpoints/` + +Each `endpoints/` example has a `YOUR_ENDPOINT_ID` placeholder. To use an +endpoint you already have, list them with `rp endpoint list`. To create one +for trying the examples, run [`api/create_endpoint.py`](api/create_endpoint.py), +which deploys `runpod/mock-worker` and prints the new id. + +`endpoints/streaming.py` needs a worker that streams: a generator handler, or +`runpod/mock-worker` started with `--generator`. + +## Cost and cleanup + +- Endpoints with `workers_min=0` cost nothing while idle. Delete them with + `rp endpoint delete ENDPOINT_ID`. +- Templates stay on your account. Delete them from the Templates page in the + console. +- `rest_wrapper.py` terminates its pod at the end, including when a step + fails. +- `rp flash dev` deletes the endpoints it created when the session ends. + Network volumes created by `apps/` examples are not deleted; list them with + `rp volume list` and remove them with `rp volume delete`. diff --git a/examples/api/create_container_registry_auth.py b/examples/api/create_container_registry_auth.py index 6ea6e7229..cd539c26b 100644 --- a/examples/api/create_container_registry_auth.py +++ b/examples/api/create_container_registry_auth.py @@ -1,15 +1,20 @@ -""" Example of creating a container registry auth with the Runpod API. """ +"""Example of creating a container registry auth with the Runpod API. + +Registry credentials let templates pull images from a private registry. The +credential stays on your account after the script exits; delete it with +`rp registry delete NAME`. +""" import runpod -# Set your global API key with `runpod config` or uncomment the line below: +# Run `rp login` once to store your API key, or uncomment the line below: # runpod.api_key = "YOUR_RUNPOD_API_KEY" try: new_container_registry_auth = runpod.create_container_registry_auth( - name="test-container-registry-auth-name", - username="test-username", - password="test-password", + name="example-registry-auth", + username="YOUR_REGISTRY_USERNAME", + password="YOUR_REGISTRY_PASSWORD", ) print(new_container_registry_auth) diff --git a/examples/api/create_endpoint.py b/examples/api/create_endpoint.py index da99d2ed7..3f4fa4a9c 100644 --- a/examples/api/create_endpoint.py +++ b/examples/api/create_endpoint.py @@ -1,20 +1,33 @@ -""" Example of creating an endpoint with the Runpod API. """ +"""Example of creating an endpoint with the Runpod API. + +The endpoint scales to zero workers when idle, so it costs nothing until it +receives requests. It stays on your account after the script exits; delete it +with `rp endpoint delete ENDPOINT_ID`, then delete its template from the +Templates page in the console. +""" + +import uuid import runpod -# Set your global API key with `runpod config` or uncomment the line below: +# Run `rp login` once to store your API key, or uncomment the line below: # runpod.api_key = "YOUR_RUNPOD_API_KEY" +# Template and endpoint names must be unique on your account. +NAME = f"example-endpoint-{uuid.uuid4().hex[:8]}" + try: + # runpod/mock-worker echoes its input back, which is enough to try the + # examples in examples/endpoints/. new_template = runpod.create_template( - name="test", image_name="runpod/base:0.4.4", is_serverless=True + name=NAME, image_name="runpod/mock-worker:latest", is_serverless=True ) print(new_template) new_endpoint = runpod.create_endpoint( - name="test", + name=NAME, template_id=new_template["id"], gpu_ids="AMPERE_16", workers_min=0, @@ -23,6 +36,8 @@ ) print(new_endpoint) + print(f"Endpoint id: {new_endpoint['id']}") + print(f"Delete it with: rp endpoint delete {new_endpoint['id']}") except runpod.error.QueryError as err: print(err) diff --git a/examples/api/create_template.py b/examples/api/create_template.py index 6b534218d..c033e9cf2 100644 --- a/examples/api/create_template.py +++ b/examples/api/create_template.py @@ -1,13 +1,24 @@ -""" Example of creating a template with the Runpod API. """ +"""Example of creating a template with the Runpod API. + +The template stays on your account after the script exits. Delete it from +the Templates page in the console when you no longer need it. +""" + +import uuid import runpod -# Set your global API key with `runpod config` or uncomment the line below: +# Run `rp login` once to store your API key, or uncomment the line below: # runpod.api_key = "YOUR_RUNPOD_API_KEY" +# Template names must be unique on your account. +TEMPLATE_NAME = f"example-template-{uuid.uuid4().hex[:8]}" + try: - new_template = runpod.create_template(name="test", image_name="runpod/base:0.1.0") + new_template = runpod.create_template( + name=TEMPLATE_NAME, image_name="runpod/base:2.0.0-ubuntu2404" + ) print(new_template) diff --git a/examples/api/get_endpoints.py b/examples/api/get_endpoints.py index 3d61ce94d..3094bdb1e 100644 --- a/examples/api/get_endpoints.py +++ b/examples/api/get_endpoints.py @@ -1,7 +1,13 @@ -""" Get all endpoints from the API """ +"""Example of listing your serverless endpoints with the Runpod API.""" import runpod +# Run `rp login` once to store your API key, or uncomment the line below: +# runpod.api_key = "YOUR_RUNPOD_API_KEY" + endpoints = runpod.get_endpoints() -print(endpoints) +# Each entry is the endpoint's full configuration, including its environment +# variables, so print only the id and name here. +for endpoint in endpoints: + print(endpoint["id"], endpoint.get("name")) diff --git a/examples/endpoints/asyncio_job_request.py b/examples/endpoints/asyncio_job_request.py index fe0c5a6e8..54ef17975 100644 --- a/examples/endpoints/asyncio_job_request.py +++ b/examples/endpoints/asyncio_job_request.py @@ -7,11 +7,10 @@ import runpod from runpod import AsyncioEndpoint, AsyncioJob, http_client -asyncio.set_event_loop_policy( - asyncio.WindowsSelectorEventLoopPolicy() -) # For Windows Users +# Run `rp login` once to store your API key, or uncomment the line below: +# runpod.api_key = "YOUR_RUNPOD_API_KEY" -runpod.api_key = "YOUR_API_KEY" +JOB_TIMEOUT = 120 # seconds async def main(): @@ -21,7 +20,7 @@ async def main(): async with http_client.AsyncClientSession() as session: # Invoke API payload = {} - endpoint = AsyncioEndpoint("ENDPOINT_ID", session) + endpoint = AsyncioEndpoint("YOUR_ENDPOINT_ID", session) job: AsyncioJob = await endpoint.run(payload) # Get current job status @@ -30,8 +29,9 @@ async def main(): # Print status print(status) - # Wait until job is completed or failed - output = await job.output() + # Wait until the job completes or fails, up to JOB_TIMEOUT seconds. + # Raises TimeoutError if the job is still running after that. + output = await job.output(timeout=JOB_TIMEOUT) # Print output print(output) diff --git a/examples/endpoints/health.py b/examples/endpoints/health.py index 1daaacd19..5752f627f 100644 --- a/examples/endpoints/health.py +++ b/examples/endpoints/health.py @@ -2,10 +2,10 @@ import runpod -# Set your global API key with `runpod config` or uncomment the line below: +# Run `rp login` once to store your API key, or uncomment the line below: # runpod.api_key = "YOUR_RUNPOD_API_KEY" -endpoint = runpod.Endpoint("gwp4kx5yd3nur1") +endpoint = runpod.Endpoint("YOUR_ENDPOINT_ID") endpoint_health = endpoint.health() diff --git a/examples/endpoints/run.py b/examples/endpoints/run.py index 06672e610..97c3d1d7c 100644 --- a/examples/endpoints/run.py +++ b/examples/endpoints/run.py @@ -4,21 +4,20 @@ import runpod -# Set your global API key with `runpod config` or uncomment the line below: +# Run `rp login` once to store your API key, or uncomment the line below: # runpod.api_key = "YOUR_RUNPOD_API_KEY" -endpoint = runpod.Endpoint("sdxl") # Where "sdxl" is the endpoint ID +endpoint = runpod.Endpoint("YOUR_ENDPOINT_ID") run_request = endpoint.run( {"input": {"prompt": "a photo of a horse the size of a Boeing 787"}} ) -# Check the status of the run request +# Check the status of the run request, for example IN_QUEUE or IN_PROGRESS. print(run_request.status()) -# Get the output of the endpoint run request. -print(run_request.output()) - -# Get the output of the endpoint run request. -# If timeout is greater than 0, this will block until the endpoint run is complete. +# Wait up to 60 seconds for the job to finish, then print its output. +# Raises TimeoutError if the job is still running after that. +# Without a timeout, output() returns immediately, and returns None if the +# job has not finished yet. print(run_request.output(timeout=60)) diff --git a/examples/endpoints/run_sync.py b/examples/endpoints/run_sync.py index 23a3c2470..ee073ed0e 100644 --- a/examples/endpoints/run_sync.py +++ b/examples/endpoints/run_sync.py @@ -2,12 +2,14 @@ Example of calling an endpoint using the Runpod Python Language Library. """ +import requests + import runpod -# Set your global API key with `runpod config` or uncomment the line below: +# Run `rp login` once to store your API key, or uncomment the line below: # runpod.api_key = "YOUR_RUNPOD_API_KEY" -endpoint = runpod.Endpoint("sdxl") # Where "sdxl" is the endpoint ID +endpoint = runpod.Endpoint("YOUR_ENDPOINT_ID") try: # Run the endpoint synchronously, blocking until the endpoint run is complete. @@ -17,5 +19,7 @@ ) print(run_request) -except TimeoutError: +except (TimeoutError, requests.exceptions.Timeout): + # TimeoutError: the job was still running when the timeout expired. + # requests.exceptions.Timeout: the HTTP request itself timed out. print("Job timed out.") diff --git a/examples/endpoints/streaming.py b/examples/endpoints/streaming.py index 363727e09..a4168b95f 100644 --- a/examples/endpoints/streaming.py +++ b/examples/endpoints/streaming.py @@ -2,10 +2,14 @@ import runpod -# Set your global API key with `runpod config` or uncomment the line below: +# Run `rp login` once to store your API key, or uncomment the line below: # runpod.api_key = "YOUR_RUNPOD_API_KEY" -endpoint = runpod.Endpoint("gwp4kx5yd3nur1") +# This input targets an endpoint running the runpod/mock-worker image started +# with --generator, which yields each item of mock_return with mock_delay +# seconds between them. Replace it with the input your own streaming +# handler expects. +endpoint = runpod.Endpoint("YOUR_ENDPOINT_ID") run_request = endpoint.run( { diff --git a/examples/rest_wrapper.py b/examples/rest_wrapper.py index b00fe19d3..beb9000ef 100644 --- a/examples/rest_wrapper.py +++ b/examples/rest_wrapper.py @@ -1,37 +1,47 @@ -"""REST v2 wrapper for the Runpod API.""" +"""Create, stop, resume, and terminate a pod with the Runpod API. + +This creates a billable GPU pod. The pod is terminated at the end, including +when a step fails. +""" import time import runpod -runpod.api_key = "YOUR_RUNPOD_API_KEY" +# Run `rp login` once to store your API key, or uncomment the line below: +# runpod.api_key = "YOUR_RUNPOD_API_KEY" + +GPU_TYPE = "NVIDIA GeForce RTX 4090" # see `rp catalog` for other GPU types +WAIT_SECONDS = 10 gpus = runpod.get_gpus() print(gpus) -gpu = runpod.get_gpu("NVIDIA GeForce RTX 4090") +gpu = runpod.get_gpu(GPU_TYPE) print(gpu) pod = runpod.create_pod( - "test", + "example-pod", "runpod/pytorch:1.0.2-cu1281-torch280-ubuntu2404", - "NVIDIA GeForce RTX 4090", + GPU_TYPE, ) print(pod) +pod_id = pod["id"] -print("Waiting for pod to be created...") -time.sleep(10) +try: + print("Waiting for pod to be created...") + time.sleep(WAIT_SECONDS) -pod = runpod.stop_pod(pod["id"]) -print(pod) + print(runpod.stop_pod(pod_id)) -print("Waiting for pod to be stopped...") -time.sleep(10) - -pod = runpod.resume_pod(pod["id"]) -print(pod) + print("Waiting for pod to be stopped...") + time.sleep(WAIT_SECONDS) -print("Waiting for pod to be resumed...") -time.sleep(10) + # Resuming can fail if no GPU of this type is free on the pod's machine. + print(runpod.resume_pod(pod_id)) -runpod.terminate_pod(pod["id"]) + print("Waiting for pod to be resumed...") + time.sleep(WAIT_SECONDS) +finally: + runpod.terminate_pod(pod_id) + print(f"Terminated pod {pod_id}") diff --git a/examples/serverless/concurrent_handler.py b/examples/serverless/concurrent_handler.py index 59e437632..a1b7c445f 100644 --- a/examples/serverless/concurrent_handler.py +++ b/examples/serverless/concurrent_handler.py @@ -1,28 +1,41 @@ -""" Concurrency Modifier Example """ +"""Concurrency Modifier Example + +An async handler lets one worker run several jobs at once. The worker calls +the concurrency modifier with its current concurrency and resizes to the value +it returns, waiting until no jobs are in flight before resizing. + +To test locally, run: +python concurrent_handler.py --test_input '{"input": {"seconds": 1}}' +""" + +import asyncio import runpod +MAX_CONCURRENCY = 4 + # ---------------------------------- Handler --------------------------------- # -async def async_generator_handler(job): +async def async_handler(job): """ - Async generator type handler. + Async handler. While it awaits, the worker can start other jobs. """ - return job + seconds = job["input"].get("seconds", 1) + await asyncio.sleep(seconds) + return {"slept": seconds} # --------------------------- Concurrency Modifier --------------------------- # -def concurrency_modifier(current_concurrency=1): +def concurrency_modifier(current_concurrency: int) -> int: """ - Concurrency modifier. - """ - desired_concurrency = current_concurrency - - # Do some logic to determine the desired concurrency. + Return the number of jobs this worker should run at once. - return desired_concurrency + Replace the fixed value with your own logic, for example based on free + GPU memory or queue depth. + """ + return MAX_CONCURRENCY runpod.serverless.start( - {"handler": async_generator_handler, "concurrency_modifier": concurrency_modifier} + {"handler": async_handler, "concurrency_modifier": concurrency_modifier} ) diff --git a/examples/serverless/logger.py b/examples/serverless/logger.py index d97738b2a..c8fc7a057 100644 --- a/examples/serverless/logger.py +++ b/examples/serverless/logger.py @@ -1,4 +1,8 @@ -""" Example of using the RunPodLogger class. """ "" +"""Example of using the RunPodLogger class. + +Running locally, the logger prints plain text. On a deployed worker, where +RUNPOD_ENDPOINT_ID is set, it prints one JSON object per line instead. +""" import runpod @@ -11,11 +15,11 @@ log.warn("A warning message") log.error("An error message") -# Output: -# DEBUG | A debug message -# INFO | An info message -# WARN | A warning message -# ERROR | An error message +# Local output: +# DEBUG | A debug message +# INFO | An info message +# WARN | A warning message +# ERROR | An error message log.debug("A debug message", request_id=JOB_ID) @@ -23,7 +27,13 @@ log.warn("A warning message", request_id=JOB_ID) log.error("An error message", request_id=JOB_ID) -# Output: +# Local output: +# DEBUG | 1234567890 | A debug message +# INFO | 1234567890 | An info message +# WARN | 1234567890 | A warning message +# ERROR | 1234567890 | An error message + +# Output on a deployed worker: # {"requestId": "1234567890", "message": "A debug message", "level": "DEBUG"} # {"requestId": "1234567890", "message": "An info message", "level": "INFO"} # {"requestId": "1234567890", "message": "A warning message", "level": "WARN"}