Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 62 additions & 43 deletions vermeer/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -452,15 +452,25 @@ Vermeer exposes a REST API on port `6688` (configurable in `master.ini`).

| Endpoint | Method | Description |
|----------|--------|-------------|
| `/api/v1/graphs` | POST | Load graph from data source |
| `/api/v1/graphs/{graph_id}` | GET | Get graph metadata |
| `/api/v1/graphs/{graph_id}` | DELETE | Unload graph from memory |
| `/api/v1/compute` | POST | Execute algorithm on loaded graph |
| `/api/v1/tasks/{task_id}` | GET | Get task status and results |
| `/api/v1/graphs` | GET | List graphs visible to the caller |
| `/api/v1/graphs/{graph_name}` | GET | Get graph metadata |
| `/api/v1/graphs/{graph_name}` | DELETE | Unload the graph |
| `/api/v1/tasks` | GET | List tasks; POST returns 405 |
| `/api/v1/tasks/create` | POST | Submit a load, reload or compute task |
| `/api/v1/tasks/create/sync` | POST | Submit and wait for a task |
| `/api/v1/task/{task_id}` | GET | Get task status |
| `/api/v1/tasks/value/{task_id}` | GET | Query retained compute results |
| `/api/v1/workers` | GET | List connected workers |
| `/api/v1/workers/availability` | GET | Read worker readiness for the authenticated space |
| `/api/v1/workers/availability` | GET | Read readiness for the authenticated space |
| `/` | GET | Web UI dashboard |

Task collection POST previously returned a success-shaped response without creating
anything. It now returns HTTP 405, `Allow: GET` and a nonzero `errcode` pointing to
`/api/v1/tasks/create`. The legacy `/tasks` route has the same behavior.
Do not treat an arbitrary HTTP 200 as proof of submission: successful creation
returns a `task` object with an `id`, which must then reach a terminal state.
The native API paths and IDs differ from Hubble's `/api/v1.3` routes and negative task IDs.

Worker availability uses the normal space credential and returns only
`{"available":true|false}` for that authenticated space's allocations or the
common pool. It accepts no caller-selected space and does not allocate workers
Expand All @@ -480,39 +490,56 @@ cooperative cancellation behavior.

### Example: Run PageRank

Use a service token with access to the graph's space. HugeGraph bulk loading under
`auth=token` requires an administrator credential; the native UI password is not a
Bearer token. Ensure PD/Store addresses are reachable and match the metadata namespace.

```bash
# 1. Load graph from HugeGraph
curl -X POST http://localhost:6688/api/v1/graphs \
-H "Content-Type: application/json" \
# 1. Submit a load task. This creates the Vermeer graph copy.
curl -X POST http://localhost:6688/api/v1/tasks/create \
-H "Authorization: Bearer $VERMEER_TOKEN" \
-H 'Content-Type: application/json' \
-d '{
"graph_name": "my_graph",
"load_type": "hugegraph",
"hugegraph": {
"pd_peers": ["127.0.0.1:8686"],
"graph_name": "hugegraph"
"graph": "my_graph",
"task_type": "load",
"params": {
"load.type": "hugegraph",
"load.hg_pd_peers": "[\"127.0.0.1:8686\"]",
"load.hugegraph_name": "DEFAULT/hugegraph/g"
}
}'

# 2. Run PageRank
curl -X POST http://localhost:6688/api/v1/compute \
-H "Content-Type: application/json" \
# 2. Poll the returned task.id; continue after successful loading.
curl -H "Authorization: Bearer $VERMEER_TOKEN" \
http://localhost:6688/api/v1/task/123

# 3. Submit a computation against the loaded copy.
curl -X POST http://localhost:6688/api/v1/tasks/create \
-H "Authorization: Bearer $VERMEER_TOKEN" \
-H 'Content-Type: application/json' \
-d '{
"graph_name": "my_graph",
"algorithm": "pagerank",
"graph": "my_graph",
"task_type": "compute",
"params": {
"max_iterations": 20,
"damping_factor": 0.85
},
"output": {
"type": "hugegraph",
"property_name": "pagerank_value"
"compute.algorithm": "pagerank",
"compute.parallel": "1",
"compute.max_step": "20",
"pagerank.damping": "0.85",
"pagerank.diff_threshold": "0.00001",
"output.type": "local",
"output.file_path": "./data/pagerank-example",
"output.need_query": "1"
}
}'

# 3. Check task status
curl http://localhost:6688/api/v1/tasks/{task_id}
```

Replace `123` with the actual positive native task ID. The load task succeeding or
the graph becoming `loaded` does not mean the computation has finished. Poll the
computation's own ID; read results through `/api/v1/tasks/value/{task_id}` only after
success. `params` is a flat string-to-string map. `algorithm`, `max_iterations` and
`damping_factor` at the old example locations are not this API's request model.
The configured output path is on the Vermeer workers, not the HTTP client's machine.

### OLAP vs OLTP Modes

- **OLAP Mode**: Load entire graph into memory, run multiple algorithms
Expand All @@ -528,19 +555,11 @@ Vermeer integrates with HugeGraph via:
2. **Data Loading**: Streams vertices/edges from HugeGraph Store via gRPC (`ScanPartition`)
3. **Result Writing**: Writes computed results back via HugeGraph REST API (adds vertex properties)

Configuration in graph load request:

```json
{
"load_type": "hugegraph",
"hugegraph": {
"pd_peers": ["127.0.0.1:8686"],
"graph_name": "hugegraph",
"vertex_label": "person",
"edge_label": "knows"
}
}
```
In the task request above, `load.hg_pd_peers` is a JSON array encoded as a string,
and `load.hugegraph_name` uses `<graphspace>/<graph>/g`. Pass these keys inside the
flat `params` map with `task_type: "load"`, not a nested `hugegraph` object.
The Server/Store metadata prefix and credentials must follow the deployment guide.
The current HugeGraph loader does not subdivide each Store partition using `load.parallel`; omit that option here.

### Local CSV Files

Expand Down Expand Up @@ -684,7 +703,7 @@ vermeer/tools/protoc/linux64/protoc vermeer/apps/protos/*.proto --go-grpc_out=ve

### Algorithm-Specific

- **PageRank**: Use `damping_factor=0.85`, `tolerance=0.0001` for faster convergence
- **PageRank**: Use `pagerank.damping=0.85`; tune `pagerank.diff_threshold` (for example, `0.0001`) to control convergence.
- **Louvain**: Enable `weighted=true` only if edge weights are meaningful
- **SSSP**: Provide source vertex ID for single-source queries

Expand Down Expand Up @@ -745,4 +764,4 @@ See the main [Contributing Guide](../README.md#contributing) for how to contribu

## License

Vermeer is part of Apache HugeGraph-Computer, licensed under [Apache 2.0 License](https://github.com/apache/hugegraph-computer/blob/master/LICENSE).
Vermeer is part of Apache HugeGraph-Computer, licensed under [Apache 2.0 License](https://github.com/apache/hugegraph-computer/blob/master/LICENSE).
6 changes: 4 additions & 2 deletions vermeer/apps/master/services/http_tasks.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,10 @@ func (th *TasksHandler) GET(ctx *gin.Context) {
}

func (th *TasksHandler) POST(ctx *gin.Context) {
ctx.JSON(http.StatusOK, gin.H{
"code": http.StatusOK,
ctx.Header("Allow", http.MethodGet)
ctx.JSON(http.StatusMethodNotAllowed, common.BaseResp{
ErrCode: -1,
Message: "Task collection is read-only; submit tasks with POST /api/v1/tasks/create",
})
}

Expand Down
94 changes: 94 additions & 0 deletions vermeer/apps/master/services/http_tasks_method_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with this
work for additional information regarding copyright ownership. The ASF
licenses this file to You under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
*/

package services

import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"vermeer/apps/common"

"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus"
)

func taskMethodRouter(t *testing.T, filters ...gin.HandlerFunc) *common.Sentinel {
t.Helper()
previousMetrics := common.PrometheusMetrics
previousRegisterer, previousGatherer := prometheus.DefaultRegisterer, prometheus.DefaultGatherer
previousMode := gin.Mode()
registry := prometheus.NewRegistry()
prometheus.DefaultRegisterer, prometheus.DefaultGatherer = registry, registry
t.Cleanup(func() {
common.PrometheusMetrics = previousMetrics
prometheus.DefaultRegisterer, prometheus.DefaultGatherer = previousRegisterer, previousGatherer
gin.SetMode(previousMode)
})
sentinel := &common.Sentinel{}
sentinel.Init("127.0.0.1:0", "release")
regVerAPI(sentinel, 1, "/tasks", map[string]common.BaseHandler{"": &TasksHandler{}}, filters...)
return sentinel
}

func TestTaskCollectionPostRejectsFalseSuccess(t *testing.T) {
router := taskMethodRouter(t)
for _, path := range []string{"/tasks", "/api/v1/tasks"} {
for _, body := range []string{
`{}`, `{"algorithm":"pagerank"}`,
`{"graph":"example","task_type":"compute","params":{"compute.algorithm":"pagerank"}}`,
} {
t.Run(path+body, func(t *testing.T) {
request := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))
request.Header.Set("Content-Type", "application/json")
response := httptest.NewRecorder()
router.ServeHTTP(response, request)
if response.Code != http.StatusMethodNotAllowed {
t.Fatalf("POST %s returned %d: %s", path, response.Code, response.Body.String())
}
if response.Header().Get("Allow") != http.MethodGet {
t.Fatalf("Allow = %q", response.Header().Get("Allow"))
}
var result common.BaseResp
if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil {
t.Fatal(err)
}
if result.ErrCode == 0 || !strings.Contains(result.Message, "/tasks/create") {
t.Fatalf("missing submission guidance: %+v", result)
}
})
}
}
}

func TestTaskCollectionPostRetainsAuthenticationFilters(t *testing.T) {
checked := false
router := taskMethodRouter(t, func(ctx *gin.Context) {
checked = true
ctx.AbortWithStatus(http.StatusUnauthorized)
})
for _, path := range []string{"/tasks", "/api/v1/tasks"} {
checked = false
response := httptest.NewRecorder()
router.ServeHTTP(response, httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{}`)))
if !checked || response.Code != http.StatusUnauthorized {
t.Fatalf("authentication bypassed for %s: checked=%v status=%d", path, checked, response.Code)
}
}
}
Loading